From c7d2d66f0c8fc213975c16836c087c2856f55b03 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Thu, 17 Sep 2026 17:38:46 -0700 Subject: [PATCH] feat(trace-viewer): show screencast frames during playback Playback and dragging the scrubber show the screencast at the current time. Stopping, releasing the scrubber, hovering an action or picking a locator snap back to the action snapshot. Traces with a video play the video itself, the thumbnails only index the film strip lanes. --- packages/trace-viewer/src/ui/ariaModeView.css | 3 +- packages/trace-viewer/src/ui/ariaModeView.tsx | 30 ++-- packages/trace-viewer/src/ui/filmStrip.tsx | 73 +++++---- .../trace-viewer/src/ui/playbackControl.tsx | 149 +++++++++--------- packages/trace-viewer/src/ui/snapshotTab.css | 10 ++ packages/trace-viewer/src/ui/snapshotTab.tsx | 47 +++++- packages/trace-viewer/src/ui/videoFrame.tsx | 79 ++++++++++ .../trace-viewer/src/ui/videoThumbnails.ts | 19 ++- packages/trace-viewer/src/ui/workbench.tsx | 12 +- tests/library/trace-viewer-scrub.spec.ts | 86 ++++++++++ 10 files changed, 376 insertions(+), 132 deletions(-) create mode 100644 packages/trace-viewer/src/ui/videoFrame.tsx diff --git a/packages/trace-viewer/src/ui/ariaModeView.css b/packages/trace-viewer/src/ui/ariaModeView.css index 852e6f8de8bf9..95800f66f2ec2 100644 --- a/packages/trace-viewer/src/ui/ariaModeView.css +++ b/packages/trace-viewer/src/ui/ariaModeView.css @@ -28,7 +28,8 @@ position: relative; } -.aria-mode-screenshot img { +.aria-mode-screenshot img, +.aria-mode-screenshot .screencast-frame { max-width: 100%; max-height: 100%; object-fit: contain; diff --git a/packages/trace-viewer/src/ui/ariaModeView.tsx b/packages/trace-viewer/src/ui/ariaModeView.tsx index da0caec2f2830..77a9dc4d5be15 100644 --- a/packages/trace-viewer/src/ui/ariaModeView.tsx +++ b/packages/trace-viewer/src/ui/ariaModeView.tsx @@ -21,7 +21,7 @@ import { renderAriaSnapshotAsYaml } from '@isomorphic/ariaSnapshotRenderer'; import { clsx, useMeasure } from '@web/uiUtils'; import { PlaceholderPanel } from './placeholderPanel'; -import type { ActionPhase, ActionTraceEvent, ScreenshotTraceEvent } from '@isomorphic/trace/trace'; +import type { ActionPhase, ActionTraceEvent } from '@isomorphic/trace/trace'; import type { AriaNodeJSON, AriaSnapshotJSON } from '@isomorphic/ariaSnapshot'; import type { TraceModel } from '@isomorphic/trace/traceModel'; @@ -144,7 +144,8 @@ export const AriaModeView: React.FunctionComponent<{ target: AriaModeTarget | undefined, point?: Point, box?: Box, -}> = ({ model, target, point, box }) => { + screencastFrame?: React.ReactNode, +}> = ({ model, target, point, box, screencastFrame }) => { const screenshot = model && target ? model.screenshotForCall(target.callId, target.phase) : undefined; const ariaSnapshot = model && target ? model.ariaSnapshotForCall(target.callId, target.phase) : undefined; const [lines, setLines] = React.useState([]); @@ -172,11 +173,16 @@ export const AriaModeView: React.FunctionComponent<{ }; }, [model, ariaSnapshot]); - if (!screenshot && !ariaSnapshot) + if (!screenshot && !ariaSnapshot && !screencastFrame) return ; return
- +
{ariaSnapshot &&
setHighlightedBox(undefined)}> {lines.map((line, index) =>
= ({ model, screenshot, highlightedBox, point, box }) => { +}> = ({ imageUrl, screencastFrame, highlightedBox, point, box }) => { const [measure, ref] = useMeasure(); const [naturalSize, setNaturalSize] = React.useState<{ width: number, height: number } | undefined>(); @@ -204,7 +210,7 @@ const AriaModeScreenshot: React.FunctionComponent<{ // coordinates. The image is scaled to fit into the available area, scale the boxes and // the action point to match the rendered image. let overlays: React.ReactNode; - if (screenshot && naturalSize && measure.width) { + if (imageUrl && naturalSize && measure.width) { const padding = 10; const availableWidth = measure.width - 2 * padding; const availableHeight = measure.height - 2 * padding; @@ -231,13 +237,13 @@ const AriaModeScreenshot: React.FunctionComponent<{ } return
- {screenshot && Screenshot setNaturalSize({ width: event.currentTarget.naturalWidth, height: event.currentTarget.naturalHeight })} />} - {!screenshot && } + {!screencastFrame && !imageUrl && } {overlays}
; }; diff --git a/packages/trace-viewer/src/ui/filmStrip.tsx b/packages/trace-viewer/src/ui/filmStrip.tsx index 4529b80d69cc1..3bf5d2b9f02dd 100644 --- a/packages/trace-viewer/src/ui/filmStrip.tsx +++ b/packages/trace-viewer/src/ui/filmStrip.tsx @@ -20,8 +20,10 @@ import * as React from 'react'; import { useMeasure, upperBound } from '@web/uiUtils'; import type { PageEntry } from '@isomorphic/trace/entries'; import { useTraceModel } from './traceModelContext'; -import { useVideoThumbnails } from './videoThumbnails'; -import type { VideoThumbnail } from './videoThumbnails'; +import { useVideoSources } from './videoThumbnails'; +import type { VideoSource, VideoThumbnail } from './videoThumbnails'; +import { VideoFrame, lastVideoIndex } from './videoFrame'; +import type * as trace from '@isomorphic/trace/trace'; export type FilmStripPreviewPoint = { x: number; @@ -41,7 +43,7 @@ export const FilmStrip: React.FunctionComponent<{ const lanesRef = React.useRef(null); const videos = (model?.videos ?? []).map(video => ({ video, url: model!.createRelativeUrl(`file/${video.file}`) })); - const videoThumbnails = useVideoThumbnails(videos); + const videoSources = useVideoSources(videos); let laneIndex = 0; if (lanesRef.current && previewPoint) { @@ -50,24 +52,30 @@ export const FilmStrip: React.FunctionComponent<{ } const pageLanes = (model?.pages ?? []).filter(page => page.screencastFrames.length); - const videoLanes = videoLanesByPage(videos.map(({ video }) => video.pageId), videoThumbnails); + const videoLanes = videoLanesByPage(videos.map(({ video }) => video), videoSources); - let previewFrames: { timestamp: number, width: number, height: number, url: string }[] | undefined; - if (laneIndex < pageLanes.length) - previewFrames = model ? pageLanes[laneIndex]?.screencastFrames.map(frame => ({ ...frame, url: model.createRelativeUrl(`file/${frame.file}`) })) : undefined; - else - previewFrames = videoLanes[laneIndex - pageLanes.length]; - - let previewImage = undefined; - let previewSize = undefined; - if (previewPoint !== undefined && previewFrames && previewFrames.length) { + // The thumbnails only index the lanes, the preview renders the video itself. + let preview: React.ReactNode | undefined; + let previewSize: Size | undefined; + if (model && previewPoint !== undefined) { const previewTime = boundaries.minimum + (boundaries.maximum - boundaries.minimum) * previewPoint.x / measure.width; - previewImage = previewFrames[upperBound(previewFrames, previewTime, timeComparator) - 1]; const fitInto = { width: Math.min(800, (window.innerWidth / 2) | 0), height: Math.min(800, (window.innerHeight / 2) | 0), }; - previewSize = previewImage ? inscribe({ width: previewImage.width, height: previewImage.height }, fitInto) : undefined; + const frames = pageLanes[laneIndex]?.screencastFrames; + const frame = frames?.[upperBound(frames, previewTime, timeComparator) - 1]; + const lane = videoLanes[laneIndex - pageLanes.length]; + const videoIndex = lane ? lastVideoIndex(lane.videos, previewTime) : -1; + const videoUrl = videoIndex === -1 ? undefined : lane.sources[videoIndex].url; + if (frame) { + previewSize = inscribe(frame, fitInto); + preview = ; + } else if (videoUrl) { + const video = lane.videos[videoIndex]; + previewSize = inscribe(video, fitInto); + preview = ; + } } return
@@ -78,37 +86,48 @@ export const FilmStrip: React.FunctionComponent<{ width={measure.width} key={index} />)} - {videoLanes.map((thumbnails, index) => )}
- {model && previewPoint && previewImage && previewSize && + {previewPoint && preview && previewSize &&
- + {preview}
}
; }; -function videoLanesByPage(pageIds: string[], thumbnails: VideoThumbnail[][]): VideoThumbnail[][] { - const lanes = new Map(); - pageIds.forEach((pageId, index) => { - let lane = lanes.get(pageId); +type VideoLane = { + videos: trace.VideoTraceEvent[]; + sources: VideoSource[]; + thumbnails: VideoThumbnail[]; +}; + +function videoLanesByPage(videos: trace.VideoTraceEvent[], sources: VideoSource[]): VideoLane[] { + const lanes = new Map(); + videos.forEach((video, index) => { + let lane = lanes.get(video.pageId); if (!lane) { - lane = []; - lanes.set(pageId, lane); + lane = { videos: [], sources: [], thumbnails: [] }; + lanes.set(video.pageId, lane); } - lane.push(...thumbnails[index]); + lane.videos.push(video); + lane.sources.push(sources[index]); + lane.thumbnails.push(...sources[index].thumbnails); + }); + return [...lanes.values()].filter(lane => lane.thumbnails.length).map(lane => { + lane.thumbnails.sort((a, b) => a.timestamp - b.timestamp); + return lane; }); - return [...lanes.values()].filter(lane => lane.length).map(lane => lane.sort((a, b) => a.timestamp - b.timestamp)); } const VideoFilmStripLane: React.FunctionComponent<{ diff --git a/packages/trace-viewer/src/ui/playbackControl.tsx b/packages/trace-viewer/src/ui/playbackControl.tsx index 35aea8ef8556f..3c7a62fd865d9 100644 --- a/packages/trace-viewer/src/ui/playbackControl.tsx +++ b/packages/trace-viewer/src/ui/playbackControl.tsx @@ -28,11 +28,14 @@ export type PlaybackState = { currentIndex: number; percent: number; animating: boolean; + screencastTime: number | undefined; togglePlay: () => void; stop: () => void; prev: () => void; next: () => void; cycleSpeed: () => void; + selectAction: (action: ActionEntry) => void; + showSelectedAction: () => void; onScrubberMouseDown: (e: React.MouseEvent) => void; scrubberRef: React.RefObject; actionsLength: number; @@ -52,8 +55,8 @@ export function usePlayback( const [playing, setPlaying] = React.useState(false); const [speedIndex, setSpeedIndex] = React.useState(1); const [dragging, setDragging] = React.useState(false); - const [dragFraction, setDragFraction] = React.useState(undefined); - const [cursorTime, setCursorTime] = React.useState(undefined); + // Set while playing or positioned between actions: the screencast is shown at this time instead of the action snapshot. + const [screencastTime, setScreencastTime] = React.useState(undefined); const speed = speeds[speedIndex]; const currentIndex = selectedAction ? actions.indexOf(selectedAction) : -1; @@ -64,7 +67,6 @@ export function usePlayback( const fullDuration = fullMax - fullMin || 1; // Playback boundaries: constrained to time window when selected. - const windowMin = timeWindow ? timeWindow.minimum : fullMin; const windowMax = timeWindow ? timeWindow.maximum : fullMax; // Actions within the effective window. @@ -86,7 +88,12 @@ export function usePlayback( const scrubberRef = React.useRef(null); - const actionIndexAtTime = React.useCallback((t: number): number => { + const clampToWindow = React.useCallback((index: number) => { + return Math.max(firstWindowIndex, Math.min(lastWindowIndex, index)); + }, [firstWindowIndex, lastWindowIndex]); + + // Last action started at or before t, or the first action when none has started yet. + const lastStartedIndex = React.useCallback((t: number): number => { let lo = 0; let hi = actions.length - 1; while (lo < hi) { @@ -96,29 +103,13 @@ export function usePlayback( else hi = mid - 1; } - if (lo < actions.length - 1) { - const distPrev = t - actions[lo].startTime; - const distNext = actions[lo + 1].startTime - t; - if (distNext < distPrev) - lo = lo + 1; - } - // Clamp to window bounds. - return Math.max(firstWindowIndex, Math.min(lastWindowIndex, lo)); - }, [actions, firstWindowIndex, lastWindowIndex]); + return lo; + }, [actions]); const selectedTime = selectedAction ? selectedAction.startTime : fullMin; + const positionTime = screencastTime ?? selectedTime; + const percent = Math.max(0, Math.min(100, ((positionTime - fullMin) / fullDuration) * 100)); - let percent: number; - if (dragging && dragFraction !== undefined) - percent = dragFraction * 100; - else if (playing && cursorTime !== undefined) - percent = Math.max(0, Math.min(100, ((cursorTime - fullMin) / fullDuration) * 100)); - else - percent = Math.max(0, Math.min(100, ((selectedTime - fullMin) / fullDuration) * 100)); - - // Refs for raf closure. - const windowMinRef = React.useRef(windowMin); - windowMinRef.current = windowMin; const windowMaxRef = React.useRef(windowMax); windowMaxRef.current = windowMax; @@ -127,13 +118,8 @@ export function usePlayback( return; let rafId: number; let lastFrameTime: number | undefined; - let traceTime = selectedTime; - // If starting from before the window, jump to window start. - if (traceTime < windowMinRef.current) - traceTime = windowMinRef.current; - let lastSelectedIndex = currentIndex; - - setCursorTime(traceTime); + let traceTime = positionTime; + let lastSelectedIndex = clampToWindow(lastStartedIndex(traceTime)); const tick = (now: number) => { if (lastFrameTime !== undefined) { @@ -141,16 +127,17 @@ export function usePlayback( traceTime = Math.min(traceTime + delta, windowMaxRef.current); } lastFrameTime = now; - setCursorTime(traceTime); + setScreencastTime(traceTime); - const idx = actionIndexAtTime(traceTime); - if (idx !== lastSelectedIndex) { - lastSelectedIndex = idx; - onActionSelectedRef.current(actionsRef.current[idx]); + const index = clampToWindow(lastStartedIndex(traceTime)); + if (index !== lastSelectedIndex) { + lastSelectedIndex = index; + onActionSelectedRef.current(actionsRef.current[index]); } if (traceTime >= windowMaxRef.current) { setPlaying(false); + setScreencastTime(undefined); return; } rafId = requestAnimationFrame(tick); @@ -160,38 +147,51 @@ export function usePlayback( // eslint-disable-next-line react-hooks/exhaustive-deps }, [playing, speed]); - React.useEffect(() => { - if (!playing) - setCursorTime(undefined); - }, [playing]); - const togglePlay = React.useCallback(() => { if (!actions.length) return; - // Always restart from the window start when at the end (or beyond the window). - const atEnd = currentIndex >= lastWindowIndex; - if (!playing && atEnd) + if (playing) { + setPlaying(false); + setScreencastTime(undefined); + return; + } + // Restart from the window start when at the end, or outside the window. + if (currentIndex >= lastWindowIndex || currentIndex < firstWindowIndex) onActionSelected(actions[firstWindowIndex]); - setPlaying(!playing); + setPlaying(true); }, [playing, actions, currentIndex, onActionSelected, firstWindowIndex, lastWindowIndex]); + const selectAction = React.useCallback((action: ActionEntry) => { + setPlaying(false); + setScreencastTime(undefined); + onActionSelectedRef.current(action); + }, []); + + // Leave the between-actions position, so that the selected action's snapshot is shown. + const showSelectedAction = React.useCallback(() => { + setPlaying(false); + setScreencastTime(undefined); + }, []); + const stop = React.useCallback(() => { setPlaying(false); + setScreencastTime(undefined); if (actions.length) onActionSelected(actions[firstWindowIndex]); }, [actions, onActionSelected, firstWindowIndex]); + const canPrev = currentIndex > firstWindowIndex; + const canNext = currentIndex < lastWindowIndex; + const prev = React.useCallback(() => { - const target = Math.max(currentIndex - 1, firstWindowIndex); - if (target !== currentIndex) - onActionSelected(actions[target]); - }, [actions, currentIndex, onActionSelected, firstWindowIndex]); + if (canPrev) + selectAction(actions[currentIndex - 1]); + }, [actions, canPrev, currentIndex, selectAction]); const next = React.useCallback(() => { - const target = Math.min(currentIndex + 1, lastWindowIndex); - if (target !== currentIndex) - onActionSelected(actions[target]); - }, [actions, currentIndex, onActionSelected, lastWindowIndex]); + if (canNext) + selectAction(actions[currentIndex + 1]); + }, [actions, canNext, currentIndex, selectAction]); const cycleSpeed = React.useCallback(() => { setSpeedIndex(i => (i + 1) % speeds.length); @@ -199,20 +199,21 @@ export function usePlayback( React.useEffect(() => { setPlaying(false); + setScreencastTime(undefined); }, [actions]); - const fractionFromMouseEvent = React.useCallback((e: MouseEvent | React.MouseEvent) => { + const seekToMouseEvent = React.useCallback((e: MouseEvent | React.MouseEvent, snap: boolean) => { const rect = scrubberRef.current!.getBoundingClientRect(); - return Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width)); - }, []); - - const selectActionAtFraction = React.useCallback((fraction: number) => { - if (!actions.length) - return; - const t = fullMin + fraction * fullDuration; - const idx = actionIndexAtTime(t); - onActionSelectedRef.current(actionsRef.current[idx]); - }, [actions, fullMin, fullDuration, actionIndexAtTime]); + const fraction = Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width)); + const time = fullMin + fraction * fullDuration; + // The screencast follows the pointer while dragging, releasing snaps to the action snapshot. + setScreencastTime(snap ? undefined : time); + let index = lastStartedIndex(time); + const next = actions[index + 1]; + if (next && next.startTime - time < time - actions[index].startTime) + ++index; + onActionSelectedRef.current(actions[clampToWindow(index)]); + }, [actions, fullMin, fullDuration, clampToWindow, lastStartedIndex]); const dragCleanupRef = React.useRef<(() => void) | null>(null); @@ -228,22 +229,16 @@ export function usePlayback( scrubberRef.current?.focus(); setDragging(true); setPlaying(false); - const fraction = fractionFromMouseEvent(e); - setDragFraction(fraction); - selectActionAtFraction(fraction); + seekToMouseEvent(e, false); const onMouseMove = (me: MouseEvent) => { - const f = fractionFromMouseEvent(me); - setDragFraction(f); - selectActionAtFraction(f); + seekToMouseEvent(me, false); }; const onMouseUp = (me: MouseEvent) => { document.removeEventListener('mousemove', onMouseMove); document.removeEventListener('mouseup', onMouseUp); dragCleanupRef.current = null; - const f = fractionFromMouseEvent(me); - selectActionAtFraction(f); - setDragFraction(undefined); + seekToMouseEvent(me, true); setDragging(false); }; document.addEventListener('mousemove', onMouseMove); @@ -252,18 +247,16 @@ export function usePlayback( document.removeEventListener('mousemove', onMouseMove); document.removeEventListener('mouseup', onMouseUp); }; - }, [actions, selectActionAtFraction, fractionFromMouseEvent]); + }, [actions, seekToMouseEvent]); const animating = !playing && !dragging; const ticks = actions.length > 0 && actions.length <= 200 ? actions.map(a => ((a.startTime - fullMin) / fullDuration) * 100) : undefined; - const canPrev = currentIndex > firstWindowIndex; - const canNext = currentIndex < lastWindowIndex; const canStop = playing || currentIndex > firstWindowIndex; return { - playing, speed, currentIndex, percent, animating, - togglePlay, stop, prev, next, cycleSpeed, + playing, speed, currentIndex, percent, animating, screencastTime, + togglePlay, stop, prev, next, cycleSpeed, selectAction, showSelectedAction, onScrubberMouseDown, scrubberRef, actionsLength: actions.length, canPrev, canNext, canStop, ticks, }; diff --git a/packages/trace-viewer/src/ui/snapshotTab.css b/packages/trace-viewer/src/ui/snapshotTab.css index 5265f4f5f152e..c895859d9a507 100644 --- a/packages/trace-viewer/src/ui/snapshotTab.css +++ b/packages/trace-viewer/src/ui/snapshotTab.css @@ -105,6 +105,16 @@ iframe.snapshot-visible[name=snapshot] { visibility: visible; } +.snapshot-switcher .screencast-frame { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + object-fit: contain; + background: light-dark(white, #333); +} + .no-snapshot { text-align: center; padding: 50px; diff --git a/packages/trace-viewer/src/ui/snapshotTab.tsx b/packages/trace-viewer/src/ui/snapshotTab.tsx index 6cd00da5ebfdc..7fdf5b9f77835 100644 --- a/packages/trace-viewer/src/ui/snapshotTab.tsx +++ b/packages/trace-viewer/src/ui/snapshotTab.tsx @@ -21,7 +21,7 @@ import { nextActionByStartTime, previousActionByEndTime } from '@isomorphic/trac import type { TraceModel } from '@isomorphic/trace/traceModel'; import { Toolbar } from '@web/components/toolbar'; import { ToolbarButton } from '@web/components/toolbarButton'; -import { clsx, useMeasure, useSetting } from '@web/uiUtils'; +import { clsx, upperBound, useMeasure, useSetting } from '@web/uiUtils'; import { InjectedScript } from '@injected/injectedScript'; import { Recorder } from '@injected/recorder/recorder'; import { asLocator } from '@isomorphic/locatorGenerators'; @@ -36,6 +36,8 @@ import yaml from 'yaml'; import { PlaybackButtons } from './playbackControl'; import type { PlaybackState } from './playbackControl'; import { AriaModeView, collectAriaModeTargets, shouldDisplayAriaMode } from './ariaModeView'; +import { useVideoSources } from './videoThumbnails'; +import { VideoFrame, lastVideoIndex } from './videoFrame'; export type HighlightedElement = { locator?: string, @@ -52,13 +54,16 @@ export const SnapshotTabsView: React.FunctionComponent<{ setIsInspecting: (isInspecting: boolean) => void, highlightedElement: HighlightedElement, setHighlightedElement: (element: HighlightedElement) => void, + // Time to show the screencast at, undefined to show the action snapshot. + screencastTime?: number, playback: PlaybackState -}> = ({ action, model, sdkLanguage, testIdAttributeName, isInspecting, setIsInspecting, highlightedElement, setHighlightedElement, playback }) => { +}> = ({ action, model, sdkLanguage, testIdAttributeName, isInspecting, setIsInspecting, highlightedElement, setHighlightedElement, screencastTime, playback }) => { const [snapshotTab, setSnapshotTab] = React.useState<'action'|'before'|'after'>('action'); const [shouldPopulateCanvasFromScreenshot] = useSetting('shouldPopulateCanvasFromScreenshot', false); const [displayAriaModeSetting] = useSetting('displayAriaMode', false); const displayAriaMode = shouldDisplayAriaMode(model, displayAriaModeSetting); + const screencastFrame = useScreencastFrame(model, screencastTime, playback.playing, playback.speed); const snapshots = React.useMemo(() => { return collectSnapshots(model, action); @@ -107,6 +112,7 @@ export const SnapshotTabsView: React.FunctionComponent<{ target={ariaModeTargets[snapshotTab]} point={snapshotTab === 'action' ? action?.point : undefined} box={snapshotTab === 'action' ? action?.box : undefined} + screencastFrame={screencastFrame} />} {!displayAriaMode && }
; }; @@ -128,7 +135,8 @@ export const SnapshotView: React.FunctionComponent<{ setIsInspecting: (isInspecting: boolean) => void, highlightedElement: HighlightedElement, setHighlightedElement: (element: HighlightedElement) => void, -}> = ({ snapshotUrls, sdkLanguage, testIdAttributeName, isInspecting, setIsInspecting, highlightedElement, setHighlightedElement }) => { + screencastFrame?: React.ReactNode, +}> = ({ snapshotUrls, sdkLanguage, testIdAttributeName, isInspecting, setIsInspecting, highlightedElement, setHighlightedElement, screencastFrame }) => { const iframeRef0 = React.useRef(null); const iframeRef1 = React.useRef(null); const [snapshotInfo, setSnapshotInfo] = React.useState({ viewport: kDefaultViewport, url: '' }); @@ -146,7 +154,8 @@ export const SnapshotView: React.FunctionComponent<{ if (loadingRef.current.iteration !== thisIteration) return; - const iframe = [iframeRef0, iframeRef1][newVisibleIframe].current; + // While the screencast frame covers the snapshot, only take the viewport and the url from it. + const iframe = screencastFrame ? undefined : [iframeRef0, iframeRef1][newVisibleIframe].current; if (iframe) { let loadedCallback = () => {}; const loadedPromise = new Promise(f => loadedCallback = f); @@ -172,10 +181,11 @@ export const SnapshotView: React.FunctionComponent<{ if (loadingRef.current.iteration !== thisIteration) return; - loadingRef.current.visibleIframe = newVisibleIframe; + if (iframe) + loadingRef.current.visibleIframe = newVisibleIframe; setSnapshotInfo(newSnapshotInfo); })(); - }, [snapshotUrls]); + }, [snapshotUrls, screencastFrame]); return
+ {screencastFrame}
; @@ -478,3 +489,27 @@ export async function fetchSnapshotInfo(snapshotInfoUrl: string | undefined) { } export const kDefaultViewport = { width: 1280, height: 720 }; + +function useScreencastFrame(model: TraceModel | undefined, time: number | undefined, playing: boolean, speed: number): React.ReactNode | undefined { + const videos = model && time !== undefined ? model.videos.map(video => ({ video, url: model.createRelativeUrl(`file/${video.file}`) })) : []; + const videoSources = useVideoSources(videos); + if (!model || time === undefined) + return undefined; + + // A trace has either screencast frames or videos, never both. + // Show the page that painted most recently. + let frame: { timestamp: number, file: string } | undefined; + for (const page of model.pages) { + const candidate = page.screencastFrames[upperBound(page.screencastFrames, time, (t, f) => t - f.timestamp) - 1]; + if (candidate && (!frame || candidate.timestamp > frame.timestamp)) + frame = candidate; + } + if (frame) + return Screencast frame; + + const index = lastVideoIndex(videos.map(({ video }) => video), time); + const url = videoSources[index]?.url; + if (!url) + return undefined; + return ; +} diff --git a/packages/trace-viewer/src/ui/videoFrame.tsx b/packages/trace-viewer/src/ui/videoFrame.tsx new file mode 100644 index 0000000000000..651971b32ca3f --- /dev/null +++ b/packages/trace-viewer/src/ui/videoFrame.tsx @@ -0,0 +1,79 @@ +/* + Copyright (c) Microsoft Corporation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import * as React from 'react'; + +import type * as trace from '@isomorphic/trace/trace'; + +// Playback runs the video on its own clock and only resyncs when it drifts this far away. +const maxDriftSeconds = 0.25; + +// Video that was recording at the given trace time, -1 when none has started yet. +export function lastVideoIndex(videos: trace.VideoTraceEvent[], time: number): number { + let result = -1; + videos.forEach((video, index) => { + if (video.timestamp <= time && (result === -1 || video.timestamp > videos[result].timestamp)) + result = index; + }); + return result; +} + +export const VideoFrame: React.FunctionComponent<{ + url: string, + // Video start in the trace time. + startTime: number, + // Trace time to render. + time: number, + playing?: boolean, + speed?: number, + className?: string, + width?: number, + height?: number, +}> = ({ url, startTime, time, playing, speed, className, width, height }) => { + const ref = React.useRef(null); + const offset = Math.max(0, (time - startTime) / 1000); + const offsetRef = React.useRef(offset); + offsetRef.current = offset; + + React.useEffect(() => { + const video = ref.current; + if (!video) + return; + if (!playing) { + video.pause(); + if (Math.abs(video.currentTime - offset) > 0.001) + video.currentTime = offset; + return; + } + video.playbackRate = speed ?? 1; + if (Math.abs(video.currentTime - offset) > maxDriftSeconds) + video.currentTime = offset; + if (video.paused) + void video.play().catch(() => {}); + }, [url, offset, playing, speed]); + + return