feat(video): add storyboard-based video hover previews (#429)

src-tauri/src/api/innertube/extractors/storyboard.rs

@@ -6,10 +6,81 @@
 //! is the template with `$L` as the level's index and `$N` as its name, plus
 //! its `sigh`; `$M` in the name numbers the sheets.
 
+use std::collections::HashMap;
+use std::sync::{Mutex, OnceLock};
+use std::time::{Duration, Instant};
+
 use serde_json::Value;
 
+use crate::api::innertube::InnertubeClient;
+use crate::api::innertube::core::clients;
+use crate::api::innertube::core::player_requests::PlayerFlights;
+use crate::errors::AppResult;
 use crate::models::video::StoryboardLevel;
 
+const STORYBOARD_CACHE_CAPACITY: usize = 100;
+const STORYBOARD_TTL: Duration = Duration::from_secs(3600); // 1 hour
+
+#[derive(Clone, Debug)]
+struct CachedStoryboard {
+    levels: Vec<StoryboardLevel>,
+    resolved_at: Instant,
+}
+
+fn storyboard_cache() -> &'static Mutex<HashMap<String, CachedStoryboard>> {
+    static CACHE: OnceLock<Mutex<HashMap<String, CachedStoryboard>>> = OnceLock::new();
+    CACHE.get_or_init(|| Mutex::new(HashMap::new()))
+}
+
+fn flights() -> &'static PlayerFlights<Option<Vec<StoryboardLevel>>> {
+    static FLIGHTS: OnceLock<PlayerFlights<Option<Vec<StoryboardLevel>>>> = OnceLock::new();
+    FLIGHTS.get_or_init(PlayerFlights::default)
+}
+
+fn cached_storyboard(video_id: &str) -> Option<Vec<StoryboardLevel>> {
+    let mut cache = storyboard_cache().lock().ok()?;
+    let entry = cache.get(video_id)?;
+    if entry.resolved_at.elapsed() >= STORYBOARD_TTL {
+        cache.remove(video_id);
+        return None;
+    }
+    Some(entry.levels.clone())
+}
+
+fn store_storyboard(video_id: &str, levels: Vec<StoryboardLevel>) {
+    let Ok(mut cache) = storyboard_cache().lock() else {
+        return;
+    };
+    cache.retain(|_, entry| entry.resolved_at.elapsed() < STORYBOARD_TTL);
+    if cache.len() >= STORYBOARD_CACHE_CAPACITY {
+        if let Some(oldest) = cache
+            .iter()
+            .min_by_key(|(_, entry)| entry.resolved_at)
+            .map(|(key, _)| key.clone())
+        {
+            cache.remove(&oldest);
+        }
+    }
+    cache.insert(
+        video_id.to_string(),
+        CachedStoryboard {
+            levels,
+            resolved_at: Instant::now(),
+        },
+    );
+}
+
+fn extract_duration_seconds(response: &Value) -> Option<u64> {
+    response["videoDetails"]["lengthSeconds"]
+        .as_str()
+        .and_then(|value| value.parse::<u64>().ok())
+        .or_else(|| {
+            response["microformat"]["playerMicroformatRenderer"]["lengthSeconds"]
+                .as_str()
+                .and_then(|value| value.parse::<u64>().ok())
+        })
+}
+
 #[must_use]
 pub fn parse_storyboard(player: &Value, duration_seconds: Option<u64>) -> Vec<StoryboardLevel> {
     player["storyboards"]["playerStoryboardSpecRenderer"]["spec"]
@@ -61,6 +132,50 @@ fn parse_spec(spec: &str, duration_seconds: Option<u64>) -> Vec<StoryboardLevel>
         .collect()
 }
 
+impl InnertubeClient {
+    /// Fetches storyboard levels for hover preview or seek scrubbing.
+    /// Deduplicated via `PlayerFlights` and cached in-memory with a 1-hour TTL.
+    /// Unplayable videos or missing specs return an empty vector gracefully.
+    pub async fn get_video_storyboard(&self, video_id: &str) -> AppResult<Vec<StoryboardLevel>> {
+        let video_id_trimmed = video_id.trim();
+        if video_id_trimmed.is_empty() {
+            return Ok(Vec::new());
+        }
+
+        if let Some(cached) = cached_storyboard(video_id_trimmed) {
+            return Ok(cached);
+        }
+
+        let flight = flights().for_video(video_id_trimmed);
+        let outcome = flight
+            .get_or_init(|| async {
+                let mut payload = serde_json::json!({
+                    "videoId": video_id_trimmed,
+                    "playbackContext": {
+                        "contentPlaybackContext": {
+                            "signatureTimestamp": clients::DEFAULT_SIGNATURE_TIMESTAMP,
+                        }
+                    },
+                    "contentCheckOk": true,
+                    "racyCheckOk": true,
+                });
+
+                let response = self
+                    .post_innertube("player", &clients::WEB, &mut payload)
+                    .await
+                    .ok()?;
+
+                let duration_seconds = extract_duration_seconds(&response);
+                let levels = parse_storyboard(&response, duration_seconds);
+                store_storyboard(video_id_trimmed, levels.clone());
+                Some(levels)
+            })
+            .await;
+
+        Ok(outcome.clone().unwrap_or_default())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -101,4 +216,30 @@ mod tests {
         assert!(parse_spec("https://x/$L|bad#level", Some(60)).is_empty());
         assert!(parse_spec("https://x/$L/$N|48#27#100#10#10#0#default#s", None).is_empty());
     }
+
+    #[test]
+    fn storyboard_cache_stores_and_evicts_oldest() {
+        let test_levels = vec![StoryboardLevel {
+            width: 160,
+            height: 90,
+            count: 10,
+            columns: 5,
+            rows: 5,
+            interval_ms: 2000,
+            url_template: "https://example.com/sheet.jpg".to_string(),
+        }];
+
+        store_storyboard("test_vid_1", test_levels.clone());
+        let cached = cached_storyboard("test_vid_1");
+        assert!(cached.is_some());
+        assert_eq!(cached.unwrap()[0].width, 160);
+
+        // Fill past capacity to verify eviction
+        for i in 0..105 {
+            store_storyboard(&format!("fill_vid_{i}"), test_levels.clone());
+        }
+
+        let cache = storyboard_cache().lock().unwrap();
+        assert!(cache.len() <= STORYBOARD_CACHE_CAPACITY);
+    }
 }

src-tauri/src/commands/youtube.rs

@@ -12,7 +12,8 @@ use crate::models::explore::{ExploreGrid, ExplorePage, ExploreTarget};
 use crate::models::playlist::PlaylistDetailsResponse;
 use crate::models::search::{SearchSuggestion, SearchVideosRequest, SearchVideosResponse};
 use crate::models::video::{
-    RelatedPage, StreamInfo, StreamVariant, VideoDetails, VideoMetadata, VideoSummary,
+    RelatedPage, StoryboardLevel, StreamInfo, StreamVariant, VideoDetails, VideoMetadata,
+    VideoSummary,
 };
 use crate::security::validation::{
     validate_browse_id, validate_channel_id, validate_page_token, validate_search_query,
@@ -389,6 +390,19 @@ pub async fn get_video_category(
         .map_err(ErrorResponse::from)
 }
 
+/// The storyboard specification for hover previews and seek thumbnails.
+#[tauri::command]
+pub async fn get_video_storyboard(
+    video_id: String,
+    youtube_service: State<'_, YoutubeService>,
+) -> Result<Vec<StoryboardLevel>, ErrorResponse> {
+    validate_video_id(&video_id).map_err(ErrorResponse::from)?;
+    youtube_service
+        .get_video_storyboard(&video_id)
+        .await
+        .map_err(ErrorResponse::from)
+}
+
 #[tauri::command]
 pub async fn get_more_related_videos(
     continuation: String,

src-tauri/src/lib.rs

@@ -145,8 +145,9 @@ use commands::youtube::{
     get_playlist_details, get_post_comments, get_related_videos, get_return_youtube_dislike,
     get_sabr_debug_state, get_search_suggestions, get_sponsorblock_segments, get_stream_info,
     get_subscription_rotation_feed, get_subscription_rss_feed, get_unplayable_video_ids,
-    get_video_category, get_video_details, get_video_metadata, parse_subscription_export,
-    resolve_channel_id, search_videos, stream_subscription_rss_feed, submit_sponsorblock_segment,
+    get_video_category, get_video_details, get_video_metadata, get_video_storyboard,
+    parse_subscription_export, resolve_channel_id, search_videos, stream_subscription_rss_feed,
+    submit_sponsorblock_segment,
 };
 use services::discord_presence::DiscordPresence;
 use services::music_service::MusicService;
@@ -600,6 +601,7 @@ pub fn run() {
             get_more_related_videos,
             get_video_category,
             get_video_metadata,
+            get_video_storyboard,
             get_stream_info,
             get_unplayable_video_ids,
             get_channel_details,

src-tauri/src/services/youtube_service.rs

@@ -10,7 +10,9 @@ use crate::models::live_chat::LiveChatResponse;
 use crate::models::playlist::PlaylistDetailsResponse;
 use crate::models::search::{SearchSuggestion, SearchVideosRequest, SearchVideosResponse};
 use crate::models::shorts::{ShortItem, ShortsFeed};
-use crate::models::video::{RelatedPage, StreamInfo, VideoBasics, VideoDetails, VideoMetadata};
+use crate::models::video::{
+    RelatedPage, StoryboardLevel, StreamInfo, VideoBasics, VideoDetails, VideoMetadata,
+};
 
 #[derive(Clone)]
 pub struct YoutubeService {
@@ -46,6 +48,10 @@ impl YoutubeService {
         self.extractor.get_video_metadata(video_id).await
     }
 
+    pub async fn get_video_storyboard(&self, video_id: &str) -> AppResult<Vec<StoryboardLevel>> {
+        self.extractor.get_video_storyboard(video_id).await
+    }
+
     pub async fn get_video_category(&self, video_id: &str) -> AppResult<Option<String>> {
         self.extractor.get_video_category(video_id).await
     }

src/App.css

@@ -247,6 +247,19 @@
   animation: fade-in 0.25s cubic-bezier(0.16, 1, 0.3, 1) forwards;
 }
 
+@keyframes section-crossfade {
+  from {
+    opacity: 0;
+  }
+  to {
+    opacity: 1;
+  }
+}
+
+.animate-section-crossfade {
+  animation: section-crossfade 0.12s ease-out forwards;
+}
+
 @keyframes shimmer {
   100% {
     transform: translateX(100%);

src/components/settings/tabs/ContentTab.tsx

@@ -21,6 +21,7 @@ import { BlockedKeywordsPanel } from '../BlockedKeywordsPanel';
 export function ContentTab() {
   const deepFlowActive = useAppSettingsStore((state) => state.values[SETTINGS.DEEP_FLOW_ACTIVE] === 'true');
   const [titleMaxLines, setTitleMaxLines] = useNumberPref(SETTINGS.VIDEO_TITLE_MAX_LINES, 1);
+  const [hoverPreviews, setHoverPreviews] = useBoolPref(SETTINGS.VIDEO_HOVER_PREVIEWS_ENABLED, true);
   const [gridColumns, setGridColumns] = useNumberPref(SETTINGS.GRID_COLUMNS, GRID_COLUMNS_DEFAULT);
   const [homeFeed, setHomeFeed] = useBoolPref(SETTINGS.HOME_FEED_ENABLED, true);
   const [appLogo, setAppLogo] = useBoolPref(SETTINGS.SHOW_APP_LOGO_ICON, true);
@@ -116,6 +117,17 @@ export function ContentTab() {
             { value: '0', label: getString('settings_option_unlimited') }, { value: '1', label: getString('settings_option_one_line') }, { value: '2', label: getString('settings_option_two_lines') }, { value: '3', label: getString('settings_option_three_lines') },
           ]} />
         </SettingItem>
+        <SettingItem
+          title={getString('settings_video_hover_previews')}
+          description={getString('settings_video_hover_previews_desc')}
+          disabled={isSettingDisabledUntilWired(SETTINGS.VIDEO_HOVER_PREVIEWS_ENABLED)}
+        >
+          <ToggleSwitch
+            checked={hoverPreviews}
+            onChange={setHoverPreviews}
+            disabled={isSettingDisabledUntilWired(SETTINGS.VIDEO_HOVER_PREVIEWS_ENABLED)}
+          />
+        </SettingItem>
       </SettingsGroup>
 
       <SettingsGroup title={getString('settings_group_feed')}>

src/components/video/VideoCard.tsx

@@ -39,6 +39,8 @@ import { CardPills, ChannelLabel, CollabAvatars, UpcomingBadge } from "./CardMet
 import { collaboratorsByline } from "../../lib/collaborators";
 import { backgroundTabClickProps, openedInBackgroundTab, routeForVideo } from '../../lib/openTarget';
 import { openActions } from '../common/openActions';
+import { useHoverPreview } from '../../lib/useHoverPreview';
+import { VideoThumbnailPreview } from './VideoThumbnailPreview';
 
 export interface VideoCardProps {
   video: VideoSummary;
@@ -196,6 +198,19 @@ function VideoCardComponent({
   );
   const { src: displayThumbnail, onError: handleThumbnailError } = useImageFallback(thumbnailCandidates);
 
+  const preview = useHoverPreview({
+    videoId: video.id,
+    isHovered,
+    durationSeconds: video.durationSeconds,
+    targetWidth: thumbnailRef.current?.offsetWidth
+      ? thumbnailRef.current.offsetWidth * (typeof window !== 'undefined' ? window.devicePixelRatio || 1 : 1)
+      : undefined,
+    targetHeight: thumbnailRef.current?.offsetHeight
+      ? thumbnailRef.current.offsetHeight * (typeof window !== 'undefined' ? window.devicePixelRatio || 1 : 1)
+      : undefined,
+    enabled: !isChannel && !isLiveVideo && !video.isUpcoming,
+  });
+
   const handleMouseEnter = useCallback(() => {
     setIsHovered(true);
     // getImageData is a synchronous pixel readback — cheap on GPU-composited
@@ -581,6 +596,12 @@ function VideoCardComponent({
           ) : (
             <div className="h-full w-full bg-chrome-zinc-800" />
           )}
+          <VideoThumbnailPreview
+            isActive={preview.isPreviewActive}
+            frame={preview.currentFrame}
+            level={preview.currentLevel}
+            progressPercent={preview.progressPercent}
+          />
           {isLiveVideo ? (
             <LiveBadge className="absolute bottom-1 right-1" />
           ) : video.isUpcoming ? (
@@ -686,6 +707,12 @@ function VideoCardComponent({
           ) : (
             <div className="h-full w-full bg-chrome-zinc-800" />
           )}
+          <VideoThumbnailPreview
+            isActive={preview.isPreviewActive}
+            frame={preview.currentFrame}
+            level={preview.currentLevel}
+            progressPercent={preview.progressPercent}
+          />
 
           {isLiveVideo ? (
             <LiveBadge className="absolute bottom-1 right-1" />
@@ -776,6 +803,12 @@ function VideoCardComponent({
         ) : (
           <div className="w-full h-full bg-chrome-zinc-800" />
         )}
+        <VideoThumbnailPreview
+          isActive={preview.isPreviewActive}
+          frame={preview.currentFrame}
+          level={preview.currentLevel}
+          progressPercent={preview.progressPercent}
+        />
 
         {isLiveVideo ? (
           <LiveBadge className={`absolute right-1 ${isHistoryCard || isContinueCard ? 'bottom-2' : 'bottom-1'}`} />