perf(web-player): reduce transmux and audio processing overhead

web-ui/src/mpegts/audio/pcm-audio-player.ts

@@ -62,6 +62,11 @@ const RATIO_DRIFT_GAIN = 0.5;
 /** Max stretch ratio deviation used for drift correction. WSOLA preserves
  *  pitch, so a transient 10% tempo offset is inaudible while it converges. */
 const RATIO_DRIFT_MAX = 0.1;
+/** At normal playback speed, let an already-synchronized chain free-run
+ *  inside this deadband so WSOLA can use its ratio=1 memcpy bypass. Enter at
+ *  half the limit and exit at the full limit to avoid toggling at the edge. */
+const WSOLA_BYPASS_ENTER_DRIFT = 0.01;
+const WSOLA_BYPASS_EXIT_DRIFT = 0.02;
 /** Initial/large-drift mode: allow stronger WSOLA correction before falling back to hard resync. */
 const SOFT_SYNC_WINDOW_SEC = 3.0;
 const SOFT_SYNC_EXIT_DRIFT = 0.08;
@@ -93,6 +98,7 @@ const RECOVERY_TIMEOUT_MS = 4000;
  *    timeupdate while visible, or seeked); then one deterministic resync.
  */
 type SyncState = "active" | "background" | "recovering";
+type ControlTickResult = "skipped" | "updated" | "pumped";
 
 interface AudioChunk {
   samples: Float32Array;
@@ -149,6 +155,7 @@ export class PCMAudioPlayer {
   private driftEma = 0;
   private hasDriftEma = false;
   private softSyncUntil = 0;
+  private wsolaBypassActive = false;
   private driftLogCounter = 0;
   private controlTimer: ReturnType<typeof setInterval> | null = null;
 
@@ -286,16 +293,15 @@ export class PCMAudioPlayer {
         this.completeRecovery("timeupdate");
         return;
       }
-      this.controlTick();
-      this.pump();
+      this.controlAndPump();
     };
     this.boundOnRateChange = () => {
       // Apply the new rate to the stretcher immediately instead of waiting for
       // drift to build through the already-scheduled pipeline. The remaining
       // mismatch (scheduled-ahead audio at the old rate) is absorbed by the
       // drift correction, so moderate rate changes (live sync 1 ↔ 1.2) cause
       // no audible interruption.
-      this.controlTick();
+      this.controlAndPump(false);
     };
     this.boundOnVideoWaiting = () => this.enterBuffering("waiting");
     this.boundOnVideoStalled = () => {
@@ -325,8 +331,7 @@ export class PCMAudioPlayer {
     }
 
     this.controlTimer = setInterval(() => {
-      this.controlTick();
-      this.pump();
+      this.controlAndPump();
     }, CONTROL_INTERVAL_MS);
   }
 
@@ -644,6 +649,7 @@ export class PCMAudioPlayer {
   private resetDriftState(): void {
     this.driftEma = 0;
     this.hasDriftEma = false;
+    this.wsolaBypassActive = false;
   }
 
   private enterBuffering(reason: "waiting" | "stalled"): void {
@@ -825,17 +831,28 @@ export class PCMAudioPlayer {
 
   // ==================== Drift control ====================
 
-  private controlTick(): void {
+  /** Run drift control and then keep the scheduling window filled exactly
+   *  once. A successful resync pumps internally, so do not repeat it.
+   *  Ratechange passes false to preserve its old no-op behavior when drift
+   *  control is not currently applicable. */
+  private controlAndPump(pumpWhenSkipped = true): void {
+    const result = this.controlTick();
+    if (result === "updated" || (result === "skipped" && pumpWhenSkipped)) {
+      this.pump();
+    }
+  }
+
+  private controlTick(): ControlTickResult {
     const ctx = this.context;
     const video = this.videoElement;
     if (!ctx || !video || ctx.state !== "running" || !this.stretcher) {
-      return;
+      return "skipped";
     }
     // Drift control and hard resync are meaningful only when the video clock
     // is trusted. BACKGROUND/RECOVERING free-run: correcting against a frozen
     // or rebuilding video clock replays audio (the "broken record" loop).
     if (this.syncState !== "active" || !this.canScheduleAudio()) {
-      return;
+      return "skipped";
     }
 
     const audioTime = this.audioStreamTimeNow();
@@ -846,10 +863,10 @@ export class PCMAudioPlayer {
         const target = video.currentTime;
         const idx = this.findChunkIndexByTime(target);
         if (idx >= 0 && this.audioBuffer[this.audioBuffer.length - 1].endTime > target + 0.1) {
-          this.resyncFromBuffer(target);
+          return this.resyncFromBuffer(target) ? "pumped" : "skipped";
         }
       }
-      return;
+      return "skipped";
     }
 
     const drift = audioTime - video.currentTime;
@@ -862,8 +879,7 @@ export class PCMAudioPlayer {
 
     if (Math.abs(drift) > HARD_RESYNC_THRESHOLD) {
       Log.v(TAG, `Emergency hard resync: drift=${drift.toFixed(3)}s`);
-      this.resyncFromBuffer(video.currentTime);
-      return;
+      return this.resyncFromBuffer(video.currentTime) ? "pumped" : "skipped";
     }
 
     // Rate matching: follow video.playbackRate, correct residual drift.
@@ -874,17 +890,23 @@ export class PCMAudioPlayer {
     const correctionMax = softSyncActive ? SOFT_SYNC_RATIO_DRIFT_MAX : RATIO_DRIFT_MAX;
     const correctionGain = softSyncActive ? SOFT_SYNC_DRIFT_GAIN : RATIO_DRIFT_GAIN;
     const correction = Math.min(correctionMax, Math.max(-correctionMax, correctionDrift * correctionGain));
-    const ratio = Math.min(2, Math.max(0.5, rate * (1 - correction)));
+    const bypassDriftLimit = this.wsolaBypassActive ? WSOLA_BYPASS_EXIT_DRIFT : WSOLA_BYPASS_ENTER_DRIFT;
+    this.wsolaBypassActive =
+      video.playbackRate === 1 &&
+      !softSyncActive &&
+      Math.abs(drift) <= bypassDriftLimit &&
+      Math.abs(this.driftEma) <= bypassDriftLimit;
+    const ratio = this.wsolaBypassActive ? 1 : Math.min(2, Math.max(0.5, rate * (1 - correction)));
     this.stretcher.setRatio(ratio);
-    this.pump();
 
     if (++this.driftLogCounter >= DRIFT_LOG_TICKS) {
       this.driftLogCounter = 0;
       Log.v(
         TAG,
-        `A/V drift=${(this.driftEma * 1000).toFixed(1)}ms, rate=${rate}, stretch ratio=${ratio.toFixed(4)}, mode=${softSyncActive ? "soft" : "steady"}`,
+        `A/V drift=${(this.driftEma * 1000).toFixed(1)}ms, rate=${rate}, stretch ratio=${ratio.toFixed(4)}, mode=${this.wsolaBypassActive ? "bypass" : softSyncActive ? "soft" : "steady"}`,
       );
     }
+    return "updated";
   }
 
   // ==================== Buffer Management ====================

web-ui/src/mpegts/demux/annexb.ts

@@ -0,0 +1,36 @@
+/**
+ * Find the next Annex-B start code at or after `startOffset`.
+ *
+ * This preserves the parsers' existing boundary semantics: a three-byte start
+ * code must have a following NAL header byte, while a four-byte start code can
+ * end at the end of the buffer. The latter is retained for compatibility with
+ * the previous scanner, even though such an empty trailing NAL is malformed.
+ */
+export function findAnnexBStartCodeOffset(data: Uint8Array, startOffset: number): number {
+  const length = data.byteLength;
+  let oneOffset = Math.max(2, startOffset + 2);
+
+  while (oneOffset < length) {
+    oneOffset = data.indexOf(1, oneOffset);
+    if (oneOffset === -1) {
+      break;
+    }
+
+    if (data[oneOffset - 1] !== 0 || data[oneOffset - 2] !== 0) {
+      oneOffset++;
+      continue;
+    }
+
+    const fourByteOffset = oneOffset - 3;
+    const startCodeOffset =
+      fourByteOffset >= startOffset && data[fourByteOffset] === 0 ? fourByteOffset : oneOffset - 2;
+
+    if (startCodeOffset >= startOffset && startCodeOffset + 3 < length) {
+      return startCodeOffset;
+    }
+
+    oneOffset++;
+  }
+
+  return length;
+}

web-ui/src/mpegts/demux/h264.ts

@@ -1,4 +1,5 @@
 import Log from "../utils/logger";
+import { findAnnexBStartCodeOffset } from "./annexb";
 
 export enum H264NaluType {
   kUnspecified = 0,
@@ -57,24 +58,11 @@ export class H264AnnexBParser {
   }
 
   private findNextStartCodeOffset(start_offset: number) {
-    let i = start_offset;
-    const data = this.data_;
-
-    while (true) {
-      if (i + 3 >= data.byteLength) {
-        this.eof_flag_ = true;
-        return data.byteLength;
-      }
-
-      // search 00 00 00 01 or 00 00 01
-      const uint32 = (data[i + 0] << 24) | (data[i + 1] << 16) | (data[i + 2] << 8) | data[i + 3];
-      const uint24 = (data[i + 0] << 16) | (data[i + 1] << 8) | data[i + 2];
-      if (uint32 === 0x00000001 || uint24 === 0x000001) {
-        return i;
-      } else {
-        i++;
-      }
+    const offset = findAnnexBStartCodeOffset(this.data_, start_offset);
+    if (offset === this.data_.byteLength) {
+      this.eof_flag_ = true;
     }
+    return offset;
   }
 
   public readNextNaluPayload(): H264NaluPayload | null {

web-ui/src/mpegts/demux/h265.ts

@@ -1,4 +1,5 @@
 import Log from "../utils/logger";
+import { findAnnexBStartCodeOffset } from "./annexb";
 
 export enum H265NaluType {
   kSliceIDR_W_RADL = 19,
@@ -49,24 +50,11 @@ export class H265AnnexBParser {
   }
 
   private findNextStartCodeOffset(start_offset: number) {
-    let i = start_offset;
-    const data = this.data_;
-
-    while (true) {
-      if (i + 3 >= data.byteLength) {
-        this.eof_flag_ = true;
-        return data.byteLength;
-      }
-
-      // search 00 00 00 01 or 00 00 01
-      const uint32 = (data[i + 0] << 24) | (data[i + 1] << 16) | (data[i + 2] << 8) | data[i + 3];
-      const uint24 = (data[i + 0] << 16) | (data[i + 1] << 8) | data[i + 2];
-      if (uint32 === 0x00000001 || uint24 === 0x000001) {
-        return i;
-      } else {
-        i++;
-      }
+    const offset = findAnnexBStartCodeOffset(this.data_, start_offset);
+    if (offset === this.data_.byteLength) {
+      this.eof_flag_ = true;
     }
+    return offset;
   }
 
   public readNextNaluPayload(): H265NaluPayload | null {

web-ui/src/mpegts/demux/ts-demuxer.ts

@@ -401,6 +401,7 @@ class TSDemuxer {
     }
 
     if (this.isAudioPid(pid)) {
+      this.flushMediaBeforeTrackDiscontinuity();
       this.resetAudioParserState();
       Log.w(this.TAG, `Audio TS discontinuity on pid ${pid}: ${reason}; resetting audio parser state`);
       this.onTrackDiscontinuity?.("audio");
@@ -1312,7 +1313,7 @@ class TSDemuxer {
     }
     if (this.isInitSegmentDispatched()) {
       if (this.video_track_.length) {
-        this.onDataAvailable?.(null, this.video_track_);
+        this.onDataAvailable?.(null, this.video_track_, true);
       }
     }
   }
@@ -1323,7 +1324,7 @@ class TSDemuxer {
     }
     if (this.isInitSegmentDispatched()) {
       if (this.audio_track_.length) {
-        this.onDataAvailable?.(this.audio_track_, null);
+        this.onDataAvailable?.(this.audio_track_, null, true);
       }
     }
   }

web-ui/src/mpegts/remux/media-batch.ts

@@ -0,0 +1,58 @@
+export const DEFAULT_MEDIA_SEGMENT_BATCH_DURATION_MS = 250;
+export const DEFAULT_MEDIA_SEGMENT_BATCH_MAX_BYTES = 512 * 1024;
+
+export function normalizeMediaBatchLimit(value: number | undefined, fallback: number): number {
+  const resolved = value ?? fallback;
+  return Number.isFinite(resolved) ? Math.max(0, resolved) : fallback;
+}
+
+interface TimedSample {
+  dts: number;
+}
+
+export interface PendingMediaTrack {
+  samples: unknown[];
+  length: number;
+}
+
+function trackDtsSpan(track: PendingMediaTrack | null | undefined): number {
+  const samples = track?.samples as TimedSample[] | undefined;
+  if (!samples || samples.length < 2) {
+    return 0;
+  }
+
+  const firstDts = samples[0].dts;
+  const lastDts = samples[samples.length - 1].dts;
+  if (!Number.isFinite(firstDts) || !Number.isFinite(lastDts)) {
+    return 0;
+  }
+  return Math.max(0, lastDts - firstDts);
+}
+
+/**
+ * Decide whether pending demuxed samples are large enough to build the next
+ * fMP4 fragment. The caller handles initial low-latency fragments separately.
+ */
+export function isMediaBatchReady(
+  audioTrack: PendingMediaTrack | null | undefined,
+  videoTrack: PendingMediaTrack | null | undefined,
+  targetDurationMs: number,
+  maxBytes: number,
+): boolean {
+  const audioSampleCount = audioTrack?.samples.length ?? 0;
+  const videoSampleCount = videoTrack?.samples.length ?? 0;
+  if (audioSampleCount === 0 && videoSampleCount === 0) {
+    return false;
+  }
+
+  if (targetDurationMs <= 0) {
+    return true;
+  }
+
+  const pendingBytes = (audioTrack?.length ?? 0) + (videoTrack?.length ?? 0);
+  if (maxBytes > 0 && pendingBytes >= maxBytes) {
+    return true;
+  }
+
+  return trackDtsSpan(audioTrack) >= targetDurationMs || trackDtsSpan(videoTrack) >= targetDurationMs;
+}