From 7a3dd2e43c6b6ed81c497a9c14976a08ba5a1506 Mon Sep 17 00:00:00 2001 From: dillonstreator Date: Wed, 18 Nov 2026 11:26:31 -0600 Subject: [PATCH] Refuse live and upcoming stream links before downloading A live stream link announces no length, so the minutes check before the download was skipped and yt-dlp recorded the stream through the download proxy until it ended, with the project stuck in ingest and no progress. The link lookup now reads is_live/live_status and throws LiveSourceError for a live or scheduled stream; ingest fails it at once without a retry, and the download itself runs with --match-filter for when the lookup failed. check_source and project creation refuse it up front on accounts that already look the link up (minutes limits). Signed-off-by: dillonstreator --- .changeset/refuse-live-streams.md | 5 +++ apps/server/src/pipeline/ingest.ts | 10 ++++- apps/server/src/projects/peek.ts | 58 +++++++++++++++++++++++------ apps/server/src/projects/service.ts | 24 +++++++++--- apps/server/test/peek.test.ts | 26 +++++++++++-- docs/TROUBLESHOOTING.md | 5 +++ docs/n8n/README.md | 3 +- website/content/guides.ts | 2 +- 8 files changed, 109 insertions(+), 24 deletions(-) create mode 100644 .changeset/refuse-live-streams.md diff --git a/.changeset/refuse-live-streams.md b/.changeset/refuse-live-streams.md new file mode 100644 index 00000000..24767032 --- /dev/null +++ b/.changeset/refuse-live-streams.md @@ -0,0 +1,5 @@ +--- +"@reelsaw/server": patch +--- + +Links to a live stream, or a stream that has not started yet, are now refused with a message saying to add the link again once the recording is up. Before, ingest recorded the stream through the download proxy until the stream ended, which could take hours and use up proxy traffic. The project sat in ingest with no progress. The recording of a stream that has ended downloads as before. Accounts with a minutes limit see the refusal as soon as they check the link or create the project (check_source, the New project form). Every other account sees it when ingest starts, before anything is downloaded. diff --git a/apps/server/src/pipeline/ingest.ts b/apps/server/src/pipeline/ingest.ts index d94c1a98..d06cdb9c 100644 --- a/apps/server/src/pipeline/ingest.ts +++ b/apps/server/src/pipeline/ingest.ts @@ -7,7 +7,7 @@ import { schema } from "../db/index.js"; import { keys } from "../storage/index.js"; import { QUEUES } from "../queue/boss.js"; import { UnresolvableHostError, UnsafeUrlError, assertPublicHttpUrl } from "../lib/safe-url.js"; -import { announcedDuration } from "../projects/peek.js"; +import { LIVE_SOURCE_MESSAGE, LiveSourceError, NOT_LIVE_FILTER, announcedDuration } from "../projects/peek.js"; import { type StageContext, DownloadFailedError, NonRetryableError, SourceUnavailableError, exists, sourceScratchMB } from "./common.js"; import { count, histogram, realtimeFactor } from "../metrics.js"; import { sha256File } from "./bumpers.js"; @@ -87,6 +87,9 @@ export async function ingest(ctx: StageContext) { "bv*[vcodec^=avc1][height<=1080][ext=mp4]+ba[ext=m4a]/bv*[height<=1080][ext=mp4]+ba[ext=m4a]/b[height<=1080][ext=mp4]/bv*[height<=1080]+ba/b", "--merge-output-format", "mp4", + // a live stream would be recorded until it ends; skip it here too, for when the lookup above could not tell + "--match-filter", + NOT_LIVE_FILTER, "--max-filesize", `${svc.settings.get().MAX_SOURCE_FILE_MB}M`, "-o", @@ -142,6 +145,8 @@ export async function ingest(ctx: StageContext) { : undefined, }; } + // --match-filter skipped the link: no info line, no file, exit 0 + if (!info) throw new NonRetryableError(LIVE_SOURCE_MESSAGE); const files = (await fs.readdir(dir)).filter((f) => f.startsWith("source.")); const chosen = files.find((f) => f.endsWith(".mp4")) ?? files[0]; if (!chosen) { @@ -233,7 +238,7 @@ export async function ingest(ctx: StageContext) { /** * `announcedDuration` for the stage, and why it is missing (the site did not say, or the lookup failed): only a - * cancelled stage or a permanently dead link stops here. + * cancelled stage, a permanently dead link or a live stream stops here. */ async function announcedDurationOrNull(ctx: StageContext, url: string): Promise<{ sec: number | null; missing: "unannounced" | "failed" | null }> { try { @@ -241,6 +246,7 @@ async function announcedDurationOrNull(ctx: StageContext, url: string): Promise< return { sec: sec || null, missing: sec ? null : "unannounced" }; } catch (err) { ctx.signal.throwIfAborted(); + if (err instanceof LiveSourceError) throw new NonRetryableError(err.message); // the lookup already walked every proxy: a permanent answer here would be the download's answer too const stderr = err instanceof CommandError ? err.stderr : ""; const text = `${err instanceof Error ? err.message : String(err)}\n${stderr}`; diff --git a/apps/server/src/projects/peek.ts b/apps/server/src/projects/peek.ts index 8eed3834..cd2be39b 100644 --- a/apps/server/src/projects/peek.ts +++ b/apps/server/src/projects/peek.ts @@ -29,19 +29,42 @@ export async function peekUrlMeta(url: string): Promise<{ title?: string | undef } } +/** What every surface says when a link is a live or scheduled stream: there is no end to download up to. */ +export const LIVE_SOURCE_MESSAGE = + "This link is a live stream (or one that has not started yet), which never finishes downloading. Add it again once the stream has ended and its recording is up, or upload the file instead (New project → Upload)."; + +/** The link is a live or scheduled stream: refused before anything is downloaded, on every attempt alike. */ +export class LiveSourceError extends Error { + constructor() { + super(LIVE_SOURCE_MESSAGE); + this.name = "LiveSourceError"; + } +} + +/** + * yt-dlp `--match-filter` that skips a live or scheduled stream instead of recording it until the stream ends. A skipped + * link exits 0 with no info line and no file, so the download step reads that as this refusal. + */ +export const NOT_LIVE_FILTER = "!is_live & live_status!=is_upcoming"; + /** - * The length yt-dlp reports for a link without downloading it, in seconds; null when it does not say (live streams, some - * sites). Throws when the lookup itself fails. Used when a link is checked (check_source, the New project form), when the - * project is created (a refusal with the upgrade offer instead of a failed project) and again before ingest downloads; the - * binding check still runs on the probed file. + * The length yt-dlp reports for a link without downloading it, in seconds; null when it does not say (some sites). + * Throws `LiveSourceError` for a live or scheduled stream (no length, and a download that would run until the stream + * ends), and the lookup's own error when it fails. Used when a link is checked (check_source, the New project form), when + * the project is created (a refusal with the upgrade offer instead of a failed project) and again before ingest downloads; + * the binding check still runs on the probed file. * * Those three ask about the same link within seconds of each other, and each lookup is a yt-dlp run of a few seconds * (through the download proxies, when set), so an answer is reused for `LENGTH_REUSE_MS`. A failed lookup is not kept: * the next caller tries again. */ export async function announcedDuration(media: Pick, url: string, signal?: AbortSignal): Promise { - const known = lengths.get(url); - if (known) return known.sec; + const known = lengths.get(url) ?? (await lookUp(media, url, signal)); + if (known.live) throw new LiveSourceError(); + return known.sec; +} + +async function lookUp(media: Pick, url: string, signal?: AbortSignal): Promise { const { stdout } = await media.ytdlp(["--no-playlist", "--no-warnings", "--no-download", "--ignore-no-formats-error", "--dump-single-json", "--", url], { signal, }); @@ -49,16 +72,29 @@ export async function announcedDuration(media: Pick, url: s .split("\n") .reverse() .find((l) => l.trim().startsWith("{")); - const duration = jsonLine ? z.object({ duration: z.number().positive() }).safeParse(JSON.parse(jsonLine)) : null; - const sec = duration?.success ? duration.data.duration : null; - lengths.set(url, { sec }); - return sec; + const parsed = jsonLine ? announcedSchema.safeParse(JSON.parse(jsonLine)) : null; + const info = parsed?.success ? parsed.data : {}; + const answer = { sec: info.duration ?? null, live: info.is_live === true || info.live_status === "is_live" || info.live_status === "is_upcoming" }; + lengths.set(url, answer); + return answer; +} + +/** The fields of a yt-dlp info dict the lookup reads; anything malformed counts as not said. */ +const announcedSchema = z.object({ + duration: z.number().positive().nullish().catch(null), + is_live: z.boolean().nullish().catch(null), + live_status: z.string().nullish().catch(null), +}); + +interface Announced { + sec: number | null; + live: boolean; } /** How long a link's looked-up length is reused: long enough to span checking a link, creating it and ingest starting. */ const LENGTH_REUSE_MS = 10 * 60_000; /** Each link's answer, boxed so a link that announces no length is kept too (`sec: null`); least recently used goes first. */ -const lengths = new LRUCache({ max: 500, ttl: LENGTH_REUSE_MS }); +const lengths = new LRUCache({ max: 500, ttl: LENGTH_REUSE_MS }); /** Drop every remembered length (tests). */ export function forgetAnnouncedDurations() { diff --git a/apps/server/src/projects/service.ts b/apps/server/src/projects/service.ts index 472a0a6c..19f51a94 100644 --- a/apps/server/src/projects/service.ts +++ b/apps/server/src/projects/service.ts @@ -58,7 +58,7 @@ import { enqueueRefine, REFINE_QUEUE } from "../pipeline/refine.js"; import { resolveProjectSettings, userDefaults } from "../pipeline/settings.js"; import { syncProjectRefs } from "../library/refs.js"; import { assetVisibleTo } from "../library/visible.js"; -import { announcedDuration, peekUrlMeta } from "./peek.js"; +import { LIVE_SOURCE_MESSAGE, LiveSourceError, announcedDuration, peekUrlMeta } from "./peek.js"; import { runFacts } from "../pipeline/common.js"; import { deadlineIn, keysetLoop } from "../lib/batch.js"; import { getLayerTemplate } from "../library/templates.js"; @@ -872,7 +872,13 @@ export async function createProject(svc: Services, userId: string, input: Create sourceMeta = { uploadId: up.id, filename: up.filename }; title ||= up.filename.replace(/\.[^.]+$/, ""); } else if (input.sourceUrl) { - const [peek, announced] = await Promise.all([peekUrlMeta(input.sourceUrl), announcedMinutesFor(svc, userId, input.sourceUrl, log)]); + const [peek, announced] = await Promise.all([ + peekUrlMeta(input.sourceUrl), + announcedMinutesFor(svc, userId, input.sourceUrl, log).catch((err: unknown) => { + if (err instanceof LiveSourceError) throw new TRPCError({ code: "BAD_REQUEST", message: err.message }); + throw err; + }), + ]); // a video the account's minutes cannot cover is refused here, with the upgrade offer, not created and failed by ingest if (announced) await checkAnnouncedMinutes(db, svc.settings.get(), userId, announced, svc.entitlements).catch(quotaReached); sourceMeta = { ...(peek.thumbnail ? { thumbnail: peek.thumbnail } : {}), ...(peek.uploader ? { uploader: peek.uploader } : {}) }; @@ -914,7 +920,7 @@ export async function createProject(svc: Services, userId: string, input: Create /** * The length a link announces, in seconds, looked up only when the account has a minutes limit (no lookup, no wait, for * everyone else). Null when the site does not say, the link is not a public address, or the lookup is slow or fails: - * ingest checks again before downloading and on the file itself. + * ingest checks again before downloading and on the file itself. Throws `LiveSourceError` for a live or scheduled stream. */ async function announcedMinutesFor(svc: Services, userId: string, url: string, log: Services["log"]): Promise { if (!(await hasMinuteLimits(svc.database.db, svc.settings.get(), userId, svc.entitlements))) return null; @@ -922,6 +928,7 @@ async function announcedMinutesFor(svc: Services, userId: string, url: string, l await assertPublicHttpUrl(url, { allowPrivate: svc.settings.get().ALLOW_PRIVATE_SOURCE_URLS }); return await announcedDuration(svc.media, url, AbortSignal.timeout(ANNOUNCED_LOOKUP_MS)); } catch (err) { + if (err instanceof LiveSourceError) throw err; log.debug({ err }, "could not look up the video's length before creating the project"); return null; } @@ -939,7 +946,7 @@ export const checkSourceInput = z export interface SourceCheck { /** the video's length in minutes, when it is known (a link whose site does not say has none) */ minutes: number | null; - /** false: creating the project now would be refused (message and quota say why and what to do); null: the length is unknown, so ingest decides */ + /** false: creating the project now would be refused (message and quota say why and what to do; no quota for a live stream); null: the length is unknown, so ingest decides */ fits: boolean | null; message: string | null; quota: QuotaRefusal | null; @@ -960,8 +967,15 @@ export async function checkSource(svc: Services, userId: string, input: z.infer< const log = svc.log.child({ module: "projects", userId }); const [allowance, durationSec] = await Promise.all([ minutesAllowance(db, s, userId, svc.entitlements), - input.durationSec ?? (input.sourceUrl ? announcedMinutesFor(svc, userId, input.sourceUrl, log) : null), + input.durationSec ?? + (input.sourceUrl + ? announcedMinutesFor(svc, userId, input.sourceUrl, log).catch((err: unknown) => { + if (err instanceof LiveSourceError) return "live" as const; + throw err; + }) + : null), ]); + if (durationSec === "live") return { minutes: null, fits: false, message: LIVE_SOURCE_MESSAGE, quota: null, allowance }; const minutes = durationSec ? Math.round((durationSec / 60) * 10) / 10 : null; const refused = (err: QuotaError): SourceCheck => { svc.telemetry.capture("quota.warned", userId, refusalEvent(err.limit, surface, err.quota, err.props)); diff --git a/apps/server/test/peek.test.ts b/apps/server/test/peek.test.ts index c9225aa9..36f6013c 100644 --- a/apps/server/test/peek.test.ts +++ b/apps/server/test/peek.test.ts @@ -1,9 +1,9 @@ import { beforeEach, describe, it } from "node:test"; import assert from "node:assert/strict"; -import { announcedDuration, forgetAnnouncedDurations } from "../src/projects/peek.js"; +import { LiveSourceError, announcedDuration, forgetAnnouncedDurations } from "../src/projects/peek.js"; -/** A yt-dlp that answers the metadata lookup with `answer` (a duration, null for none, or an error) and counts its runs. */ -function fakeYtdlp(answer: () => number | null | Error) { +/** A yt-dlp that answers the metadata lookup with `answer` (a duration, null for none, an info dict, or an error) and counts its runs. */ +function fakeYtdlp(answer: () => number | null | Error | Record) { const runs: string[] = []; return { runs, @@ -12,7 +12,8 @@ function fakeYtdlp(answer: () => number | null | Error) { runs.push(args[args.length - 1]!); const a = answer(); if (a instanceof Error) throw a; - const stdout = `[info] noise\n${JSON.stringify(a === null ? { id: "x" } : { id: "x", duration: a })}\n`; + const info = a === null ? { id: "x" } : typeof a === "number" ? { id: "x", duration: a } : { id: "x", ...a }; + const stdout = `[info] noise\n${JSON.stringify(info)}\n`; return { stdout, stderr: "", proxyId: "direct" }; }, }, @@ -54,4 +55,21 @@ describe("announcedDuration", () => { assert.equal(await announcedDuration(y.media as never, LINK), 90); assert.equal(y.runs.length, 2); }); + + it("refuses a live stream, remembered like any other answer", async () => { + const y = fakeYtdlp(() => ({ is_live: true, live_status: "is_live", duration: null })); + await assert.rejects(announcedDuration(y.media as never, LINK), LiveSourceError); + await assert.rejects(announcedDuration(y.media as never, LINK), LiveSourceError); + assert.equal(y.runs.length, 1); + }); + + it("refuses a stream that has not started yet", async () => { + const y = fakeYtdlp(() => ({ is_live: false, live_status: "is_upcoming" })); + await assert.rejects(announcedDuration(y.media as never, LINK), LiveSourceError); + }); + + it("accepts the recording of a stream that has ended", async () => { + const y = fakeYtdlp(() => ({ is_live: false, live_status: "was_live", was_live: true, duration: 5400 })); + assert.equal(await announcedDuration(y.media as never, LINK), 5400); + }); }); diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index de318461..d666103d 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -334,6 +334,11 @@ whisper-1 on a 10-minute chunk looped `"no"` 2,270 times after a spoken "No, no, - **Deploys interrupt jobs.** Rebuilding the app container kills in-flight renders; pg-boss retries them, and each interruption consumes a retry (render limit is 4 for this reason; `updateQueue` syncs limits on boot). Check `reelsaw status` for live jobs before deploying while someone is watching a project. +- **Live streams are refused.** A link to a stream that is still live or not yet started fails ingest at once + ("This link is a live stream…") instead of recording it until the stream ends through the download proxy. The + link lookup reads `is_live` / `live_status`, and the download itself runs with `--match-filter` for when the + lookup failed. On a minutes-limited account, check_source and creating the project refuse it up front too. + The recording of an ended stream downloads like any video. - **YouTube bot-check from a datacentre / `YTDLP_PROXY`.** yt-dlp takes one `--proxy`; Reelsaw walks an operator-ranked pool in the same variable (one URL still works). **Priority** is yours and never changes at runtime (put the cheap home Pi first). **Reliability** is a circuit breaker per candidate diff --git a/docs/n8n/README.md b/docs/n8n/README.md index fc374915..c7fb2897 100644 --- a/docs/n8n/README.md +++ b/docs/n8n/README.md @@ -214,7 +214,8 @@ tools and webhooks as a Zapier app. ## Notes - The RSS trigger fires once per new feed entry. Shorts and live streams appear in the feed too; add - an IF node on `title` or `link` before **Settings** to skip them. + an IF node on `title` or `link` before **Settings** to skip them. A stream that is still live (or not + started) is refused, not recorded: add it again once its recording is up. - Every new video costs source minutes on a hosted plan; quota errors fail the start execution. - Checked on n8n 1.118.2 (Docker) against a mock of `/api/mcp`, Buffer's endpoint and a share link: the start workflow's `create_project` call and metadata, the `create_webhook` branch, and signed diff --git a/website/content/guides.ts b/website/content/guides.ts index 59ad0316..1dd85e5a 100644 --- a/website/content/guides.ts +++ b/website/content/guides.ts @@ -392,7 +392,7 @@ export const GUIDES: Guide[] = [ }, { q: "Will it clip my Shorts and live streams too?", - a: "The channel feed lists every upload, including Shorts and finished streams. Add an IF node on the title or link before the Settings node to skip what you do not want clipped.", + a: "The channel feed lists every upload, including Shorts and streams. Reelsaw clips a stream once it has ended and its recording is up; a link to a stream that is still live, or has not started, is refused with a message saying so. Add an IF node on the title or link before the Settings node to skip what you do not want clipped.", }, { q: "Can I use it with Make, Zapier or a script instead of n8n?",