diff --git a/packages/timeline/src/ClapTimeline.tsx b/packages/timeline/src/ClapTimeline.tsx index cfaeb1aa..76a26730 100644 --- a/packages/timeline/src/ClapTimeline.tsx +++ b/packages/timeline/src/ClapTimeline.tsx @@ -5,6 +5,7 @@ import { Stats } from "@react-three/drei" import { TimelineControls, HorizontalScroller, + TimelineQuickActions, Timeline } from "@/components" import { ClapProject, isValidNumber } from "@aitube/clap" @@ -20,7 +21,7 @@ import { cn } from "./utils" import { TimelineCamera } from "./components/camera" import { useTimeline } from "./hooks" import { topBarTimeScaleHeight } from "./constants/themes" -import { TimelineStore } from "./types" +import { SegmentEditionStatus, TimelineStore } from "./types" export function ClapTimeline({ clap, @@ -69,12 +70,52 @@ export function ClapTimeline({ const handleMouseMove = (event: React.MouseEvent | React.TouchEvent) => { const timeline: TimelineStore = useTimeline.getState() - const { editedSegment } = timeline - - // do something based on the current status of the edited segment - // for instance if the edited segment is being grabbed, - // we are going to want to display the segments that are around it - // console.log(`TODO @julian: implement edit here`) + const { + editedSegment, + cellWidth, + durationInMsPerStep, + getVerticalCellPosition, + tracks, + scrollX, + moveSegment, + } = timeline + + if (editedSegment?.editionStatus === SegmentEditionStatus.DRAGGING && canvas) { + const rect = canvas.getBoundingClientRect() + const clientX = "touches" in event ? event.touches[0]?.clientX : event.clientX + const clientY = "touches" in event ? event.touches[0]?.clientY : event.clientY + + if (typeof clientX === "number" && typeof clientY === "number") { + const localX = clientX - rect.left + const localY = clientY - rect.top + const pointerTimeInMs = Math.round(((localX + scrollX) / cellWidth) * durationInMsPerStep) + const segmentDurationInMs = editedSegment.endTimeInMs - editedSegment.startTimeInMs + const startTimeInMs = Math.max( + 0, + pointerTimeInMs - Math.round(segmentDurationInMs / 2) + ) + + let track = editedSegment.track + const trackY = Math.max(0, localY - topBarTimeScaleHeight) + let top = 0 + for (const candidate of tracks) { + const height = timeline.getCellHeight(candidate.id) + const trackTop = top + const trackBottom = top + height + if (trackY >= trackTop && trackY <= trackBottom) { + track = candidate.id + break + } + top = getVerticalCellPosition(0, candidate.id + 1) + } + + moveSegment({ + segment: editedSegment, + startTimeInMs, + track, + }) + } + } // since we are un frameloop="demand" mode, we need to manual invalidate the scene invalidate() @@ -83,6 +124,13 @@ export function ClapTimeline({ return false } + const handlePointerRelease = () => { + const { editedSegment, setEditedSegment } = useTimeline.getState() + if (editedSegment?.editionStatus === SegmentEditionStatus.DRAGGING) { + setEditedSegment({ segment: undefined }) + } + } + const handleWheel = (event: React.WheelEvent) => { const rect = canvas?.getBoundingClientRect() if (!rect) { return } @@ -110,7 +158,7 @@ export function ClapTimeline({ return (
@@ -121,6 +169,7 @@ export function ClapTimeline({ {({ height, width }: Size) => (
+ { @@ -155,6 +204,8 @@ export function ClapTimeline({ onWheel={handleWheel} onMouseMove={handleMouseMove} onTouchMove={handleMouseMove} + onMouseUp={handlePointerRelease} + onTouchEnd={handlePointerRelease} > (ClapSegmentCategory.VIDEO) + const addTrack = useTimeline((s) => s.addTrack) + const createClip = useTimeline((s) => s.createClip) + const cursorTimestampAtInMs = useTimeline((s) => s.cursorTimestampAtInMs) + + return ( +
event.stopPropagation()} + onClick={(event) => event.stopPropagation()} + > + + + +
+ ) +} diff --git a/packages/timeline/src/components/controls/index.ts b/packages/timeline/src/components/controls/index.ts index 05c2eb0e..108d9032 100644 --- a/packages/timeline/src/components/controls/index.ts +++ b/packages/timeline/src/components/controls/index.ts @@ -1 +1,2 @@ -export { TimelineControls } from "./TimelineControls" \ No newline at end of file +export { TimelineControls } from "./TimelineControls" +export { TimelineQuickActions } from "./TimelineQuickActions" diff --git a/packages/timeline/src/components/index.ts b/packages/timeline/src/components/index.ts index f5ff2aad..d4184ed1 100644 --- a/packages/timeline/src/components/index.ts +++ b/packages/timeline/src/components/index.ts @@ -7,6 +7,6 @@ export { type SpecializedCellProps } from "./cells" -export { TimelineControls } from "./controls" +export { TimelineControls, TimelineQuickActions } from "./controls" export { HorizontalScroller, VerticalScroller } from "./scroller" -export { Timeline, TopBarTimeScale, Cells, Grid, type JumpAt } from "./timeline" \ No newline at end of file +export { Timeline, TopBarTimeScale, Cells, Grid, type JumpAt } from "./timeline" diff --git a/packages/timeline/src/hooks/useTimeline.ts b/packages/timeline/src/hooks/useTimeline.ts index c5276f4a..d48c925e 100644 --- a/packages/timeline/src/hooks/useTimeline.ts +++ b/packages/timeline/src/hooks/useTimeline.ts @@ -1,7 +1,7 @@ import { create } from "zustand" import * as THREE from "three" import type { ThreeEvent } from "@react-three/fiber" -import { ClapProject, ClapSegment, ClapSegmentCategory, isValidNumber, newClap, serializeClap, ClapTracks, ClapEntity, ClapMeta } from "@aitube/clap" +import { ClapProject, ClapSegment, ClapSegmentCategory, ClapOutputType, isValidNumber, newClap, newSegment, serializeClap, ClapTracks, ClapEntity, ClapMeta } from "@aitube/clap" import { TimelineSegment, SegmentEditionStatus, SegmentVisibility, TimelineStore, SegmentArea, SegmentPointerEvent, SegmentEventCallbackHandler, Invalidate } from "@/types/timeline" import { getDefaultProjectState, getDefaultState } from "@/utils/getDefaultState" @@ -14,6 +14,27 @@ import { IsPlaying, JumpAt, TimelineCursorImpl, TogglePlayback } from "@/compone import { computeContentSizeMetrics } from "@/compute/computeContentSizeMetrics" import { topBarTimeScaleHeight } from "@/constants/themes" +const DEFAULT_NEW_CLIP_DURATION_IN_MS = 2000 + +function isPreviewCategory(category: ClapSegmentCategory): boolean { + return category === ClapSegmentCategory.IMAGE || category === ClapSegmentCategory.VIDEO +} + +function outputTypeForCategory(category: ClapSegmentCategory): ClapOutputType { + switch (category) { + case ClapSegmentCategory.IMAGE: + return ClapOutputType.IMAGE + case ClapSegmentCategory.VIDEO: + return ClapOutputType.VIDEO + case ClapSegmentCategory.MUSIC: + case ClapSegmentCategory.SOUND: + case ClapSegmentCategory.DIALOGUE: + return ClapOutputType.AUDIO + default: + return ClapOutputType.TEXT + } +} + export const useTimeline = create((set, get) => ({ ...getDefaultState(), @@ -616,10 +637,14 @@ export const useTimeline = create((set, get) => ({ segment, status: SegmentEditionStatus.EDITING }) + } else if (eventType === SegmentPointerEvent.MOVE) { + setHoveredSegment({ + segment, + area + }) } else if ( eventType === SegmentPointerEvent.CLICK || - eventType === SegmentPointerEvent.DOWN || - eventType === SegmentPointerEvent.MOVE + eventType === SegmentPointerEvent.DOWN ) { setHoveredSegment({ segment, @@ -730,6 +755,53 @@ export const useTimeline = create((set, get) => ({ }) }) }, + addTrack: ({ + category = ClapSegmentCategory.GENERIC + }: { + category?: ClapSegmentCategory + } = {}): number => { + const { + width, + height, + tracks, + cellWidth, + defaultSegmentDurationInSteps, + durationInMsPerStep, + durationInMs, + defaultPreviewHeight, + defaultCellHeight, + atLeastOneSegmentChanged: previousAtLeastOneSegmentChanged, + allSegmentsChanged: previousAllSegmentsChanged, + } = get() + + const newTrackId = tracks.length + const isPreview = isPreviewCategory(category) + const newTracks = tracks.concat({ + id: newTrackId, + name: `${category}`, + isPreview, + height: isPreview ? defaultPreviewHeight : defaultCellHeight, + hue: 0, + occupied: false, + visible: true, + }) + + set({ + atLeastOneSegmentChanged: previousAtLeastOneSegmentChanged + 1, + allSegmentsChanged: previousAllSegmentsChanged + 1, + ...computeContentSizeMetrics({ + width, + height, + tracks: newTracks, + cellWidth, + defaultSegmentDurationInSteps, + durationInMsPerStep, + durationInMs, + }), + }) + + return newTrackId + }, setContainerSize: ({ width, height }: { width: number; height: number }) => { const { containerWidth: previousWidth, containerHeight: previousHeight } = get() const changed = @@ -901,12 +973,10 @@ export const useTimeline = create((set, get) => ({ let nbTracks = tracks.length + const isPreview = isPreviewCategory(segment.category) + // add the track if it is missing if (!tracks[segment.track]) { - const isPreview = - segment.category === ClapSegmentCategory.IMAGE || - segment.category === ClapSegmentCategory.VIDEO - tracks[segment.track] = { id: segment.track, // name: `Track ${s.track}`, @@ -920,6 +990,24 @@ export const useTimeline = create((set, get) => ({ occupied: true, visible: true, } + } else { + const targetTrack = tracks[segment.track] + const targetTrackIsEmpty = !targetTrack.occupied || targetTrack.name === "(empty)" + if (targetTrackIsEmpty) { + tracks[segment.track] = { + ...targetTrack, + name: `${segment.category}`, + isPreview, + height: isPreview ? defaultPreviewHeight : defaultCellHeight, + occupied: true, + } + } else if (targetTrack.name !== segment.category) { + tracks[segment.track] = { + ...targetTrack, + name: "(misc)", + occupied: true, + } + } } if (triggerChange) { @@ -1038,6 +1126,159 @@ export const useTimeline = create((set, get) => ({ }) }) }, + createClip: async ({ + category = ClapSegmentCategory.GENERIC, + startTimeInMs, + track, + }: { + category?: ClapSegmentCategory + startTimeInMs?: number + track?: number + } = {}): Promise => { + const { + addSegment, + addTrack, + cursorTimestampAtInMs, + durationInMsPerStep, + defaultSegmentDurationInSteps, + segments, + tracks, + } = get() + + const safeStartTimeInMs = isValidNumber(startTimeInMs) + ? startTimeInMs! + : cursorTimestampAtInMs + const durationInMs = durationInMsPerStep * defaultSegmentDurationInSteps || DEFAULT_NEW_CLIP_DURATION_IN_MS + const safeEndTimeInMs = safeStartTimeInMs + durationInMs + const trackHasCollision = (candidateTrack: number) => segments.some((existingSegment) => ( + existingSegment.track === candidateTrack && + safeStartTimeInMs < existingSegment.endTimeInMs && + safeEndTimeInMs > existingSegment.startTimeInMs + )) + const matchingTrack = tracks.find((candidateTrack) => ( + candidateTrack.name === category && + !trackHasCollision(candidateTrack.id) + )) + const emptyTrack = tracks.find((candidateTrack) => ( + (!candidateTrack.occupied || candidateTrack.name === "(empty)") && + !trackHasCollision(candidateTrack.id) + )) + const requestedTrack = isValidNumber(track) ? tracks[track!] : undefined + const requestedTrackIsAllowed = requestedTrack + ? ( + !trackHasCollision(requestedTrack.id) && + (!requestedTrack.occupied || requestedTrack.name === "(empty)" || requestedTrack.name === category) + ) + : false + const safeTrack = requestedTrackIsAllowed + ? requestedTrack!.id + : matchingTrack?.id ?? emptyTrack?.id ?? addTrack({ category }) + + const segment = await clapSegmentToTimelineSegment(newSegment({ + track: safeTrack, + startTimeInMs: safeStartTimeInMs, + endTimeInMs: safeEndTimeInMs, + assetDurationInMs: durationInMs, + category, + outputType: outputTypeForCategory(category), + prompt: `New ${category.toLowerCase()} clip`, + label: `New ${category}`, + createdBy: "user", + editedBy: "user", + })) + + await addSegment({ + segment, + startTimeInMs: safeStartTimeInMs, + track: safeTrack, + }) + + return segment + }, + moveSegment: ({ + segment, + startTimeInMs, + track, + }: { + segment: TimelineSegment + startTimeInMs: number + track: number + }): void => { + const { + width, + height, + tracks, + cellWidth, + defaultSegmentDurationInSteps, + durationInMsPerStep, + durationInMs, + segments, + atLeastOneSegmentChanged: previousAtLeastOneSegmentChanged, + allSegmentsChanged: previousAllSegmentsChanged, + silentChangesInSegment, + defaultPreviewHeight, + defaultCellHeight, + } = get() + + const targetTrack = tracks[track] + if (!targetTrack) { + return + } + + const targetTrackIsEmpty = !targetTrack.occupied || targetTrack.name === "(empty)" + const targetTrackMatchesCategory = targetTrack.name === segment.category + if (!targetTrackIsEmpty && !targetTrackMatchesCategory) { + return + } + + const previousTrack = segment.track + const duration = segment.endTimeInMs - segment.startTimeInMs + segment.startTimeInMs = Math.max(0, startTimeInMs) + segment.endTimeInMs = segment.startTimeInMs + duration + segment.track = track + + if (previousTrack !== track && tracks[previousTrack]) { + const previousTrackStillHasSegments = segments.some((existingSegment) => ( + existingSegment.id !== segment.id && existingSegment.track === previousTrack + )) + if (!previousTrackStillHasSegments) { + tracks[previousTrack] = { + ...tracks[previousTrack], + occupied: false, + } + } + } + + if (targetTrackIsEmpty) { + const isPreview = isPreviewCategory(segment.category) + tracks[track] = { + ...targetTrack, + name: `${segment.category}`, + isPreview, + height: isPreview ? defaultPreviewHeight : defaultCellHeight, + occupied: true, + } + } + + silentChangesInSegment[segment.id] = 1 + (silentChangesInSegment[segment.id] || 0) + + set({ + segments: [...segments], + silentChangesInSegment, + atLeastOneSegmentChanged: previousAtLeastOneSegmentChanged + 1, + allSegmentsChanged: previousAllSegmentsChanged + 1, + durationInMs: Math.max(durationInMs, segment.endTimeInMs), + ...computeContentSizeMetrics({ + width, + height, + tracks, + cellWidth, + defaultSegmentDurationInSteps, + durationInMsPerStep, + durationInMs: Math.max(durationInMs, segment.endTimeInMs), + }), + }) + }, findFreeTrack: ({ startTimeInMs, endTimeInMs diff --git a/packages/timeline/src/types/timeline.ts b/packages/timeline/src/types/timeline.ts index 71f9a4f7..a8b53b73 100644 --- a/packages/timeline/src/types/timeline.ts +++ b/packages/timeline/src/types/timeline.ts @@ -1,6 +1,6 @@ import * as THREE from "three" import type { ThreeEvent } from "@react-three/fiber" -import { ClapEntity, ClapImageRatio, ClapMeta, ClapProject, ClapScene, ClapSegment, ClapTracks } from "@aitube/clap" +import { ClapEntity, ClapImageRatio, ClapMeta, ClapProject, ClapScene, ClapSegment, ClapSegmentCategory, ClapTracks } from "@aitube/clap" import { ClapSegmentColorScheme, ClapTimelineTheme } from "./theme" import { TimelineControlsImpl } from "@/components/controls/types" @@ -327,6 +327,7 @@ export type TimelineStoreModifiers = { setScrollX: (scrollX: number) => void handleMouseWheel: ({ deltaX, deltaY }: { deltaX: number; deltaY: number }) => void toggleTrackVisibility: (trackId: number) => void + addTrack: (params?: { category?: ClapSegmentCategory }) => number setContainerSize: ({ width, height }: { width: number; height: number }) => void setTimelineCursor: (timelineCursor?: TimelineCursorImpl) => void setIsDraggingCursor: (isDraggingCursor: boolean) => void @@ -378,6 +379,16 @@ export type TimelineStoreModifiers = { startTimeInMs?: number track?: number }) => Promise + createClip: (params?: { + category?: ClapSegmentCategory + startTimeInMs?: number + track?: number + }) => Promise + moveSegment: (params: { + segment: TimelineSegment + startTimeInMs: number + track: number + }) => void /** * Find an available free track diff --git a/packages/timeline/src/utils/clapSegmentToTimelineSegment.ts b/packages/timeline/src/utils/clapSegmentToTimelineSegment.ts index b90181fe..d4f62ddb 100644 --- a/packages/timeline/src/utils/clapSegmentToTimelineSegment.ts +++ b/packages/timeline/src/utils/clapSegmentToTimelineSegment.ts @@ -36,7 +36,7 @@ export async function clapSegmentToTimelineSegment(clapSegment: ClapSegment): Pr segment.colors = getSegmentColorScheme(segment) - if (!segment.audioBuffer) { + if (!segment.audioBuffer && segment.assetUrl) { if (segment.outputType === ClapOutputType.AUDIO) { try { segment.audioBuffer = await getAudioBuffer(segment.assetUrl) @@ -47,4 +47,4 @@ export async function clapSegmentToTimelineSegment(clapSegment: ClapSegment): Pr } return segment -} \ No newline at end of file +}