Skip to content

Commit 40367b3

Browse files
charliewwdevclaude
charliewwdev
and
claude
committed
fix I2V pipeline selection and F5-TTS-MLX API for trailer V2 production
- wan22 backend: use WanImageToVideoPipeline for I2V mode on all model variants, not just A14B (fixes TI2V-5B I2V generation) - audio.py: fix F5-TTS-MLX API params (generation_text, ref_audio_path, output_path) matching current f5-tts-mlx package - produce_trailer_v2.py: split pipeline into T2V (portraits) + I2V (shots) phases; add deploy_cloud.sh for Vast.ai GPU automation Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 8304438 commit 40367b3

4 files changed

Lines changed: 935 additions & 10 deletions

File tree

‎animatediff/backends/wan22.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,10 +103,9 @@ def load(
103103
load_kwargs["quantization_config"] = quant_config
104104

105105
# Choose pipeline class
106-
if mode == "i2v" and model_variant == "A14B":
106+
if mode == "i2v":
107107
PipelineClass = WanImageToVideoPipeline
108108
else:
109-
# TI2V-5B uses WanPipeline for both T2V and I2V
110109
PipelineClass = WanPipeline
111110

112111
pipe = PipelineClass.from_pretrained(model_path, **load_kwargs)

‎animatediff/postprocess/audio.py‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -127,15 +127,19 @@ def _generate_f5_mlx(self, text, output_path, ref_audio, ref_text):
127127
"""Generate speech using F5-TTS-MLX (Apple Silicon native)."""
128128
from f5_tts_mlx.generate import generate
129129

130-
audio = generate(
131-
text=text,
132-
ref_audio=ref_audio,
133-
ref_text=ref_text or "",
134-
)
130+
kwargs = dict(generation_text=text, output_path=output_path)
131+
if ref_audio:
132+
kwargs["ref_audio_path"] = ref_audio
133+
if ref_text:
134+
kwargs["ref_audio_text"] = ref_text
135+
136+
audio = generate(**kwargs)
137+
138+
# generate() saves to output_path if provided, but also returns audio
139+
if audio is not None and not os.path.exists(output_path):
140+
import soundfile as sf
141+
sf.write(output_path, audio, samplerate=24000)
135142

136-
# Save to file
137-
import soundfile as sf
138-
sf.write(output_path, audio, samplerate=24000)
139143
logger.info(f"Generated speech (MLX): {output_path}")
140144
return output_path
141145

‎scripts/deploy_cloud.sh‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
#!/bin/bash
2+
# ============================================================================
3+
# 凡人修仙传 Trailer V2 — Cloud Deployment (Vast.ai)
4+
#
5+
# Automated workflow:
6+
# 1. Search for RTX 5090 / 4090 instance
7+
# 2. Create instance with PyTorch + diffusers
8+
# 3. Upload code + narration audio
9+
# 4. Run portrait generation + I2V shot generation
10+
# 5. Download results
11+
# 6. Destroy instance
12+
#
13+
# Prerequisites:
14+
# vastai set api-key YOUR_KEY
15+
#
16+
# Usage:
17+
# bash scripts/deploy_cloud.sh
18+
# ============================================================================
19+
20+
set -euo pipefail
21+
22+
PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
23+
OUTPUT_DIR="${PROJECT_ROOT}/output/fanren-trailer-v2"
24+
REMOTE_WORK="/workspace/AnimateDiff"
25+
26+
# Colors
27+
RED='\033[0;31m'
28+
GREEN='\033[0;32m'
29+
YELLOW='\033[1;33m'
30+
NC='\033[0m'
31+
32+
echo -e "${GREEN}============================================${NC}"
33+
echo -e "${GREEN} 凡人修仙传 V2 — Cloud GPU Deployment${NC}"
34+
echo -e "${GREEN}============================================${NC}"
35+
36+
# ---- Step 1: Check API key ----
37+
if ! .venv/bin/vastai show user 2>/dev/null | grep -q "email"; then
38+
echo -e "${RED}Error: Vast.ai API key not set.${NC}"
39+
echo "Run: .venv/bin/vastai set api-key YOUR_KEY"
40+
exit 1
41+
fi
42+
echo -e "${GREEN}✓ Vast.ai authenticated${NC}"
43+
44+
# ---- Step 2: Find a GPU instance ----
45+
echo -e "\n${YELLOW}Searching for RTX 5090/4090 instances...${NC}"
46+
47+
# Search: RTX 5090 preferred, RTX 4090 fallback, >=24GB VRAM, cuda 12+
48+
INSTANCE_ID=""
49+
for gpu_name in "RTX_5090" "RTX_4090"; do
50+
echo " Trying ${gpu_name}..."
51+
OFFERS=$(.venv/bin/vastai search offers \
52+
"gpu_name=${gpu_name} num_gpus=1 gpu_ram>=24 inet_down>=500 disk_space>=80 reliability>0.95" \
53+
--order "dph_total" --limit 3 --raw 2>/dev/null || true)
54+
55+
if [ -n "$OFFERS" ] && [ "$OFFERS" != "[]" ]; then
56+
echo -e " ${GREEN}Found ${gpu_name} offers${NC}"
57+
echo "$OFFERS" | head -5
58+
break
59+
fi
60+
done
61+
62+
if [ -z "$OFFERS" ] || [ "$OFFERS" = "[]" ]; then
63+
echo -e "${RED}No suitable GPU instances found. Try again later.${NC}"
64+
exit 1
65+
fi
66+
67+
# ---- Step 3: Create instance ----
68+
echo -e "\n${YELLOW}Creating instance...${NC}"
69+
70+
# Use PyTorch nightly image with CUDA 12.8
71+
OFFER_ID=$(echo "$OFFERS" | python3 -c "import sys,json; data=json.load(sys.stdin); print(data[0]['id'])" 2>/dev/null)
72+
73+
if [ -z "$OFFER_ID" ]; then
74+
echo -e "${RED}Failed to parse offer ID. Please create instance manually:${NC}"
75+
echo " .venv/bin/vastai search offers 'gpu_name=RTX_4090 num_gpus=1 gpu_ram>=24'"
76+
echo " .venv/bin/vastai create instance <OFFER_ID> --image pytorch/pytorch:2.6.0-cuda12.6-cudnn9-devel --disk 80"
77+
echo ""
78+
echo "Then run the remote script manually:"
79+
echo " scp scripts/remote_generate.py <instance>:/workspace/"
80+
echo " ssh <instance> 'cd /workspace && python remote_generate.py'"
81+
exit 1
82+
fi
83+
84+
echo " Offer ID: ${OFFER_ID}"
85+
RESULT=$(.venv/bin/vastai create instance "${OFFER_ID}" \
86+
--image "pytorch/pytorch:2.6.0-cuda12.6-cudnn9-devel" \
87+
--disk 80 \
88+
--onstart-cmd "pip install diffusers transformers accelerate safetensors soundfile pillow" \
89+
--raw 2>/dev/null)
90+
91+
INSTANCE_ID=$(echo "$RESULT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('new_contract', ''))" 2>/dev/null)
92+
echo -e " ${GREEN}Instance created: ${INSTANCE_ID}${NC}"
93+
94+
# ---- Step 4: Wait for instance to start ----
95+
echo -e "\n${YELLOW}Waiting for instance to start...${NC}"
96+
for i in $(seq 1 60); do
97+
STATUS=$(.venv/bin/vastai show instance "${INSTANCE_ID}" --raw 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('actual_status', 'unknown'))" 2>/dev/null || echo "unknown")
98+
if [ "$STATUS" = "running" ]; then
99+
echo -e " ${GREEN}Instance running!${NC}"
100+
break
101+
fi
102+
echo " Status: ${STATUS} (${i}/60)..."
103+
sleep 10
104+
done
105+
106+
# Get SSH details
107+
SSH_CMD=$(.venv/bin/vastai ssh-url "${INSTANCE_ID}" 2>/dev/null)
108+
echo " SSH: ${SSH_CMD}"
109+
110+
# ---- Step 5: Upload files ----
111+
echo -e "\n${YELLOW}Uploading code and narration audio...${NC}"
112+
113+
# Create tarball of necessary files
114+
TARBALL="/tmp/animatediff_deploy.tar.gz"
115+
cd "${PROJECT_ROOT}"
116+
tar czf "${TARBALL}" \
117+
animatediff/ \
118+
scripts/produce_trailer_v2.py \
119+
scripts/generate_portraits.py \
120+
examples/fanren_trailer_v2.json \
121+
output/fanren-trailer-v2/narration/ \
122+
output/fanren-trailer-v2/storyboard_computed.json \
123+
output/fanren-trailer-v2/shot_timing.json \
124+
2>/dev/null
125+
126+
echo " Tarball: $(du -h ${TARBALL} | cut -f1)"
127+
128+
# Upload via SCP (extract SSH host/port from SSH URL)
129+
.venv/bin/vastai copy "${TARBALL}" "${INSTANCE_ID}:/workspace/deploy.tar.gz"
130+
.venv/bin/vastai execute "${INSTANCE_ID}" "cd /workspace && tar xzf deploy.tar.gz && echo 'Upload complete'"
131+
132+
# ---- Step 6: Run generation ----
133+
echo -e "\n${YELLOW}Running Phase 2 on GPU...${NC}"
134+
.venv/bin/vastai execute "${INSTANCE_ID}" \
135+
"cd /workspace && python scripts/produce_trailer_v2.py --phase 2" 2>&1
136+
137+
# ---- Step 7: Download results ----
138+
echo -e "\n${YELLOW}Downloading generated shots...${NC}"
139+
140+
.venv/bin/vastai execute "${INSTANCE_ID}" \
141+
"cd /workspace && tar czf /workspace/results.tar.gz output/fanren-trailer-v2/portraits/ output/fanren-trailer-v2/shots/"
142+
143+
.venv/bin/vastai copy "${INSTANCE_ID}:/workspace/results.tar.gz" "/tmp/results.tar.gz"
144+
cd "${PROJECT_ROOT}" && tar xzf /tmp/results.tar.gz
145+
146+
echo -e "${GREEN}Downloaded portraits and shots to ${OUTPUT_DIR}${NC}"
147+
148+
# ---- Step 8: Destroy instance ----
149+
echo -e "\n${YELLOW}Destroying instance to stop billing...${NC}"
150+
.venv/bin/vastai destroy instance "${INSTANCE_ID}"
151+
echo -e "${GREEN}Instance destroyed.${NC}"
152+
153+
# ---- Summary ----
154+
echo -e "\n${GREEN}============================================${NC}"
155+
echo -e "${GREEN} Phase 2 Complete!${NC}"
156+
echo -e "${GREEN}============================================${NC}"
157+
echo ""
158+
echo "Portraits: ${OUTPUT_DIR}/portraits/"
159+
echo "Shots: ${OUTPUT_DIR}/shots/"
160+
echo ""
161+
echo "Next: Run Phase 3 locally:"
162+
echo " python scripts/produce_trailer_v2.py --phase 3"

0 commit comments

Comments
 (0)