From 8c95f3da99f0ee375d441c6f0468dd45465dfd5b Mon Sep 17 00:00:00 2001 From: calderbuild Date: Thu, 19 Feb 2026 15:50:17 +0800 Subject: [PATCH] feat: complete AgentCut multi-agent video production pipeline 6 specialized AI agents collaborate through a structured SOP pipeline: Director (planning) -> Script (writing) -> Visual+Voice+Music (parallel generation) -> Editor (ffmpeg compositing) Backend: FastAPI with SSE streaming, MiniMax API integration (Hailuo video, Speech-2.6-HD TTS, Music-2.0, M1 LLM) Frontend: single-page app with real-time pipeline visualization --- .gitignore | 5 + README.md | 103 +++++++++++ backend/__init__.py | 0 backend/agents/__init__.py | 0 backend/agents/director.py | 71 ++++++++ backend/agents/editor.py | 166 ++++++++++++++++++ backend/agents/music.py | 45 +++++ backend/agents/script.py | 68 ++++++++ backend/agents/visual.py | 92 ++++++++++ backend/agents/voice.py | 75 ++++++++ backend/config.py | 16 ++ backend/main.py | 166 ++++++++++++++++++ backend/pipeline.py | 107 ++++++++++++ backend/requirements.txt | 6 + backend/test_minimax.py | 140 +++++++++++++++ frontend/index.html | 346 +++++++++++++++++++++++++++++++++++++ 16 files changed, 1406 insertions(+) create mode 100644 README.md create mode 100644 backend/__init__.py create mode 100644 backend/agents/__init__.py create mode 100644 backend/agents/director.py create mode 100644 backend/agents/editor.py create mode 100644 backend/agents/music.py create mode 100644 backend/agents/script.py create mode 100644 backend/agents/visual.py create mode 100644 backend/agents/voice.py create mode 100644 backend/config.py create mode 100644 backend/main.py create mode 100644 backend/pipeline.py create mode 100644 backend/requirements.txt create mode 100644 backend/test_minimax.py create mode 100644 frontend/index.html diff --git a/.gitignore b/.gitignore index 4d214f2..49dd8c3 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,8 @@ __pycache__/ *.pyc .next/ output/ +*.egg-info/ +dist/ +build/ +.venv/ +venv/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..6910535 --- /dev/null +++ b/README.md @@ -0,0 +1,103 @@ +# AgentCut - Multi-Agent AI Video Production + +AgentCut turns a single text prompt into a fully produced video using 6 specialized AI agents that collaborate through a structured pipeline. + +## Architecture + +``` +User Prompt + | + v +[Director Agent] -- Analyzes creative vision, plans shots + | + v +[Script Agent] -- Writes production-ready script with video prompts, narration, subtitles + | + v +[Visual Agent] ----+---- [Voice Agent] ----+---- [Music Agent] + (Hailuo Video) | (Speech-2.6-HD) | (Music-2.0) + v v + [Editor Agent] -- ffmpeg compositing: concat, mix, subtitle burn-in + | + v + Final MP4 +``` + +**Pipeline:** Director -> Script -> Visual + Voice + Music (parallel) -> Editor + +## Tech Stack + +- **LLM:** MiniMax M1 (chat completions for Director & Script agents) +- **Video:** MiniMax Hailuo 2.3 (1080P, 6s clips, async task polling) +- **Voice:** MiniMax Speech-2.6-HD (TTS with emotion control) +- **Music:** MiniMax Music-2.0 (instrumental background music) +- **Composition:** ffmpeg (concat, audio mixing, subtitle burn-in) +- **Backend:** Python FastAPI + SSE streaming +- **Frontend:** HTML + Tailwind CSS + +## Quick Start + +### Prerequisites + +- Python 3.10+ +- ffmpeg installed (`brew install ffmpeg` or `apt install ffmpeg`) +- MiniMax API key + +### Setup + +```bash +git clone https://github.com/calderbuild/agentcut.git +cd agentcut + +pip install -r backend/requirements.txt + +echo "MINIMAX_API_KEY=your-key-here" > .env +``` + +### Run + +```bash +python -m backend.main +``` + +Open http://localhost:8000 in your browser. + +### Usage + +1. Enter a video description (or pick a template) +2. Choose duration and shot count +3. Click "Start Production" +4. Watch the 6 agents work in real-time via the pipeline visualization +5. Download the final MP4 + +## API + +| Endpoint | Method | Description | +|---|---|---| +| `/api/create` | POST | Start a production job | +| `/api/stream/{job_id}` | GET | SSE event stream for real-time progress | +| `/api/status/{job_id}` | GET | Poll job status | +| `/api/download/{job_id}` | GET | Download final video | +| `/api/health` | GET | Health check | + +## Project Structure + +``` +backend/ + agents/ + director.py -- Creative director: prompt -> shot outline + script.py -- Scriptwriter: outline -> production script + visual.py -- Visual artist: script -> video clips (parallel) + voice.py -- Voice artist: script -> narration audio + music.py -- Composer: style+mood -> background music + editor.py -- Editor: ffmpeg composition + pipeline.py -- Agent orchestration + main.py -- FastAPI server + config.py -- Configuration +frontend/ + index.html -- Single-page app +``` + +## Built for + +[Return of the Agents Hackathon](https://aforehacks.org) by Afore Capital & AI Valley (Feb 2026) diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/agents/__init__.py b/backend/agents/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/agents/director.py b/backend/agents/director.py new file mode 100644 index 0000000..20c4866 --- /dev/null +++ b/backend/agents/director.py @@ -0,0 +1,71 @@ +"""Director Agent: understands user intent and creates a shot-by-shot outline.""" +import json +import requests +from backend.config import MINIMAX_API_KEY, MINIMAX_BASE_URL, HEADERS + + +SYSTEM_PROMPT = """You are the Director of an AI film production team. Your job is to take the user's creative idea and break it down into a detailed shot-by-shot outline for a short video. + +For each shot, provide: +1. shot_number: sequential number +2. duration: in seconds (total should match requested length) +3. visual_description: detailed description for AI video generation (style, camera angle, lighting, motion, subject) +4. narration: voiceover text for this shot (in English) +5. mood: emotional tone (e.g. "inspiring", "dramatic", "calm", "energetic") + +Also provide: +- title: a short title for the video +- style: overall visual style (e.g. "cinematic sci-fi", "warm documentary", "minimal modern") +- total_duration: total video length in seconds + +Respond ONLY with valid JSON. No markdown, no explanation. Example format: +{ + "title": "AI Writing Assistant", + "style": "cinematic sci-fi with neon accents", + "total_duration": 30, + "shots": [ + { + "shot_number": 1, + "duration": 10, + "visual_description": "A writer staring at a blank screen...", + "narration": "Every great story starts with a blank page.", + "mood": "contemplative" + } + ] +}""" + + +def run(user_prompt: str, duration: int = 30, num_shots: int = 3) -> dict: + """Generate a shot outline from user's creative prompt.""" + user_message = ( + f"Create a {duration}-second video with {num_shots} shots.\n" + f"Theme: {user_prompt}\n" + f"Each shot should be approximately {duration // num_shots} seconds." + ) + + url = f"{MINIMAX_BASE_URL}/text/chatcompletion_v2" + payload = { + "model": "MiniMax-M1", + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": user_message}, + ], + "temperature": 0.7, + "max_tokens": 2000, + } + + resp = requests.post(url, headers=HEADERS, json=payload) + resp.raise_for_status() + data = resp.json() + + content = data.get("choices", [{}])[0].get("message", {}).get("content", "") + + # Parse JSON from response (handle potential markdown wrapping) + content = content.strip() + if content.startswith("```"): + content = content.split("\n", 1)[1] if "\n" in content else content[3:] + if content.endswith("```"): + content = content[:-3] + content = content.strip() + + return json.loads(content) diff --git a/backend/agents/editor.py b/backend/agents/editor.py new file mode 100644 index 0000000..3479d35 --- /dev/null +++ b/backend/agents/editor.py @@ -0,0 +1,166 @@ +"""Editor Agent: composites video clips, voiceover, music, and subtitles via ffmpeg.""" +import os +import subprocess +import json +from backend.config import OUTPUT_DIR + + +def get_media_duration(filepath: str) -> float: + """Get duration of a media file in seconds using ffprobe.""" + result = subprocess.run( + ["ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", filepath], + capture_output=True, text=True, + ) + info = json.loads(result.stdout) + return float(info["format"]["duration"]) + + +def create_subtitle_file(shots: list, video_durations: list) -> str: + """Create an SRT subtitle file from shot data.""" + srt_path = os.path.join(OUTPUT_DIR, "subtitles.srt") + offset = 0.0 + + with open(srt_path, "w", encoding="utf-8") as f: + for i, shot in enumerate(shots): + duration = video_durations[i] if i < len(video_durations) else shot.get("duration", 6) + start = offset + end = offset + duration + subtitle_text = shot.get("subtitle", shot.get("narration", "")) + + f.write(f"{i + 1}\n") + f.write(f"{_format_time(start)} --> {_format_time(end)}\n") + f.write(f"{subtitle_text}\n\n") + + offset = end + + return srt_path + + +def _format_time(seconds: float) -> str: + """Format seconds to SRT time format HH:MM:SS,mmm.""" + h = int(seconds // 3600) + m = int((seconds % 3600) // 60) + s = int(seconds % 60) + ms = int((seconds % 1) * 1000) + return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" + + +def concat_videos(video_paths: list) -> str: + """Concatenate video clips into a single video.""" + concat_list = os.path.join(OUTPUT_DIR, "concat_list.txt") + with open(concat_list, "w") as f: + for path in video_paths: + f.write(f"file '{os.path.abspath(path)}'\n") + + concat_path = os.path.join(OUTPUT_DIR, "concat.mp4") + subprocess.run([ + "ffmpeg", "-y", "-f", "concat", "-safe", "0", + "-i", concat_list, + "-c", "copy", + concat_path, + ], check=True, capture_output=True) + return concat_path + + +def concat_audios(audio_paths: list) -> str: + """Concatenate audio clips into a single audio track.""" + concat_list = os.path.join(OUTPUT_DIR, "audio_concat_list.txt") + with open(concat_list, "w") as f: + for path in audio_paths: + f.write(f"file '{os.path.abspath(path)}'\n") + + concat_path = os.path.join(OUTPUT_DIR, "narration.mp3") + subprocess.run([ + "ffmpeg", "-y", "-f", "concat", "-safe", "0", + "-i", concat_list, + "-c", "copy", + concat_path, + ], check=True, capture_output=True) + return concat_path + + +def run( + video_results: list, + voice_results: list, + script: dict, + bgm_path: str = None, + on_progress=None, +) -> str: + """Composite final video with voiceover, music, and subtitles. Returns output path.""" + if on_progress: + on_progress("Starting video composition...", 0, 4) + + # Step 1: Concatenate video clips + video_paths = [r["video_path"] for r in video_results] + concat_video = concat_videos(video_paths) + if on_progress: + on_progress("Video clips merged", 1, 4) + + # Step 2: Concatenate voiceover audio + audio_paths = [r["audio_path"] for r in voice_results] + concat_audio = concat_audios(audio_paths) + if on_progress: + on_progress("Voiceover merged", 2, 4) + + # Step 3: Create subtitles + video_durations = [] + for vpath in video_paths: + try: + video_durations.append(get_media_duration(vpath)) + except Exception: + video_durations.append(6.0) + + srt_path = create_subtitle_file(script["shots"], video_durations) + if on_progress: + on_progress("Subtitles created", 3, 4) + + # Step 4: Final composition + output_path = os.path.join(OUTPUT_DIR, "final.mp4") + video_duration = get_media_duration(concat_video) + + filter_parts = [] + inputs = ["-i", concat_video, "-i", concat_audio] + audio_mix = "[1:a]" + + if bgm_path and os.path.exists(bgm_path): + inputs.extend(["-i", bgm_path]) + # Mix narration (louder) with BGM (quieter), trim BGM to video length + audio_mix = "[mixed]" + filter_parts.append( + f"[2:a]atrim=0:{video_duration},asetpts=PTS-STARTPTS,volume=0.2[bgm];" + f"[1:a]volume=1.0[narr];" + f"[narr][bgm]amix=inputs=2:duration=first[mixed]" + ) + + # Burn in subtitles + subtitle_filter = f"subtitles={srt_path}:force_style='FontSize=24,PrimaryColour=&HFFFFFF,OutlineColour=&H000000,BorderStyle=3,Outline=2,Shadow=0,MarginV=30'" + + if filter_parts: + full_filter = ";".join(filter_parts) + f";[0:v]{subtitle_filter}[outv]" + cmd = [ + "ffmpeg", "-y", + *inputs, + "-filter_complex", full_filter, + "-map", "[outv]", "-map", audio_mix, + "-c:v", "libx264", "-preset", "fast", + "-c:a", "aac", "-b:a", "192k", + "-shortest", + output_path, + ] + else: + cmd = [ + "ffmpeg", "-y", + *inputs, + "-vf", subtitle_filter, + "-map", "0:v", "-map", "1:a", + "-c:v", "libx264", "-preset", "fast", + "-c:a", "aac", "-b:a", "192k", + "-shortest", + output_path, + ] + + subprocess.run(cmd, check=True, capture_output=True) + if on_progress: + on_progress("Final video ready!", 4, 4) + + return output_path diff --git a/backend/agents/music.py b/backend/agents/music.py new file mode 100644 index 0000000..d2b9d7d --- /dev/null +++ b/backend/agents/music.py @@ -0,0 +1,45 @@ +"""Music Agent: generates background music via MiniMax Music API.""" +import os +import json +import requests +from backend.config import MINIMAX_BASE_URL, HEADERS, OUTPUT_DIR + + +def run(style: str, mood: str, total_duration: int, on_progress=None) -> str: + """Generate background music matching the video mood. Returns local file path.""" + if on_progress: + on_progress("Generating background music...", 0, 1) + + url = f"{MINIMAX_BASE_URL}/music_generation" + payload = { + "model": "music-2.0", + "prompt": ( + f"An instrumental background music track for a {total_duration}-second video. " + f"Style: {style}. Mood: {mood}. " + f"No vocals, purely instrumental. Suitable as background music for a short video. " + f"Cinematic, polished production quality." + ), + "lyrics": "[Instrumental]\nBackground music\nCinematic instrumental\nSmooth and polished", + "audio_setting": { + "sample_rate": 44100, + "bitrate": 128000, + "format": "mp3", + }, + } + + resp = requests.post(url, headers=HEADERS, json=payload) + resp.raise_for_status() + data = resp.json() + + if "data" not in data or "audio" not in data["data"]: + raise RuntimeError(f"Music generation failed: {json.dumps(data)[:500]}") + + audio_bytes = bytes.fromhex(data["data"]["audio"]) + out_path = os.path.join(OUTPUT_DIR, "bgm.mp3") + with open(out_path, "wb") as f: + f.write(audio_bytes) + + if on_progress: + on_progress("Background music ready", 1, 1) + + return out_path diff --git a/backend/agents/script.py b/backend/agents/script.py new file mode 100644 index 0000000..12422ea --- /dev/null +++ b/backend/agents/script.py @@ -0,0 +1,68 @@ +"""Script Agent: refines the Director's outline into production-ready prompts.""" +import json +import requests +from backend.config import MINIMAX_BASE_URL, HEADERS + + +SYSTEM_PROMPT = """You are the Scriptwriter of an AI film production team. You take the Director's shot outline and refine each shot into production-ready specifications. + +For each shot, enhance: +1. video_prompt: a highly detailed, specific prompt optimized for AI video generation (Hailuo). Include: subject, action, camera movement, lighting, color palette, style. Be vivid and specific. Max 200 words. +2. narration: polished voiceover text, timed to fit the shot duration. Keep it concise and impactful. +3. subtitle: clean subtitle text (shorter than narration if needed) + +Respond ONLY with valid JSON. Format: +{ + "shots": [ + { + "shot_number": 1, + "duration": 10, + "video_prompt": "Detailed AI video generation prompt...", + "narration": "Polished voiceover text...", + "subtitle": "Clean subtitle text...", + "mood": "inspiring" + } + ] +}""" + + +def run(outline: dict) -> dict: + """Refine Director's outline into production-ready shot specifications.""" + user_message = ( + f"Video title: {outline['title']}\n" + f"Visual style: {outline['style']}\n" + f"Total duration: {outline['total_duration']}s\n\n" + f"Director's outline:\n{json.dumps(outline['shots'], indent=2)}\n\n" + f"Refine each shot with detailed video prompts optimized for AI video generation." + ) + + url = f"{MINIMAX_BASE_URL}/text/chatcompletion_v2" + payload = { + "model": "MiniMax-M1", + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": user_message}, + ], + "temperature": 0.7, + "max_tokens": 3000, + } + + resp = requests.post(url, headers=HEADERS, json=payload) + resp.raise_for_status() + data = resp.json() + + content = data.get("choices", [{}])[0].get("message", {}).get("content", "") + content = content.strip() + if content.startswith("```"): + content = content.split("\n", 1)[1] if "\n" in content else content[3:] + if content.endswith("```"): + content = content[:-3] + content = content.strip() + + result = json.loads(content) + + # Carry over metadata from outline + result["title"] = outline["title"] + result["style"] = outline["style"] + result["total_duration"] = outline["total_duration"] + return result diff --git a/backend/agents/visual.py b/backend/agents/visual.py new file mode 100644 index 0000000..4578059 --- /dev/null +++ b/backend/agents/visual.py @@ -0,0 +1,92 @@ +"""Visual Agent: generates video clips via MiniMax Hailuo API.""" +import os +import time +import requests +from concurrent.futures import ThreadPoolExecutor, as_completed +from backend.config import MINIMAX_BASE_URL, HEADERS, OUTPUT_DIR + + +def create_video_task(prompt: str, duration: int = 6) -> str: + """Submit a video generation task, return task_id.""" + url = f"{MINIMAX_BASE_URL}/video_generation" + payload = { + "prompt": prompt, + "model": "MiniMax-Hailuo-2.3", + "duration": min(duration, 6), # Hailuo supports 6s or 10s + "resolution": "1080P", + } + resp = requests.post(url, headers=HEADERS, json=payload) + resp.raise_for_status() + data = resp.json() + task_id = data.get("task_id", "") + if not task_id: + raise RuntimeError(f"Video task creation failed: {data}") + return task_id + + +def poll_task(task_id: str, max_wait: int = 300) -> str: + """Poll until video generation completes, return file_id.""" + url = f"{MINIMAX_BASE_URL}/query/video_generation" + start = time.time() + while time.time() - start < max_wait: + time.sleep(10) + resp = requests.get(url, headers=HEADERS, params={"task_id": task_id}) + data = resp.json() + status = data.get("status", "Unknown") + if status == "Success": + return data["file_id"] + elif status == "Fail": + raise RuntimeError(f"Video generation failed: {data.get('error_message')}") + raise TimeoutError(f"Video generation timed out after {max_wait}s") + + +def download_file(file_id: str, filename: str) -> str: + """Download video file by file_id, return local path.""" + url = f"{MINIMAX_BASE_URL}/files/retrieve" + resp = requests.get(url, headers=HEADERS, params={"file_id": file_id}) + resp.raise_for_status() + download_url = resp.json()["file"]["download_url"] + + out_path = os.path.join(OUTPUT_DIR, filename) + video_resp = requests.get(download_url) + with open(out_path, "wb") as f: + f.write(video_resp.content) + return out_path + + +def generate_single_shot(shot: dict, index: int) -> dict: + """Generate video for a single shot. Returns dict with shot info and local path.""" + prompt = shot["video_prompt"] + duration = shot.get("duration", 6) + + task_id = create_video_task(prompt, duration) + file_id = poll_task(task_id) + filename = f"shot_{index + 1}.mp4" + local_path = download_file(file_id, filename) + + return { + "shot_number": shot["shot_number"], + "video_path": local_path, + "duration": duration, + } + + +def run(script: dict, on_progress=None) -> list: + """Generate video clips for all shots in parallel. Returns list of shot results.""" + shots = script["shots"] + results = [None] * len(shots) + + with ThreadPoolExecutor(max_workers=3) as executor: + futures = {} + for i, shot in enumerate(shots): + future = executor.submit(generate_single_shot, shot, i) + futures[future] = i + + for future in as_completed(futures): + idx = futures[future] + result = future.result() + results[idx] = result + if on_progress: + on_progress(f"Shot {idx + 1} video ready", idx + 1, len(shots)) + + return results diff --git a/backend/agents/voice.py b/backend/agents/voice.py new file mode 100644 index 0000000..f9891a1 --- /dev/null +++ b/backend/agents/voice.py @@ -0,0 +1,75 @@ +"""Voice Agent: generates voiceover audio via MiniMax TTS API.""" +import os +import json +import requests +from backend.config import MINIMAX_BASE_URL, HEADERS, OUTPUT_DIR + + +def generate_tts(text: str, filename: str, voice_id: str = "English_expressive_narrator", mood: str = "neutral") -> str: + """Generate TTS audio for text, return local file path.""" + url = f"{MINIMAX_BASE_URL}/t2a_v2" + + # Map mood to emotion parameter + emotion_map = { + "inspiring": "happy", + "dramatic": "sad", + "calm": "neutral", + "energetic": "happy", + "contemplative": "neutral", + "exciting": "happy", + "professional": "neutral", + "warm": "happy", + } + emotion = emotion_map.get(mood, "neutral") + + payload = { + "model": "speech-2.6-hd", + "text": text, + "stream": False, + "voice_setting": { + "voice_id": voice_id, + "speed": 1, + "vol": 1, + "pitch": 0, + "emotion": emotion, + }, + "audio_setting": { + "sample_rate": 32000, + "bitrate": 128000, + "format": "mp3", + "channel": 1, + }, + } + + resp = requests.post(url, headers=HEADERS, json=payload) + resp.raise_for_status() + data = resp.json() + + if "data" not in data or "audio" not in data["data"]: + raise RuntimeError(f"TTS failed: {json.dumps(data)[:500]}") + + audio_bytes = bytes.fromhex(data["data"]["audio"]) + out_path = os.path.join(OUTPUT_DIR, filename) + with open(out_path, "wb") as f: + f.write(audio_bytes) + return out_path + + +def run(script: dict, on_progress=None) -> list: + """Generate voiceover for all shots. Returns list of audio file paths.""" + shots = script["shots"] + results = [] + + for i, shot in enumerate(shots): + narration = shot["narration"] + mood = shot.get("mood", "neutral") + filename = f"voice_{i + 1}.mp3" + path = generate_tts(narration, filename, mood=mood) + results.append({ + "shot_number": shot["shot_number"], + "audio_path": path, + }) + if on_progress: + on_progress(f"Shot {i + 1} voiceover ready", i + 1, len(shots)) + + return results diff --git a/backend/config.py b/backend/config.py new file mode 100644 index 0000000..30fee07 --- /dev/null +++ b/backend/config.py @@ -0,0 +1,16 @@ +"""Configuration and shared constants.""" +import os +from dotenv import load_dotenv + +load_dotenv(os.path.join(os.path.dirname(__file__), '..', '.env')) + +MINIMAX_API_KEY = os.getenv("MINIMAX_API_KEY", "") +MINIMAX_BASE_URL = "https://api.minimax.io/v1" + +OUTPUT_DIR = os.path.join(os.path.dirname(__file__), "output") +os.makedirs(OUTPUT_DIR, exist_ok=True) + +HEADERS = { + "Authorization": f"Bearer {MINIMAX_API_KEY}", + "Content-Type": "application/json", +} diff --git a/backend/main.py b/backend/main.py new file mode 100644 index 0000000..8c04532 --- /dev/null +++ b/backend/main.py @@ -0,0 +1,166 @@ +"""FastAPI server for AgentCut - AI Director Video Production Pipeline.""" +import os +import json +import uuid +import asyncio +from typing import AsyncGenerator +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import StreamingResponse, FileResponse +from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel + +from backend.pipeline import run_pipeline, PipelineEvent +from backend.config import OUTPUT_DIR + +app = FastAPI(title="AgentCut", description="AI Director - Multi-Agent Video Production") + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + +# In-memory job storage +jobs: dict = {} + + +class CreateVideoRequest(BaseModel): + prompt: str + duration: int = 30 + num_shots: int = 3 + include_music: bool = True + + +class JobStatus(BaseModel): + job_id: str + status: str # "pending", "running", "completed", "failed" + progress: float + events: list + output_path: str = None + error: str = None + + +@app.post("/api/create") +async def create_video(req: CreateVideoRequest): + """Start a new video production job.""" + job_id = str(uuid.uuid4())[:8] + jobs[job_id] = { + "status": "pending", + "progress": 0.0, + "events": [], + "output_path": None, + "error": None, + } + + # Run pipeline in background + asyncio.get_event_loop().run_in_executor( + None, _run_job, job_id, req.prompt, req.duration, req.num_shots, req.include_music, + ) + + return {"job_id": job_id} + + +def _run_job(job_id: str, prompt: str, duration: int, num_shots: int, include_music: bool): + """Execute the pipeline in a background thread.""" + jobs[job_id]["status"] = "running" + + def on_event(event: PipelineEvent): + jobs[job_id]["progress"] = event.progress + jobs[job_id]["events"].append({ + "stage": event.stage, + "message": event.message, + "progress": event.progress, + "data": event.data, + "timestamp": event.timestamp, + }) + + try: + result = run_pipeline( + prompt, duration, num_shots, include_music, on_event=on_event, + ) + jobs[job_id]["status"] = "completed" + jobs[job_id]["progress"] = 1.0 + jobs[job_id]["output_path"] = result["output_path"] + except Exception as e: + jobs[job_id]["status"] = "failed" + jobs[job_id]["error"] = str(e) + + +@app.get("/api/status/{job_id}") +async def get_status(job_id: str): + """Get current job status and progress.""" + if job_id not in jobs: + raise HTTPException(status_code=404, detail="Job not found") + return jobs[job_id] + + +@app.get("/api/stream/{job_id}") +async def stream_events(job_id: str): + """SSE stream for real-time progress updates.""" + if job_id not in jobs: + raise HTTPException(status_code=404, detail="Job not found") + + async def event_generator() -> AsyncGenerator[str, None]: + last_idx = 0 + while True: + job = jobs[job_id] + events = job["events"] + + # Send new events + while last_idx < len(events): + event = events[last_idx] + yield f"data: {json.dumps(event)}\n\n" + last_idx += 1 + + if job["status"] in ("completed", "failed"): + final = { + "stage": "done", + "message": "Pipeline finished" if job["status"] == "completed" else f"Error: {job['error']}", + "progress": job["progress"], + "data": {"status": job["status"], "output_path": job.get("output_path")}, + } + yield f"data: {json.dumps(final)}\n\n" + break + + await asyncio.sleep(1) + + return StreamingResponse( + event_generator(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "Connection": "keep-alive"}, + ) + + +@app.get("/api/download/{job_id}") +async def download_video(job_id: str): + """Download the final video.""" + if job_id not in jobs: + raise HTTPException(status_code=404, detail="Job not found") + + job = jobs[job_id] + if job["status"] != "completed" or not job["output_path"]: + raise HTTPException(status_code=400, detail="Video not ready") + + return FileResponse( + job["output_path"], + media_type="video/mp4", + filename="agentcut_output.mp4", + ) + + +@app.get("/api/health") +async def health(): + return {"status": "ok", "service": "AgentCut"} + + +# Serve frontend static files (must be last - catches all unmatched routes) +_frontend_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "frontend") +if os.path.isdir(_frontend_dir): + app.mount("/", StaticFiles(directory=_frontend_dir, html=True), name="frontend") + + +if __name__ == "__main__": + import uvicorn + uvicorn.run("backend.main:app", host="0.0.0.0", port=8000, reload=True) diff --git a/backend/pipeline.py b/backend/pipeline.py new file mode 100644 index 0000000..30284e5 --- /dev/null +++ b/backend/pipeline.py @@ -0,0 +1,107 @@ +"""Agent orchestration pipeline: Director -> Script -> Visual+Voice+Music -> Editor.""" +import time +import json +from dataclasses import dataclass, field +from typing import Callable + +from backend.agents import director, script, visual, voice, music, editor + + +@dataclass +class PipelineEvent: + stage: str + message: str + progress: float # 0.0 to 1.0 + data: dict = field(default_factory=dict) + timestamp: float = field(default_factory=time.time) + + +def run_pipeline( + user_prompt: str, + duration: int = 30, + num_shots: int = 3, + include_music: bool = True, + on_event: Callable[[PipelineEvent], None] = None, +) -> dict: + """Run the full video production pipeline. + + Args: + user_prompt: User's creative description + duration: Target video duration in seconds + num_shots: Number of shots to generate + include_music: Whether to generate background music + on_event: Callback for progress events + + Returns: + Dict with output_path and metadata + """ + def emit(stage, message, progress, data=None): + if on_event: + on_event(PipelineEvent(stage, message, progress, data or {})) + + result = {"stages": {}} + + # Stage 1: Director Agent + emit("director", "Director is analyzing your creative vision...", 0.05) + outline = director.run(user_prompt, duration, num_shots) + result["stages"]["director"] = outline + emit("director", f"Outline ready: {outline['title']} ({len(outline['shots'])} shots)", 0.15, + {"title": outline["title"], "shots": len(outline["shots"])}) + + # Stage 2: Script Agent + emit("script", "Scriptwriter is crafting detailed production specs...", 0.20) + production_script = script.run(outline) + result["stages"]["script"] = production_script + emit("script", "Production script finalized", 0.30) + + # Stage 3: Visual + Voice + Music (parallel where possible) + emit("visual", "Visual team is generating video clips...", 0.35) + + def visual_progress(msg, current, total): + p = 0.35 + (current / total) * 0.25 + emit("visual", msg, p) + + def voice_progress(msg, current, total): + p = 0.60 + (current / total) * 0.10 + emit("voice", msg, p) + + # Generate videos (parallel, takes longest) + video_results = visual.run(production_script, on_progress=visual_progress) + result["stages"]["visual"] = [ + {"shot": r["shot_number"], "path": r["video_path"]} for r in video_results + ] + + # Generate voiceover + emit("voice", "Voice artist is recording narration...", 0.60) + voice_results = voice.run(production_script, on_progress=voice_progress) + result["stages"]["voice"] = [ + {"shot": r["shot_number"], "path": r["audio_path"]} for r in voice_results + ] + + # Generate music (optional) + bgm_path = None + if include_music: + emit("music", "Composer is creating background music...", 0.72) + try: + mood = outline["shots"][0].get("mood", "inspiring") + bgm_path = music.run(outline["style"], mood, outline["total_duration"]) + result["stages"]["music"] = {"path": bgm_path} + emit("music", "Background music composed", 0.78) + except Exception as e: + emit("music", f"Music generation skipped: {e}", 0.78) + + # Stage 4: Editor Agent + emit("editor", "Editor is compositing the final video...", 0.80) + + def editor_progress(msg, current, total): + p = 0.80 + (current / total) * 0.18 + emit("editor", msg, p) + + output_path = editor.run( + video_results, voice_results, production_script, + bgm_path=bgm_path, on_progress=editor_progress, + ) + result["output_path"] = output_path + emit("complete", "Your video is ready!", 1.0, {"output_path": output_path}) + + return result diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..a4572f9 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.115.0 +uvicorn==0.32.0 +requests==2.32.0 +python-dotenv==1.0.1 +httpx==0.27.0 +openai==1.50.0 diff --git a/backend/test_minimax.py b/backend/test_minimax.py new file mode 100644 index 0000000..8abc647 --- /dev/null +++ b/backend/test_minimax.py @@ -0,0 +1,140 @@ +"""Test MiniMax API connectivity: video generation, TTS, and music.""" +import os +import sys +import json +import time +import requests +from dotenv import load_dotenv + +load_dotenv(os.path.join(os.path.dirname(__file__), '..', '.env')) + +API_KEY = os.getenv("MINIMAX_API_KEY") +HEADERS = { + "Authorization": f"Bearer {API_KEY}", + "Content-Type": "application/json", +} + + +def test_tts(): + """Test TTS API (synchronous, fastest to verify).""" + print("\n=== Testing TTS (Speech-2.6-HD) ===") + url = "https://api.minimax.io/v1/t2a_v2" + payload = { + "model": "speech-2.6-hd", + "text": "Welcome to AgentCut, your AI film crew.", + "stream": False, + "voice_setting": { + "voice_id": "English_expressive_narrator", + "speed": 1, + "vol": 1, + "pitch": 0, + }, + "audio_setting": { + "sample_rate": 32000, + "bitrate": 128000, + "format": "mp3", + "channel": 1, + }, + } + resp = requests.post(url, headers=HEADERS, json=payload) + if resp.status_code != 200: + print(f"FAIL: status={resp.status_code}, body={resp.text[:500]}") + return False + + data = resp.json() + if "data" in data and "audio" in data["data"]: + audio_bytes = bytes.fromhex(data["data"]["audio"]) + out_path = os.path.join(os.path.dirname(__file__), "output", "test_tts.mp3") + with open(out_path, "wb") as f: + f.write(audio_bytes) + print(f"OK: saved {len(audio_bytes)} bytes to {out_path}") + return True + else: + print(f"FAIL: unexpected response: {json.dumps(data, indent=2)[:500]}") + return False + + +def test_video_create(): + """Test video generation task creation (does not wait for completion).""" + print("\n=== Testing Video Generation (Hailuo) ===") + url = "https://api.minimax.io/v1/video_generation" + payload = { + "prompt": "A futuristic city skyline at sunset with flying cars and neon lights, cinematic style, smooth camera pan from left to right.", + "model": "MiniMax-Hailuo-2.3", + "duration": 6, + "resolution": "1080P", + } + resp = requests.post(url, headers=HEADERS, json=payload) + if resp.status_code != 200: + print(f"FAIL: status={resp.status_code}, body={resp.text[:500]}") + return None + + data = resp.json() + task_id = data.get("task_id") + if task_id: + print(f"OK: task_id={task_id}") + return task_id + else: + print(f"FAIL: no task_id in response: {json.dumps(data, indent=2)[:500]}") + return None + + +def poll_video(task_id, max_wait=300): + """Poll video generation status until done.""" + print(f"\n=== Polling Video Task {task_id} ===") + url = "https://api.minimax.io/v1/query/video_generation" + start = time.time() + while time.time() - start < max_wait: + time.sleep(10) + resp = requests.get(url, headers=HEADERS, params={"task_id": task_id}) + data = resp.json() + status = data.get("status", "Unknown") + print(f" status={status} ({int(time.time()-start)}s elapsed)") + if status == "Success": + return data.get("file_id") + elif status == "Fail": + print(f" FAIL: {data.get('error_message', 'unknown error')}") + return None + print(f" TIMEOUT after {max_wait}s") + return None + + +def download_video(file_id): + """Download video by file_id.""" + print(f"\n=== Downloading Video (file_id={file_id}) ===") + url = "https://api.minimax.io/v1/files/retrieve" + resp = requests.get(url, headers=HEADERS, params={"file_id": file_id}) + data = resp.json() + download_url = data.get("file", {}).get("download_url") + if not download_url: + print(f"FAIL: no download_url: {json.dumps(data, indent=2)[:500]}") + return False + + video_resp = requests.get(download_url) + out_path = os.path.join(os.path.dirname(__file__), "output", "test_video.mp4") + with open(out_path, "wb") as f: + f.write(video_resp.content) + print(f"OK: saved {len(video_resp.content)} bytes to {out_path}") + return True + + +if __name__ == "__main__": + if not API_KEY: + print("ERROR: MINIMAX_API_KEY not set in .env") + sys.exit(1) + + print(f"API Key: {API_KEY[:10]}...{API_KEY[-4:]}") + + # Test 1: TTS (fast, synchronous) + tts_ok = test_tts() + + # Test 2: Video generation (async, takes a few minutes) + task_id = test_video_create() + if task_id: + file_id = poll_video(task_id) + if file_id: + download_video(file_id) + + print("\n=== Summary ===") + print(f"TTS: {'PASS' if tts_ok else 'FAIL'}") + print(f"Video: {'task created' if task_id else 'FAIL'}") diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..ff55126 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,346 @@ + + + + + + AgentCut - AI Director + + + + + + + +
+
+
+
AC
+

AgentCut

+ Multi-Agent Video Production +
+ GitHub +
+
+ +
+ +
+
+

Describe your video, let the AI crew handle the rest.

+

6 specialized agents collaborate to produce your video from a single prompt.

+
+ +
+ + +
+
+ + +
+ + +
+
+ + +
+ + + +
+
+ + + + + + +
+ + + + + + +