新增动作迁移板块,修改部分图生视频&数字人配置文件内容

This commit is contained in:
1152450298 committed 2026-01-21 16:21:57 +08:00
1 parent f6c561db35
commit 62df8f8cbe
9 files changed
+2241 -1716

No files matched your search

+1
View File
@@ -26,6 +26,7 @@ dependencies = [
"python-multipart>=0.0.12",
"comfykit>=0.1.12",
"beautifulsoup4>=4.14.2",
"moviepy==1.0.3",
]
[project.optional-dependencies]
Generated
+1771 -1690
View File
File diff suppressed because it is too large. Load diff
+39 -15
View File
@@ -18,13 +18,13 @@
"mode.digital": "💻 Product placement mode",
"mode.customize": "🧐 Custom mode",
"input.topic": "Topic",
"input.topic_placeholder": "AI automatically creates specified number of narrations\nExample: How to build passive income, 如何增加被动收入",
"input.topic_placeholder": "AI automatically creates specified number of narrations\nExample: How to build passive income",
"input.topic_help": "Enter a topic, AI will generate content based on it",
"input.text": "Text Input",
"input.text_help_generate": "Enter topic or theme (AI will create narrations)",
"input.text_help_fixed": "Enter complete narration script (used directly without modification)",
"input.text_help_digital": "Enter product description. If a product description is provided, you do not need to enter the product name.",
"input.text_help_audio": "Enter prompts/descriptions, describing the desired visual content in as much detail as possible.",
"input.text_help_digital": "Enter product description. If a product description is provided, you do not need to enter the product name",
"input.text_help_audio": "Enter prompts/descriptions, describing the desired visual content in as much detail as possible",
"split.mode_label": "Split Strategy",
"split.mode_help": "Choose how to split the text into video segments",
"split.mode_paragraph": "📄 By Paragraph (\\n\\n)",
@@ -375,8 +375,10 @@
"pipeline.custom_media.description": "Use your own photos/videos, AI adds narration and voiceover",
"pipeline.digital_human.name": "Digital Human Broadcast",
"pipeline.digital_human.description": "Use a piece of text, two images, and an audio clip, and AI will generate a digital human video for you",
"pipeline.audio-visual.name": "Image-to-Video",
"pipeline.audio-visual.description": "Enter an image and a prompt, and AI will instantly generate a video",
"pipeline.i2v.name": "Image To Video",
"pipeline.i2v.description": "Enter an image and a prompt, and AI will instantly generate a video",
"pipeline.action_transfer.name": "Action Transfer",
"pipeline.action_transfer.description": "One image, one video, recreate amazing moves",
"asset_based.section.assets": "📦 Asset Upload",
"asset_based.section.video_info": "📝 Video Information",
"asset_based.section.source": "⚙️ Service Configuration",
@@ -446,16 +448,38 @@
"digital_tts.what": "Convert narration text to natural human-like speech",
"i2v.video_generation": "🛠️ Configuration parameters",
"i2v.assets.image_what": "Reference image of the first frame generated from the uploaded video",
"i2v.assets.how": "Supports JPG/PNG/WebP and other image formats. Clear and relevant images are recommended. Accurate and specific video prompts will enhance the video's quality.",
"i2v.input.topic_placeholder": "Enter creative prompts (e.g., audio and rhythm, camera movement rules, overall visual style, color, color grading, etc.)",
"i2v.assets.upload": "Upload video first frame image",
"i2v.assets.upload_help": "Only single image upload is supported",
"i2v.assets.character_sucess": "Video asset uploaded successfully",
"i2v.assets.preview": "📷 Asset Preview",
"i2v.assets.character_empty_hint": "💡 Please upload a first frame image for your video",
"i2v.input_text": "Video prompt",
"i2v.assets.image_warning": "Upload the first frame image of your video",
"i2v.assets.prompt_warning": "Please enter a video prompt",
"i2v.assets.how": "Supports JPG/PNG/WebP and other image formats. Clear and relevant images are recommended. Accurate and specific video prompts will enhance the video effect",
"i2v.input.topic_placeholder": "Enter creative prompts (e.g., audio and rhythm, shot and motion rules, overall visual style, color and color grading, etc.)",
"i2v.assets.upload": "Upload the first frame image of the video",
"i2v.assets.upload_help": "Single image upload only",
"i2v.assets.character_sucess": "Upload successful",
"i2v.assets.preview": "📷 Material preview",
"i2v.assets.character_empty_hint": "💡 Please upload the first frame image of the video you want to generate",
"i2v.input_text": "Video prompt text",
"i2v.assets.image_warning": "Upload the first frame image of the video",
"i2v.assets.prompt_warning": "Please enter the video prompt text",
"iv2.workflow_select": "Selection of image-to-video workflow (including local and runninghub)",
"action_transfer.video_upload": "🎞️Video footage",
"action_transfer.assets.video_what": "Upload the reference video used for the migration action",
"action_transfer.assets.video_how": "Supports MP4/MKV/MOV image formats. The reference video only supports single-person action migration. It is recommended that the video action be clearly displayed",
"action_transfer.assets.video_upload": "Upload the reference action video",
"action_transfer.assets.video_upload_help": "Only supports single video uploads, and the video can only show one person. The video length must be less than or equal to 30 seconds. If the video length is longer than 30 seconds, only the first 30 seconds will be used as a reference",
"action_transfer.assets.video_sucess": "Uploaded material successfully",
"action_transfer.assets.preview": "👀 Material preview",
"action_transfer.assets.video_empty_hint": "💡 Please upload a reference video for the action transfer",
"action_transfer.image_upload": "✏️ Configuration parameters",
"action_transfer.assets.image_what": "Upload the image to be used for the action to be transferred",
"action_transfer.assets.image_how": "Supports JPG/PNG/WebP and other image formats. Clear and relevant images are recommended",
"action_transfer.assets.image_upload": "Upload the image for the transfer action",
"action_transfer.assets.image_upload_help": "Single image uploads are supported only; images featuring a single person will produce the best results",
"action_transfer.assets.image_sucess": "Upload successful",
"action_transfer.assets.image_empty_hint": "💡 Please upload a reference video for motion transfer",
"action_transfer.input_text": "Video prompt",
"action_transfer.input.topic_placeholder": "Enter a creative prompt (e.g., imitate the dance in the reference video, keeping the position constant, with consistent and synchronized steps, etc.)",
"action_transfer.workflow_select": "Motion transfer workflow selection (including local and runninghub)",
"action_transfer.assets.image_warning": "Upload an image for the transfer motion",
"action_transfer.assets.video_warning": "Upload a reference video for motion transfer",
"action_transfer.assets.prompt_warning": "Please enter a video prompt",
"faq.expand_to_view": "FAQ",
"faq.load_error": "Failed to load FAQ content",
+28 -4
View File
@@ -375,8 +375,10 @@
"pipeline.custom_media.description": "用你自己的照片/视频,AI帮你配文案和配音",
"pipeline.digital_human.name": "数字人口播",
"pipeline.digital_human.description": "用一段文本、两张图片、一段音频,AI帮你生成一个数字人视频",
"pipeline.audio-visual.name": "图生视频",
"pipeline.audio-visual.description": "输入图片和提示词,AI即刻生成视频",
"pipeline.i2v.name": "图生视频",
"pipeline.i2v.description": "输入图片和提示词,AI即刻生成视频",
"pipeline.action_transfer.name": "动作迁移",
"pipeline.action_transfer.description": "一张图、一段视频,复刻精彩动作",
"asset_based.section.assets": "📦 素材上传",
"asset_based.section.video_info": "📝 视频信息",
"asset_based.section.source": "⚙️ 服务配置",
@@ -450,13 +452,35 @@
"i2v.input.topic_placeholder" : "输入创作提示词(例如:音频与节奏、镜头与运动规则、整体视觉风格、色彩与调色等)",
"i2v.assets.upload": "上传视频首帧图像",
"i2v.assets.upload_help": "仅支持单张图片上传",
"i2v.assets.character_sucess": "上传视频素材成功",
"i2v.assets.character_sucess": "上传素材成功",
"i2v.assets.preview": "📷 素材预览",
"i2v.assets.character_empty_hint": "💡 请上传一个想要生成视频的首帧图像",
"i2v.input_text": "视频提示词",
"i2v.assets.image_warning": "上传视频首帧图片",
"i2v.assets.prompt_warning": "请输入视频提示词",
"iv2.workflow_select": "图生视频工作流选择(包含本地和runninghub)",
"action_transfer.video_upload": "🎞️视频素材",
"action_transfer.assets.video_what": "上传迁移动作使用的参考视频",
"action_transfer.assets.video_how": "支持 MP4/MKV/MOV 等格式图片,参考视频仅支持单人动作迁移,建议视频动作表现明显",
"action_transfer.assets.video_upload": "上传参考动作视频",
"action_transfer.assets.video_upload_help": "仅支持单个视频上传,且视频画面只能有一人,视频时长小于等于30秒,若视频长度大于30秒则只取视频的前30秒作为参考",
"action_transfer.assets.video_sucess": "上传素材成功",
"action_transfer.assets.preview": "👀 素材预览",
"action_transfer.assets.video_empty_hint": "💡 请上传一个动作迁移的参考视频",
"action_transfer.image_upload": "✏️配置参数",
"action_transfer.assets.image_what": "上传需要被迁移动作使用的图片",
"action_transfer.assets.image_how": "支持 JPG/PNG/WebP 等格式图片,建议素材清晰且内容相关",
"action_transfer.assets.image_upload": "上传迁移动作的图片",
"action_transfer.assets.image_upload_help": "仅支持单张图片上传,图片为单人时效果最佳",
"action_transfer.assets.image_sucess": "上传素材成功",
"action_transfer.assets.image_empty_hint": "💡 请上传一个动作迁移的参考视频",
"action_transfer.input_text": "视频提示词",
"action_transfer.input.topic_placeholder" : "输入创作提示词(例如:模仿参考视频跳舞,位置不变,步伐一致且整齐等)",
"action_transfer.workflow_select": "动作迁移工作流选择(包含本地和runninghub)",
"action_transfer.assets.image_warning": "上传迁移动作的图片",
"action_transfer.assets.video_warning": "上传动作迁移的参考视频",
"action_transfer.assets.prompt_warning": "请输入视频提示词",
"faq.expand_to_view": "常见问题",
"faq.load_error": "无法加载常见问题内容",
"faq.more_help": "需要更多帮助?"
+1
View File
@@ -28,6 +28,7 @@ from web.pipelines import standard
from web.pipelines import asset_based
from web.pipelines import digital_human
from web.pipelines import i2v
from web.pipelines import action_transfer
__all__ = [
"PipelineUI",
+390
View File
@@ -0,0 +1,390 @@
import os
import time
from pathlib import Path
from typing import Any
from moviepy.editor import VideoFileClip
import streamlit as st
from loguru import logger
import httpx
from web.i18n import tr, get_language
from web.pipelines.base import PipelineUI, register_pipeline_ui
from web.components.content_input import render_version_info
from web.utils.async_helpers import run_async
from pixelle_video.config import config_manager
from pixelle_video.utils.os_util import create_task_output_dir
class ActionTransferPipelineUI(PipelineUI):
"""
UI for the Action transfer Video Generation Pipeline.
Generates videos from user-provided assets (images&text&video).
"""
name = "action_transfer"
icon = "💃"
@property
def display_name(self):
return tr("pipeline.action_transfer.name")
@property
def description(self):
return tr("pipeline.action_transfer.description")
def render(self, pixelle_video: Any):
# Three-column layout
left_col,middle_col,right_col = st.columns([1, 1, 1])
# ====================================================================
# Left Column: Video Upload
# ====================================================================
with left_col:
video_params = self.render_action_transfer_video_input(pixelle_video)
render_version_info()
# ====================================================================
# Middle Column: Image Upload & Prompt
# ====================================================================
with middle_col:
assets_params = self.render_action_transfer_assets_input(pixelle_video)
# ====================================================================
# Right Column: Output Preview
# ====================================================================
with right_col:
video_params = {
**video_params,
**assets_params
}
self._render_output_preview(pixelle_video, video_params)
def render_action_transfer_video_input(self, pixelle_video) -> dict:
with st.container(border=True):
st.markdown(f"**{tr('action_transfer.video_upload')}**")
with st.expander(tr("help.feature_description"), expanded=False):
st.markdown(f"**{tr('help.what')}**")
st.markdown(tr("action_transfer.assets.video_what"))
st.markdown(f"**{tr('help.how')}**")
st.markdown(tr("action_transfer.assets.video_how"))
# File uploader for multiple files
uploaded_files = st.file_uploader(
tr("action_transfer.assets.video_upload"),
type=["mp4","mkv","mov"],
accept_multiple_files=True,
help=tr("action_transfer.assets.video_upload_help"),
key="action_reference_files"
)
# Save uploaded files to temp directory with unique session ID
video_asset_paths = []
if uploaded_files:
import uuid
session_id = str(uuid.uuid4()).replace('-', '')[:12]
temp_dir = Path(f"temp/assets_{session_id}")
temp_dir.mkdir(parents=True, exist_ok=True)
for uploaded_file in uploaded_files:
file_path = temp_dir / uploaded_file.name
with open(file_path, "wb") as f:
f.write(uploaded_file.getbuffer())
video_asset_paths.append(str(file_path.absolute()))
st.success(tr("action_transfer.assets.video_sucess"))
# Preview uploaded assets
with st.expander(tr("action_transfer.assets.preview"), expanded=True):
# Show in a grid (3 columns)
cols = st.columns(3)
for i, (file, path) in enumerate(zip(uploaded_files, video_asset_paths)):
with cols[i % 3]:
# Check if image
ext = Path(path).suffix.lower()
if ext in [".mp4", ".mkv", ".mov"]:
st.video(file)
else:
st.info(tr("action_transfer.assets.video_empty_hint"))
# Get the video length (rounded down).
if video_asset_paths:
clip = VideoFileClip(video_asset_paths[0])
int_duration = int(clip.duration)
duration = min(int_duration, 30)
else:
duration = 0
return {
"video_assets": video_asset_paths,
"duration": duration
}
def render_action_transfer_assets_input(self, pixelle_video) -> dict:
with st.container(border=True):
st.markdown(f"**{tr('action_transfer.image_upload')}**")
with st.expander(tr("help.feature_description"), expanded=False):
st.markdown(f"**{tr('help.what')}**")
st.markdown(tr("action_transfer.assets.image_what"))
st.markdown(f"**{tr('help.how')}**")
st.markdown(tr("action_transfer.assets.image_how"))
# File uploader for multiple files
uploaded_files = st.file_uploader(
tr("action_transfer.assets.image_upload"),
type=["jpg", "jpeg", "png", "webp"],
accept_multiple_files=True,
help=tr("action_transfer.assets.image_upload_help"),
key="image_files"
)
# Save uploaded files to temp directory with unique session ID
image_asset_paths = []
if uploaded_files:
import uuid
session_id = str(uuid.uuid4()).replace('-', '')[:12]
temp_dir = Path(f"temp/assets_{session_id}")
temp_dir.mkdir(parents=True, exist_ok=True)
for uploaded_file in uploaded_files:
file_path = temp_dir / uploaded_file.name
with open(file_path, "wb") as f:
f.write(uploaded_file.getbuffer())
image_asset_paths.append(str(file_path.absolute()))
st.success(tr("action_transfer.assets.image_sucess"))
# Preview uploaded assets
with st.expander(tr("action_transfer.assets.preview"), expanded=True):
# Show in a grid (3 columns)
cols = st.columns(3)
for i, (file, path) in enumerate(zip(uploaded_files, image_asset_paths)):
with cols[i % 3]:
# Check if image
ext = Path(path).suffix.lower()
if ext in [".jpg", ".jpeg", ".png", ".webp"]:
st.image(file, caption=file.name, use_container_width=True)
else:
st.info(tr("action_transfer.assets.image_empty_hint"))
def list_action_transfer_workflows():
result = []
for source in ("runninghub", "selfhost"):
dir_path = os.path.join("workflows", source)
if not os.path.isdir(dir_path):
continue
for fname in os.listdir(dir_path):
if fname.startswith("af_") and fname.endswith(".json"):
display = f"{fname} - {'Runninghub' if source == 'runninghub' else 'Selfhost'}"
result.append({
"key": f"{source}/{fname}",
"display_name": display
})
return result
prompt_text = st.text_area(
tr("action_transfer.input_text"),
placeholder=tr("action_transfer.input.topic_placeholder"),
height=200,
help=tr("input.text_help_audio"),
key="prompt_box"
)
iv2_workflows = list_action_transfer_workflows()
workflow_options = [wf["display_name"] for wf in iv2_workflows]
workflow_keys = [wf["key"] for wf in iv2_workflows]
default_workflow_index = 0
workflow_display = st.selectbox(
tr("action_transfer.workflow_select"),
workflow_options if workflow_options else ["No workflow found"],
index=default_workflow_index,
label_visibility="collapsed",
key="action_transfer_workflow_select"
)
if workflow_options:
workflow_selected_index = workflow_options.index(workflow_display)
workflow_key = workflow_keys[workflow_selected_index]
else:
workflow_key = None
return {
"image_assets": image_asset_paths,
"prompt_text": prompt_text,
"workflow_key": workflow_key
}
def _render_output_preview(self, pixelle_video: Any, video_params: dict):
"""Render output preview section"""
with st.container(border=True):
st.markdown(f"**{tr('section.video_generation')}**")
# Check configuration
if not config_manager.validate():
st.warning(tr("settings.not_configured"))
image_assets = video_params.get("image_assets", [])
video_assets = video_params.get("video_assets", [])
prompt_text = video_params.get("prompt_text", "")
duration = video_params.get("duration")
workflow_key = video_params.get("workflow_key")
logger.info(f" - video_params: {video_params}")
if not image_assets:
st.info(tr("action_transfer.assets.image_warning"))
st.button(
tr("btn.generate"),
type="primary",
use_container_width=True,
disabled=True,
key="action_transfer_generate_image_disabled"
)
return
if not video_assets:
st.info(tr("action_transfer.assets.video_warning"))
st.button(
tr("btn.generate"),
type="primary",
use_container_width=True,
disabled=True,
key="action_transfer_generate_video_disabled"
)
return
if not prompt_text:
st.info(tr("action_transfer.assets.prompt_warning"))
st.button(
tr("btn.generate"),
type="primary",
use_container_width=True,
disabled=True,
key="action_transfer_generate"
)
return
# Generate button
if st.button(tr("btn.generate"), type="primary", use_container_width=True, key="iv2_generate"):
if not config_manager.validate():
st.error(tr("settings.not_configured"))
st.stop()
progress_bar = st.progress(0)
status_text = st.empty()
start_time = time.time()
try:
async def generate_audio_visual_video():
task_dir, task_id = create_task_output_dir()
logger.info(f"[Initialization] Task Directory: {task_dir}")
kit = await pixelle_video._get_or_create_comfykit()
import json
from pathlib import Path
status_text.text(tr("progress.generation"))
progress_bar.progress(10)
image_path = image_assets[0]
video_path = video_assets[0]
second = duration
prompt = prompt_text
workflow_path = Path("workflows") / workflow_key
if not workflow_path.exists():
raise Exception(f"The workflow file does not exist: {workflow_path}")
with open(workflow_path, 'r', encoding='utf-8') as f:
workflow_config = json.load(f)
workflow_params = {
"video": video_path,
"image": image_path,
"prompt": prompt,
"second": second
}
if workflow_config.get("source") == "runninghub" and "workflow_id" in workflow_config:
workflow_input = workflow_config["workflow_id"]
else:
workflow_input = str(workflow_path)
video_result = await kit.execute(workflow_input, workflow_params)
generated_video_url = None
if hasattr(video_result, 'videos') and video_result.videos:
generated_video_url = video_result.videos[0]
elif hasattr(video_result, 'outputs') and video_result.outputs:
for node_id, node_output in video_result.outputs.items():
if isinstance(node_output, dict) and 'videos' in node_output:
videos = node_output['videos']
if videos and len(videos) > 0:
generated_video_url = videos[0]
break
if not generated_video_url:
raise Exception("The workflow did not return a video. Please check the workflow configuration.")
final_video_path = os.path.join(task_dir, "final.mp4")
timeout = httpx.Timeout(300.0)
async with httpx.AsyncClient(timeout=timeout) as client:
response = await client.get(generated_video_url)
response.raise_for_status()
with open(final_video_path, 'wb') as f:
f.write(response.content)
progress_bar.progress(100)
status_text.text(tr("status.success"))
return final_video_path
# Execute async generation
final_video_path = run_async(generate_audio_visual_video())
total_time = time.time() - start_time
progress_bar.progress(100)
status_text.text(tr("status.success"))
# Display result
st.success(tr("status.video_generated", path=final_video_path))
st.markdown("---")
# Video info
if os.path.exists(final_video_path):
file_size_mb = os.path.getsize(final_video_path) / (1024 * 1024)
info_text = (
f"⏱️ {tr('info.generation_time')} {total_time:.1f}s "
f"📦 {file_size_mb:.2f}MB"
)
st.caption(info_text)
st.markdown("---")
# Video preview
st.video(final_video_path)
# Download button
with open(final_video_path, "rb") as video_file:
video_bytes = video_file.read()
video_filename = os.path.basename(final_video_path)
st.download_button(
label="⬇️ 下载视频" if get_language() == "zh_CN" else "⬇️ Download Video",
data=video_bytes,
file_name=video_filename,
mime="video/mp4",
use_container_width=True
)
else:
st.error(tr("status.video_not_found", path=final_video_path))
except Exception as e:
logger.exception(e)
status_text.text("")
progress_bar.empty()
st.error(tr("status.error", error=str(e)))
st.stop()
register_pipeline_ui(ActionTransferPipelineUI)
+3 -3
View File
@@ -51,9 +51,9 @@ class DigitalHumanPipelineUI(PipelineUI):
workflow_path = self.workflow_path_config()
mode_params = self.render_digital_human_mode(asset_params["character_assets"])
# # ====================================================================
# # Right Column: Output Preview
# # ====================================================================
# ====================================================================
# Right Column: Output Preview
# ====================================================================
with right_col:
# Combine all parameters
video_params = {
+4 -4
View File
@@ -15,19 +15,19 @@ from pixelle_video.utils.os_util import create_task_output_dir
class ImageToVideoPipelineUI(PipelineUI):
"""
UI for the Audio-visual synchronization Video Generation Pipeline.
UI for the Image To Video Video Generation Pipeline.
Generates videos from user-provided assets (images&text).
"""
name = "voice_synchronization"
name = "image_to_video"
icon = "🎥"
@property
def display_name(self):
return tr("pipeline.audio-visual.name")
return tr("pipeline.i2v.name")
@property
def description(self):
return tr("pipeline.audio-visual.description")
return tr("pipeline.i2v.description")
def render(self, pixelle_video: Any):
# Three-column layout
+4
View File
@@ -0,0 +1,4 @@
{
"source": "runninghub",
"workflow_id": "2013073105194852353"
}