diff --git a/launcher2.py b/launcher2.py index 99db2e1..f449e0f 100644 --- a/launcher2.py +++ b/launcher2.py @@ -48,7 +48,10 @@ default_arg = { 'frame_rate_limit': '30', 'sr': 'anime4k_x2', 'use_tensorrt': False, - 'preset': 'Low' + 'preset': 'Low', + 'mouse_audio_input': False, + 'audio_sensitivity': '0.02', + 'audio_threshold': '10.0' } try: @@ -211,14 +214,14 @@ class OptionPanel(wx.Panel): try: if default is not None: self.control.SetValue(default) - self.valueLabel.SetLabelText(f"{mapper(default):.2f}") + self.valueLabel.SetLabelText(f"{mapper(default):.4f}") except: pass # Update label when slider changes def onSliderChange(event): val = mapper(self.control.GetValue()) - self.valueLabel.SetLabelText(f"{val:.2f}") + self.valueLabel.SetLabelText(f"{val:.4f}") self.control.Bind(wx.EVT_SLIDER, onSliderChange) sliderSizer.Add(self.control, 1, wx.ALIGN_CENTER_VERTICAL) @@ -302,9 +305,12 @@ class LauncherPanel(wx.Panel): addOption('is_eyebrow', title='Eyebrow', desc='使用眉毛输入，对性能有一定影响\n仅支持iFacialMocap', type=1, default=True) addOption('osf', title='OpenSeeFace IP:Port', desc='输入OpenSeeFace连接使用的IP:端口号', type=2) + addOption('mouse_audio_input', title='Audio Input', desc='启用WASAPI音频输入控制嘴部动作', type=1) + addOption('audio_sensitivity', title='Audio Sensitivity', desc='音频灵敏度，控制音频对嘴部动作的影响程度', type=2) + addOption('audio_threshold', title='Audio Threshold', desc='音频阈值，低于此值的音频将被忽略', type=2) addOption('min_cutoff', title='Min CutOff', desc='输入滤波频率截断，\n越小越平滑，越大静止时越灵敏', type=3, mapper=min_cutoff_mapper) - addOption('beta', title='Beta', desc='输入滤波速度补偿，\n越小越平滑，越大快速运动时越灵敏', + addOption('beta', title='Beta', desc='输入滤波速度补偿，\n越小越平滑，越大运动时越灵敏', type=3, mapper=beta_mapper) addOption('output', title='Output', desc='选择输出目标', @@ -379,12 +385,35 @@ class LauncherPanel(wx.Panel): else: self.optionSizer.Show(self.optionDict['min_cutoff']) self.optionSizer.Show(self.optionDict['beta']) - + # Show/hide audio input options for Mouse Input (s == 3) + if s != 3: + self.optionSizer.Hide(self.optionDict['mouse_audio_input']) + self.optionSizer.Hide(self.optionDict['audio_sensitivity']) + self.optionSizer.Hide(self.optionDict['audio_threshold']) + else: + self.optionSizer.Show(self.optionDict['mouse_audio_input']) + # Update audio fields based on checkbox state + audioInputChoice() self.frame.fSizer.Layout() self.frame.Fit() + def audioInputChoice(e=None): + """Handle mouse_audio_input checkbox changes""" + enabled = self.optionDict['mouse_audio_input'].GetValue() + if enabled: + self.optionSizer.Show(self.optionDict['audio_sensitivity']) + self.optionSizer.Show(self.optionDict['audio_threshold']) + self.optionDict['audio_sensitivity'].control.Enable(True) + self.optionDict['audio_threshold'].control.Enable(True) + else: + self.optionSizer.Hide(self.optionDict['audio_sensitivity']) + self.optionSizer.Hide(self.optionDict['audio_threshold']) + self.frame.fSizer.Layout() + self.frame.Fit() + self.optionDict['input'].Bind(wx.EVT_CHOICE, inputChoice) + self.optionDict['mouse_audio_input'].Bind(wx.EVT_CHECKBOX, audioInputChoice) inputChoice() def presetChoice(e=None): @@ -495,6 +524,15 @@ class LauncherPanel(wx.Panel): elif args['input'] == 3: run_args.append('--mouse_input') run_args.append('0,0,' + str(wx.GetDisplaySize().width) + ',' + str(wx.GetDisplaySize().height)) + # Add audio input options for mouse input + if args['mouse_audio_input']: + run_args.append('--mouse_audio_input') + if args['audio_sensitivity']: + run_args.append('--audio_sensitivity') + run_args.append(str(args['audio_sensitivity'])) + if args['audio_threshold']: + run_args.append('--audio_threshold') + run_args.append(str(args['audio_threshold'])) elif args['input'] == 4: if len(args['osf']): run_args.append('--osf_input') diff --git a/src/args.py b/src/args.py index 22ed258..d7ad390 100644 --- a/src/args.py +++ b/src/args.py @@ -21,6 +21,10 @@ parser.add_argument('--mouse_input', type=str) parser.add_argument('--ifm_input', type=str) parser.add_argument('--osf_input', type=str) +parser.add_argument('--mouse_audio_input', action='store_true') +parser.add_argument('--audio_sensitivity', type=float, default=0.02) +parser.add_argument('--audio_threshold', type=float, default=10.0) + parser.add_argument('--output_virtual_cam', action='store_true') parser.add_argument('--output_spout2', action='store_true') parser.add_argument('--output_debug', action='store_true') diff --git a/src/mouse_client.py b/src/mouse_client.py index a61436b..c08d38b 100644 --- a/src/mouse_client.py +++ b/src/mouse_client.py @@ -2,15 +2,40 @@ from multiprocessing import Process, shared_memory, Value from pynput.mouse import Controller import numpy as np import time +import sounddevice as sd from .args import args from .utils.shared_mem_guard import SharedMemoryGuard from .utils.timer_wait import wait_until +from OneEuroFilter import OneEuroFilter +from .utils.fps import FPS +from .utils.timer_wait import wait_until class MouseClientProcess(Process): def __init__(self, pose_position_shm: shared_memory.SharedMemory): super().__init__() self.pose_position_shm = pose_position_shm self.fps = Value('f', 60.0) + self.audio_volume = Value('f', 0.0) + self.audio_running = False + self.audio_callback_fps = FPS(60) + + def audio_callback(self, indata, frames, time_info, status): + """音频回调函数，计算音量""" + if status: + print(f"Audio status: {status}") + + self.audio_callback_fps() + # 计算 RMS (均方根) 音量 + volume_norm = np.linalg.norm(indata) * 10 + + # 应用阈值和灵敏度 + if volume_norm < args.audio_threshold: + volume_norm = 0 + else: + volume_norm = (volume_norm - args.audio_threshold) * args.audio_sensitivity + # 限制在 0-1 范围 + volume_norm = np.clip(volume_norm, 0, 1) + self.audio_volume.value = volume_norm def run(self): mouse = Controller() @@ -20,6 +45,29 @@ class MouseClientProcess(Process): # posLimit = [0, 0, 1920, 1080] posLimit = [int(x) for x in args.mouse_input.split(',')] + + if args.mouse_audio_input: + # 启动音频流 (使用 WASAPI) + try: + # 获取默认输入设备（麦克风或系统音频） + # 如果要捕获系统音频输出，需要使用 loopback 设备 + audio_stream = sd.InputStream( + callback=self.audio_callback, + channels=1, + samplerate=16000, + blocksize=1024, + ) + audio_stream.start() + self.audio_running = True + wait_until(time.perf_counter() + 1.0) # 等待一秒以稳定音频输入 + print("Audio capture started (WASAPI)") + except Exception as e: + print(f"Failed to start audio capture: {e}") + print("Continuing without audio input...") + self.audio_running = False + + audio_filter = OneEuroFilter(freq=self.audio_callback_fps.view(), mincutoff=10.0, beta=0.0) + prev = { 'eye_l_h_temp': 0, 'eye_r_h_temp': 0, @@ -41,10 +89,13 @@ class MouseClientProcess(Process): eye_limit = [0.8, 0.5] head_eye_reduce = 0.6 head_slowness = 0.2 + + # 获取音频音量并映射到 mouth_ratio + current_mouth_ratio = audio_filter(self.audio_volume.value, time.perf_counter()) if self.audio_running else 0 mouse_data = { 'eye_l_h_temp': 0, 'eye_r_h_temp': 0, - 'mouth_ratio': 0, + 'mouth_ratio': current_mouth_ratio, 'eye_y_ratio': np.interp(pos[1], [posLimit[1], posLimit[3]], [1, -1]) * eye_limit[1], 'eye_x_ratio': np.interp(pos[0], [posLimit[0], posLimit[2]], [1, -1]) * eye_limit[0], 'x_angle': np.interp(pos[1], [posLimit[1], posLimit[3]], [1, -1]),