feat:debug output handle

src/args.py

@@ -80,8 +80,9 @@ if not args.cam_input and args.mouse_input is None \
 
 if args.use_sr:
     args.anime4k = False  # Disable anime4k if SR is enabled
-if args.anime4k:
-    args.use_sr = False  # Disable SR if anime4k is enabled
+
+if args.output_spout2:
+    args.alpha_split = False  # Disable alpha split for spout2 output
 
 args.model_output_size = 1024 if args.use_sr or args.anime4k else 512
 

src/face_mesh_client.py

@@ -7,6 +7,8 @@ from .utils.channel_shared_mem import SharedMemoryExclusiveChannel
 from .utils.pose import get_pose
 from .utils.fps import FPS
 from .utils.filter import OneEuroFilterNumpy
+from OneEuroFilter import OneEuroFilter
+import time
 
 
 class FaceMeshClientProcess(Process):
@@ -38,7 +40,6 @@ class FaceMeshClientProcess(Process):
 
         
         pose_filter = OneEuroFilterNumpy(freq=input_fps.view(), mincutoff=args.filter_min_cutoff, beta=args.filter_beta)
-
         position_offset = None
         print("Webcam Input Running at {:.2f} FPS".format(input_fps.view()))
         position_vector = np.array([0, 0, 0, 1], dtype=np.float32)
@@ -66,14 +67,13 @@ class FaceMeshClientProcess(Process):
             mouth_eye_vector = [0.0] * 27
             pose_vector = [0.0] * 6
 
-            mouth_eye_vector[2] = (eye_l_h_temp + eye_r_h_temp) / 2.0
-            mouth_eye_vector[3] = (eye_l_h_temp + eye_r_h_temp) / 2.0
+            mouth_eye_vector[2] = min(eye_l_h_temp, eye_r_h_temp)
+            mouth_eye_vector[3] = min(eye_l_h_temp, eye_r_h_temp)
 
             mouth_eye_vector[14] = mouth_ratio * 1.5
 
-            mouth_eye_vector[25] = (eye_y_ratio + eye_x_ratio) / 2.0
-            mouth_eye_vector[26] = (eye_y_ratio + eye_x_ratio) / 2.0
-
+            mouth_eye_vector[25] = 0.0 # keep iris stable for user demo
+            mouth_eye_vector[26] = 0.0
             if position_offset is None:
                 position_offset = [(x_angle - 1.5) * 1.6, y_angle * 2.0 , (z_angle + 1.5) * 2]
             pose_vector[0] = (x_angle - 1.5) * 1.6 - position_offset[0]

src/main.py

@@ -63,14 +63,16 @@ def main():
     infer_process.daemon = True
     infer_process.start()
 
+    cam_width_scale = 2 if args.alpha_split else 1
+    ret_channels = 3 if args.output_virtual_cam or args.output_debug else 4
     ret_batch_shm_channels = [
         SharedMemoryExclusiveChannel(infer_process.ret_shared_mem, ctrl_name=f"ret_shm_ctrl_batch_{i}")
         for i in range(args.interpolation_scale)
     ]
     np_ret_shms = [
-        np.ndarray((args.model_output_size, args.model_output_size, 4), dtype=np.uint8,
-                    buffer=infer_process.ret_shared_mem.buf[i * args.model_output_size * args.model_output_size * 4:
-                                                    (i + 1) * args.model_output_size * args.model_output_size * 4])
+        np.ndarray((args.model_output_size, cam_width_scale * args.model_output_size, ret_channels), dtype=np.uint8,
+                    buffer=infer_process.ret_shared_mem.buf[i * cam_width_scale * args.model_output_size * args.model_output_size * ret_channels:
+                                                    (i + 1) * cam_width_scale * args.model_output_size * args.model_output_size * ret_channels])
         for i in range(args.interpolation_scale)
     ]
 
@@ -89,9 +91,13 @@ def main():
         for i in range(args.interpolation_scale):
             wait_until(last_time + interval)
             last_time += interval
-            cv2.imshow("Output Frame Batch {}".format(i), np_ret_shms[i])
+
+            if args.output_virtual_cam:
+                pass
+            else:
+                cv2.imshow("Output Frame Batch {}".format(i), np_ret_shms[i])
+                cv2.waitKey(1)
             ret_batch_shm_channels[i].release()
-            cv2.waitKey(1)
         print("pipeline FPS: {:.2f}, Input FPS: {:.2f}, Model Avg Interval: {:.2f} ms, Cache Hit Ratio: {:.2f}%, GPU Cache Hit Ratio: {:.2f}%, current time {:.5f}".format(
             infer_process.pipeline_fps_number.value,
             input_fps.value,

src/model_infer_client.py

@@ -17,8 +17,10 @@ class ModelClientProcess(Process):
         self.input_image = input_image
         self.pose_position_shm = pose_position_shm  # 45 floats for pose, 4 floats for position
         
-        self.ret_shape = (args.interpolation_scale, args.model_output_size, args.model_output_size, 4)
-        self.ret_nbytes = args.interpolation_scale * args.model_output_size * args.model_output_size * 4 # RGBA
+        self.alpha_width_scale = 2 if args.alpha_split else 1
+        self.ret_channels = 3 if args.output_virtual_cam or args.output_debug else 4
+        self.ret_shape = (args.interpolation_scale, args.model_output_size, self.alpha_width_scale * args.model_output_size, self.ret_channels)
+        self.ret_nbytes = self.alpha_width_scale * args.interpolation_scale * args.model_output_size * args.model_output_size * self.ret_channels # RGBA
         self.ret_shared_mem = shared_memory.SharedMemory(create=True, size=self.ret_nbytes)
         
         self.last_model_interval = Value('f', 0.0)
@@ -40,9 +42,9 @@ class ModelClientProcess(Process):
             for i in range(args.interpolation_scale)
         ]
         np_ret_shms = [
-            np.ndarray((args.model_output_size, args.model_output_size, 4), dtype=np.uint8,
-                        buffer=self.ret_shared_mem.buf[i * args.model_output_size * args.model_output_size * 4:
-                                                       (i + 1) * args.model_output_size * args.model_output_size * 4])
+            np.ndarray((args.model_output_size, self.alpha_width_scale * args.model_output_size, self.ret_channels), dtype=np.uint8,
+                        buffer=self.ret_shared_mem.buf[i * self.alpha_width_scale * args.model_output_size * args.model_output_size * self.ret_channels:
+                                                       (i + 1) * self.alpha_width_scale * args.model_output_size * args.model_output_size * self.ret_channels])
             for i in range(args.interpolation_scale)
         ]
 
@@ -151,7 +153,7 @@ class ModelClientProcess(Process):
                 rm,
                 (bgra_image.shape[1], bgra_image.shape[0]))
 
-            if args.debug_input:
+            if args.output_debug:
                 cv2.putText(bgra_image, str('OUT_FPS:%.1f' % self.pipeline_fps_number.value), (0, 16), cv2.FONT_HERSHEY_PLAIN, 1,
                             (0, 255, 0), 1)
                 cv2.putText(bgra_image, str(
@@ -180,19 +182,25 @@ class ModelClientProcess(Process):
                 rgb_channels = np.ascontiguousarray(rgba_image[:,:,:3]) 
 
             if args.anime4k:
-                rgb_output = self.a4k_processor(rgb_channels)
-                alpha_output = cv2.resize(alpha_channel, None, fx=2, fy=2)
-                rgba_image = cv2.merge((rgb_output, alpha_output))
+                rgb_channels = self.a4k_processor(rgb_channels)
+                alpha_channel = cv2.resize(alpha_channel, None, fx=2, fy=2)
+                rgba_image = cv2.merge((rgb_channels, alpha_channel))
             if args.alpha_split:
                 alpha_image = cv2.cvtColor(alpha_channel, cv2.COLOR_GRAY2RGB)
-                rgba_image = cv2.hconcat([rgba_image, alpha_image])
+                rgb_channels = cv2.hconcat([rgb_channels, alpha_image])
 
-            
-            if args.debug_input:
-                if args.anime4k or args.alpha_split:
-                    bgra_image = cv2.cvtColor(rgba_image, cv2.COLOR_RGBA2BGRA)
-                ret.append(bgra_image)
-            else: 
+            if args.output_debug:
+                if args.alpha_split:
+                    bgr_channels = cv2.cvtColor(rgb_channels, cv2.COLOR_RGB2BGR)
+                else:
+                    if args.anime4k:
+                        bgr_channels = cv2.cvtColor(rgba_image, cv2.COLOR_RGBA2BGR)
+                    else:
+                        bgr_channels = cv2.cvtColor(bgra_image, cv2.COLOR_BGRA2BGR)
+                ret.append(bgr_channels)
+            elif args.output_virtual_cam:
+                ret.append(rgb_channels)
+            else:
                 ret.append(rgba_image)
         return ret
     

src/open_see_face_client.py

@@ -6,13 +6,15 @@ import numpy as np
 from .utils.channel_shared_mem import SharedMemoryExclusiveChannel
 from .utils.fps import FPS
 from .utils.filter import OneEuroFilterNumpy
+from OneEuroFilter import OneEuroFilter
+import time
 
 class OSFClientProcess(Process):
     def __init__(self, pose_position_shm: shared_memory.SharedMemory):
         super().__init__()
         self.pose_position_shm = pose_position_shm
-        self.address = args.osf.split(':')[0]
-        self.port = int(args.osf.split(':')[1])
+        self.address = args.osf_input.split(':')[0]
+        self.port = int(args.osf_input.split(':')[1])
         self.fps = Value('f', 0.0)
 
     def run(self):
@@ -40,7 +42,8 @@ class OSFClientProcess(Process):
 
         pose_filter = OneEuroFilterNumpy(freq=input_fps.view(), mincutoff=args.filter_min_cutoff, beta=args.filter_beta)
         position_filter = OneEuroFilterNumpy(freq=input_fps.view(), mincutoff=args.filter_min_cutoff, beta=args.filter_beta)
-
+        iris_x_filter = OneEuroFilter(freq=input_fps.view(), mincutoff=0.1, beta=0.3) # extra filter for iris movement
+        iris_y_filter = OneEuroFilter(freq=input_fps.view(), mincutoff=0.1, beta=0.3) # extra filter for iris movement
         rotation_offset = None
         print("OpenSeeFace Input Running at {:.2f} FPS".format(input_fps.view()))
         while True:
@@ -132,11 +135,11 @@ class OSFClientProcess(Process):
             mouth_eye_vector[2] = 1 - data['leftEyeOpen']
             mouth_eye_vector[3] = 1 - data['rightEyeOpen']
 
-            mouth_eye_vector[14] = max(data['MouthOpen'], 0) * 2 # Open larger mouth
+            mouth_eye_vector[14] = max(data['MouthOpen'], 0) * 1.5 # Open larger mouth
             # print(mouth_eye_vector[14])
 
-            mouth_eye_vector[25] = -data['eyeRotationY'] * 3 - (data['rotationX']) / 57.3 * 1.5
-            mouth_eye_vector[26] = data['eyeRotationX'] * 3 + (data['rotationY']) / 57.3
+            mouth_eye_vector[25] = iris_x_filter(-data['eyeRotationY'] * 3 - (data['rotationX']) / 57.3 * 1.5, timestamp=time.perf_counter())
+            mouth_eye_vector[26] = iris_y_filter(data['eyeRotationX'] * 3 + (data['rotationY']) / 57.3, timestamp=time.perf_counter())
             # print(mouth_eye_vector[25:27])
             eyebrow_vector[6] = data['EyebrowUpDownLeft']
             eyebrow_vector[7] = data['EyebrowUpDownRight']