diff --git a/src/args.py b/src/args.py index 6d2a18f..fe52506 100644 --- a/src/args.py +++ b/src/args.py @@ -80,8 +80,9 @@ if not args.cam_input and args.mouse_input is None \ if args.use_sr: args.anime4k = False # Disable anime4k if SR is enabled -if args.anime4k: - args.use_sr = False # Disable SR if anime4k is enabled + +if args.output_spout2: + args.alpha_split = False # Disable alpha split for spout2 output args.model_output_size = 1024 if args.use_sr or args.anime4k else 512 diff --git a/src/face_mesh_client.py b/src/face_mesh_client.py index 387400f..5b67cfa 100644 --- a/src/face_mesh_client.py +++ b/src/face_mesh_client.py @@ -7,6 +7,8 @@ from .utils.channel_shared_mem import SharedMemoryExclusiveChannel from .utils.pose import get_pose from .utils.fps import FPS from .utils.filter import OneEuroFilterNumpy +from OneEuroFilter import OneEuroFilter +import time class FaceMeshClientProcess(Process): @@ -38,7 +40,6 @@ class FaceMeshClientProcess(Process): pose_filter = OneEuroFilterNumpy(freq=input_fps.view(), mincutoff=args.filter_min_cutoff, beta=args.filter_beta) - position_offset = None print("Webcam Input Running at {:.2f} FPS".format(input_fps.view())) position_vector = np.array([0, 0, 0, 1], dtype=np.float32) @@ -66,14 +67,13 @@ class FaceMeshClientProcess(Process): mouth_eye_vector = [0.0] * 27 pose_vector = [0.0] * 6 - mouth_eye_vector[2] = (eye_l_h_temp + eye_r_h_temp) / 2.0 - mouth_eye_vector[3] = (eye_l_h_temp + eye_r_h_temp) / 2.0 + mouth_eye_vector[2] = min(eye_l_h_temp, eye_r_h_temp) + mouth_eye_vector[3] = min(eye_l_h_temp, eye_r_h_temp) mouth_eye_vector[14] = mouth_ratio * 1.5 - mouth_eye_vector[25] = (eye_y_ratio + eye_x_ratio) / 2.0 - mouth_eye_vector[26] = (eye_y_ratio + eye_x_ratio) / 2.0 - + mouth_eye_vector[25] = 0.0 # keep iris stable for user demo + mouth_eye_vector[26] = 0.0 if position_offset is None: position_offset = [(x_angle - 1.5) * 1.6, y_angle * 2.0 , (z_angle + 1.5) * 2] pose_vector[0] = (x_angle - 1.5) * 1.6 - position_offset[0] diff --git a/src/main.py b/src/main.py index b775cde..d6f28f8 100644 --- a/src/main.py +++ b/src/main.py @@ -63,14 +63,16 @@ def main(): infer_process.daemon = True infer_process.start() + cam_width_scale = 2 if args.alpha_split else 1 + ret_channels = 3 if args.output_virtual_cam or args.output_debug else 4 ret_batch_shm_channels = [ SharedMemoryExclusiveChannel(infer_process.ret_shared_mem, ctrl_name=f"ret_shm_ctrl_batch_{i}") for i in range(args.interpolation_scale) ] np_ret_shms = [ - np.ndarray((args.model_output_size, args.model_output_size, 4), dtype=np.uint8, - buffer=infer_process.ret_shared_mem.buf[i * args.model_output_size * args.model_output_size * 4: - (i + 1) * args.model_output_size * args.model_output_size * 4]) + np.ndarray((args.model_output_size, cam_width_scale * args.model_output_size, ret_channels), dtype=np.uint8, + buffer=infer_process.ret_shared_mem.buf[i * cam_width_scale * args.model_output_size * args.model_output_size * ret_channels: + (i + 1) * cam_width_scale * args.model_output_size * args.model_output_size * ret_channels]) for i in range(args.interpolation_scale) ] @@ -89,9 +91,13 @@ def main(): for i in range(args.interpolation_scale): wait_until(last_time + interval) last_time += interval - cv2.imshow("Output Frame Batch {}".format(i), np_ret_shms[i]) + + if args.output_virtual_cam: + pass + else: + cv2.imshow("Output Frame Batch {}".format(i), np_ret_shms[i]) + cv2.waitKey(1) ret_batch_shm_channels[i].release() - cv2.waitKey(1) print("pipeline FPS: {:.2f}, Input FPS: {:.2f}, Model Avg Interval: {:.2f} ms, Cache Hit Ratio: {:.2f}%, GPU Cache Hit Ratio: {:.2f}%, current time {:.5f}".format( infer_process.pipeline_fps_number.value, input_fps.value, diff --git a/src/model_infer_client.py b/src/model_infer_client.py index cddfe06..4d2a6c0 100644 --- a/src/model_infer_client.py +++ b/src/model_infer_client.py @@ -17,8 +17,10 @@ class ModelClientProcess(Process): self.input_image = input_image self.pose_position_shm = pose_position_shm # 45 floats for pose, 4 floats for position - self.ret_shape = (args.interpolation_scale, args.model_output_size, args.model_output_size, 4) - self.ret_nbytes = args.interpolation_scale * args.model_output_size * args.model_output_size * 4 # RGBA + self.alpha_width_scale = 2 if args.alpha_split else 1 + self.ret_channels = 3 if args.output_virtual_cam or args.output_debug else 4 + self.ret_shape = (args.interpolation_scale, args.model_output_size, self.alpha_width_scale * args.model_output_size, self.ret_channels) + self.ret_nbytes = self.alpha_width_scale * args.interpolation_scale * args.model_output_size * args.model_output_size * self.ret_channels # RGBA self.ret_shared_mem = shared_memory.SharedMemory(create=True, size=self.ret_nbytes) self.last_model_interval = Value('f', 0.0) @@ -40,9 +42,9 @@ class ModelClientProcess(Process): for i in range(args.interpolation_scale) ] np_ret_shms = [ - np.ndarray((args.model_output_size, args.model_output_size, 4), dtype=np.uint8, - buffer=self.ret_shared_mem.buf[i * args.model_output_size * args.model_output_size * 4: - (i + 1) * args.model_output_size * args.model_output_size * 4]) + np.ndarray((args.model_output_size, self.alpha_width_scale * args.model_output_size, self.ret_channels), dtype=np.uint8, + buffer=self.ret_shared_mem.buf[i * self.alpha_width_scale * args.model_output_size * args.model_output_size * self.ret_channels: + (i + 1) * self.alpha_width_scale * args.model_output_size * args.model_output_size * self.ret_channels]) for i in range(args.interpolation_scale) ] @@ -151,7 +153,7 @@ class ModelClientProcess(Process): rm, (bgra_image.shape[1], bgra_image.shape[0])) - if args.debug_input: + if args.output_debug: cv2.putText(bgra_image, str('OUT_FPS:%.1f' % self.pipeline_fps_number.value), (0, 16), cv2.FONT_HERSHEY_PLAIN, 1, (0, 255, 0), 1) cv2.putText(bgra_image, str( @@ -180,19 +182,25 @@ class ModelClientProcess(Process): rgb_channels = np.ascontiguousarray(rgba_image[:,:,:3]) if args.anime4k: - rgb_output = self.a4k_processor(rgb_channels) - alpha_output = cv2.resize(alpha_channel, None, fx=2, fy=2) - rgba_image = cv2.merge((rgb_output, alpha_output)) + rgb_channels = self.a4k_processor(rgb_channels) + alpha_channel = cv2.resize(alpha_channel, None, fx=2, fy=2) + rgba_image = cv2.merge((rgb_channels, alpha_channel)) if args.alpha_split: alpha_image = cv2.cvtColor(alpha_channel, cv2.COLOR_GRAY2RGB) - rgba_image = cv2.hconcat([rgba_image, alpha_image]) + rgb_channels = cv2.hconcat([rgb_channels, alpha_image]) - - if args.debug_input: - if args.anime4k or args.alpha_split: - bgra_image = cv2.cvtColor(rgba_image, cv2.COLOR_RGBA2BGRA) - ret.append(bgra_image) - else: + if args.output_debug: + if args.alpha_split: + bgr_channels = cv2.cvtColor(rgb_channels, cv2.COLOR_RGB2BGR) + else: + if args.anime4k: + bgr_channels = cv2.cvtColor(rgba_image, cv2.COLOR_RGBA2BGR) + else: + bgr_channels = cv2.cvtColor(bgra_image, cv2.COLOR_BGRA2BGR) + ret.append(bgr_channels) + elif args.output_virtual_cam: + ret.append(rgb_channels) + else: ret.append(rgba_image) return ret diff --git a/src/open_see_face_client.py b/src/open_see_face_client.py index 7b27ea5..18a70bb 100644 --- a/src/open_see_face_client.py +++ b/src/open_see_face_client.py @@ -6,13 +6,15 @@ import numpy as np from .utils.channel_shared_mem import SharedMemoryExclusiveChannel from .utils.fps import FPS from .utils.filter import OneEuroFilterNumpy +from OneEuroFilter import OneEuroFilter +import time class OSFClientProcess(Process): def __init__(self, pose_position_shm: shared_memory.SharedMemory): super().__init__() self.pose_position_shm = pose_position_shm - self.address = args.osf.split(':')[0] - self.port = int(args.osf.split(':')[1]) + self.address = args.osf_input.split(':')[0] + self.port = int(args.osf_input.split(':')[1]) self.fps = Value('f', 0.0) def run(self): @@ -40,7 +42,8 @@ class OSFClientProcess(Process): pose_filter = OneEuroFilterNumpy(freq=input_fps.view(), mincutoff=args.filter_min_cutoff, beta=args.filter_beta) position_filter = OneEuroFilterNumpy(freq=input_fps.view(), mincutoff=args.filter_min_cutoff, beta=args.filter_beta) - + iris_x_filter = OneEuroFilter(freq=input_fps.view(), mincutoff=0.1, beta=0.3) # extra filter for iris movement + iris_y_filter = OneEuroFilter(freq=input_fps.view(), mincutoff=0.1, beta=0.3) # extra filter for iris movement rotation_offset = None print("OpenSeeFace Input Running at {:.2f} FPS".format(input_fps.view())) while True: @@ -132,11 +135,11 @@ class OSFClientProcess(Process): mouth_eye_vector[2] = 1 - data['leftEyeOpen'] mouth_eye_vector[3] = 1 - data['rightEyeOpen'] - mouth_eye_vector[14] = max(data['MouthOpen'], 0) * 2 # Open larger mouth + mouth_eye_vector[14] = max(data['MouthOpen'], 0) * 1.5 # Open larger mouth # print(mouth_eye_vector[14]) - mouth_eye_vector[25] = -data['eyeRotationY'] * 3 - (data['rotationX']) / 57.3 * 1.5 - mouth_eye_vector[26] = data['eyeRotationX'] * 3 + (data['rotationY']) / 57.3 + mouth_eye_vector[25] = iris_x_filter(-data['eyeRotationY'] * 3 - (data['rotationX']) / 57.3 * 1.5, timestamp=time.perf_counter()) + mouth_eye_vector[26] = iris_y_filter(data['eyeRotationX'] * 3 + (data['rotationY']) / 57.3, timestamp=time.perf_counter()) # print(mouth_eye_vector[25:27]) eyebrow_vector[6] = data['EyebrowUpDownLeft'] eyebrow_vector[7] = data['EyebrowUpDownRight']