diff --git a/.gitignore b/.gitignore index b6e4761..47fa1bc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,10 @@ __pycache__/ # C extensions *.so +#Pytorch Models / Config +models/*.tar +models/*.yaml + # Distribution / packaging .Python build/ @@ -127,3 +131,10 @@ dmypy.json # Pyre type checker .pyre/ +model/vox-adv-cpk.pth.tar +media/rodrigo.jpg +media/natorf.jpg +media/dani.png +media/caio2.png +media/bruno.png +media/ale.png diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..c6255e8 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "first-order-model"] + path = first-order-model + url = https://github.com/AliaksandrSiarohin/first-order-model diff --git a/README.md b/README.md index 13a97d2..3f4aef5 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,112 @@ # faceit_live3 -This is an update to faceit_live using first order model +This is an update to http://github.com/faceit_live using [first order model](https://github.com/AliaksandrSiarohin/first-order-model) by Aliaksandr Siarohin to generate the images. This model only requires a single image, so no training is needed and things are much easier. + +# Setup + +## Requirements +This has been tested on **Ubuntu 18.04 with a Titan RTX/X GPU**. +You will need the following to make it work: + + Linux host OS + NVidia fast GPU (GTX 1080, GTX 1080i, Titan, etc ...) + Fast Desktop CPU (Quad Core or more) + NVidia CUDA 10 and cuDNN 7 libraries installed + Webcam + + +## Setup Host System +To use the fake webcam feature to enter conferences with our stream we need to insert the **v4l2loopback** kernel module in order to create */dev/video1*. Follow the install instructions at (https://github.com/umlaeute/v4l2loopback), then let's setup our fake webcam: + +``` +$ git clone https://github.com/umlaeute/v4l2loopback.git +$ make && sudo make install +$ sudo depmod -a +$ sudo modprobe v4l2loopback devices=1 +$ sudo modprobe v4l2loopback exclusive_caps=1 card_label="faceit_live" video_nr=1 +$ v4l2-ctl -d /dev/video1 -c timeout=1000 +``` + +# v4l2loopback-ctl set-timeout-image caio.png /dev/video1 + + +Change the video_nr above in case you already have a webcam running on /dev/video1 + +To check if things are working, try running an mp4 to generate a video the */dev/video1* (replace ale.mp4 with your own video). +``` +$ ffmpeg -re -i media/ale.mp4 -f v4l2 /dev/video1 -loop 10 +``` +And view it +``` +$ ffplay -f v4l2 /dev/video1 +``` + +On Ubuntu 18, I had to make a minor change to the source code of v4l2loopback.c to get loopback working. In case the above doesn't work, you can try this change before running *make* : + +``` +# v4l2loopback.c +from +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29) + +to +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,7,0) +``` + +You can also inspect your /dev/video* devices: + +``` +$ v4l2-ctl --list-devices +$ v4l2-ctl --list-formats -d /dev/video1 + +``` + + +If you have more than one GPU, you might need to set some environment variables: +``` +# specify which display to use for rendering +$ export DISPLAY=:1 + +# which CUDA DEVICE to use (run nvidia-smi to discover the ID) +$ export CUDA_VISIBLE_DEVICES = 0 + +``` + +## Clone this repository +Don't forget to use the *--recurse-submodules* parameter to checkout all dependencies. + + $ git clone --recurse-submodules https://github.com/alew3/faceit_live3.git /local_path/ + +## Create an Anaconda environment and install requirments +``` +$ conda create -n faceit_live3 python=3.8 +$ source activate faceit_live3 +$ conda install pytorch=1.4 torchvision=0.5 cudatoolkit=10.1 -c pytorch +$ pip install -r requirements.txt + +``` + +## Download 'vox-adv-cpk.pth.tar' to /models folder + +You can find it at: [google-drive](https://drive.google.com/open?id=1PyQJmkdCsAkOYwUyaj_l-l0as-iLDgeH) or [yandex-disk](https://yadi.sk/d/lEw8uRm140L_eQ). + + +# Usage + +Put in the `./media/` directory the images in jpg/png you want to play with. + + +# Run the program + +``` +$ python faceit_live.py +``` + +## Parameters + --webcam # the videoid of the Webcam e.g. 0 if /dev/video0 (default is 0) + --image # the face to use for transformations, put the files inside media (by default it loads the first image in the folder) + --streamto # the /dev/video number to stream to (default is 1) + +## Example +``` +$ python faceit_live.py --webcam 0 --stream 1 --image oliver.jpg +``` + diff --git a/faceit_live.py b/faceit_live.py new file mode 100644 index 0000000..e6a0ece --- /dev/null +++ b/faceit_live.py @@ -0,0 +1,212 @@ +import imageio +import numpy as np +import pandas as pd +from skimage.transform import resize +import warnings +import sys +import cv2 +import time +import PIL.Image as Image +import PIL.ImageFilter +import io +from io import BytesIO +import pyfakewebcam +import pyautogui +import os +import glob +warnings.filterwarnings("ignore") + +############## setup #### +stream = True +media_path = './media/' +model_path = 'model/' +webcam_id = 2 +webcam_height = 480 +webcam_width = 640 +screen_width, screen_height = pyautogui.size() + +stream_id = 1 +first_order_path = 'first-order-model/' +sys.path.insert(0,first_order_path) +reset = True + +# import methods from first-order-model +import demo +from demo import load_checkpoints, make_animation, tqdm + +# prevent tqdm from outputting to console +demo.tqdm = lambda *i, **kwargs: i[0] + +img_list = [] +for filename in os.listdir(media_path): + if filename.endswith(".jpg") or filename.endswith(".jpeg") or filename.endswith(".png"): + img_list.append(os.path.join(media_path, filename)) + print(os.path.join(media_path, filename)) + +print(img_list, len(img_list)) + + + + + +############## end setup #### + +def main(): + global source_image + source_image = readnextimage(0) + + # start streaming + camera = pyfakewebcam.FakeWebcam(f'/dev/video{stream_id}', webcam_width, webcam_height) + camera.print_capabilities() + print(f"Fake webcam created on /dev/video{stream}. Use Firefox and join a Google Meeting to test.") + + # capture webcam + video_capture = cv2.VideoCapture(webcam_id) + time.sleep(1) + width = video_capture.get(3) # float + height = video_capture.get(4) # float + print("webcam dimensions = {} x {}".format(width,height)) + + # load models + previous = None + net = load_face_model() + generator, kp_detector = demo.load_checkpoints(config_path=f'{first_order_path}config/vox-adv-256.yaml', checkpoint_path=f'{model_path}/vox-adv-cpk.pth.tar') + + + # create windows + cv2.namedWindow('Face', cv2.WINDOW_GUI_NORMAL) # extracted face + cv2.moveWindow('Face', int(screen_width/2)-150, 100) + cv2.resizeWindow('Face', 256,256) + + cv2.namedWindow('DeepFake', cv2.WINDOW_GUI_NORMAL) # face transformation + cv2.moveWindow('DeepFake', int(screen_width/2)+150, 100) + cv2.resizeWindow('DeepFake', 256,256) + + + cv2.namedWindow('Stream', cv2.WINDOW_GUI_NORMAL) # rendered to fake webcam + cv2.moveWindow('Stream', int(screen_width/2)-int(webcam_width/2), 400) + cv2.resizeWindow('Stream', webcam_width,webcam_width) + + + print("Press C to center Webcam, Press N for next image in media directory") + + while True: + ret, frame = video_capture.read() + frame = cv2.resize(frame, (640, 480)) + frame = cv2.flip(frame,1) + + if (previous is None or reset is True): + x1,y1,x2,y2 = find_face_cut(net,frame) + previous = cut_face_window(x1,y1,x2,y2,source_image) + reset = False + + deep_fake = process_image(previous,cut_face_window(x1,y1,x2,y2,frame),net, generator, kp_detector) + deep_fake = cv2.cvtColor(deep_fake, cv2.COLOR_RGB2BGR) + + #cv2.imshow('Webcam', frame) - get face + cv2.imshow('Face', cut_face_window(x1,y1,x2,y2,frame)) + cv2.imshow('DeepFake', deep_fake) + + + rgb = cv2.resize(deep_fake,(480,480)) + # pad image + stream_v = cv2.copyMakeBorder( rgb, 0, 0, 80, 80, cv2.BORDER_CONSTANT) + cv2.imshow('Stream',stream_v) + + #time.sleep(1/30.0) + stream_v = cv2.flip(stream_v,1) + stream_v = cv2.cvtColor(stream_v, cv2.COLOR_BGR2RGB) + stream_v = (stream_v*255).astype(np.uint8) + + # stream to fakewebcam + camera.schedule_frame(stream_v) + + + k = cv2.waitKey(1) + # Hit 'q' on the keyboard to quit! + if k & 0xFF == ord('q'): + video_capture.release() + break + elif k==ord('c'): + # center + reset = True + elif k==ord('n'): + # rotate images + source_image = readnextimage() + reset = True + + cv2.destroyAllWindows() + exit() + + +# transform face with first-order-model +def process_image(base,current,net, generator,kp_detector): + predictions = make_animation(source_image, [base,current], generator, kp_detector, relative=False, adapt_movement_scale=False) + return predictions[1] + +def load_face_model(): + modelFile = f"{model_path}/res10_300x300_ssd_iter_140000.caffemodel" + configFile = f"{model_path}./deploy.prototxt.txt" + net = cv2.dnn.readNetFromCaffe(configFile, modelFile) + return net + +def cut_face_window(x1,y1,x2,y2,face): + cut_x1 = x1 + cut_y1 = y1 + cut_x2 = x2 + cut_y2 = y2 + face = face[cut_y1:cut_y2,cut_x1:cut_x2] + face = resize(face, (256, 256))[..., :3] + + return face + +# find the face in webcam stream and center a 256x256 window +def find_face_cut(net,face,previous=False): + blob = cv2.dnn.blobFromImage(face, 1.0, (300, 300), [104, 117, 123], False, False) + frameWidth = 640 + frameHeight = 480 + net.setInput(blob) + detections = net.forward() + bboxes = [] + for i in range(detections.shape[2]): + confidence = detections[0, 0, i, 2] + if confidence > 0.8: + x1 = int(detections[0, 0, i, 3] * frameWidth) + y1 = int(detections[0, 0, i, 4] * frameHeight) + x2 = int(detections[0, 0, i, 5] * frameWidth) + y2 = int(detections[0, 0, i, 6] * frameHeight) + + face_margin_w = int(256 - (abs(x1-x2) -.5)) + face_margin_h = int(256 - (abs(y1-y2) -.5)) + + cut_x1 = (x1 - int(face_margin_w/2)) + if cut_x1<0: cut_x1=0 + cut_y1 = y1 - int(2*face_margin_h/3) + if cut_y1<0: cut_y1=0 + cut_x2 = x2 + int(face_margin_w/2) + cut_y2 = y2 + int(face_margin_h/3) + + if range(detections.shape[2]) == 0: + print("face not found in video") + exit() + else: + print(f'Found face at: ({x1,y1}) ({x2},{y2} width:{abs(x2-x1)} height: {abs(y2-y1)})') + print(f'Cutting at: ({cut_x1,cut_y1}) ({cut_x2},{cut_y2} width:{abs(cut_x2-cut_x1)} height: {abs(cut_y2-cut_y1)})') + + + return cut_x1,cut_y1,cut_x2,cut_y2 + +def readnextimage(position=-1): + global img_list,pos + if (position != -1): + pos = position + else: + if pos