First Commit
parent
d235c2493b
commit
2bb7f61cf4
@ -0,0 +1,3 @@
|
||||
[submodule "first-order-model"]
|
||||
path = first-order-model
|
||||
url = https://github.com/AliaksandrSiarohin/first-order-model
|
@ -1,2 +1,112 @@
|
||||
# faceit_live3
|
||||
This is an update to faceit_live using first order model
|
||||
This is an update to http://github.com/faceit_live using [first order model](https://github.com/AliaksandrSiarohin/first-order-model) by Aliaksandr Siarohin to generate the images. This model only requires a single image, so no training is needed and things are much easier.
|
||||
|
||||
# Setup
|
||||
|
||||
## Requirements
|
||||
This has been tested on **Ubuntu 18.04 with a Titan RTX/X GPU**.
|
||||
You will need the following to make it work:
|
||||
|
||||
Linux host OS
|
||||
NVidia fast GPU (GTX 1080, GTX 1080i, Titan, etc ...)
|
||||
Fast Desktop CPU (Quad Core or more)
|
||||
NVidia CUDA 10 and cuDNN 7 libraries installed
|
||||
Webcam
|
||||
|
||||
|
||||
## Setup Host System
|
||||
To use the fake webcam feature to enter conferences with our stream we need to insert the **v4l2loopback** kernel module in order to create */dev/video1*. Follow the install instructions at (https://github.com/umlaeute/v4l2loopback), then let's setup our fake webcam:
|
||||
|
||||
```
|
||||
$ git clone https://github.com/umlaeute/v4l2loopback.git
|
||||
$ make && sudo make install
|
||||
$ sudo depmod -a
|
||||
$ sudo modprobe v4l2loopback devices=1
|
||||
$ sudo modprobe v4l2loopback exclusive_caps=1 card_label="faceit_live" video_nr=1
|
||||
$ v4l2-ctl -d /dev/video1 -c timeout=1000
|
||||
```
|
||||
|
||||
# v4l2loopback-ctl set-timeout-image caio.png /dev/video1
|
||||
|
||||
|
||||
Change the video_nr above in case you already have a webcam running on /dev/video1
|
||||
|
||||
To check if things are working, try running an mp4 to generate a video the */dev/video1* (replace ale.mp4 with your own video).
|
||||
```
|
||||
$ ffmpeg -re -i media/ale.mp4 -f v4l2 /dev/video1 -loop 10
|
||||
```
|
||||
And view it
|
||||
```
|
||||
$ ffplay -f v4l2 /dev/video1
|
||||
```
|
||||
|
||||
On Ubuntu 18, I had to make a minor change to the source code of v4l2loopback.c to get loopback working. In case the above doesn't work, you can try this change before running *make* :
|
||||
|
||||
```
|
||||
# v4l2loopback.c
|
||||
from
|
||||
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
|
||||
|
||||
to
|
||||
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,7,0)
|
||||
```
|
||||
|
||||
You can also inspect your /dev/video* devices:
|
||||
|
||||
```
|
||||
$ v4l2-ctl --list-devices
|
||||
$ v4l2-ctl --list-formats -d /dev/video1
|
||||
|
||||
```
|
||||
|
||||
|
||||
If you have more than one GPU, you might need to set some environment variables:
|
||||
```
|
||||
# specify which display to use for rendering
|
||||
$ export DISPLAY=:1
|
||||
|
||||
# which CUDA DEVICE to use (run nvidia-smi to discover the ID)
|
||||
$ export CUDA_VISIBLE_DEVICES = 0
|
||||
|
||||
```
|
||||
|
||||
## Clone this repository
|
||||
Don't forget to use the *--recurse-submodules* parameter to checkout all dependencies.
|
||||
|
||||
$ git clone --recurse-submodules https://github.com/alew3/faceit_live3.git /local_path/
|
||||
|
||||
## Create an Anaconda environment and install requirments
|
||||
```
|
||||
$ conda create -n faceit_live3 python=3.8
|
||||
$ source activate faceit_live3
|
||||
$ conda install pytorch=1.4 torchvision=0.5 cudatoolkit=10.1 -c pytorch
|
||||
$ pip install -r requirements.txt
|
||||
|
||||
```
|
||||
|
||||
## Download 'vox-adv-cpk.pth.tar' to /models folder
|
||||
|
||||
You can find it at: [google-drive](https://drive.google.com/open?id=1PyQJmkdCsAkOYwUyaj_l-l0as-iLDgeH) or [yandex-disk](https://yadi.sk/d/lEw8uRm140L_eQ).
|
||||
|
||||
|
||||
# Usage
|
||||
|
||||
Put in the `./media/` directory the images in jpg/png you want to play with.
|
||||
|
||||
|
||||
# Run the program
|
||||
|
||||
```
|
||||
$ python faceit_live.py
|
||||
```
|
||||
|
||||
## Parameters
|
||||
--webcam # the videoid of the Webcam e.g. 0 if /dev/video0 (default is 0)
|
||||
--image # the face to use for transformations, put the files inside media (by default it loads the first image in the folder)
|
||||
--streamto # the /dev/video number to stream to (default is 1)
|
||||
|
||||
## Example
|
||||
```
|
||||
$ python faceit_live.py --webcam 0 --stream 1 --image oliver.jpg
|
||||
```
|
||||
|
||||
|
@ -0,0 +1,212 @@
|
||||
import imageio
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from skimage.transform import resize
|
||||
import warnings
|
||||
import sys
|
||||
import cv2
|
||||
import time
|
||||
import PIL.Image as Image
|
||||
import PIL.ImageFilter
|
||||
import io
|
||||
from io import BytesIO
|
||||
import pyfakewebcam
|
||||
import pyautogui
|
||||
import os
|
||||
import glob
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
############## setup ####
|
||||
stream = True
|
||||
media_path = './media/'
|
||||
model_path = 'model/'
|
||||
webcam_id = 2
|
||||
webcam_height = 480
|
||||
webcam_width = 640
|
||||
screen_width, screen_height = pyautogui.size()
|
||||
|
||||
stream_id = 1
|
||||
first_order_path = 'first-order-model/'
|
||||
sys.path.insert(0,first_order_path)
|
||||
reset = True
|
||||
|
||||
# import methods from first-order-model
|
||||
import demo
|
||||
from demo import load_checkpoints, make_animation, tqdm
|
||||
|
||||
# prevent tqdm from outputting to console
|
||||
demo.tqdm = lambda *i, **kwargs: i[0]
|
||||
|
||||
img_list = []
|
||||
for filename in os.listdir(media_path):
|
||||
if filename.endswith(".jpg") or filename.endswith(".jpeg") or filename.endswith(".png"):
|
||||
img_list.append(os.path.join(media_path, filename))
|
||||
print(os.path.join(media_path, filename))
|
||||
|
||||
print(img_list, len(img_list))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
############## end setup ####
|
||||
|
||||
def main():
|
||||
global source_image
|
||||
source_image = readnextimage(0)
|
||||
|
||||
# start streaming
|
||||
camera = pyfakewebcam.FakeWebcam(f'/dev/video{stream_id}', webcam_width, webcam_height)
|
||||
camera.print_capabilities()
|
||||
print(f"Fake webcam created on /dev/video{stream}. Use Firefox and join a Google Meeting to test.")
|
||||
|
||||
# capture webcam
|
||||
video_capture = cv2.VideoCapture(webcam_id)
|
||||
time.sleep(1)
|
||||
width = video_capture.get(3) # float
|
||||
height = video_capture.get(4) # float
|
||||
print("webcam dimensions = {} x {}".format(width,height))
|
||||
|
||||
# load models
|
||||
previous = None
|
||||
net = load_face_model()
|
||||
generator, kp_detector = demo.load_checkpoints(config_path=f'{first_order_path}config/vox-adv-256.yaml', checkpoint_path=f'{model_path}/vox-adv-cpk.pth.tar')
|
||||
|
||||
|
||||
# create windows
|
||||
cv2.namedWindow('Face', cv2.WINDOW_GUI_NORMAL) # extracted face
|
||||
cv2.moveWindow('Face', int(screen_width/2)-150, 100)
|
||||
cv2.resizeWindow('Face', 256,256)
|
||||
|
||||
cv2.namedWindow('DeepFake', cv2.WINDOW_GUI_NORMAL) # face transformation
|
||||
cv2.moveWindow('DeepFake', int(screen_width/2)+150, 100)
|
||||
cv2.resizeWindow('DeepFake', 256,256)
|
||||
|
||||
|
||||
cv2.namedWindow('Stream', cv2.WINDOW_GUI_NORMAL) # rendered to fake webcam
|
||||
cv2.moveWindow('Stream', int(screen_width/2)-int(webcam_width/2), 400)
|
||||
cv2.resizeWindow('Stream', webcam_width,webcam_width)
|
||||
|
||||
|
||||
print("Press C to center Webcam, Press N for next image in media directory")
|
||||
|
||||
while True:
|
||||
ret, frame = video_capture.read()
|
||||
frame = cv2.resize(frame, (640, 480))
|
||||
frame = cv2.flip(frame,1)
|
||||
|
||||
if (previous is None or reset is True):
|
||||
x1,y1,x2,y2 = find_face_cut(net,frame)
|
||||
previous = cut_face_window(x1,y1,x2,y2,source_image)
|
||||
reset = False
|
||||
|
||||
deep_fake = process_image(previous,cut_face_window(x1,y1,x2,y2,frame),net, generator, kp_detector)
|
||||
deep_fake = cv2.cvtColor(deep_fake, cv2.COLOR_RGB2BGR)
|
||||
|
||||
#cv2.imshow('Webcam', frame) - get face
|
||||
cv2.imshow('Face', cut_face_window(x1,y1,x2,y2,frame))
|
||||
cv2.imshow('DeepFake', deep_fake)
|
||||
|
||||
|
||||
rgb = cv2.resize(deep_fake,(480,480))
|
||||
# pad image
|
||||
stream_v = cv2.copyMakeBorder( rgb, 0, 0, 80, 80, cv2.BORDER_CONSTANT)
|
||||
cv2.imshow('Stream',stream_v)
|
||||
|
||||
#time.sleep(1/30.0)
|
||||
stream_v = cv2.flip(stream_v,1)
|
||||
stream_v = cv2.cvtColor(stream_v, cv2.COLOR_BGR2RGB)
|
||||
stream_v = (stream_v*255).astype(np.uint8)
|
||||
|
||||
# stream to fakewebcam
|
||||
camera.schedule_frame(stream_v)
|
||||
|
||||
|
||||
k = cv2.waitKey(1)
|
||||
# Hit 'q' on the keyboard to quit!
|
||||
if k & 0xFF == ord('q'):
|
||||
video_capture.release()
|
||||
break
|
||||
elif k==ord('c'):
|
||||
# center
|
||||
reset = True
|
||||
elif k==ord('n'):
|
||||
# rotate images
|
||||
source_image = readnextimage()
|
||||
reset = True
|
||||
|
||||
cv2.destroyAllWindows()
|
||||
exit()
|
||||
|
||||
|
||||
# transform face with first-order-model
|
||||
def process_image(base,current,net, generator,kp_detector):
|
||||
predictions = make_animation(source_image, [base,current], generator, kp_detector, relative=False, adapt_movement_scale=False)
|
||||
return predictions[1]
|
||||
|
||||
def load_face_model():
|
||||
modelFile = f"{model_path}/res10_300x300_ssd_iter_140000.caffemodel"
|
||||
configFile = f"{model_path}./deploy.prototxt.txt"
|
||||
net = cv2.dnn.readNetFromCaffe(configFile, modelFile)
|
||||
return net
|
||||
|
||||
def cut_face_window(x1,y1,x2,y2,face):
|
||||
cut_x1 = x1
|
||||
cut_y1 = y1
|
||||
cut_x2 = x2
|
||||
cut_y2 = y2
|
||||
face = face[cut_y1:cut_y2,cut_x1:cut_x2]
|
||||
face = resize(face, (256, 256))[..., :3]
|
||||
|
||||
return face
|
||||
|
||||
# find the face in webcam stream and center a 256x256 window
|
||||
def find_face_cut(net,face,previous=False):
|
||||
blob = cv2.dnn.blobFromImage(face, 1.0, (300, 300), [104, 117, 123], False, False)
|
||||
frameWidth = 640
|
||||
frameHeight = 480
|
||||
net.setInput(blob)
|
||||
detections = net.forward()
|
||||
bboxes = []
|
||||
for i in range(detections.shape[2]):
|
||||
confidence = detections[0, 0, i, 2]
|
||||
if confidence > 0.8:
|
||||
x1 = int(detections[0, 0, i, 3] * frameWidth)
|
||||
y1 = int(detections[0, 0, i, 4] * frameHeight)
|
||||
x2 = int(detections[0, 0, i, 5] * frameWidth)
|
||||
y2 = int(detections[0, 0, i, 6] * frameHeight)
|
||||
|
||||
face_margin_w = int(256 - (abs(x1-x2) -.5))
|
||||
face_margin_h = int(256 - (abs(y1-y2) -.5))
|
||||
|
||||
cut_x1 = (x1 - int(face_margin_w/2))
|
||||
if cut_x1<0: cut_x1=0
|
||||
cut_y1 = y1 - int(2*face_margin_h/3)
|
||||
if cut_y1<0: cut_y1=0
|
||||
cut_x2 = x2 + int(face_margin_w/2)
|
||||
cut_y2 = y2 + int(face_margin_h/3)
|
||||
|
||||
if range(detections.shape[2]) == 0:
|
||||
print("face not found in video")
|
||||
exit()
|
||||
else:
|
||||
print(f'Found face at: ({x1,y1}) ({x2},{y2} width:{abs(x2-x1)} height: {abs(y2-y1)})')
|
||||
print(f'Cutting at: ({cut_x1,cut_y1}) ({cut_x2},{cut_y2} width:{abs(cut_x2-cut_x1)} height: {abs(cut_y2-cut_y1)})')
|
||||
|
||||
|
||||
return cut_x1,cut_y1,cut_x2,cut_y2
|
||||
|
||||
def readnextimage(position=-1):
|
||||
global img_list,pos
|
||||
if (position != -1):
|
||||
pos = position
|
||||
else:
|
||||
if pos<len(img_list)-1:
|
||||
pos=pos+1
|
||||
else:
|
||||
pos=0
|
||||
source_image = imageio.imread(img_list[pos])
|
||||
source_image = resize(source_image, (256, 256))[..., :3]
|
||||
return source_image
|
||||
|
||||
main()
|
@ -0,0 +1 @@
|
||||
Subproject commit 83d8d7be6a24bba8cf985d126c1eabab906d85bf
|
Binary file not shown.
After Width: | Height: | Size: 680 KiB |
Binary file not shown.
After Width: | Height: | Size: 69 KiB |
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
@ -0,0 +1,90 @@
|
||||
astroid==2.3.3
|
||||
attrs==19.3.0
|
||||
backcall==0.1.0
|
||||
bleach==3.1.4
|
||||
certifi==2019.11.28
|
||||
cffi==1.14.0
|
||||
chardet==3.0.4
|
||||
cryptography==2.8
|
||||
cycler==0.10.0
|
||||
decorator==4.4.2
|
||||
defusedxml==0.6.0
|
||||
entrypoints==0.3
|
||||
idna==2.9
|
||||
imageio==2.8.0
|
||||
imageio-ffmpeg==0.4.1
|
||||
importlib-metadata==1.6.0
|
||||
ipykernel==5.2.0
|
||||
ipython==7.13.0
|
||||
ipython-genutils==0.2.0
|
||||
ipywebrtc==0.5.0
|
||||
ipywidgets==7.5.1
|
||||
isort==4.3.21
|
||||
jedi==0.16.0
|
||||
Jinja2==2.11.1
|
||||
joblib==0.14.1
|
||||
json5==0.9.4
|
||||
jsonschema==3.2.0
|
||||
jupyter-client==6.1.2
|
||||
jupyter-core==4.6.3
|
||||
jupyterlab==2.0.1
|
||||
jupyterlab-server==1.1.0
|
||||
kiwisolver==1.2.0
|
||||
lazy-object-proxy==1.4.3
|
||||
MarkupSafe==1.1.1
|
||||
matplotlib==3.2.1
|
||||
mccabe==0.6.1
|
||||
mistune==0.8.4
|
||||
mkl-fft==1.0.15
|
||||
mkl-random==1.1.0
|
||||
mkl-service==2.3.0
|
||||
nbconvert==5.6.1
|
||||
nbformat==5.0.5
|
||||
networkx==2.4
|
||||
notebook==6.0.3
|
||||
numpy==1.18.1
|
||||
olefile==0.46
|
||||
opencv-python==4.2.0.32
|
||||
pandas==1.0.3
|
||||
pandocfilters==1.4.2
|
||||
parso==0.6.2
|
||||
pexpect==4.8.0
|
||||
pickleshare==0.7.5
|
||||
Pillow==7.0.0
|
||||
prometheus-client==0.7.1
|
||||
prompt-toolkit==3.0.5
|
||||
ptyprocess==0.6.0
|
||||
pycparser==2.18
|
||||
pyfakewebcam==0.1.0
|
||||
Pygments==2.6.1
|
||||
pylint==2.4.4
|
||||
pyOpenSSL==19.1.0
|
||||
pyparsing==2.4.6
|
||||
pyrsistent==0.16.0
|
||||
PySocks==1.7.1
|
||||
python-dateutil==2.8.1
|
||||
pytz==2019.3
|
||||
PyWavelets==1.1.1
|
||||
PyYAML==5.3.1
|
||||
pyzmq==19.0.0
|
||||
requests==2.23.0
|
||||
scikit-image==0.16.2
|
||||
scikit-learn==0.22.2.post1
|
||||
scipy==1.4.1
|
||||
Send2Trash==1.5.0
|
||||
six==1.14.0
|
||||
sklearn==0.0
|
||||
terminado==0.8.3
|
||||
testpath==0.4.4
|
||||
torch==1.4.0
|
||||
torchvision==0.5.0
|
||||
tornado==6.0.4
|
||||
tqdm==4.45.0
|
||||
traitlets==4.3.3
|
||||
urllib3==1.25.7
|
||||
wcwidth==0.1.9
|
||||
webencodings==0.5.1
|
||||
widgetsnbextension==3.5.1
|
||||
wrapt==1.12.1
|
||||
zipp==3.1.0
|
||||
PyAutoGUI==0.9.50
|
Loading…
Reference in New Issue