Update Dependencies (#950)

* 1st Round update for Python 3.7, TF1.15, Keras2.3
    Move Tensorflow logging verbosity prior to first tensorflow import
    Keras Optimizers and nn_block update
    lib.logger - Change tf deprecation messages from WARNING to DEBUG
    Raise Tensorflow Max version check to 1.15
    Update requirements and conda check for python 3.7+
    Update install scripts, travis and documentation to Python 3.7

* Revert Keras to 2.2.4

.install/linux/faceswap_setup_x64.sh

@@ -336,10 +336,10 @@ delete_env() {
 }
 
 create_env() {
-    # Create Python 3.6 env for faceswap
+    # Create Python 3.7 env for faceswap
     delete_env
     info "Creating Conda Virtual Environment..."
-    yellow ; "$CONDA_EXECUTABLE" create -n "$ENV_NAME" -q python=3.6 -y
+    yellow ; "$CONDA_EXECUTABLE" create -n "$ENV_NAME" -q python=3.7 -y
 }
 
 

.install/windows/install.nsi

@@ -22,7 +22,7 @@ InstallDir $PROFILE\faceswap
 # Install cli flags
 !define flagsConda "/S /RegisterPython=0 /AddToPath=0 /D=$PROFILE\MiniConda3"
 !define flagsRepo "--depth 1 --no-single-branch ${wwwRepo}"
-!define flagsEnv "-y python=3.6"
+!define flagsEnv "-y python=3.7"
 
 # Folders
 Var ProgramData

.travis.yml

@@ -4,7 +4,7 @@ language: shell
 
 env:
   global:
-    - CONDA_PYTHON=3.6
+    - CONDA_PYTHON=3.7
     - CONDA_BLD_PATH=${HOME}/conda-bld
 
 os:

INSTALL.md

@@ -93,8 +93,8 @@ Reboot your PC, so that everything you have just installed gets registered.
 - Select "Create" at the bottom
 - In the pop up:
     - Give it the name: faceswap
-    - **IMPORTANT**: Select python version 3.6
-    - Hit "Create" (NB: This may take a while as it will need to download Python 3.6)
+    - **IMPORTANT**: Select python version 3.7
+    - Hit "Create" (NB: This may take a while as it will need to download Python 3.7)
 ![Anaconda virtual env setup](https://i.imgur.com/59RHnLs.png)
 
 #### Entering your virtual environment

lib/cli.py

@@ -15,7 +15,7 @@ from importlib import import_module
 
 from lib.logger import crash_log, log_setup
 from lib.model.masks import get_available_masks, get_default_mask
-from lib.utils import FaceswapError, get_backend, safe_shutdown
+from lib.utils import FaceswapError, get_backend, safe_shutdown, set_system_verbosity
 from plugins.plugin_loader import PluginLoader
 
 logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
@@ -46,7 +46,7 @@ class ScriptExecutor():
     def test_for_tf_version():
         """ Check that the minimum required Tensorflow version is installed """
         min_ver = 1.12
-        max_ver = 1.14
+        max_ver = 1.15
         try:
             # Ensure tensorflow doesn't pin all threads to one core when using tf-mkl
             os.environ["KMP_AFFINITY"] = "disabled"
@@ -113,6 +113,7 @@ class ScriptExecutor():
 
     def execute_script(self, arguments):
         """ Run the script for called command """
+        set_system_verbosity(arguments.loglevel)
         is_gui = hasattr(arguments, "redirect_gui") and arguments.redirect_gui
         log_setup(arguments.loglevel, arguments.logfile, self.command, is_gui)
         logger.debug("Executing: %s. PID: %s", self.command, os.getpid())

lib/logger.py

@@ -42,6 +42,7 @@ class FaceswapFormatter(logging.Formatter):
 
     def format(self, record):
         record.message = record.getMessage()
+        record = self.rewrite_tf_deprecation(record)
         # strip newlines
         if "\n" in record.message or "\r" in record.message:
             record.message = record.message.replace("\n", "\\n").replace("\r", "\\r")
@@ -64,6 +65,15 @@ class FaceswapFormatter(logging.Formatter):
             msg = msg + self.formatStack(record.stack_info)
         return msg
 
+    @staticmethod
+    def rewrite_tf_deprecation(record):
+        """ Change TF deprecation messages from WARNING to DEBUG """
+        if record.levelno == 30 and (record.funcName == "_tfmw_add_deprecation_warning" or
+                                     record.module == "deprecation"):
+            record.levelno = 10
+            record.levelname = "DEBUG"
+        return record
+
 
 class RollingBuffer(collections.deque):
     """File-like that keeps a certain number of lines of text in memory."""

lib/sysinfo.py

@@ -44,7 +44,8 @@ class SysInfo():
     @property
     def is_conda(self):
         """ Boolean for whether in a conda environment """
-        return "conda" in sys.version.lower()
+        return ("conda" in sys.version.lower() or
+                os.path.exists(os.path.join(sys.prefix, 'conda-meta')))
 
     @property
     def is_linux(self):

lib/vgg_face2_keras.py

@@ -9,7 +9,7 @@ import psutil
 import cv2
 import numpy as np
 from fastcluster import linkage, linkage_vector
-from lib.utils import GetModel, set_system_verbosity, FaceswapError
+from lib.utils import GetModel, FaceswapError
 
 logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
@@ -41,7 +41,6 @@ class VGGFace2():
     def __init__(self, backend="GPU", loglevel="INFO"):
         logger.debug("Initializing %s: (backend: %s, loglevel: %s)",
                      self.__class__.__name__, backend, loglevel)
-        set_system_verbosity(loglevel)
         backend = backend.upper()
         git_model_id = 10
         model_filename = ["vggface2_resnet50_v2.h5"]

requirements.txt

@@ -1,15 +1,15 @@
 tqdm
 psutil
 pathlib
-numpy==1.16.2
-opencv-python==4.1.1.26
+numpy==1.17.4
+opencv-python==4.1.2.30
 scikit-image
-Pillow==6.1.0
+Pillow==6.2.1
 scikit-learn
 toposort
 fastcluster
-matplotlib==2.2.2
-imageio==2.5.0
+matplotlib==3.1.1
+imageio==2.6.1
 imageio-ffmpeg
 ffmpy==0.2.2
 # Revert back to nvidia-ml-py3 when windows/system32 patch is implemented
@@ -18,7 +18,7 @@ git+https://github.com/deepfakes/nvidia-ml-py3.git
 h5py==2.9.0
 Keras==2.2.4
 pywin32 ; sys_platform == "win32"
-pynvx==0.0.4 ; sys_platform == "darwin"
+pynvx==1.0.0 ; sys_platform == "darwin"
 
 # tensorflow is included within the docker image.
 # If you are looking for dependencies for a manual install,

scripts/convert.py

@@ -34,7 +34,6 @@ class Convert():
     def __init__(self, arguments):
         logger.debug("Initializing %s: (args: %s)", self.__class__.__name__, arguments)
         self.args = arguments
-        Utils.set_verbosity(self.args.loglevel)
 
         self.patch_threads = None
         self.images = Images(self.args)

scripts/extract.py

@@ -38,7 +38,6 @@ class Extract():
     def __init__(self, arguments):
         logger.debug("Initializing %s: (args: %s", self.__class__.__name__, arguments)
         self._args = arguments
-        Utils.set_verbosity(self._args.loglevel)
 
         self._output_dir = str(get_folder(self._args.output_dir))
 

scripts/fsmedia.py

@@ -15,7 +15,7 @@ import imageio
 from lib.alignments import Alignments as AlignmentsBase
 from lib.face_filter import FaceFilter as FilterFunc
 from lib.image import count_frames, read_image
-from lib.utils import (camel_case_split, get_image_paths, set_system_verbosity, _video_extensions)
+from lib.utils import (camel_case_split, get_image_paths, _video_extensions)
 
 logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
@@ -24,11 +24,6 @@ class Utils():
     """ Holds utility functions that are required by more than one media
         object """
 
-    @staticmethod
-    def set_verbosity(loglevel):
-        """ Set the system output verbosity """
-        set_system_verbosity(loglevel)
-
     @staticmethod
     def finalize(images_found, num_faces_detected, verify_output):
         """ Finalize the image processing """

scripts/gui.py

@@ -9,7 +9,6 @@ from tkinter import messagebox, ttk
 from lib.gui import (TaskBar, CliOptions, CommandNotebook, ConsoleOut, Session, DisplayNotebook,
                      get_images, initialize_images, initialize_config, LastSession,
                      MainMenuBar, ProcessWrapper, StatusBar)
-from lib.utils import set_system_verbosity
 
 logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
@@ -208,7 +207,6 @@ class FaceswapGui(tk.Tk):
 class Gui():  # pylint: disable=too-few-public-methods
     """ The GUI process. """
     def __init__(self, arguments):
-        set_system_verbosity(arguments.loglevel)
         self.root = FaceswapGui(arguments.debug)
 
     def process(self):

scripts/train.py

@@ -15,7 +15,7 @@ from keras.backend.tensorflow_backend import set_session
 from lib.image import read_image
 from lib.keypress import KBHit
 from lib.multithreading import MultiThread
-from lib.utils import get_folder, get_image_paths, set_system_verbosity, deprecation_warning
+from lib.utils import get_folder, get_image_paths, deprecation_warning
 from plugins.plugin_loader import PluginLoader
 
 logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
@@ -151,7 +151,6 @@ class Train():
             deprecation_warning("`-nac`, ``--no-augment-color``",
                                 additional_info="This option will be available within training "
                                                 "config settings (/config/train.ini).")
-        set_system_verbosity(self._args.loglevel)
         thread = self._start_thread()
         # from lib.queue_manager import queue_manager; queue_manager.debug_monitor(1)
 

setup.py

@@ -15,7 +15,7 @@ from subprocess import CalledProcessError, run, PIPE, Popen
 INSTALL_FAILED = False
 # Revisions of tensorflow-gpu and cuda/cudnn requirements
 TENSORFLOW_REQUIREMENTS = {"==1.12.0": ["9.0", "7.2"],
-                           ">=1.13.1,<1.15": ["10.0", "7.4"]}  # TF 2.0 Not currently supported
+                           ">=1.13.1,<1.16": ["10.0", "7.4"]}  # TF 2.0 Not currently supported
 # Mapping of Python packages to their conda names if different from pypi or in non-default channel
 CONDA_MAPPING = {
     # "opencv-python": ("opencv", "conda-forge"),  # Periodic issues with conda-forge opencv
@@ -74,7 +74,8 @@ class Environment():
     @property
     def is_conda(self):
         """ Check whether using Conda """
-        return bool("conda" in sys.version.lower())
+        return ("conda" in sys.version.lower() or
+                os.path.exists(os.path.join(sys.prefix, 'conda-meta')))
 
     @property
     def ld_library_path(self):
@@ -220,7 +221,7 @@ class Environment():
             return
 
         if not self.enable_cuda:
-            self.required_packages.append("tensorflow==1.14.0")
+            self.required_packages.append("tensorflow==1.15.0")
             return
 
         tf_ver = None
@@ -266,9 +267,9 @@ class Environment():
     def update_tf_dep_conda(self):
         """ Update Conda TF Dependency """
         if not self.enable_cuda:
-            self.required_packages.append("tensorflow==1.14.0")
+            self.required_packages.append("tensorflow==1.15.0")
         else:
-            self.required_packages.append("tensorflow-gpu==1.14.0")
+            self.required_packages.append("tensorflow-gpu==1.15.0")
 
     def update_amd_dep(self):
         """ Update amd dependency for AMD cards """

tools/alignments.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python3
 """ Tools for manipulating the alignments seralized file """
 import logging
-
-from lib.utils import set_system_verbosity
 from .lib_alignments import (AlignmentData, Check, Dfl, Draw, # noqa pylint: disable=unused-import
                              Extract, Fix, Manual, Merge, Rename,
                              RemoveAlignments, Sort, Spatial, UpdateHashes)
@@ -15,7 +13,6 @@ class Alignments():
     def __init__(self, arguments):
         logger.debug("Initializing %s: (arguments: '%s'", self.__class__.__name__, arguments)
         self.args = arguments
-        set_system_verbosity(self.args.loglevel)
         self.alignments = self.load_alignments()
         logger.debug("Initialized %s", self.__class__.__name__)
 

tools/mask.py

@@ -12,7 +12,7 @@ from lib.faces_detect import DetectedFace
 from lib.image import ImagesLoader, ImagesSaver
 
 from lib.multithreading import MultiThread
-from lib.utils import set_system_verbosity, get_folder
+from lib.utils import get_folder
 from plugins.extract.pipeline import Extractor, ExtractMedia
 
 
@@ -33,7 +33,6 @@ class Mask():
     """
     def __init__(self, arguments):
         logger.debug("Initializing %s: (arguments: %s", self.__class__.__name__, arguments)
-        set_system_verbosity(arguments.loglevel)
         self._update_type = arguments.processing
         self._input_is_faces = arguments.input_type == "faces"
         self._mask_type = arguments.masker

tools/preview.py

@@ -24,7 +24,7 @@ from lib.convert import Converter
 from lib.faces_detect import DetectedFace
 from lib.model.masks import get_available_masks
 from lib.multithreading import MultiThread
-from lib.utils import FaceswapError, set_system_verbosity
+from lib.utils import FaceswapError
 from lib.queue_manager import queue_manager
 from scripts.fsmedia import Alignments, Images
 from scripts.convert import Predict
@@ -42,7 +42,6 @@ class Preview():
 
     def __init__(self, arguments):
         logger.debug("Initializing %s: (arguments: '%s'", self.__class__.__name__, arguments)
-        set_system_verbosity(arguments.loglevel)
         self.config_tools = ConfigTools()
         self.lock = Lock()
         self.trigger_patch = Event()