├── .gitattributes ├── .gitignore ├── README.md ├── data ├── dataset_util.py ├── paired_dataset.py ├── processing │ ├── celebahq_crop.py │ ├── export_tfrecord_to_img.py │ ├── faceforensics_process_frames.py │ ├── glow_tf.py │ ├── pgan_tf.py │ └── sgan_tf.py ├── transforms.py └── unpaired_dataset.py ├── environment_basic.yml ├── environment_data.yml ├── img └── classifier.jpeg ├── models ├── __init__.py ├── base_model.py ├── basic_discriminator_model.py ├── networks │ ├── customnet.py │ ├── netutils.py │ ├── networks.py │ └── xception.py └── patch_discriminator_model.py ├── notebooks ├── histograms.ipynb ├── ipynb_drop_output.py ├── models ├── overlay_heatmap.ipynb ├── setup_notebooks.sh ├── tables.ipynb └── utils ├── options ├── base_options.py ├── test_options.py └── train_options.py ├── patches.py ├── resources ├── download_resources_basic.sh └── download_resources_data.sh ├── scripts ├── 00_data_processing_export_tfrecord_to_img.sh ├── 00_data_processing_faceforensics_aligned_frames.sh ├── 00_data_processing_sample_celebahq_models.sh ├── 00_data_processing_sample_ffhq_models.sh ├── 01_train_gan_xception_patches_invonly.sh ├── 01_train_gan_xception_patches_samplesonly.sh ├── 01_train_gan_xception_patches_winversion.sh ├── 01_train_gan_xception_patches_winversion_randcrop.sh ├── 01_train_gan_xception_patches_winversion_randresizecrop.sh ├── 02_train_faceforensics_DF_xception_patches_aligned.sh ├── 02_train_faceforensics_F2F_xception_patches_aligned.sh ├── 02_train_faceforensics_FS_xception_patches_aligned.sh ├── 02_train_faceforensics_NT_xception_patches_aligned.sh ├── 03_baseline_resnet_full.sh ├── 03_baseline_xception_full.sh ├── 03_train_resnet_block1.sh ├── 04_eval_checkpoint.sh ├── 04_eval_patches_faceforensics_DF.sh ├── 04_eval_patches_faceforensics_F2F.sh ├── 04_eval_patches_gen_models.sh ├── 04_eval_visualize_faceforensics_DF.sh ├── 04_eval_visualize_faceforensics_F2F.sh └── 04_eval_visualize_gen_models.sh ├── segmenter.py ├── test.py ├── test_runs.py ├── train.py └── utils ├── imutil.py ├── lightbox.html ├── logging.py ├── options.py ├── pbar.py ├── pidfile.py ├── renormalize.py ├── rfutil.py ├── show.py ├── tensorboard_utils.py ├── util.py └── visualizer.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.ipynb filter=clean_ipynb 2 | client/* linguist-vendored 3 | client_dist/* linguist-vendored 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # Jupyter Notebook 7 | .ipynb_checkpoints 8 | 9 | dataset/ 10 | checkpoints/ 11 | results/ 12 | runs/ 13 | runs_finished/ 14 | resources/* 15 | !resources/download_resources* 16 | utils/rf_cache/ 17 | notebooks/plots/ 18 | 19 | # extras/extra_resources/ 20 | # extras/caricatures/ 21 | # extras/finetune_G/ 22 | # extras/for_figure* 23 | extras/ 24 | 25 | *.mp4 26 | # dataset-temp/ 27 | notes/ 28 | website/ 29 | -------------------------------------------------------------------------------- /data/dataset_util.py: -------------------------------------------------------------------------------- 1 | """ 2 | modified from PyTorch image folder (https://github.com/pytorch/vision/blob/master/torchvision/datasets/folder.py) 3 | """ 4 | 5 | import torch.utils.data as data 6 | 7 | from PIL import Image 8 | import os 9 | import os.path 10 | from utils import util 11 | 12 | IMG_EXTENSIONS = [ 13 | '.jpg', '.JPG', '.jpeg', '.JPEG', 14 | '.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', 15 | '.tif', '.TIF', '.tiff', '.TIFF', 16 | ] 17 | 18 | def is_image_file(filename): 19 | return any(filename.endswith(extension) for extension in IMG_EXTENSIONS) 20 | 21 | def make_dataset(dir, max_dataset_size=float("inf")): 22 | cache = dir.rstrip('/') + '.txt' 23 | if os.path.isfile(cache): 24 | print("Using filelist cached at %s" % cache) 25 | with open(cache) as f: 26 | images = [line.strip() for line in f] 27 | # patch up image list with new loading method 28 | if images[0].startswith(dir): 29 | print("Using image list from older version") 30 | image_list = [] 31 | for image in images: 32 | image_list.append(image) 33 | else: 34 | print("Adding prefix to saved image list") 35 | image_list = [] 36 | prefix = os.path.dirname(dir.rstrip('/')) 37 | for image in images: 38 | image_list.append(os.path.join(prefix, image)) 39 | return image_list 40 | print("Walking directory ...") 41 | images = [] 42 | assert os.path.isdir(dir), '%s is not a valid directory' % dir 43 | for root, _, fnames in sorted(os.walk(dir, followlinks=True)): 44 | for fname in fnames: 45 | if is_image_file(fname): 46 | path = os.path.join(root, fname) 47 | images.append(path) 48 | image_list = images[:min(max_dataset_size, len(images))] 49 | with open(cache, 'w') as f: 50 | prefix = os.path.dirname(dir.rstrip('/')) + '/' 51 | for i in image_list: 52 | f.write('%s\n' % util.remove_prefix(i, prefix)) 53 | return image_list 54 | 55 | def default_loader(path): 56 | return Image.open(path).convert('RGB') 57 | -------------------------------------------------------------------------------- /data/paired_dataset.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torch.utils.data as data 3 | from .dataset_util import make_dataset 4 | from PIL import Image 5 | import numpy as np 6 | import torch 7 | from . import transforms 8 | import random 9 | 10 | class PairedDataset(data.Dataset): 11 | """A dataset class for paired images 12 | e.g. corresponding real and manipulated images 13 | """ 14 | 15 | def __init__(self, opt, im_path_real, im_path_fake, is_val=False): 16 | """Initialize this dataset class. 17 | 18 | Parameters: 19 | opt -- experiment options 20 | im_path_real -- path to folder of real images 21 | im_path_fake -- path to folder of fake images 22 | is_val -- is this training or validation? used to determine 23 | transform 24 | """ 25 | super().__init__() 26 | self.dir_real = im_path_real 27 | self.dir_fake = im_path_fake 28 | 29 | # if pairs are named in the same order 30 | # e.g. real/train/face1.png, real/train/face2.png ... 31 | # fake/train/face1.png, fake/train/face2.png ... 32 | # then this will align them in a batch unless 33 | # --no_serial_batches is specified 34 | self.real_paths = sorted(make_dataset(self.dir_real, 35 | opt.max_dataset_size)) 36 | self.fake_paths = sorted(make_dataset(self.dir_fake, 37 | opt.max_dataset_size)) 38 | self.real_size = len(self.real_paths) 39 | self.fake_size = len(self.fake_paths) 40 | self.transform = transforms.get_transform(opt, for_val=is_val) 41 | self.opt = opt 42 | 43 | def __getitem__(self, index): 44 | """Return a data point and its metadata information. 45 | 46 | Parameters: 47 | index - - a random integer for data indexing 48 | """ 49 | # read a image given a random integer index 50 | real_path = self.real_paths[index % self.real_size] # make sure index is within then range 51 | if self.opt.no_serial_batches: 52 | # randomize the index for one domain to avoid fixed pairs 53 | index_fake = random.randint(0, self.fake_size - 1) 54 | else: 55 | # make sure index is within range 56 | index_fake = index % self.fake_size 57 | 58 | fake_path = self.fake_paths[index_fake] 59 | real_img = Image.open(real_path).convert('RGB') 60 | fake_img = Image.open(fake_path).convert('RGB') 61 | 62 | # apply image transformation 63 | real = self.transform(real_img) 64 | fake = self.transform(fake_img) 65 | 66 | return {'manipulated': fake, 67 | 'original': real, 68 | 'path_manipulated': fake_path, 69 | 'path_original': real_path 70 | } 71 | 72 | def __len__(self): 73 | return max(self.real_size, self.fake_size) 74 | -------------------------------------------------------------------------------- /data/processing/celebahq_crop.py: -------------------------------------------------------------------------------- 1 | from PIL import Image 2 | import dlib 3 | import cv2 4 | import scipy.ndimage 5 | import numpy as np 6 | 7 | detector = dlib.get_frontal_face_detector() 8 | predictor = dlib.shape_predictor('resources/shape_predictor_68_face_landmarks.dat') 9 | 10 | def rot90(v): 11 | return np.array([-v[1], v[0]]) 12 | 13 | def shape_to_np(shape, dtype="int"): 14 | # initialize the list of (x, y)-coordinates 15 | coords = np.zeros((68, 2), dtype=dtype) 16 | 17 | # loop over the 68 facial landmarks and convert them 18 | # to a 2-tuple of (x, y)-coordinates 19 | for i in range(0, 68): 20 | coords[i] = (shape.part(i).x, shape.part(i).y) 21 | 22 | # return the list of (x, y)-coordinates 23 | return coords 24 | 25 | def rect_to_bb(rect): 26 | # take a bounding predicted by dlib and convert it 27 | # to the format (x, y, w, h) as we would normally do 28 | # with OpenCV 29 | x = rect.left() 30 | y = rect.top() 31 | w = rect.right() - x 32 | h = rect.bottom() - y 33 | 34 | # return a tuple of (x, y, w, h) 35 | return (x, y, w, h) 36 | 37 | def celebahq_crop(im, landmarks=None): 38 | if landmarks is None: 39 | gray = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY) 40 | rects = detector(gray, 1) 41 | if not rects: 42 | return None 43 | shape = predictor(gray, rects[0]) 44 | shape = shape_to_np(shape) 45 | 46 | lefteye = np.mean(shape[[37, 38, 40, 41], :], axis=0) 47 | righteye = np.mean(shape[[43, 44, 46, 47], :], axis=0) 48 | nose = shape[30] 49 | leftmouth = shape[48] 50 | rightmouth = shape[54] 51 | lm = np.stack([lefteye, righteye, nose, leftmouth, rightmouth]) 52 | else: 53 | lm = landmarks 54 | 55 | img = Image.fromarray(im) 56 | 57 | # Choose oriented crop rectangle. 58 | eye_avg = (lm[0] + lm[1]) * 0.5 + 0.5 59 | mouth_avg = (lm[3] + lm[4]) * 0.5 + 0.5 60 | eye_to_eye = lm[1] - lm[0] 61 | eye_to_mouth = mouth_avg - eye_avg 62 | x = eye_to_eye - rot90(eye_to_mouth) 63 | x /= np.hypot(*x) 64 | x *= max(np.hypot(*eye_to_eye) * 2.0, np.hypot(*eye_to_mouth) * 1.8) 65 | y = rot90(x) 66 | c = eye_avg + eye_to_mouth * 0.1 67 | quad = np.stack([c - x - y, c - x + y, c + x + y, c + x - y]) 68 | zoom = 128 / (np.hypot(*x) * 2) 69 | 70 | # Shrink. 71 | shrink = int(np.floor(0.5 / zoom)) 72 | if shrink > 1: 73 | size = (int(np.round(float(img.size[0]) / shrink)), int(np.round(float(img.size[1]) / shrink))) 74 | img = img.resize(size, Image.ANTIALIAS) 75 | quad /= shrink 76 | zoom *= shrink 77 | 78 | # Crop. 79 | border = max(int(np.round(128 * 0.1 / zoom)), 3) 80 | crop = (int(np.floor(min(quad[:,0]))), int(np.floor(min(quad[:,1]))), int(np.ceil(max(quad[:,0]))), int(np.ceil(max(quad[:,1])))) 81 | crop = (max(crop[0] - border, 0), max(crop[1] - border, 0), min(crop[2] + border, img.size[0]), min(crop[3] + border, img.size[1])) 82 | if crop[2] - crop[0] < img.size[0] or crop[3] - crop[1] < img.size[1]: 83 | img = img.crop(crop) 84 | quad -= crop[0:2] 85 | 86 | # Simulate super-resolution. 87 | superres = int(np.exp2(np.ceil(np.log2(zoom)))) 88 | if superres > 1: 89 | img = img.resize((img.size[0] * superres, img.size[1] * superres), Image.ANTIALIAS) 90 | quad *= superres 91 | zoom /= superres 92 | 93 | # Pad. 94 | pad = (int(np.floor(min(quad[:,0]))), int(np.floor(min(quad[:,1]))), int(np.ceil(max(quad[:,0]))), int(np.ceil(max(quad[:,1])))) 95 | pad = (max(-pad[0] + border, 0), max(-pad[1] + border, 0), max(pad[2] - img.size[0] + border, 0), max(pad[3] - img.size[1] + border, 0)) 96 | if max(pad) > border - 4: 97 | pad = np.maximum(pad, int(np.round(128 * 0.3 / zoom))) 98 | img = np.pad(np.float32(img), ((pad[1], pad[3]), (pad[0], pad[2]), (0, 0)), 'reflect') 99 | h, w, _ = img.shape 100 | y, x, _ = np.mgrid[:h, :w, :1] 101 | mask = 1.0 - np.minimum(np.minimum(np.float32(x) / pad[0], np.float32(y) / pad[1]), np.minimum(np.float32(w-1-x) / pad[2], np.float32(h-1-y) / pad[3])) 102 | blur = 128 * 0.02 / zoom 103 | img += (scipy.ndimage.gaussian_filter(img, [blur, blur, 0]) - img) * np.clip(mask * 3.0 + 1.0, 0.0, 1.0) 104 | img += (np.median(img, axis=(0,1)) - img) * np.clip(mask, 0.0, 1.0) 105 | img = Image.fromarray(np.uint8(np.clip(np.round(img), 0, 255)), 'RGB') 106 | quad += pad[0:2] 107 | 108 | # Transform. 109 | img = img.transform((512, 512), Image.QUAD, (quad + 0.5).flatten(), Image.BILINEAR) 110 | img = img.resize((1024, 1024), Image.ANTIALIAS) 111 | return img, lm 112 | -------------------------------------------------------------------------------- /data/processing/export_tfrecord_to_img.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import argparse 3 | import os 4 | import numpy as np 5 | from tqdm import tqdm 6 | from PIL import Image 7 | 8 | parser = argparse.ArgumentParser(description='Save tfrecord format as image format') 9 | parser.add_argument('--tfrecord', required=True, help='Path to tfrecord file') 10 | parser.add_argument('--outdir', required=True, help='Path to save png output') 11 | parser.add_argument('--dataset', required=True, help='which tf record dataset [celebahq|ffhq]') 12 | parser.add_argument('--outsize', type=int, help='resize to this size') 13 | parser.add_argument('--format', type=str, default='png', help='image format') 14 | parser.add_argument('--resize_method', type=str, default='lanczos', help='image format') 15 | 16 | args = parser.parse_args() 17 | 18 | os.makedirs(args.outdir, exist_ok=True) 19 | os.makedirs(os.path.join(args.outdir, 'train'), exist_ok=True) 20 | os.makedirs(os.path.join(args.outdir, 'val'), exist_ok=True) 21 | os.makedirs(os.path.join(args.outdir, 'test'), exist_ok=True) 22 | 23 | tfr_opt = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.NONE) 24 | tfr_file = args.tfrecord 25 | 26 | resize_methods = { 27 | 'lanczos': Image.LANCZOS, 28 | 'bilinear': Image.BILINEAR 29 | } 30 | 31 | if args.dataset == 'celebahq': 32 | # this will save images in the same order as original celebahq images 33 | image_list_file = 'resources/celebahq_image_list.txt' 34 | with open(image_list_file, 'rt') as file: 35 | lines = [line.split() for line in file] 36 | fields = dict() 37 | for idx, field in enumerate(lines[0]): 38 | type = int if field.endswith('idx') else str 39 | fields[field] = [type(line[idx]) for line in lines[1:]] 40 | 41 | # determine the train/test/val partitions 42 | partition_list = 'resources/celeba_list_eval_partition.txt' 43 | partition_keys={'0':'train', '1':'val', '2':'test'} 44 | with open(partition_list) as f: 45 | celeba_partitions={} 46 | for line in f: 47 | img, split = line.strip().split() 48 | img = img.split('.')[0] 49 | celeba_partitions[img] = partition_keys[split] 50 | 51 | indices = np.array(fields['idx']) 52 | order = np.arange(len(indices)) 53 | np.random.RandomState(123).shuffle(order) 54 | for o, record in tqdm(zip(order, tf.python_io.tf_record_iterator(tfr_file, tfr_opt)), total=len(order)): 55 | ex = tf.train.Example() 56 | ex.ParseFromString(record) 57 | shape = ex.features.feature['shape'].int64_list.value 58 | data = ex.features.feature['data'].bytes_list.value[0] 59 | im = np.fromstring(data, np.uint8).reshape(shape) 60 | im = Image.fromarray(np.transpose(im, (1, 2, 0)), 'RGB') 61 | if args.outsize: 62 | resizer = resize_methods[args.resize_method] 63 | im = im.resize((args.outsize, args.outsize), resizer) 64 | orig_number = fields['orig_file'][o].split('.')[0] 65 | partition = celeba_partitions[str(orig_number)] 66 | im.save(os.path.join(args.outdir, partition, 67 | '%s.%s' % (orig_number, args.format))) 68 | elif args.dataset == 'ffhq': 69 | total_count = 70000 70 | for i, record in tqdm(enumerate(tf.python_io.tf_record_iterator( 71 | tfr_file, tfr_opt)),total=total_count): 72 | ex = tf.train.Example() 73 | ex.ParseFromString(record) 74 | shape = ex.features.feature['shape'].int64_list.value 75 | data = ex.features.feature['data'].bytes_list.value[0] 76 | im = np.fromstring(data, np.uint8).reshape(shape) 77 | im = Image.fromarray(np.transpose(im, (1, 2, 0)), 'RGB') 78 | if args.outsize: 79 | resizer = resize_methods[args.resize_method] 80 | im = im.resize((args.outsize, args.outsize), resizer) 81 | if i < 60000: 82 | partition = 'train' 83 | elif i < 65000: 84 | partition = 'val' 85 | else: 86 | partition = 'test' 87 | im.save(os.path.join(args.outdir, partition, '%09d.%s' % (i, args.format))) 88 | else: 89 | raise NotImplementedError 90 | -------------------------------------------------------------------------------- /data/processing/faceforensics_process_frames.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import numpy as np 4 | from tqdm import tqdm 5 | from PIL import Image 6 | import skvideo.io 7 | import json 8 | import sys 9 | from utils import pidfile 10 | import numpy as np 11 | from data.processing.celebahq_crop import celebahq_crop 12 | 13 | 14 | parser = argparse.ArgumentParser(description='Process and align face forensics frames') 15 | parser.add_argument('--source_dir_manipulated', required=True, help='source videos directory, e.g. manipulated_sequences/Deepfakes/c23/videos') 16 | parser.add_argument('--source_dir_original', required=True, help='source videos directory, e.g. original_sequences/youtube/c23/videos') 17 | parser.add_argument('--outsize', type=int, default=128, help='resize to this size') 18 | parser.add_argument('--output_dir', required=True, help='output directory') 19 | parser.add_argument('--split', default='val.json', help='Path to split json file') 20 | 21 | args = parser.parse_args() 22 | rnd = np.random.RandomState(0) 23 | 24 | 25 | with open(args.split) as f: 26 | split = json.load(f) 27 | split_name = os.path.splitext(os.path.basename(args.split))[0] 28 | 29 | outdir = args.output_dir 30 | os.makedirs(outdir, exist_ok=True) 31 | os.makedirs(os.path.join(outdir, 'original', split_name), exist_ok=True) 32 | os.makedirs(os.path.join(outdir, 'manipulated', split_name), exist_ok=True) 33 | os.makedirs(os.path.join(outdir, 'detections', split_name), exist_ok=True) 34 | 35 | for i, s in enumerate(tqdm(split)): 36 | vidname = '_'.join(s) 37 | vidname_orig = s[0] # take target sequence for original videos 38 | # print("%d: %s" % (i, vidname)) 39 | vidpath = os.path.join(args.source_dir_manipulated, vidname + '.mp4') 40 | vidpath_orig = os.path.join(args.source_dir_original, vidname_orig + '.mp4') 41 | 42 | if not os.path.isfile(vidpath): 43 | print("Video not found: %s" % vidpath) 44 | continue 45 | 46 | manipulated_video = skvideo.io.vreader(vidpath) 47 | original_video = skvideo.io.vreader(vidpath_orig) 48 | 49 | counter = 0 50 | for j, (frame, orig) in enumerate(zip(manipulated_video, original_video)): 51 | if os.path.isfile(os.path.join(outdir, 'detections', split_name, 52 | '%s_%03d.npz' % (vidname, j))): 53 | print('Found existing %s_%03d.npz' % (vidname, j)) 54 | counter += 1 55 | continue 56 | try: 57 | # might return none or out of bounds error 58 | if rnd.uniform() > 0.5: 59 | # use manipulated landmarks 60 | cropped, landmarks = celebahq_crop(frame) 61 | cropped = cropped.resize((args.outsize, args.outsize), Image.LANCZOS) 62 | cropped_orig = (celebahq_crop(orig, landmarks)[0] 63 | .resize((args.outsize, args.outsize), Image.LANCZOS)) 64 | else: 65 | # use original landmarks 66 | cropped_orig, landmarks = celebahq_crop(orig) 67 | cropped_orig = cropped_orig.resize((args.outsize, args.outsize), Image.LANCZOS) 68 | cropped = (celebahq_crop(frame, landmarks)[0] 69 | .resize((args.outsize, args.outsize), Image.LANCZOS)) 70 | # save the results 71 | cropped.save(os.path.join(outdir, 'manipulated', split_name, 72 | '%s_%03d.png' % (vidname, j))) 73 | cropped_orig.save(os.path.join(outdir, 'original', split_name, 74 | '%s_%03d.png' % (vidname, j))) 75 | np.savez(os.path.join(outdir, 'detections', split_name, 76 | '%s_%03d.npz' % (vidname, j)), 77 | lm=landmarks) 78 | counter += 1 79 | 80 | # for val/test partitions, just take 100 detected frames per video 81 | if counter == 100 and split_name in ['test', 'val']: 82 | print("Processed 100 frames of %s" % vidname) 83 | print("Moving to next video") 84 | break 85 | except: 86 | print("Error:", sys.exc_info()[0]) 87 | -------------------------------------------------------------------------------- /data/processing/glow_tf.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import argparse 3 | import os 4 | import utils.pbar as pbar 5 | import utils.pidfile as pidfile 6 | import numpy as np 7 | import pickle 8 | import tensorflow as tf 9 | from tqdm import tqdm 10 | import PIL.Image 11 | import sys 12 | 13 | 14 | def parse_args(): 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument('--model_path', help='path to pretrained model') 17 | parser.add_argument('--pretrained', help='downloads pretrained model [celebahq]') 18 | parser.add_argument('--output_path', required=True, help='path to save generated samples') 19 | parser.add_argument('--num_samples', type=int, default=100, help='number of samples') 20 | parser.add_argument('--seed', type=int, default=0, help='random seed for sampling') 21 | parser.add_argument('--batch_size', type=int, default=64, help='batch size for generating samples') 22 | parser.add_argument("--gpu", default="", type=str, help='GPUs to use (leave blank for CPU only)') 23 | parser.add_argument("--manipulate", action="store_true", help='add random manipulations to face') 24 | parser.add_argument("--format", default="jpg", type=str, help='file format to save generated images') 25 | parser.add_argument("--resize", type=int, help='resizes images to this size before saving') 26 | 27 | opt = parser.parse_args() 28 | print(opt) 29 | return opt 30 | 31 | def sample(opt): 32 | 33 | # Initialize TensorFlow session. 34 | tf.InteractiveSession() 35 | 36 | assert(opt.model_path or opt.pretrained), 'specify weights path or pretrained model' 37 | 38 | if opt.model_path: 39 | raise NotImplementedError 40 | elif opt.pretrained: 41 | assert(opt.pretrained == 'celebahq') 42 | # make sure to git clone glow repository first 43 | sys.path.append('resources/glow/demo') 44 | import model 45 | eps_std = 0.7 46 | eps_size = model.eps_size 47 | 48 | rng = np.random.RandomState(opt.seed) 49 | attr = np.random.RandomState(opt.seed+1) 50 | tags = [] 51 | amts = [] 52 | 53 | for batch_start in tqdm(range(0, opt.num_samples, opt.batch_size)): 54 | # Generate latent vectors. 55 | bs = min(opt.num_samples, batch_start + opt.batch_size) - batch_start 56 | feps = rng.normal(scale=eps_std, size=[bs, eps_size]) 57 | 58 | if opt.manipulate: 59 | tag = attr.randint(len(model._TAGS), size=bs) 60 | amt = attr.uniform(-1, 1, size=(bs, 1)) 61 | dzs = model.z_manipulate[tag] 62 | feps = feps + amt * dzs 63 | tags.append(tag) 64 | amts.append(amt) 65 | 66 | images = model.decode(feps) 67 | 68 | # Save images as PNG. 69 | for idx in range(images.shape[0]): 70 | filename = os.path.join(opt.output_path, 'seed%03d_sample%06d.%s' 71 | % (opt.seed, batch_start + idx, 72 | opt.format)) 73 | im = PIL.Image.fromarray(images[idx], 'RGB') 74 | if opt.resize: 75 | im = im.resize((opt.resize, opt.resize), PIL.Image.LANCZOS) 76 | im.save(filename) 77 | 78 | if opt.manipulate: 79 | outfile = os.path.join(opt.output_path, 'manipulations.npz') 80 | np.savez(outfile, tags=np.concatenate(tags), amts=np.concatenate(amts)) 81 | 82 | 83 | if __name__ == '__main__': 84 | opt = parse_args() 85 | if os.environ.get('CUDA_VISIBLE_DEVICES') is None: 86 | os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpu 87 | expdir = opt.output_path 88 | os.makedirs(expdir, exist_ok=True) 89 | 90 | pidfile.exit_if_job_done(expdir) 91 | sample(opt) 92 | pidfile.mark_job_done(expdir) 93 | -------------------------------------------------------------------------------- /data/processing/pgan_tf.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import argparse 3 | import os 4 | import utils.pbar as pbar 5 | import utils.pidfile as pidfile 6 | import numpy as np 7 | import dill as pickle 8 | import tensorflow as tf 9 | import PIL.Image 10 | from tqdm import tqdm 11 | import sys 12 | import albumentations as A 13 | sys.path.append('resources/progressive_growing_of_gans') 14 | 15 | 16 | def parse_args(): 17 | parser = argparse.ArgumentParser() 18 | parser.add_argument('--model_path', required=True, help='path to pretrained model') 19 | parser.add_argument('--output_path', required=True, help='path to save generated samples') 20 | parser.add_argument('--num_samples', type=int, default=100, help='number of samples') 21 | parser.add_argument('--seed', type=int, default=0, help='random seed for sampling') 22 | parser.add_argument('--batch_size', type=int, default=64, help='batch size for generating samples') 23 | parser.add_argument("--gpu", default="", type=str, help='GPUs to use (leave blank for CPU only)') 24 | parser.add_argument("--format", default="jpg", type=str, help='file format to save generated images') 25 | parser.add_argument("--resize", type=int, help='resizes images to this size before saving') 26 | parser.add_argument("--quality", type=int, help='compression quality') 27 | 28 | opt = parser.parse_args() 29 | print(opt) 30 | return opt 31 | 32 | def sample(opt): 33 | 34 | # Initialize TensorFlow session. 35 | tf.InteractiveSession() 36 | 37 | # Import official CelebA-HQ networks. 38 | with open(opt.model_path, 'rb') as file: 39 | G, D, Gs = pickle.load(file) 40 | 41 | rng = np.random.RandomState(opt.seed) 42 | 43 | for batch_start in tqdm(range(0, opt.num_samples, opt.batch_size)): 44 | # Generate latent vectors. 45 | bs = min(opt.num_samples, batch_start + opt.batch_size) - batch_start 46 | latents = rng.randn(bs, *Gs.input_shapes[0][1:]) 47 | # Generate dummy labels (not used by the official networks). 48 | labels = np.zeros([latents.shape[0]] + Gs.input_shapes[1][1:]) 49 | 50 | # Run the generator to produce a set of images. 51 | images = Gs.run(latents, labels) 52 | 53 | # Convert images to PIL-compatible format. 54 | images = np.clip(np.rint((images + 1.0) / 2.0 * 255.0), 0.0, 55 | 255.0).astype(np.uint8) # [-1,1] => [0,255] 56 | images = images.transpose(0, 2, 3, 1) # NCHW => NHWC 57 | 58 | # Save images as PNG. 59 | for idx in range(images.shape[0]): 60 | filename = os.path.join(opt.output_path, 'seed%03d_sample%06d.%s' 61 | % (opt.seed, batch_start + idx, 62 | opt.format)) 63 | im = PIL.Image.fromarray(images[idx], 'RGB') 64 | if opt.resize: 65 | im = im.resize((opt.resize, opt.resize), PIL.Image.LANCZOS) 66 | if opt.quality: 67 | aug = A.augmentations.transforms.JpegCompression(p=1) 68 | w, h = im.size 69 | im_np = np.asarray(im.resize((1024, 1024), PIL.Image.LANCZOS)) 70 | im = PIL.Image.fromarray(aug.apply(im_np, quality=opt.quality)) 71 | im = im.resize((w, h), PIL.Image.LANCZOS) 72 | im.save(filename) 73 | 74 | 75 | if __name__ == '__main__': 76 | opt = parse_args() 77 | if os.environ.get('CUDA_VISIBLE_DEVICES') is None: 78 | os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpu 79 | expdir = opt.output_path 80 | os.makedirs(expdir, exist_ok=True) 81 | 82 | pidfile.exit_if_job_done(expdir) 83 | sample(opt) 84 | pidfile.mark_job_done(expdir) 85 | -------------------------------------------------------------------------------- /data/processing/sgan_tf.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import argparse 3 | import os 4 | import utils.pbar as pbar 5 | import utils.pidfile as pidfile 6 | import numpy as np 7 | import dill as pickle 8 | import tensorflow as tf 9 | import PIL.Image 10 | from tqdm import tqdm 11 | import sys 12 | sys.path.append('resources/stylegan') 13 | import dnnlib 14 | import dnnlib.tflib as tflib 15 | import config 16 | 17 | 18 | def parse_args(): 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument('--model_path', help='path to pretrained model') 21 | parser.add_argument('--pretrained', help='downloads pretrained model [ffhq, celebahq]') 22 | parser.add_argument('--output_path', required=True, help='path to save generated samples') 23 | parser.add_argument('--num_samples', type=int, default=100, help='number of samples') 24 | parser.add_argument('--seed', type=int, default=0, help='random seed for sampling') 25 | parser.add_argument('--batch_size', type=int, default=64, help='batch size for generating samples') 26 | parser.add_argument("--gpu", default="", type=str, help='GPUs to use (leave blank for CPU only)') 27 | parser.add_argument("--format", default="jpg", type=str, help='file format to save generated images') 28 | parser.add_argument("--resize", type=int, help='resizes images to this size before saving') 29 | 30 | opt = parser.parse_args() 31 | print(opt) 32 | return opt 33 | 34 | def sample(opt): 35 | 36 | # Initialize TensorFlow session. 37 | tf.InteractiveSession() 38 | 39 | assert(opt.model_path or opt.pretrained), 'specify weights path or pretrained model' 40 | 41 | if opt.model_path: 42 | # Import official CelebA-HQ networks. 43 | with open(opt.model_path, 'rb') as file: 44 | G, D, Gs = pickle.load(file) 45 | elif opt.pretrained: 46 | urls = dict( 47 | ffhq='https://drive.google.com/uc?id=1MEGjdvVpUsu1jB4zrXZN7Y4kBBOzizDQ', 48 | celebahq='https://drive.google.com/uc?id=1MGqJl28pN4t7SAtSrPdSRJSQJqahkzUf') 49 | url = urls[opt.pretrained] 50 | with dnnlib.util.open_url(url, cache_dir=config.cache_dir) as f: 51 | _G, _D, Gs = pickle.load(f) 52 | 53 | rng = np.random.RandomState(opt.seed) 54 | 55 | for batch_start in tqdm(range(0, opt.num_samples, opt.batch_size)): 56 | # Generate latent vectors. 57 | bs = min(opt.num_samples, batch_start + opt.batch_size) - batch_start 58 | latents = rng.randn(bs, *Gs.input_shapes[0][1:]) 59 | # Generate dummy labels (not used by the official networks). 60 | labels = np.zeros([latents.shape[0]] + Gs.input_shapes[1][1:]) 61 | 62 | # Run the generator to produce a set of images. 63 | images = Gs.run(latents, labels) 64 | 65 | # Convert images to PIL-compatible format. 66 | images = np.clip(np.rint((images + 1.0) / 2.0 * 255.0), 0.0, 67 | 255.0).astype(np.uint8) # [-1,1] => [0,255] 68 | images = images.transpose(0, 2, 3, 1) # NCHW => NHWC 69 | 70 | # Save images as PNG. 71 | for idx in range(images.shape[0]): 72 | filename = os.path.join(opt.output_path, 'seed%03d_sample%06d.%s' 73 | % (opt.seed, batch_start + idx, 74 | opt.format)) 75 | im = PIL.Image.fromarray(images[idx], 'RGB') 76 | if opt.resize: 77 | im = im.resize((opt.resize, opt.resize), PIL.Image.LANCZOS) 78 | im.save(filename) 79 | 80 | 81 | if __name__ == '__main__': 82 | opt = parse_args() 83 | if os.environ.get('CUDA_VISIBLE_DEVICES') is None: 84 | os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpu 85 | expdir = opt.output_path 86 | os.makedirs(expdir, exist_ok=True) 87 | 88 | pidfile.exit_if_job_done(expdir) 89 | sample(opt) 90 | pidfile.mark_job_done(expdir) 91 | -------------------------------------------------------------------------------- /data/transforms.py: -------------------------------------------------------------------------------- 1 | import torchvision.transforms as transforms 2 | import torchvision.transforms.functional as F 3 | import logging 4 | import PIL.Image 5 | import numpy as np 6 | 7 | def get_transform(opt, for_val=False): 8 | transform_list = [] 9 | 10 | if for_val: 11 | transform_list.append(transforms.Resize( 12 | opt.loadSize, interpolation=PIL.Image.LANCZOS)) 13 | if opt.model == 'patch_discriminator': 14 | # patch discriminators have receptive field < whole image 15 | # so patch ensembling should use all patches in image 16 | transform_list.append(transforms.CenterCrop(opt.loadSize)) 17 | else: 18 | transform_list.append(transforms.CenterCrop(opt.fineSize)) 19 | 20 | # we can add other test time augmentations here 21 | if hasattr(opt, 'test_flip') and opt.test_flip: 22 | transform_list.append(transforms.RandomHorizontalFlip(p=1.0)) 23 | if hasattr(opt, 'test_compression') and opt.test_compression: 24 | transform_list.append(JPEGCompression(opt.compression)) 25 | if hasattr(opt, 'test_blur') and opt.test_blur: 26 | transform_list.append(Blur(opt.blur)) 27 | if hasattr(opt, 'test_gamma') and opt.test_gamma: 28 | transform_list.append(Gamma(opt.gamma)) 29 | else: 30 | if opt.random_resized_crop: 31 | transform_list.append(transforms.RandomResizedCrop( 32 | opt.fineSize, interpolation=PIL.Image.LANCZOS)) 33 | elif opt.random_crop: 34 | transform_list.append(transforms.Resize( 35 | opt.loadSize, interpolation=PIL.Image.LANCZOS)) 36 | transform_list.append(transforms.RandomCrop(opt.fineSize)) 37 | else: 38 | transform_list.append(transforms.Resize( 39 | opt.loadSize, interpolation=PIL.Image.LANCZOS)) 40 | transform_list.append(transforms.CenterCrop(opt.fineSize)) 41 | 42 | if opt.cnn_detection_augment: 43 | transform_list.append(CNNDetectionAugmentations( 44 | opt.cnn_detection_augment)) 45 | 46 | if opt.color_augment: 47 | transform_list.append(ColorAugmentations()) 48 | 49 | if opt.all_augment: 50 | transform_list.append(AllAugmentations()) 51 | 52 | # add horizonal flip 53 | if not opt.no_flip: 54 | transform_list.append(transforms.RandomHorizontalFlip(p=0.5)) 55 | 56 | transform_list.append(transforms.ToTensor()) 57 | transform_list.append(transforms.Normalize((0.5, 0.5, 0.5), 58 | (0.5, 0.5, 0.5))) 59 | transform = transforms.Compose(transform_list) 60 | print(transform) 61 | logging.info(transform) 62 | return transform 63 | 64 | ### additional augmentations ### 65 | 66 | class AllAugmentations(object): 67 | def __init__(self): 68 | import albumentations 69 | self.transform = albumentations.Compose([ 70 | albumentations.Blur(blur_limit=3), 71 | albumentations.JpegCompression(quality_lower=30, quality_upper=100, p=0.5), 72 | albumentations.RandomBrightnessContrast(), 73 | albumentations.RandomGamma(gamma_limit=(80, 120)), 74 | albumentations.CLAHE(), 75 | ]) 76 | 77 | def __call__(self, image): 78 | image_np = np.array(image) 79 | augmented = self.transform(image=image_np) 80 | image_pil = PIL.Image.fromarray(augmented['image']) 81 | return image_pil 82 | 83 | class CNNDetectionAugmentations(object): 84 | def __init__(self, prob=0.5): 85 | import albumentations 86 | self.transform = albumentations.Compose([ 87 | albumentations.Blur(blur_limit=3, p=prob), 88 | albumentations.JpegCompression(quality_lower=30, quality_upper=100, p=prob), 89 | ]) 90 | def __call__(self, image): 91 | image_np = np.array(image) 92 | augmented = self.transform(image=image_np) 93 | image_pil = PIL.Image.fromarray(augmented['image']) 94 | return image_pil 95 | 96 | class JPEGCompression(object): 97 | def __init__(self, level): 98 | import albumentations as A 99 | self.level = level 100 | self.transform = A.augmentations.transforms.JpegCompression(p=1) 101 | 102 | def __call__(self, image): 103 | image_np = np.array(image) 104 | image_out = self.transform.apply(image_np, quality=self.level) 105 | image_pil = PIL.Image.fromarray(image_out) 106 | return image_pil 107 | 108 | class Blur(object): 109 | def __init__(self, level): 110 | import albumentations as A 111 | self.level = level 112 | self.transform = A.Blur(blur_limit=(self.level, self.level), always_apply=True) 113 | 114 | def __call__(self, image): 115 | image_np = np.array(image) 116 | augmented = self.transform(image=image_np) 117 | image_pil = PIL.Image.fromarray(augmented['image']) 118 | return image_pil 119 | 120 | class Gamma(object): 121 | def __init__(self, level): 122 | import albumentations as A 123 | self.level = level 124 | self.transform = A.augmentations.transforms.RandomGamma(p=1) 125 | 126 | def __call__(self, image): 127 | image_np = np.array(image) 128 | image_out = self.transform.apply(image_np, gamma=self.level/100) 129 | image_pil = PIL.Image.fromarray(image_out) 130 | return image_pil 131 | 132 | class ColorAugmentations(object): 133 | def __init__(self): 134 | import albumentations 135 | self.transform = albumentations.Compose([ 136 | albumentations.RandomBrightnessContrast(), 137 | albumentations.RandomGamma(gamma_limit=(80, 120)), 138 | albumentations.CLAHE(), 139 | ]) 140 | 141 | def __call__(self, image): 142 | image_np = np.array(image) 143 | augmented = self.transform(image=image_np) 144 | image_pil = PIL.Image.fromarray(augmented['image']) 145 | return image_pil 146 | 147 | -------------------------------------------------------------------------------- /data/unpaired_dataset.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torch.utils.data as data 3 | from .dataset_util import make_dataset 4 | from PIL import Image 5 | import numpy as np 6 | import torch 7 | from . import transforms 8 | 9 | class UnpairedDataset(data.Dataset): 10 | """A dataset class for loading images within a single folder 11 | """ 12 | def __init__(self, opt, im_path, is_val=False): 13 | """Initialize this dataset class. 14 | 15 | Parameters: 16 | opt -- experiment options 17 | im_path -- path to folder of images 18 | is_val -- is this training or validation? used to determine 19 | transform 20 | """ 21 | super().__init__() 22 | self.dir = im_path 23 | self.paths = sorted(make_dataset(self.dir, opt.max_dataset_size)) 24 | self.size = len(self.paths) 25 | assert(self.size > 0) 26 | self.transform = transforms.get_transform(opt, for_val=is_val) 27 | self.opt = opt 28 | 29 | def __getitem__(self, index): 30 | """Return a data point and its metadata information. 31 | 32 | Parameters: 33 | index - - a random integer for data indexing 34 | """ 35 | # read a image given a random integer index 36 | path = self.paths[index] 37 | img = Image.open(path).convert('RGB') 38 | 39 | # apply image transformation 40 | img = self.transform(img) 41 | 42 | return {'img': img, 43 | 'path': path 44 | } 45 | 46 | def __len__(self): 47 | return self.size 48 | -------------------------------------------------------------------------------- /environment_basic.yml: -------------------------------------------------------------------------------- 1 | name: patch-forensics 2 | channels: 3 | - pytorch 4 | - ostrokach-forge 5 | - conda-forge 6 | dependencies: 7 | - python=3.6 8 | - cudatoolkit=10.1 9 | - cudnn=7.6.0 10 | - pytorch=1.4 11 | - torchvision 12 | - albumentations 13 | - numpy 14 | - scipy 15 | - scikit-learn 16 | - matplotlib 17 | - pandas 18 | - seaborn 19 | - jupyter 20 | - oyaml 21 | - opencv 22 | - tqdm 23 | - pip 24 | - pillow 25 | - tensorboard 26 | - tensorboardX 27 | - pip: 28 | - dill 29 | - gdown 30 | 31 | -------------------------------------------------------------------------------- /environment_data.yml: -------------------------------------------------------------------------------- 1 | name: patch-forensics-data 2 | channels: 3 | - pytorch 4 | - ostrokach-forge 5 | - conda-forge 6 | dependencies: 7 | - python=3.6 8 | - cudatoolkit=10.1 9 | - cudnn=7.6.0 10 | - pytorch=1.4 11 | - torchvision 12 | - albumentations 13 | - numpy 14 | - scipy 15 | - scikit-learn 16 | - matplotlib 17 | - pandas 18 | - seaborn 19 | - jupyter 20 | - oyaml 21 | - opencv 22 | - tqdm 23 | - pip 24 | - pillow 25 | - tensorboard 26 | - tensorboardX 27 | - sk-video 28 | - dlib 29 | - pip: 30 | - tensorflow-gpu==1.14.0 31 | - dill 32 | - gdown 33 | -------------------------------------------------------------------------------- /img/classifier.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chail/patch-forensics/c87fc9d76ebf82d1495ad17bd1e0cff34ca44132/img/classifier.jpeg -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from models.base_model import BaseModel 3 | 4 | def find_model_using_name(model_name): 5 | # Given the option --model [modelname], 6 | # the file "models/modelname_model.py" 7 | # will be imported. 8 | model_filename = "models." + model_name + "_model" 9 | modellib = importlib.import_module(model_filename) 10 | 11 | # In the file, the class called ModelNameModel() will 12 | # be instantiated. It has to be a subclass of BaseModel, 13 | # and it is case-insensitive. 14 | model = None 15 | target_model_name = model_name.replace('_', '') + 'model' 16 | for name, cls in modellib.__dict__.items(): 17 | if name.lower() == target_model_name.lower() \ 18 | and issubclass(cls, BaseModel): 19 | model = cls 20 | 21 | if model is None: 22 | print("In %s.py, there should be a subclass of BaseModel with class name that matches %s in lowercase." % (model_filename, target_model_name)) 23 | exit(0) 24 | 25 | return model 26 | 27 | def get_option_setter(model_name): 28 | model_class = find_model_using_name(model_name) 29 | return model_class.modify_commandline_options 30 | 31 | def create_model(opt, **kwargs): 32 | model = find_model_using_name(opt.model) 33 | instance = model(opt, **kwargs) 34 | print("model [%s] was created" % (instance.name())) 35 | return instance 36 | -------------------------------------------------------------------------------- /models/base_model.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | from collections import OrderedDict 4 | from .networks import netutils 5 | from .networks import networks 6 | import logging 7 | 8 | 9 | class BaseModel(): 10 | @staticmethod 11 | def modify_commandline_options(parser): 12 | networks.modify_commandline_options(parser) 13 | return parser 14 | 15 | def __init__(self, opt): 16 | self.opt = opt 17 | self.gpu_ids = opt.gpu_ids 18 | self.isTrain = opt.isTrain 19 | self.device = torch.device('cuda:{}'.format(self.gpu_ids[0])) if self.gpu_ids else torch.device('cpu') 20 | self.save_dir = os.path.join(opt.checkpoints_dir, opt.name) 21 | torch.backends.cudnn.benchmark = True 22 | self.loss_names = [] 23 | self.model_names = [] 24 | self.visual_names = [] 25 | self.image_paths = [] 26 | self.optimizers = {} 27 | 28 | def name(self): 29 | return 'BaseModel' 30 | 31 | def set_input(self, input): 32 | self.input = input 33 | 34 | def forward(self): 35 | pass 36 | 37 | # load and print networks; create schedulers 38 | def setup(self, opt, parser=None): 39 | current_ep = 0 40 | best_val_metric, best_val_ep = 0, 0 41 | self.print_networks() 42 | if self.isTrain: 43 | self.schedulers = {k: netutils.get_scheduler(optim, opt) for 44 | (k, optim) in self.optimizers.items()} 45 | if not self.isTrain or opt.load_model: 46 | current_ep, best_val_metric, best_val_ep = self.load_networks(opt.which_epoch) 47 | if opt.which_epoch not in ['latest', 'bestval']: 48 | # checkpoint was saved at end of epoch 49 | current_ep += 1 50 | return current_ep, best_val_metric, best_val_ep 51 | 52 | # make models eval mode 53 | def eval(self): 54 | for name in self.model_names: 55 | if isinstance(name, str): 56 | net = getattr(self, 'net_' + name) 57 | net.eval() 58 | 59 | # make models train mode 60 | def train(self): 61 | for name in self.model_names: 62 | if isinstance(name, str): 63 | net = getattr(self, 'net_' + name) 64 | net.train() 65 | 66 | # used in test time, wrapping `forward` in no_grad() so we don't save 67 | # intermediate steps for backprop 68 | def test(self, compute_losses=False): 69 | with torch.no_grad(): 70 | self.forward() 71 | if(compute_losses): 72 | self.compute_losses_D() 73 | 74 | # get image paths 75 | def get_image_paths(self): 76 | return self.image_paths 77 | 78 | def optimize_parameters(self): 79 | pass 80 | 81 | # update learning rate (called once every epoch) 82 | def update_learning_rate(self, metric=None): 83 | for k, scheduler in self.schedulers.items(): 84 | if metric is not None: 85 | assert self.opt.lr_policy in ['plateau', 'constant'] 86 | scheduler.step(metric) 87 | else: 88 | scheduler.step() 89 | for k, optim in self.optimizers.items(): 90 | logging.info('learning rate net_%s = %0.7f' % (k, optim.param_groups[0]['lr'])) 91 | 92 | # return visualization images. train.py will display these images 93 | def get_current_visuals(self): 94 | visual_ret = OrderedDict() 95 | for name in self.visual_names: 96 | assert(isinstance(name, str)) 97 | visual_ret[name] = getattr(self, name) 98 | return visual_ret 99 | 100 | # return training losses/errors. train.py will print out these errors as debugging information 101 | def get_current_losses(self): 102 | errors_ret = OrderedDict() 103 | for name in self.loss_names: 104 | assert(isinstance(name, str)) 105 | # float(...) works for both scalar tensor and float number 106 | errors_ret[name] = float(getattr(self, name)) 107 | return errors_ret 108 | 109 | # save models to the disk 110 | def save_networks(self, save_name, current_ep, 111 | best_val_metric, best_val_ep): 112 | for name in self.model_names: 113 | assert(isinstance(name, str)) 114 | save_filename = '%s_net_%s.pth' % (save_name, name) 115 | save_path = os.path.join(self.save_dir, save_filename) 116 | net = getattr(self, 'net_' + name) 117 | if isinstance(net, torch.nn.DataParallel): 118 | sd = net.module.state_dict() 119 | else: 120 | sd = net.state_dict() 121 | 122 | optim = self.optimizers[name].state_dict() 123 | sched = self.schedulers[name].state_dict() 124 | 125 | checkpoint = dict(state_dict=sd, optimizer=optim, 126 | scheduler=sched, epoch=current_ep, 127 | best_val_metric=best_val_metric, 128 | best_val_ep=best_val_ep) 129 | torch.save(checkpoint, save_path) 130 | 131 | # load models from the disk 132 | def load_networks(self, save_name): 133 | for name in self.model_names: 134 | assert(isinstance(name, str)) 135 | load_filename = '%s_net_%s.pth' % (save_name, name) 136 | load_path = os.path.join(self.save_dir, load_filename) 137 | net = getattr(self, 'net_' + name) 138 | if isinstance(net, torch.nn.DataParallel): 139 | net = net.module 140 | print('loading the model from %s' % load_path) 141 | checkpoint = torch.load(load_path, map_location=str(self.device)) 142 | state_dict = checkpoint['state_dict'] 143 | if hasattr(state_dict, '_metadata'): 144 | del state_dict._metadata 145 | net.load_state_dict(state_dict) 146 | 147 | if self.isTrain: 148 | print('restoring optimizer and scheduler for %s' % name) 149 | self.optimizers[name].load_state_dict(checkpoint['optimizer']) 150 | self.schedulers[name].load_state_dict(checkpoint['scheduler']) 151 | current_ep = checkpoint['epoch'] 152 | best_val_metric = checkpoint['best_val_metric'] 153 | best_val_ep = checkpoint['best_val_ep'] 154 | return current_ep, best_val_metric, best_val_ep 155 | 156 | # print network information 157 | def print_networks(self): 158 | print('---------- Networks initialized -------------') 159 | for name in self.model_names: 160 | if isinstance(name, str): 161 | net = getattr(self, 'net_' + name) 162 | num_params = 0 163 | for param in net.parameters(): 164 | num_params += param.numel() 165 | print(net) 166 | print('[Network %s] Total number of parameters : %.3f M' % (name, num_params / 1e6)) 167 | print('-----------------------------------------------') 168 | 169 | # set requires_grad=False to avoid computation 170 | def set_requires_grad(self, nets, requires_grad=False): 171 | if not isinstance(nets, list): 172 | nets = [nets] 173 | for net in nets: 174 | if net is not None: 175 | for param in net.parameters(): 176 | param.requires_grad = requires_grad 177 | -------------------------------------------------------------------------------- /models/basic_discriminator_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from collections import OrderedDict 4 | from utils import renormalize, imutil 5 | from .base_model import BaseModel 6 | from .networks import networks 7 | import numpy as np 8 | import logging 9 | from collections import namedtuple 10 | 11 | class BasicDiscriminatorModel(BaseModel): 12 | 13 | def name(self): 14 | return 'BasicDiscriminatorModel' 15 | 16 | def __init__(self, opt): 17 | BaseModel.__init__(self, opt) 18 | 19 | # specify the training losses you want to print out. 20 | self.loss_names = ['loss_D'] 21 | self.loss_names += ['acc_D'] 22 | self.val_metric = 'acc_D' 23 | 24 | # specify the images you want to save/display. 25 | self.visual_names = ['fake_0', 'fake_1', 'fake_2', 'fake_3', 'fake_4', 26 | 'real_0', 'real_1', 'real_2', 'real_3', 'real_4'] 27 | 28 | # specify the models you want to save to the disk. 29 | self.model_names = ['D'] 30 | 31 | # load/define networks 32 | torch.manual_seed(opt.seed) # set model seed 33 | self.net_D = networks.define_D(opt.which_model_netD, 34 | opt.init_type, self.gpu_ids) 35 | self.criterionCE = nn.CrossEntropyLoss().to(self.device) 36 | self.softmax = torch.nn.Softmax(dim=1) 37 | 38 | if self.isTrain: 39 | self.optimizers['D'] = torch.optim.Adam( 40 | self.net_D.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) 41 | 42 | def set_input(self, input): 43 | self.ims = input['ims'].to(self.device) 44 | self.labels = input['labels'].to(self.device) 45 | 46 | def forward(self): 47 | self.pred_logit = self.net_D(self.ims) 48 | 49 | def compute_losses_D(self): 50 | self.loss_D = self.criterionCE(self.pred_logit, self.labels) 51 | self.acc_D = torch.mean(torch.eq(self.labels, torch.argmax( 52 | self.pred_logit, dim=1)).float()) 53 | 54 | def backward_D(self): 55 | self.compute_losses_D() 56 | self.loss_D.backward() 57 | 58 | def optimize_parameters(self): 59 | self.optimizers['D'].zero_grad() 60 | self.forward() 61 | self.backward_D() 62 | self.optimizers['D'].step() 63 | 64 | def get_current_visuals(self): 65 | from collections import OrderedDict 66 | visual_ret = OrderedDict() 67 | fake_ims = self.ims[self.labels == self.opt.fake_class_id] 68 | real_ims = self.ims[self.labels != self.opt.fake_class_id] 69 | for i in range(min(5, len(fake_ims))): 70 | visual_ret['fake_%d' % i] = renormalize.as_tensor( 71 | fake_ims[[i], :, :, :], source='zc', target='pt') 72 | for i in range(min(5, len(real_ims))): 73 | visual_ret['real_%d' % i] = renormalize.as_tensor( 74 | real_ims[[i], :, :, :], source='zc', target='pt') 75 | return visual_ret 76 | 77 | def reset(self): 78 | # for debugging .. clear all the cached variables 79 | self.loss_D = None 80 | self.acc_D = None 81 | self.ims = None 82 | self.labels = None 83 | self.pred_logit = None 84 | 85 | def get_predictions(self, *args): 86 | # makes it consistent with patch discriminator outputs 87 | Predictions = namedtuple('predictions', ['vote', 'before_softmax', 88 | 'after_softmax', 'raw']) 89 | with torch.no_grad(): 90 | predictions = self.softmax(self.pred_logit).cpu().numpy() 91 | return Predictions(None, None, None, predictions) 92 | 93 | def visualize(self, pred_outputs, pred_paths, labels, transform, 94 | target_label, dirname, n=100, **kwargs): 95 | print("Visualization only implemented for patch-based models") 96 | raise NotImplementedError 97 | -------------------------------------------------------------------------------- /models/networks/netutils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.nn import init 4 | import functools 5 | from torch.optim import lr_scheduler 6 | from IPython import embed 7 | 8 | ############################################################################### 9 | # Helper Functions 10 | ############################################################################### 11 | 12 | def get_scheduler(optimizer, opt): 13 | if opt.lr_policy == 'plateau': 14 | scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 15 | mode='max', 16 | factor=0.1, 17 | threshold=0.0001, 18 | patience=opt.patience, 19 | eps=1e-6) 20 | elif opt.lr_policy == 'constant': 21 | # dummy scheduler: min lr threshold (eps) is set as the original learning rate 22 | scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', 23 | factor=0.1, threshold=0.0001, 24 | patience=1000, eps=opt.lr) 25 | else: 26 | return NotImplementedError('learning rate policy [%s] is not implemented', opt.lr_policy) 27 | return scheduler 28 | 29 | def init_weights(net, init_type='xavier', gain=0.02): 30 | def init_func(m): 31 | classname = m.__class__.__name__ 32 | if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1): 33 | if init_type == 'normal': 34 | init.normal_(m.weight.data, 0.0, gain) 35 | elif init_type == 'xavier': 36 | init.xavier_normal_(m.weight.data, gain=gain) 37 | elif init_type == 'kaiming': 38 | init.kaiming_normal_(m.weight.data, a=0, mode='fan_in') 39 | elif init_type == 'orthogonal': 40 | init.orthogonal_(m.weight.data, gain=gain) 41 | else: 42 | raise NotImplementedError('initialization method [%s] is not implemented' % init_type) 43 | if hasattr(m, 'bias') and m.bias is not None: 44 | init.constant_(m.bias.data, 0.0) 45 | elif classname.find('BatchNorm2d') != -1: 46 | init.normal_(m.weight.data, 1.0, gain) 47 | init.constant_(m.bias.data, 0.0) 48 | 49 | print('initialize network with %s' % init_type) 50 | net.apply(init_func) 51 | 52 | def init_net(net, init_type='xavier', gpu_ids=[]): 53 | if len(gpu_ids) > 0: 54 | assert(torch.cuda.is_available()) 55 | net.to(gpu_ids[0]) 56 | net = torch.nn.DataParallel(net, gpu_ids) 57 | if init_type is None: 58 | return net 59 | init_weights(net, init_type) 60 | return net 61 | -------------------------------------------------------------------------------- /models/networks/networks.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from IPython import embed 4 | from . import netutils 5 | 6 | def modify_commandline_options(parser): 7 | opt, _ = parser.parse_known_args() 8 | if 'xception' in opt.which_model_netD: 9 | parser.set_defaults(loadSize=333, fineSize=299) 10 | elif 'resnet' in opt.which_model_netD: 11 | parser.set_defaults(loadSize=256, fineSize=224) 12 | else: 13 | raise NotImplementedError 14 | 15 | def define_D(which_model_netD, init_type, gpu_ids=[]): 16 | if 'resnet' in which_model_netD: 17 | from torchvision.models import resnet 18 | model = getattr(resnet, which_model_netD) 19 | netD = model(pretrained=False, num_classes=2) 20 | elif 'xception' in which_model_netD: 21 | from . import xception 22 | netD = xception.xception(num_classes=2) 23 | else: 24 | raise NotImplementedError('Discriminator model name [%s] is not recognized' % 25 | which_model_netD) 26 | return netutils.init_net(netD, init_type, gpu_ids=gpu_ids) 27 | 28 | def define_patch_D(which_model_netD, init_type, gpu_ids=[]): 29 | if which_model_netD.startswith('resnet'): 30 | # e.g. which_model_netD = resnet18_layer1 31 | from . import customnet 32 | depth = int(which_model_netD.split('_')[0][6:]) 33 | layer = which_model_netD.split('_')[1] 34 | netD = customnet.make_patch_resnet(depth, layer) 35 | return netutils.init_net(netD, init_type, gpu_ids=gpu_ids) 36 | elif which_model_netD.startswith('widenet'): 37 | # e.g. which_model_netD = widenet_kw7_d1 38 | splits = which_model_netD.split('_') 39 | kernel_size = int(splits[1][2:]) 40 | dilation = int(splits[2][1:]) 41 | netD = WideNet(kernel_size, dilation) 42 | return netutils.init_net(netD, init_type, gpu_ids=gpu_ids) 43 | elif which_model_netD.startswith('xception'): 44 | # e.g. which_model_netD = xceptionnet_block2 45 | from . import customnet 46 | splits = which_model_netD.split('_') 47 | layer = splits[1] 48 | netD = customnet.make_patch_xceptionnet(layer) 49 | return netutils.init_net(netD, init_type, gpu_ids=gpu_ids) 50 | elif which_model_netD.startswith('longxception'): 51 | from . import customnet 52 | netD = customnet.make_xceptionnet_long() 53 | return netutils.init_net(netD, init_type, gpu_ids=gpu_ids) 54 | else: 55 | raise NotImplementedError('Discriminator model name [%s] is not recognized' % 56 | which_model_netD) 57 | 58 | class WideNet(nn.Module): 59 | # a shallow network based off initial layers of resnet with 60 | # a few 1x1 conv layers added on 61 | def __init__(self, kernel_size=7, dilation=1): 62 | super().__init__() 63 | sequence = [ 64 | nn.Conv2d(3, 256, kernel_size=kernel_size, dilation=dilation, 65 | stride=2, padding=kernel_size//2, bias=False), 66 | nn.BatchNorm2d(256), 67 | nn.ReLU(inplace=True), 68 | nn.MaxPool2d(3, stride=2, padding=1), 69 | # linear layers 70 | nn.Conv2d(256, 256, kernel_size=1), 71 | nn.ReLU(inplace=True), 72 | nn.Conv2d(256, 256, kernel_size=1), 73 | nn.ReLU(inplace=True), 74 | nn.Conv2d(256, 2, kernel_size=1), 75 | ] 76 | self.model = nn.Sequential(*sequence) 77 | 78 | def forward(self, x): 79 | return self.model(x) 80 | 81 | -------------------------------------------------------------------------------- /notebooks/ipynb_drop_output.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Suppress output and prompt numbers in git version control. 5 | 6 | This script will tell git to ignore prompt numbers and cell output 7 | when looking at ipynb files UNLESS their metadata contains: 8 | 9 | "git" : { "keep_output" : true } 10 | 11 | The notebooks themselves are not changed. 12 | 13 | See also this blogpost: http://pascalbugnion.net/blog/ipython-notebooks-and-git.html. 14 | 15 | Usage instructions 16 | ================== 17 | 18 | 1. Put this script in a directory that is on the system's path. 19 | For future reference, I will assume you saved it in 20 | `~/scripts/ipynb_drop_output`. 21 | 2. Make sure it is executable by typing the command 22 | `chmod +x ~/scripts/ipynb_drop_output`. 23 | 3. Register a filter for ipython notebooks by 24 | putting the following line in `~/.config/git/attributes`: 25 | `*.ipynb filter=clean_ipynb` 26 | 4. Connect this script to the filter by running the following 27 | git commands: 28 | 29 | git config --global filter.clean_ipynb.clean ipynb_drop_output 30 | git config --global filter.clean_ipynb.smudge cat 31 | 32 | To tell git NOT to ignore the output and prompts for a notebook, 33 | open the notebook's metadata (Edit > Edit Notebook Metadata). A 34 | panel should open containing the lines: 35 | 36 | { 37 | "name" : "", 38 | "signature" : "some very long hash" 39 | } 40 | 41 | Add an extra line so that the metadata now looks like: 42 | 43 | { 44 | "name" : "", 45 | "signature" : "don't change the hash, but add a comma at the end of the line", 46 | "git" : { "keep_outputs" : true } 47 | } 48 | 49 | You may need to "touch" the notebooks for git to actually register a change, if 50 | your notebooks are already under version control. 51 | 52 | Notes 53 | ===== 54 | 55 | Changed by David Bau to make stripping output the default. 56 | 57 | This script is inspired by http://stackoverflow.com/a/20844506/827862, but 58 | lets the user specify whether the ouptut of a notebook should be kept 59 | in the notebook's metadata, and works for IPython v3.0. 60 | """ 61 | 62 | import sys 63 | import json 64 | 65 | nb = sys.stdin.read() 66 | 67 | json_in = json.loads(nb) 68 | nb_metadata = json_in["metadata"] 69 | keep_output = False 70 | if "git" in nb_metadata: 71 | if "keep_outputs" in nb_metadata["git"] and nb_metadata["git"]["keep_outputs"]: 72 | keep_output = True 73 | if keep_output: 74 | sys.stdout.write(nb) 75 | exit() 76 | 77 | 78 | ipy_version = int(json_in["nbformat"])-1 # nbformat is 1 more than actual version. 79 | 80 | def strip_output_from_cell(cell): 81 | if "outputs" in cell: 82 | cell["outputs"] = [] 83 | if "prompt_number" in cell: 84 | del cell["prompt_number"] 85 | if "execution_count" in cell: 86 | cell["execution_count"] = None 87 | 88 | 89 | if ipy_version == 2: 90 | for sheet in json_in["worksheets"]: 91 | for cell in sheet["cells"]: 92 | strip_output_from_cell(cell) 93 | else: 94 | for cell in json_in["cells"]: 95 | strip_output_from_cell(cell) 96 | 97 | json.dump(json_in, sys.stdout, sort_keys=True, indent=1, separators=(",",": ")) 98 | -------------------------------------------------------------------------------- /notebooks/models: -------------------------------------------------------------------------------- 1 | ../models/ -------------------------------------------------------------------------------- /notebooks/setup_notebooks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Start from directory of script 4 | cd "$(dirname "${BASH_SOURCE[0]}")" 5 | 6 | # Set up git config filters so huge output of notebooks is not committed. 7 | git config filter.clean_ipynb.clean "$(pwd)/ipynb_drop_output.py" 8 | git config filter.clean_ipynb.smudge cat 9 | git config filter.clean_ipynb.required true 10 | 11 | # Set up symlinks for the example notebooks 12 | ln -sfn ../utils . 13 | ln -sfn ../models . 14 | 15 | -------------------------------------------------------------------------------- /notebooks/tables.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Notebook for displaying results tables\n", 8 | "Note: these will only run once you've run `test.py` on the corresponding test sets,\n", 9 | "e.g. see `scripts/04_eval_checkpoint.sh`, as it pulls the numbers from the respective experiment directories. \n", 10 | "\n", 11 | "To run the face forensics tables, you'll need to have processed the face forensics dataset accordingly, following `scripts/00_data_processing_faceforensics_aligned_frames.sh`, and then evalute the corresponding experiments in `scripts/04_eval_checkpoint.sh`." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "import os\n", 21 | "import numpy as np\n", 22 | "import pandas as pd\n", 23 | "from IPython.display import display\n", 24 | "import glob\n", 25 | "import matplotlib.pyplot as plt\n", 26 | "import seaborn as sns\n", 27 | "import matplotlib\n", 28 | "pd.options.display.float_format = '{:,.2f}'.format # print 2 decimal places" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "dataset_names_list = {\n", 38 | " 'gen_models': ['celebahq-pgan-pretrained', 'celebahq-sgan-pretrained',\n", 39 | " 'celebahq-glow-pretrained', 'celeba-gmm', 'ffhq-pgan', \n", 40 | " 'ffhq-sgan', 'ffhq-sgan2'],\n", 41 | " 'faceforensics': ['DF', 'NT', 'F2F', 'FS'],\n", 42 | "}" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "def make_table(model_paths, model_names, dataset_names, ens_type, partition='val', epoch='epoch_bestval'):\n", 52 | " # ens types: voted, avg_before_softmax, avg_after_softmax\n", 53 | " metrics_ap = dict()\n", 54 | " metrics_acc = dict()\n", 55 | " counts = dict()\n", 56 | " for model_path, model_name in zip(model_paths, model_names):\n", 57 | " metrics_ap[model_name] = []\n", 58 | " metrics_acc[model_name] = []\n", 59 | " counts[model_name] = []\n", 60 | " for dataset in dataset_names:\n", 61 | " metric_file = os.path.join(model_path, partition, epoch, dataset, 'metrics_%s.npz' % ens_type)\n", 62 | " if os.path.isfile(metric_file):\n", 63 | " metrics = np.load(metric_file)\n", 64 | " metrics_ap[model_name].append(metrics['ap'])\n", 65 | " metrics_acc[model_name].append(metrics['acc'])\n", 66 | " counts[model_name].append(metrics['n'])\n", 67 | " else:\n", 68 | " metric_file = os.path.join(model_path, partition, epoch, dataset, 'metrics.npz')\n", 69 | " if os.path.isfile(metric_file):\n", 70 | " # print(\"Using metrics.npz for %s\" % model_name)\n", 71 | " metrics = np.load(metric_file)\n", 72 | " metrics_ap[model_name].append(metrics['ap'])\n", 73 | " metrics_acc[model_name].append(metrics['acc'])\n", 74 | " counts[model_name].append(metrics['n'])\n", 75 | " else:\n", 76 | " metrics_ap[model_name].append(np.nan)\n", 77 | " metrics_acc[model_name].append(np.nan)\n", 78 | " counts[model_name].append(0)\n", 79 | " df_ap = pd.DataFrame(data=metrics_ap, index=dataset_names).T * 100\n", 80 | " df_acc = pd.DataFrame(data=metrics_acc, index=dataset_names).T * 100\n", 81 | " df_counts = pd.DataFrame(data=counts, index=dataset_names).T\n", 82 | " # df_counts is for sanity checking the number of images processed\n", 83 | " return df_ap, df_acc, df_counts" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "def get_listing_for_group(grp_string):\n", 93 | " # grp_string = gp1 etc\n", 94 | " expts = sorted(glob.glob('../results/%s-*block*' % grp_string))\n", 95 | " names = ['-'.join(x.split('_')[-4:-2]) for x in expts]\n", 96 | " baseline_expts = sorted(glob.glob('../results/%s-*baseline*' % grp_string))\n", 97 | " baseline_names = ['-'.join(x.split('_')[-2:]) for x in baseline_expts]\n", 98 | " # print(baseline_expts)\n", 99 | " # print(baseline_names)\n", 100 | " \n", 101 | " if baseline_expts:\n", 102 | " order = [4,0,3,5,1,2]\n", 103 | " baseline_expts = [baseline_expts[i] for i in order]\n", 104 | " baseline_names = [baseline_names[i] for i in order]\n", 105 | " else:\n", 106 | " print(\"No baselines found\")\n", 107 | " \n", 108 | " return expts+baseline_expts, names+baseline_names" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "### xception gen models tables" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "expts, names = get_listing_for_group('gp1')\n", 125 | "t1, t2, n = make_table(expts, names, dataset_names_list['gen_models'], 'avg_after_softmax', partition='test')\n", 126 | "display(t1)" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "# random crop\n", 136 | "expts, names = get_listing_for_group('gp1b') # random crop\n", 137 | "t1, t2, n = make_table(expts, names, dataset_names_list['gen_models'], 'avg_after_softmax', partition='test')\n", 138 | "display(t1)\n", 139 | "\n", 140 | "# random resize crop\n", 141 | "expts, names = get_listing_for_group('gp1a') # random resize crop\n", 142 | "t1, t2, n = make_table(expts, names, dataset_names_list['gen_models'], 'avg_after_softmax', partition='test')\n", 143 | "display(t1)\n" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": null, 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "# samples only\n", 153 | "expts, names = get_listing_for_group('gp1d') # samples only\n", 154 | "t1, t2, n = make_table(expts, names, dataset_names_list['gen_models'], 'avg_after_softmax', partition='test')\n", 155 | "display(t1)\n", 156 | "\n", 157 | "# inverses only\n", 158 | "expts, names = get_listing_for_group('gp1c') # inverse only\n", 159 | "t1, t2, n = make_table(expts, names, dataset_names_list['gen_models'], 'avg_after_softmax', partition='test')\n", 160 | "display(t1)" 161 | ] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "metadata": {}, 166 | "source": [ 167 | "# face forensics" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": null, 173 | "metadata": {}, 174 | "outputs": [], 175 | "source": [ 176 | "expts, names = get_listing_for_group('gp5') # train on F2F\n", 177 | "t1, t2, n = make_table(expts, names, dataset_names_list['faceforensics'], 'avg_after_softmax', 'test')\n", 178 | "display(t1)" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [ 187 | "expts, names = get_listing_for_group('gp4') # train on FS\n", 188 | "t1, t2, n = make_table(expts, names, dataset_names_list['faceforensics'], 'avg_after_softmax', 'test')\n", 189 | "display(t1)" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [ 198 | "expts, names = get_listing_for_group('gp3') # train on NT\n", 199 | "t1, t2, n = make_table(expts, names, dataset_names_list['faceforensics'], 'avg_after_softmax', 'test')\n", 200 | "display(t1)\n", 201 | "table_nt = t1" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": null, 207 | "metadata": {}, 208 | "outputs": [], 209 | "source": [ 210 | "expts, names = get_listing_for_group('gp2') # train on DF\n", 211 | "t1, t2, n = make_table(expts, names, dataset_names_list['faceforensics'], 'avg_after_softmax', 'test')\n", 212 | "display(t1)" 213 | ] 214 | } 215 | ], 216 | "metadata": { 217 | "kernelspec": { 218 | "display_name": "forgery", 219 | "language": "python", 220 | "name": "forgery" 221 | }, 222 | "language_info": { 223 | "codemirror_mode": { 224 | "name": "ipython", 225 | "version": 3 226 | }, 227 | "file_extension": ".py", 228 | "mimetype": "text/x-python", 229 | "name": "python", 230 | "nbconvert_exporter": "python", 231 | "pygments_lexer": "ipython3", 232 | "version": "3.7.5" 233 | } 234 | }, 235 | "nbformat": 4, 236 | "nbformat_minor": 2 237 | } -------------------------------------------------------------------------------- /notebooks/utils: -------------------------------------------------------------------------------- 1 | ../utils/ -------------------------------------------------------------------------------- /options/base_options.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | from utils import util 4 | from utils import options 5 | import torch 6 | import models 7 | 8 | class BaseOptions(options.Options): 9 | def __init__(self, print_opt=True): 10 | options.Options.__init__(self) 11 | self.isTrain = False # train_options will change this 12 | self.print_opt = print_opt 13 | parser = self.parser 14 | 15 | # model setup 16 | parser.add_argument('--model', type=str, default='basic_discriminator', help='chooses which model to use') 17 | parser.add_argument('--which_model_netD', type=str, default='resnet18', help='selects model to use for netD') 18 | parser.add_argument('--fake_class_id', type=int, default=0, help='class id of fake ims') 19 | parser.add_argument('--gpu_ids', type=str, default='0', help='gpu ids: e.g. 0 0,1,2, 0,2. use -1 for CPU') 20 | parser.add_argument('--which_epoch', type=str, default='latest', help='which epoch to load? set to latest to use latest cached model') 21 | parser.add_argument('--load_model', action='store_true', help='load the latest model') 22 | parser.add_argument('--seed', type=int, default=0, help='torch.manual_seed value') 23 | parser.add_argument('--init_type', type=str, default='xavier', help='network initialization [normal|xavier|kaiming|orthogonal]') 24 | 25 | # image loading 26 | parser.add_argument('--loadSize', type=int, default=256, help='scale images to this size') 27 | parser.add_argument('--fineSize', type=int, default=256, help='then crop to this size') 28 | parser.add_argument('--nThreads', default=4, type=int, help='# threads for loading data') 29 | parser.add_argument('--batch_size', type=int, default=32, help='input batch size') 30 | parser.add_argument('--real_im_path', type=str, help='path to real images') 31 | parser.add_argument('--fake_im_path', type=str, help='path to fake images') 32 | parser.add_argument('--no_serial_batches', action='store_true', help='if not specified, takes images in order to make batches, otherwise takes them randomly') 33 | parser.add_argument('--max_dataset_size', type=int, default=float("inf"), help="Maximum number of samples to use in dataset") 34 | 35 | # checkpoint saving and naming 36 | parser.add_argument('--name', type=str, default='', help='name of the experiment. it decides where to store samples and models') 37 | parser.add_argument('--suffix', default='', type=str, help='customized suffix: opt.name = opt.name + suffix: e.g., {model}_{which_model_netG}_size{loadSize}') 38 | parser.add_argument('--prefix', default='', type=str, help='customized prefix: opt.name = prefix + opt.name: e.g., {model}_{which_model_netG}_size{loadSize}') 39 | parser.add_argument('--checkpoints_dir', type=str, default='./checkpoints', help='models are saved here') 40 | parser.add_argument('--results_dir', type=str, default='./results/', help='saves results here.') 41 | 42 | def parse(self): 43 | 44 | opt = options.Options.parse(self, print_opt=False) 45 | 46 | # modify model-related parser options 47 | model_name = opt.model 48 | model_option_setter = models.get_option_setter(model_name) 49 | self.parser = model_option_setter(self.parser) 50 | opt = options.Options.parse(self, print_opt=False) 51 | 52 | opt.isTrain = self.isTrain 53 | 54 | # default model name 55 | if opt.name == '': 56 | opt.name = '{model}_{which_model_netD}_size{fineSize}'.format(**vars(opt)) 57 | else: 58 | opt.name = opt.name.format(**vars(opt)) 59 | 60 | # process opt.suffix 61 | if opt.suffix: 62 | suffix = ('_' + opt.suffix.format(**vars(opt))) if opt.suffix != '' else '' 63 | opt.name = opt.name + suffix 64 | opt.suffix = '' 65 | 66 | # process opt.prefix 67 | if opt.prefix: 68 | prefix = (opt.prefix.format(**vars(opt))) if opt.prefix != '' else '' 69 | prefix += '-' 70 | opt.name = prefix + opt.name 71 | opt.prefix = '' 72 | 73 | # print options after name/prefix/suffix is modified 74 | if self.print_opt: 75 | self.print_options(opt) 76 | 77 | # set gpu ids 78 | str_ids = opt.gpu_ids 79 | if isinstance(opt.gpu_ids, str): 80 | str_ids = opt.gpu_ids.split(',') 81 | opt.gpu_ids = [] 82 | for str_id in str_ids: 83 | id = int(str_id) 84 | if id >= 0: 85 | opt.gpu_ids.append(id) 86 | if len(opt.gpu_ids) > 0 and torch.cuda.is_available(): 87 | torch.cuda.set_device(opt.gpu_ids[0]) 88 | 89 | # check both image paths are specified 90 | assert(opt.real_im_path and opt.fake_im_path) 91 | return opt 92 | -------------------------------------------------------------------------------- /options/test_options.py: -------------------------------------------------------------------------------- 1 | from .base_options import BaseOptions 2 | import oyaml as yaml 3 | import argparse 4 | import sys 5 | 6 | class TestOptions(BaseOptions): 7 | 8 | def __init__(self): 9 | BaseOptions.__init__(self, print_opt=False) 10 | parser = self.parser 11 | parser.add_argument('--train_config', type=argparse.FileType(mode='r'), required=True, help='config file saved from model training') 12 | parser.add_argument('--partition', type=str, default='val', help='val or test') 13 | parser.add_argument('--dataset_name', type=str, required=True, help="name to describe test dataset when saving results, e.g. celebahq_pgan") 14 | parser.add_argument('--force_redo', action='store_true', help="force recompute results") 15 | 16 | # for testing model robustness (additional augmentations) 17 | parser.add_argument('--test_compression', type=int, help='jpeg compression level') 18 | parser.add_argument('--test_gamma', type=int, help='gamma adjustment level') 19 | parser.add_argument('--test_blur', type=int, help='blur level') 20 | parser.add_argument('--test_flip', action='store_true', help='flip all test images') 21 | 22 | # visualizations 23 | parser.add_argument('--visualize', action='store_true', help='save visualizations when running test') 24 | parser.add_argument('--average_mode', help='which kind of patch averaging to use for visualizations [vote, before_softmax, after_softmax]') 25 | parser.add_argument('--topn', type=int, default=100, help='visualize top n') 26 | 27 | def parse(self): 28 | opt = super().parse() 29 | train_conf = yaml.load(opt.train_config, Loader=yaml.FullLoader) 30 | 31 | # determine which options were specified 32 | # explicitly with command line args 33 | option_strings = {} 34 | for action_group in self.parser._action_groups: 35 | for action in action_group._group_actions: 36 | for option in action.option_strings: 37 | option_strings[option] = action.dest 38 | specified_options = set([option_strings[x] for x in 39 | sys.argv if x in option_strings]) 40 | 41 | # make the val options consistent with the train options 42 | # (e.g. the specified model architecture) 43 | # but avoid overwriting anything specified in command line 44 | options_from_train = [] 45 | for k, v in train_conf.items(): 46 | if k in ['real_im_path', 'fake_im_path', 'gpu_ids']: 47 | # don't overwrite these 48 | continue 49 | if getattr(opt, k, None) is None: 50 | # if the attr is in train but not in base options 51 | # e.g. learning rate, then skip them 52 | continue 53 | if k not in specified_options: 54 | # overwrite the option if it exists in train 55 | setattr(opt, k, v) 56 | options_from_train.append((k, v)) 57 | 58 | print("Using the following options from the train configuration file:") 59 | print(options_from_train) 60 | 61 | # sanity check: make sure partition and data paths is consistent 62 | # and do some cleaning 63 | if opt.real_im_path: 64 | assert(opt.partition in opt.real_im_path) 65 | opt.real_im_path = opt.real_im_path.rstrip('/') 66 | if opt.fake_im_path: 67 | assert(opt.partition in opt.fake_im_path) 68 | opt.fake_im_path = opt.fake_im_path.rstrip('/') 69 | 70 | opt.load_model = True 71 | opt.model_seed = 0 72 | opt.isTrain = False 73 | return opt 74 | 75 | 76 | -------------------------------------------------------------------------------- /options/train_options.py: -------------------------------------------------------------------------------- 1 | from .base_options import BaseOptions 2 | 3 | 4 | class TrainOptions(BaseOptions): 5 | 6 | def __init__(self, print_opt=True): 7 | BaseOptions.__init__(self, print_opt) 8 | parser = self.parser 9 | parser.add_argument('--display_freq', type=int, default=1000, help='frequency of showing training results visualization') 10 | parser.add_argument('--print_freq', type=int, default=100, help='frequency of showing training results on console') 11 | parser.add_argument('--save_latest_freq', type=int, default=1000, help='frequency of saving the latest results') 12 | parser.add_argument('--save_epoch_freq', type=int, default=100, help='frequency of saving checkpoints at the end of epochs') 13 | parser.add_argument('--beta1', type=float, default=0.9, help='momentum term of adam') 14 | parser.add_argument('--lr', type=float, default=0.001, help='initial learning rate for adam') 15 | parser.add_argument('--lr_policy', default='constant', help='lr schedule [constant|plateau]') 16 | parser.add_argument('--patience', type=int, default=10, help='will stop training if val metric does not improve for this many epochs') 17 | parser.add_argument('--max_epochs', type=int, help='maximum epochs to train, if not specified, will stop based on patience, or whichever is sooner') 18 | 19 | # augmentations 20 | parser.add_argument('--random_crop', action='store_true', help='use random crop instead of center crop') 21 | parser.add_argument('--random_resized_crop', action='store_true', help='use random resized crop instead of center crop') 22 | parser.add_argument('--no_flip', action='store_true', help='do not flip the images for data augmentation') 23 | parser.add_argument('--color_augment', action='store_true', help='performs color augmentations, see utils/transforms.py') 24 | parser.add_argument('--all_augment', action='store_true', help='performs various augmentations, see utils/transforms.py') 25 | parser.add_argument('--cnn_detection_augment', type=float, help='use augmentations from https://arxiv.org/abs/1912.11035, recommended values are 0.1 or 0.5') 26 | 27 | self.isTrain = True 28 | -------------------------------------------------------------------------------- /patches.py: -------------------------------------------------------------------------------- 1 | from options.test_options import TestOptions 2 | from models import create_model 3 | import numpy as np 4 | import os 5 | import torch 6 | from utils import pidfile 7 | from utils import rfutil 8 | from utils import imutil 9 | from utils import util 10 | from data.unpaired_dataset import UnpairedDataset 11 | from torch.utils.data import DataLoader 12 | import heapq 13 | from collections import namedtuple, OrderedDict, Counter 14 | from itertools import count 15 | import matplotlib.pyplot as plt 16 | from PIL import Image 17 | 18 | 19 | torch.backends.cudnn.benchmark = True 20 | 21 | def run_patch_topn(opt, output_dir): 22 | assert opt.model == 'patch_discriminator' # only works on patch models 23 | model = create_model(opt) 24 | model.setup(opt) 25 | model.eval() 26 | 27 | # find patch indices 28 | rf = rfutil.find_rf_model(opt.which_model_netD) 29 | print("Receptive field %d" % rf) 30 | patches = rfutil.find_rf_patches(opt.which_model_netD, opt.fineSize) 31 | 32 | fake_label = opt.fake_class_id 33 | real_label = 1 - fake_label 34 | 35 | # operations 36 | softmax = torch.nn.Softmax(dim=1) 37 | PatchInfo = namedtuple('PatchInfo', ['patch', 'pos', 'file', 'value']) 38 | 39 | for data_path, label, name in zip([opt.real_im_path, opt.fake_im_path], 40 | [real_label, fake_label], 41 | ['reals', 'fakes']): 42 | dset = UnpairedDataset(opt, data_path, is_val=True) 43 | dl = DataLoader(dset, batch_size=opt.batch_size, shuffle=False, 44 | num_workers=opt.nThreads, pin_memory=False) 45 | transform = dset.transform 46 | heap_easiest = [] 47 | 48 | # heappush will error if there are ties in value 49 | # use counter to break ties in the heap 50 | tiebreaker = count() 51 | 52 | for i, data in enumerate(dl): 53 | # set model inputs 54 | ims = data['img'].to(opt.gpu_ids[0]) 55 | assert(ims.shape[-1] == opt.fineSize) 56 | pred_labels = label * torch.ones(ims.shape[0], dtype=torch.long).cuda() 57 | inputs = dict(ims=ims, labels=pred_labels) 58 | 59 | # forward pass 60 | model.reset() 61 | model.set_input(inputs) 62 | model.test(True) 63 | 64 | # get model outputs 65 | with torch.no_grad(): 66 | model_out = softmax(model.pred_logit).detach().cpu().numpy() 67 | assert(np.ndim(model_out) == 4) # for patch model 68 | 69 | for pred, path, img in zip(model_out, data['path'], ims): 70 | img = img.cpu().numpy() 71 | pred = pred[label, :, :] # get class prediction 72 | patch_values = np.sort(pred, axis=None) 73 | random_tiebreak = np.random.random(pred.size) 74 | # if values are the same, take a random patch among 75 | # everything that has the same values 76 | # lexsort does second entry then first entry for sort order 77 | tiebreak_argsort = np.lexsort((random_tiebreak.ravel(), 78 | pred.ravel())) 79 | ylocs, xlocs = np.unravel_index(tiebreak_argsort, 80 | pred.shape) 81 | num = 1 if opt.unique else opt.topn 82 | # just iterate through top predictions for efficiency 83 | for value, yloc, xloc in zip(patch_values[-num:], 84 | ylocs[-num:], 85 | xlocs[-num:]): 86 | assert(pred[yloc, xloc] == value) 87 | if len(heap_easiest) < opt.topn or value > heap_easiest[0][0]: 88 | patch_pos = (yloc, xloc) 89 | patch_file = path 90 | slices = patches[(yloc, xloc)] 91 | patch_img = rfutil.get_patch_from_img(img, slices, rf) 92 | patch_info = PatchInfo(patch_img, patch_pos, 93 | patch_file, value) 94 | if len(heap_easiest) < opt.topn: 95 | heapq.heappush(heap_easiest, (value, next(tiebreaker), patch_info)) 96 | else: 97 | heapq.heappushpop(heap_easiest, (value, next(tiebreaker), patch_info)) 98 | 99 | # aggregate and save results (easiest) 100 | heap_easiest_sorted = sorted(heap_easiest) 101 | infos = OrderedDict( 102 | patch=np.array([h[2].patch for h in heap_easiest_sorted]), 103 | pos=np.array([h[2].pos for h in heap_easiest_sorted]), 104 | value=np.array([h[2].value for h in heap_easiest_sorted]), 105 | outsize=pred.shape, rf=rf, finesize=opt.fineSize, 106 | which_model_netD=opt.which_model_netD) 107 | np.savez(os.path.join(output_dir, name + '_easiest.npz'), **infos) 108 | 109 | with open(os.path.join(output_dir, name+'_easiest_files.txt'), 'w') as f: 110 | [f.write('%s\n' % h[2].file) for h in heap_easiest_sorted] 111 | 112 | # grid image of the easiest patches 113 | normalized = (infos['patch'] * 0.5) + 0.5 114 | grid = imutil.imgrid(np.uint8(normalized * 255), pad=0, cols= 115 | int(np.ceil(np.sqrt(normalized.shape[0])))) 116 | im = Image.fromarray(grid) 117 | im.save(os.path.join(output_dir, name + '_easiest_grid.jpg')) 118 | 119 | 120 | if __name__ == '__main__': 121 | options = TestOptions() 122 | # additional options for top n patches 123 | options.parser.add_argument('--unique', action='store_true', help='take only 1 patch per image when computing top n') 124 | opt = options.parse() 125 | print("Calculating patches from model: %s epoch %s" % (opt.name, opt.which_epoch)) 126 | print("On dataset (real): %s" % (opt.real_im_path)) 127 | print("And dataset (fake): %s" % (opt.fake_im_path)) 128 | expdir = opt.name 129 | dataset_name = opt.dataset_name 130 | output_dir = os.path.join(opt.results_dir, expdir, opt.partition, 131 | 'epoch_%s' % opt.which_epoch, dataset_name, 132 | 'patches_top%d' % opt.topn) 133 | print(output_dir) 134 | os.makedirs(output_dir, exist_ok=True) 135 | 136 | # check if checkpoint is out of date 137 | redo = opt.force_redo 138 | ckpt_path = os.path.join(opt.checkpoints_dir, opt.name, '%s_net_D.pth' % opt.which_epoch) 139 | timestamp_path = os.path.join(output_dir, 'timestamp_%s_net_D.txt' % opt.which_epoch) 140 | if util.check_timestamp(ckpt_path, timestamp_path): 141 | redo = True 142 | util.update_timestamp(ckpt_path, timestamp_path) 143 | pidfile.exit_if_job_done(output_dir, redo=True) # redo=redo) 144 | run_patch_topn(opt, output_dir) 145 | pidfile.mark_job_done(output_dir) 146 | 147 | -------------------------------------------------------------------------------- /resources/download_resources_basic.sh: -------------------------------------------------------------------------------- 1 | # face segmentation model 2 | git clone https://github.com/zllrunning/face-parsing.PyTorch.git face_parsing_pytorch 3 | gdown https://drive.google.com/uc?id=154JgKpzCPW82qINcVieuPH3fZ2e0P812 4 | mkdir -p face_parsing_pytorch/res/cp 5 | mv 79999_iter.pth face_parsing_pytorch/res/cp 6 | -------------------------------------------------------------------------------- /resources/download_resources_data.sh: -------------------------------------------------------------------------------- 1 | # stylegan 2 | git clone https://github.com/NVlabs/stylegan.git 3 | 4 | # progressive gan 5 | git clone https://github.com/tkarras/progressive_growing_of_gans.git 6 | 7 | # glow -- note: will also need to download glow pretrained weights from 8 | # glow/demo/script.sh 9 | git clone https://github.com/openai/glow.git 10 | 11 | # celebahq progressive gan 12 | gdown https://drive.google.com/uc?id=188K19ucknC6wg1R6jbuPEhTq9zoufOx4 13 | 14 | # celebahq image indices: original source from https://drive.google.com/drive/folders/0B4qLcYyJmiz0TXY1NG02bzZVRGs?resourcekey=0-arAVTUfW9KRhN-irJchVKQ 15 | # (previous) 16 | # gdown https://drive.google.com/uc?id=0B4qLcYyJmiz0U25vdEVIU3NvNFk 17 | # mv image_list.txt celebahq_image_list.txt 18 | # (alternate link) 19 | wget http://latent-composition.csail.mit.edu/other_projects/patch_forensics/resources/celebahq_image_list.txt 20 | 21 | # celeba train/test/val partitions 22 | # from celeba google drive -> eval 23 | # (previous) 24 | # gdown https://drive.google.com/uc?id=0B7EVK8r0v71pY0NSMzRuSXJEVkk 25 | # mv list_eval_partition.txt celeba_list_eval_partition.txt 26 | # (alternate link) 27 | wget http://latent-composition.csail.mit.edu/other_projects/patch_forensics/resources/celeba_list_eval_partition.txt 28 | 29 | # dlib facial landmarks predictor 30 | wget https://github.com/davisking/dlib-models/raw/master/shape_predictor_68_face_landmarks.dat.bz2 31 | bzip2 -d shape_predictor_68_face_landmarks.dat.bz2 32 | 33 | # face parsing pytorch repository - rename it 34 | git clone https://github.com/zllrunning/face-parsing.PyTorch.git face_parsing_pytorch 35 | # also need to download the weights for the face parser; following the steps from that repo 36 | gdown --id 154JgKpzCPW82qINcVieuPH3fZ2e0P812 # pretrained model 37 | mkdir -p face_parsing_pytorch/res/cp 38 | mv 79999_iter.pth face_parsing_pytorch/res/cp 39 | -------------------------------------------------------------------------------- /scripts/00_data_processing_export_tfrecord_to_img.sh: -------------------------------------------------------------------------------- 1 | python -m data.processing.export_tfrecord_to_img \ 2 | --tfrecord resources/tfrecords/celebahq/celeba-hq-r10.tfrecords \ 3 | --outdir dataset/faces/celebahq/real-tfr-1024-resized128 \ 4 | --outsize 128 --dataset celebahq 5 | 6 | python -m data.processing.export_tfrecord_to_img \ 7 | --tfrecord resources/tfrecords/ffhq/ffhq-r10.tfrecords \ 8 | --outdir dataset/faces/ffhq/real-tfr-1024-resized-128 \ 9 | --outsize 128 --dataset ffhq 10 | 11 | -------------------------------------------------------------------------------- /scripts/00_data_processing_faceforensics_aligned_frames.sh: -------------------------------------------------------------------------------- 1 | ### Deepfakes processing ### 2 | 3 | # train 4 | python -m data.processing.faceforensics_process_frames \ 5 | --source_dir_manipulated resources/faceforensics_raw/manipulated_sequences/Deepfakes/c23/videos \ 6 | --source_dir_original resources/faceforensics_raw/original_sequences/youtube/c23/videos \ 7 | --outsize 128 --split resources/faceforensics_raw/train.json \ 8 | --output_dir dataset/faces/faceforensics_aligned/Deepfakes 9 | 10 | # val 11 | python -m data.processing.faceforensics_process_frames \ 12 | --source_dir_manipulated resources/faceforensics_raw/manipulated_sequences/Deepfakes/c23/videos \ 13 | --source_dir_original resources/faceforensics_raw/original_sequences/youtube/c23/videos \ 14 | --outsize 128 --split resources/faceforensics_raw/val.json \ 15 | --output_dir dataset/faces/faceforensics_aligned/Deepfakes 16 | 17 | # test 18 | python -m data.processing.faceforensics_process_frames \ 19 | --source_dir_manipulated resources/faceforensics_raw/manipulated_sequences/Deepfakes/c23/videos \ 20 | --source_dir_original resources/faceforensics_raw/original_sequences/youtube/c23/videos \ 21 | --outsize 128 --split resources/faceforensics_raw/test.json \ 22 | --output_dir dataset/faces/faceforensics_aligned/Deepfakes 23 | 24 | ### Neural Textures processing ### 25 | 26 | # train 27 | python -m data.processing.faceforensics_process_frames \ 28 | --source_dir_manipulated resources/faceforensics_raw/manipulated_sequences/NeuralTextures/c23/videos \ 29 | --source_dir_original resources/faceforensics_raw/original_sequences/youtube/c23/videos \ 30 | --outsize 128 --split resources/faceforensics_raw/train.json \ 31 | --output_dir dataset/faces/faceforensics_aligned/NeuralTextures 32 | 33 | # val 34 | python -m data.processing.faceforensics_process_frames \ 35 | --source_dir_manipulated resources/faceforensics_raw/manipulated_sequences/NeuralTextures/c23/videos \ 36 | --source_dir_original resources/faceforensics_raw/original_sequences/youtube/c23/videos \ 37 | --outsize 128 --split resources/faceforensics_raw/val.json \ 38 | --output_dir dataset/faces/faceforensics_aligned/NeuralTextures 39 | 40 | # test 41 | python -m data.processing.faceforensics_process_frames \ 42 | --source_dir_manipulated resources/faceforensics_raw/manipulated_sequences/NeuralTextures/c23/videos \ 43 | --source_dir_original resources/faceforensics_raw/original_sequences/youtube/c23/videos \ 44 | --outsize 128 --split resources/faceforensics_raw/test.json \ 45 | --output_dir dataset/faces/faceforensics_aligned/NeuralTextures 46 | 47 | ### Face2Face processing ### 48 | 49 | # train 50 | python -m data.processing.faceforensics_process_frames \ 51 | --source_dir_manipulated resources/faceforensics_raw/manipulated_sequences/Face2Face/c23/videos \ 52 | --source_dir_original resources/faceforensics_raw/original_sequences/youtube/c23/videos \ 53 | --outsize 128 --split resources/faceforensics_raw/train.json \ 54 | --output_dir dataset/faces/faceforensics_aligned/Face2Face 55 | 56 | # val 57 | python -m data.processing.faceforensics_process_frames \ 58 | --source_dir_manipulated resources/faceforensics_raw/manipulated_sequences/Face2Face/c23/videos \ 59 | --source_dir_original resources/faceforensics_raw/original_sequences/youtube/c23/videos \ 60 | --outsize 128 --split resources/faceforensics_raw/val.json \ 61 | --output_dir dataset/faces/faceforensics_aligned/Face2Face 62 | 63 | # test 64 | python -m data.processing.faceforensics_process_frames \ 65 | --source_dir_manipulated resources/faceforensics_raw/manipulated_sequences/Face2Face/c23/videos \ 66 | --source_dir_original resources/faceforensics_raw/original_sequences/youtube/c23/videos \ 67 | --outsize 128 --split resources/faceforensics_raw/test.json \ 68 | --output_dir dataset/faces/faceforensics_aligned/Face2Face 69 | 70 | ### FaceSwap processing ### 71 | 72 | # train 73 | python -m data.processing.faceforensics_process_frames \ 74 | --source_dir_manipulated resources/faceforensics_raw/manipulated_sequences/FaceSwap/c23/videos \ 75 | --source_dir_original resources/faceforensics_raw/original_sequences/youtube/c23/videos \ 76 | --outsize 128 --split resources/faceforensics_raw/train.json \ 77 | --output_dir dataset/faces/faceforensics_aligned/FaceSwap 78 | 79 | # val 80 | python -m data.processing.faceforensics_process_frames \ 81 | --source_dir_manipulated resources/faceforensics_raw/manipulated_sequences/FaceSwap/c23/videos \ 82 | --source_dir_original resources/faceforensics_raw/original_sequences/youtube/c23/videos \ 83 | --outsize 128 --split resources/faceforensics_raw/val.json \ 84 | --output_dir dataset/faces/faceforensics_aligned/FaceSwap 85 | 86 | # test 87 | python -m data.processing.faceforensics_process_frames \ 88 | --source_dir_manipulated resources/faceforensics_raw/manipulated_sequences/FaceSwap/c23/videos \ 89 | --source_dir_original resources/faceforensics_raw/original_sequences/youtube/c23/videos \ 90 | --outsize 128 --split resources/faceforensics_raw/test.json \ 91 | --output_dir dataset/faces/faceforensics_aligned/FaceSwap 92 | -------------------------------------------------------------------------------- /scripts/00_data_processing_sample_celebahq_models.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CELEBAHQ_TRAIN=24183 4 | CELEBAHQ_VAL=2993 5 | CELEBAHQ_TEST=2824 6 | 7 | OUTPUT_ROOT=dataset/faces/celebahq 8 | 9 | ### PGAN CELEBAHQ SAMPLES ### 10 | PGAN_MODEL=resources/karras2018iclr-celebahq-1024x1024.pkl 11 | 12 | # pgan train 13 | python -m data.processing.pgan_tf \ 14 | --num_samples $CELEBAHQ_TRAIN --seed 0 --batch_size 32 --gpu 0 \ 15 | --model_path $PGAN_MODEL --format png --resize 128 \ 16 | --output_path $OUTPUT_ROOT/pgan-pretrained-128-png/train 17 | 18 | # pgan test 19 | python -m data.processing.pgan_tf \ 20 | --num_samples $CELEBAHQ_TEST --seed 1 --batch_size 32 --gpu 0 \ 21 | --model_path $PGAN_MODEL --format png --resize 128 \ 22 | --output_path $OUTPUT_ROOT/pgan-pretrained-128-png/test 23 | 24 | # pgan val 25 | python -m data.processing.pgan_tf \ 26 | --num_samples $CELEBAHQ_VAL --seed 2 --batch_size 32 --gpu 0 \ 27 | --model_path $PGAN_MODEL --format png --resize 128 \ 28 | --output_path $OUTPUT_ROOT/pgan-pretrained-128-png/val 29 | 30 | # SGAN CELEBAHQ SAMPLES ### 31 | 32 | # sgan train 33 | python -m data.processing.sgan_tf \ 34 | --num_samples $CELEBAHQ_TRAIN --seed 0 --batch_size 32 --gpu 0 \ 35 | --pretrained celebahq --format png --resize 128 \ 36 | --output_path $OUTPUT_ROOT/sgan-pretrained-128-png/train 37 | 38 | # sgan test 39 | python -m data.processing.sgan_tf \ 40 | --num_samples $CELEBAHQ_TEST --seed 1 --batch_size 32 --gpu 0 \ 41 | --pretrained celebahq --format png --resize 128 \ 42 | --output_path $OUTPUT_ROOT/sgan-pretrained-128-png/test 43 | 44 | # sgan val 45 | python -m data.processing.sgan_tf \ 46 | --num_samples $CELEBAHQ_VAL --seed 2 --batch_size 32 --gpu 0 \ 47 | --pretrained celebahq --format png --resize 128 \ 48 | --output_path $OUTPUT_ROOT/sgan-pretrained-128-png/val 49 | 50 | ### GLOW CELEBAHQ SAMPLES ### 51 | 52 | # glow train 53 | python -m data.processing.glow_tf \ 54 | --num_samples $CELEBAHQ_TRAIN --seed 0 --batch_size 16 --gpu 0 \ 55 | --manipulate --pretrained celebahq --format png --resize 128 \ 56 | --output_path $OUTPUT_ROOT/glow-pretrained-128-png/train 57 | 58 | # glow test 59 | python -m data.processing.glow_tf \ 60 | --num_samples $CELEBAHQ_TEST --seed 1 --batch_size 16 --gpu 0 \ 61 | --manipulate --pretrained celebahq --format png --resize 128 \ 62 | --output_path $OUTPUT_ROOT/glow-pretrained-128-png/test 63 | 64 | # glow val 65 | python -m data.processing.glow_tf \ 66 | --num_samples $CELEBAHQ_VAL --seed 2 --batch_size 16 --gpu 0 \ 67 | --manipulate --pretrained celebahq --format png --resize 128 \ 68 | --output_path $OUTPUT_ROOT/glow-pretrained-128-png/val 69 | 70 | -------------------------------------------------------------------------------- /scripts/00_data_processing_sample_ffhq_models.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | OUTPUT_ROOT=dataset/faces/ffhq 4 | 5 | python -m data.processing.sgan_tf \ 6 | --num_samples 5000 --seed 1 --batch_size 8 --gpu 0 \ 7 | --pretrained ffhq --format png --resize 128 \ 8 | --output_path $OUTPUT_ROOT/sgan-pretrained-128-png/test 9 | 10 | python -m data.processing.sgan_tf \ 11 | --num_samples 5000 --seed 2 --batch_size 8 --gpu 0 \ 12 | --pretrained ffhq --format png --resize 128 \ 13 | --output_path $OUTPUT_ROOT/sgan-pretrained-128-png/val 14 | -------------------------------------------------------------------------------- /scripts/01_train_gan_xception_patches_invonly.sh: -------------------------------------------------------------------------------- 1 | ### celebahq-pgan inverted real pairs vs the real image ### 2 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 3 | --name gp1c-gan-invonly --save_epoch_freq 200 \ 4 | --real_im_path dataset/faces/celebahq/inverted_real \ 5 | --fake_im_path dataset/faces/celebahq/inverted_fake \ 6 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 7 | --which_model_netD xception_block1 --model patch_discriminator \ 8 | --patience 50 --lr_policy constant --max_epochs 1000 9 | 10 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 11 | --name gp1c-gan-invonly --save_epoch_freq 200 \ 12 | --real_im_path dataset/faces/celebahq/inverted_real \ 13 | --fake_im_path dataset/faces/celebahq/inverted_fake \ 14 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 15 | --which_model_netD xception_block2 --model patch_discriminator \ 16 | --patience 20 --lr_policy constant --max_epochs 1000 17 | 18 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 19 | --name gp1c-gan-invonly --save_epoch_freq 50 \ 20 | --real_im_path dataset/faces/celebahq/inverted_real \ 21 | --fake_im_path dataset/faces/celebahq/inverted_fake \ 22 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 23 | --which_model_netD xception_block3 --model patch_discriminator \ 24 | --patience 10 --lr_policy constant --max_epochs 1000 25 | 26 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 27 | --name gp1c-gan-invonly --save_epoch_freq 50 \ 28 | --real_im_path dataset/faces/celebahq/inverted_real \ 29 | --fake_im_path dataset/faces/celebahq/inverted_fake \ 30 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 31 | --which_model_netD xception_block4 --model patch_discriminator \ 32 | --patience 10 --lr_policy constant --max_epochs 1000 33 | 34 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 35 | --name gp1c-gan-invonly --save_epoch_freq 50 \ 36 | --real_im_path dataset/faces/celebahq/inverted_real \ 37 | --fake_im_path dataset/faces/celebahq/inverted_fake \ 38 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 39 | --which_model_netD xception_block5 --model patch_discriminator \ 40 | --patience 10 --lr_policy constant --max_epochs 1000 41 | -------------------------------------------------------------------------------- /scripts/01_train_gan_xception_patches_samplesonly.sh: -------------------------------------------------------------------------------- 1 | ### celebahq-pgan raw samples from the gan vs real images ### 2 | ### uses --no_serial_batches as samples are not paired ### 3 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 4 | --name gp1d-gan-samplesonly --save_epoch_freq 200 \ 5 | --real_im_path dataset/faces/celebahq/real-tfr-1024-resized128 \ 6 | --fake_im_path dataset/faces/celebahq/pgan-pretrained-128-png \ 7 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 8 | --which_model_netD xception_block1 --model patch_discriminator \ 9 | --patience 50 --lr_policy constant --max_epochs 1000 \ 10 | --no_serial_batches 11 | 12 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 13 | --name gp1d-gan-samplesonly --save_epoch_freq 200 \ 14 | --real_im_path dataset/faces/celebahq/real-tfr-1024-resized128 \ 15 | --fake_im_path dataset/faces/celebahq/pgan-pretrained-128-png \ 16 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 17 | --which_model_netD xception_block2 --model patch_discriminator \ 18 | --patience 20 --lr_policy constant --max_epochs 1000 \ 19 | --no_serial_batches 20 | 21 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 22 | --name gp1d-gan-samplesonly --save_epoch_freq 50 \ 23 | --real_im_path dataset/faces/celebahq/real-tfr-1024-resized128 \ 24 | --fake_im_path dataset/faces/celebahq/pgan-pretrained-128-png \ 25 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 26 | --which_model_netD xception_block3 --model patch_discriminator \ 27 | --patience 10 --lr_policy constant --max_epochs 1000 \ 28 | --no_serial_batches 29 | 30 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 31 | --name gp1d-gan-samplesonly --save_epoch_freq 50 \ 32 | --real_im_path dataset/faces/celebahq/real-tfr-1024-resized128 \ 33 | --fake_im_path dataset/faces/celebahq/pgan-pretrained-128-png \ 34 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 35 | --which_model_netD xception_block4 --model patch_discriminator \ 36 | --patience 10 --lr_policy constant --max_epochs 1000 \ 37 | --no_serial_batches 38 | 39 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 40 | --name gp1d-gan-samplesonly --save_epoch_freq 50 \ 41 | --real_im_path dataset/faces/celebahq/real-tfr-1024-resized128 \ 42 | --fake_im_path dataset/faces/celebahq/pgan-pretrained-128-png \ 43 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 44 | --which_model_netD xception_block5 --model patch_discriminator \ 45 | --patience 10 --lr_policy constant --max_epochs 1000 \ 46 | --no_serial_batches 47 | -------------------------------------------------------------------------------- /scripts/01_train_gan_xception_patches_winversion.sh: -------------------------------------------------------------------------------- 1 | ### celebahq-pgan generated faces and inverted real pairs ### 2 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 3 | --name gp1-gan-winversion --save_epoch_freq 200 \ 4 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 5 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 6 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 7 | --which_model_netD xception_block1 --model patch_discriminator \ 8 | --patience 50 --lr_policy constant --max_epochs 1000 9 | 10 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 11 | --name gp1-gan-winversion --save_epoch_freq 200 \ 12 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 13 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 14 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 15 | --which_model_netD xception_block2 --model patch_discriminator \ 16 | --patience 20 --lr_policy constant --max_epochs 1000 17 | 18 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 19 | --name gp1-gan-winversion --save_epoch_freq 50 \ 20 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 21 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 22 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 23 | --which_model_netD xception_block3 --model patch_discriminator \ 24 | --patience 10 --lr_policy constant --max_epochs 1000 25 | 26 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 27 | --name gp1-gan-winversion --save_epoch_freq 50 \ 28 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 29 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 30 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 31 | --which_model_netD xception_block4 --model patch_discriminator \ 32 | --patience 10 --lr_policy constant --max_epochs 1000 33 | 34 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 35 | --name gp1-gan-winversion --save_epoch_freq 50 \ 36 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 37 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 38 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 39 | --which_model_netD xception_block5 --model patch_discriminator \ 40 | --patience 10 --lr_policy constant --max_epochs 1000 41 | -------------------------------------------------------------------------------- /scripts/01_train_gan_xception_patches_winversion_randcrop.sh: -------------------------------------------------------------------------------- 1 | ### celebahq-pgan generated faces and inverted real pairs ### 2 | python train.py --gpu_ids 0 --seed 0 --loadSize 333 --fineSize 299 \ 3 | --name gp1b-gan-winversion --save_epoch_freq 200 \ 4 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 5 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 6 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience}_randcrop \ 7 | --which_model_netD xception_block1 --model patch_discriminator \ 8 | --patience 50 --lr_policy constant --max_epochs 1000 --random_crop 9 | 10 | python train.py --gpu_ids 0 --seed 0 --loadSize 333 --fineSize 299 \ 11 | --name gp1b-gan-winversion --save_epoch_freq 200 \ 12 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 13 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 14 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience}_randcrop \ 15 | --which_model_netD xception_block2 --model patch_discriminator \ 16 | --patience 20 --lr_policy constant --max_epochs 1000 --random_crop 17 | 18 | python train.py --gpu_ids 0 --seed 0 --loadSize 333 --fineSize 299 \ 19 | --name gp1b-gan-winversion --save_epoch_freq 50 \ 20 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 21 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 22 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience}_randcrop \ 23 | --which_model_netD xception_block3 --model patch_discriminator \ 24 | --patience 10 --lr_policy constant --max_epochs 1000 --random_crop 25 | 26 | python train.py --gpu_ids 0 --seed 0 --loadSize 333 --fineSize 299 \ 27 | --name gp1b-gan-winversion --save_epoch_freq 50 \ 28 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 29 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 30 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience}_randcrop \ 31 | --which_model_netD xception_block4 --model patch_discriminator \ 32 | --patience 10 --lr_policy constant --max_epochs 1000 --random_crop 33 | 34 | python train.py --gpu_ids 0 --seed 0 --loadSize 333 --fineSize 299 \ 35 | --name gp1b-gan-winversion --save_epoch_freq 50 \ 36 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 37 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 38 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience}_randcrop \ 39 | --which_model_netD xception_block5 --model patch_discriminator \ 40 | --patience 10 --lr_policy constant --max_epochs 1000 --random_resized_crop 41 | -------------------------------------------------------------------------------- /scripts/01_train_gan_xception_patches_winversion_randresizecrop.sh: -------------------------------------------------------------------------------- 1 | ### celebahq-pgan generated faces and inverted real pairs ### 2 | python train.py --gpu_ids 0 --seed 0 --loadSize 333 --fineSize 299 \ 3 | --name gp1a-gan-winversion --save_epoch_freq 200 \ 4 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 5 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 6 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience}_randresizecrop \ 7 | --which_model_netD xception_block1 --model patch_discriminator \ 8 | --patience 50 --lr_policy constant --max_epochs 1000 --random_resized_crop 9 | 10 | python train.py --gpu_ids 0 --seed 0 --loadSize 333 --fineSize 299 \ 11 | --name gp1a-gan-winversion --save_epoch_freq 200 \ 12 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 13 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 14 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience}_randresizecrop \ 15 | --which_model_netD xception_block2 --model patch_discriminator \ 16 | --patience 20 --lr_policy constant --max_epochs 1000 --random_resized_crop 17 | 18 | python train.py --gpu_ids 0 --seed 0 --loadSize 333 --fineSize 299 \ 19 | --name gp1a-gan-winversion --save_epoch_freq 50 \ 20 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 21 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 22 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience}_randresizecrop \ 23 | --which_model_netD xception_block3 --model patch_discriminator \ 24 | --patience 10 --lr_policy constant --max_epochs 1000 --random_resized_crop 25 | 26 | python train.py --gpu_ids 0 --seed 0 --loadSize 333 --fineSize 299 \ 27 | --name gp1a-gan-winversion --save_epoch_freq 50 \ 28 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 29 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 30 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience}_randresizecrop \ 31 | --which_model_netD xception_block4 --model patch_discriminator \ 32 | --patience 10 --lr_policy constant --max_epochs 1000 --random_resized_crop 33 | 34 | python train.py --gpu_ids 0 --seed 0 --loadSize 333 --fineSize 299 \ 35 | --name gp1a-gan-winversion --save_epoch_freq 50 \ 36 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 37 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 38 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience}_randresizecrop \ 39 | --which_model_netD xception_block5 --model patch_discriminator \ 40 | --patience 10 --lr_policy constant --max_epochs 1000 --random_resized_crop 41 | -------------------------------------------------------------------------------- /scripts/02_train_faceforensics_DF_xception_patches_aligned.sh: -------------------------------------------------------------------------------- 1 | ### training on deepfakes ### 2 | 3 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 4 | --name gp2-faceforensics-df --save_epoch_freq 200 \ 5 | --real_im_path dataset/faces/faceforensics_aligned/Deepfakes/original \ 6 | --fake_im_path dataset/faces/faceforensics_aligned/Deepfakes/manipulated \ 7 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 8 | --which_model_netD xception_block1 --model patch_discriminator \ 9 | --patience 50 --lr_policy constant --max_epochs 1000 10 | 11 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 12 | --name gp2-faceforensics-df --save_epoch_freq 200 \ 13 | --real_im_path dataset/faces/faceforensics_aligned/Deepfakes/original \ 14 | --fake_im_path dataset/faces/faceforensics_aligned/Deepfakes/manipulated \ 15 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 16 | --which_model_netD xception_block2 --model patch_discriminator \ 17 | --patience 20 --lr_policy constant --max_epochs 1000 18 | 19 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 20 | --name gp2-faceforensics-df --save_epoch_freq 200 \ 21 | --real_im_path dataset/faces/faceforensics_aligned/Deepfakes/original \ 22 | --fake_im_path dataset/faces/faceforensics_aligned/Deepfakes/manipulated \ 23 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 24 | --which_model_netD xception_block3 --model patch_discriminator \ 25 | --patience 10 --lr_policy constant --max_epochs 1000 26 | 27 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 28 | --name gp2-faceforensics-df --save_epoch_freq 200 \ 29 | --real_im_path dataset/faces/faceforensics_aligned/Deepfakes/original \ 30 | --fake_im_path dataset/faces/faceforensics_aligned/Deepfakes/manipulated \ 31 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 32 | --which_model_netD xception_block4 --model patch_discriminator \ 33 | --patience 10 --lr_policy constant --max_epochs 1000 34 | 35 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 36 | --name gp2-faceforensics-df --save_epoch_freq 200 \ 37 | --real_im_path dataset/faces/faceforensics_aligned/Deepfakes/original \ 38 | --fake_im_path dataset/faces/faceforensics_aligned/Deepfakes/manipulated \ 39 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 40 | --which_model_netD xception_block5 --model patch_discriminator \ 41 | --patience 10 --lr_policy constant --max_epochs 1000 42 | -------------------------------------------------------------------------------- /scripts/02_train_faceforensics_F2F_xception_patches_aligned.sh: -------------------------------------------------------------------------------- 1 | ### training on F2f ### 2 | 3 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 4 | --name gp5-faceforensics-f2f --save_epoch_freq 200 \ 5 | --real_im_path dataset/faces/faceforensics_aligned/Face2Face/original \ 6 | --fake_im_path dataset/faces/faceforensics_aligned/Face2Face/manipulated \ 7 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 8 | --which_model_netD xception_block1 --model patch_discriminator \ 9 | --patience 50 --lr_policy constant --max_epochs 1000 10 | 11 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 12 | --name gp5-faceforensics-f2f --save_epoch_freq 200 \ 13 | --real_im_path dataset/faces/faceforensics_aligned/Face2Face/original \ 14 | --fake_im_path dataset/faces/faceforensics_aligned/Face2Face/manipulated \ 15 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 16 | --which_model_netD xception_block2 --model patch_discriminator \ 17 | --patience 20 --lr_policy constant --max_epochs 1000 18 | 19 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 20 | --name gp5-faceforensics-f2f --save_epoch_freq 200 \ 21 | --real_im_path dataset/faces/faceforensics_aligned/Face2Face/original \ 22 | --fake_im_path dataset/faces/faceforensics_aligned/Face2Face/manipulated \ 23 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 24 | --which_model_netD xception_block3 --model patch_discriminator \ 25 | --patience 10 --lr_policy constant --max_epochs 1000 26 | 27 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 28 | --name gp5-faceforensics-f2f --save_epoch_freq 200 \ 29 | --real_im_path dataset/faces/faceforensics_aligned/Face2Face/original \ 30 | --fake_im_path dataset/faces/faceforensics_aligned/Face2Face/manipulated \ 31 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 32 | --which_model_netD xception_block4 --model patch_discriminator \ 33 | --patience 10 --lr_policy constant --max_epochs 1000 34 | 35 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 36 | --name gp5-faceforensics-f2f --save_epoch_freq 200 \ 37 | --real_im_path dataset/faces/faceforensics_aligned/Face2Face/original \ 38 | --fake_im_path dataset/faces/faceforensics_aligned/Face2Face/manipulated \ 39 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 40 | --which_model_netD xception_block5 --model patch_discriminator \ 41 | --patience 10 --lr_policy constant --max_epochs 1000 42 | -------------------------------------------------------------------------------- /scripts/02_train_faceforensics_FS_xception_patches_aligned.sh: -------------------------------------------------------------------------------- 1 | ### training on FS ### 2 | 3 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 4 | --name gp4-faceforensics-fs --save_epoch_freq 200 \ 5 | --real_im_path dataset/faces/faceforensics_aligned/FaceSwap/original \ 6 | --fake_im_path dataset/faces/faceforensics_aligned/FaceSwap/manipulated \ 7 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 8 | --which_model_netD xception_block1 --model patch_discriminator \ 9 | --patience 50 --lr_policy constant --max_epochs 1000 10 | 11 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 12 | --name gp4-faceforensics-fs --save_epoch_freq 200 \ 13 | --real_im_path dataset/faces/faceforensics_aligned/FaceSwap/original \ 14 | --fake_im_path dataset/faces/faceforensics_aligned/FaceSwap/manipulated \ 15 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 16 | --which_model_netD xception_block2 --model patch_discriminator \ 17 | --patience 20 --lr_policy constant --max_epochs 1000 18 | 19 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 20 | --name gp4-faceforensics-fs --save_epoch_freq 200 \ 21 | --real_im_path dataset/faces/faceforensics_aligned/FaceSwap/original \ 22 | --fake_im_path dataset/faces/faceforensics_aligned/FaceSwap/manipulated \ 23 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 24 | --which_model_netD xception_block3 --model patch_discriminator \ 25 | --patience 10 --lr_policy constant --max_epochs 1000 26 | 27 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 28 | --name gp4-faceforensics-fs --save_epoch_freq 200 \ 29 | --real_im_path dataset/faces/faceforensics_aligned/FaceSwap/original \ 30 | --fake_im_path dataset/faces/faceforensics_aligned/FaceSwap/manipulated \ 31 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 32 | --which_model_netD xception_block4 --model patch_discriminator \ 33 | --patience 10 --lr_policy constant --max_epochs 1000 34 | 35 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 36 | --name gp4-faceforensics-fs --save_epoch_freq 200 \ 37 | --real_im_path dataset/faces/faceforensics_aligned/FaceSwap/original \ 38 | --fake_im_path dataset/faces/faceforensics_aligned/FaceSwap/manipulated \ 39 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 40 | --which_model_netD xception_block5 --model patch_discriminator \ 41 | --patience 10 --lr_policy constant --max_epochs 1000 42 | -------------------------------------------------------------------------------- /scripts/02_train_faceforensics_NT_xception_patches_aligned.sh: -------------------------------------------------------------------------------- 1 | ### training on NT ### 2 | 3 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 4 | --name gp3-faceforensics-nt --save_epoch_freq 200 \ 5 | --real_im_path dataset/faces/faceforensics_aligned/NeuralTextures/original \ 6 | --fake_im_path dataset/faces/faceforensics_aligned/NeuralTextures/manipulated \ 7 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 8 | --which_model_netD xception_block1 --model patch_discriminator \ 9 | --patience 50 --lr_policy constant --max_epochs 1000 10 | 11 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 12 | --name gp3-faceforensics-nt --save_epoch_freq 200 \ 13 | --real_im_path dataset/faces/faceforensics_aligned/NeuralTextures/original \ 14 | --fake_im_path dataset/faces/faceforensics_aligned/NeuralTextures/manipulated \ 15 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 16 | --which_model_netD xception_block2 --model patch_discriminator \ 17 | --patience 20 --lr_policy constant --max_epochs 1000 18 | 19 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 20 | --name gp3-faceforensics-nt --save_epoch_freq 200 \ 21 | --real_im_path dataset/faces/faceforensics_aligned/NeuralTextures/original \ 22 | --fake_im_path dataset/faces/faceforensics_aligned/NeuralTextures/manipulated \ 23 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 24 | --which_model_netD xception_block3 --model patch_discriminator \ 25 | --patience 10 --lr_policy constant --max_epochs 1000 26 | 27 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 28 | --name gp3-faceforensics-nt --save_epoch_freq 200 \ 29 | --real_im_path dataset/faces/faceforensics_aligned/NeuralTextures/original \ 30 | --fake_im_path dataset/faces/faceforensics_aligned/NeuralTextures/manipulated \ 31 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 32 | --which_model_netD xception_block4 --model patch_discriminator \ 33 | --patience 10 --lr_policy constant --max_epochs 1000 34 | 35 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 36 | --name gp3-faceforensics-nt --save_epoch_freq 200 \ 37 | --real_im_path dataset/faces/faceforensics_aligned/NeuralTextures/original \ 38 | --fake_im_path dataset/faces/faceforensics_aligned/NeuralTextures/manipulated \ 39 | --suffix seed{seed}_{which_model_netD}_{lr_policy}_p{patience} \ 40 | --which_model_netD xception_block5 --model patch_discriminator \ 41 | --patience 10 --lr_policy constant --max_epochs 1000 42 | -------------------------------------------------------------------------------- /scripts/03_baseline_resnet_full.sh: -------------------------------------------------------------------------------- 1 | # pretrained pgan with inversions 2 | python train.py --gpu_ids 0 --seed 0 --loadSize 224 --fineSize 224 \ 3 | --name gp1-gan-winversion --save_epoch_freq 200 \ 4 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 5 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 6 | --suffix baseline_resnet18_full \ 7 | --which_model_netD resnet18 --model basic_discriminator \ 8 | --patience 10 --lr_policy constant --max_epochs 1000 9 | 10 | # faceforensics DF 11 | python train.py --gpu_ids 0 --seed 0 --loadSize 224 --fineSize 224 \ 12 | --name gp2-faceforensics-df --save_epoch_freq 200 \ 13 | --real_im_path dataset/faces/faceforensics_aligned/Deepfakes/original \ 14 | --fake_im_path dataset/faces/faceforensics_aligned/Deepfakes/manipulated \ 15 | --suffix baseline_resnet18_full \ 16 | --which_model_netD resnet18 --model basic_discriminator \ 17 | --patience 10 --lr_policy constant --max_epochs 1000 18 | 19 | # faceforensics NT 20 | python train.py --gpu_ids 0 --seed 0 --loadSize 224 --fineSize 224 \ 21 | --name gp3-faceforensics-nt --save_epoch_freq 200 \ 22 | --real_im_path dataset/faces/faceforensics_aligned/NeuralTextures/original \ 23 | --fake_im_path dataset/faces/faceforensics_aligned/NeuralTextures/manipulated \ 24 | --suffix baseline_resnet18_full \ 25 | --which_model_netD resnet18 --model basic_discriminator \ 26 | --patience 10 --lr_policy constant --max_epochs 1000 27 | 28 | # faceforensics FS 29 | python train.py --gpu_ids 0 --seed 0 --loadSize 224 --fineSize 224 \ 30 | --name gp4-faceforensics-fs --save_epoch_freq 200 \ 31 | --real_im_path dataset/faces/faceforensics_aligned/FaceSwap/original \ 32 | --fake_im_path dataset/faces/faceforensics_aligned/FaceSwap/manipulated \ 33 | --suffix baseline_resnet18_full \ 34 | --which_model_netD resnet18 --model basic_discriminator \ 35 | --patience 10 --lr_policy constant --max_epochs 1000 36 | 37 | # faceforensics F2F 38 | python train.py --gpu_ids 0 --seed 0 --loadSize 224 --fineSize 224 \ 39 | --name gp5-faceforensics-f2f --save_epoch_freq 200 \ 40 | --real_im_path dataset/faces/faceforensics_aligned/Face2Face/original \ 41 | --fake_im_path dataset/faces/faceforensics_aligned/Face2Face/manipulated \ 42 | --suffix baseline_resnet18_full \ 43 | --which_model_netD resnet18 --model basic_discriminator \ 44 | --patience 10 --lr_policy constant --max_epochs 1000 45 | 46 | -------------------------------------------------------------------------------- /scripts/03_baseline_xception_full.sh: -------------------------------------------------------------------------------- 1 | # pretrained pgan with inversions 2 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 3 | --name gp1-gan-winversion --save_epoch_freq 200 \ 4 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 5 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 6 | --suffix baseline_xception_full \ 7 | --which_model_netD xception --model basic_discriminator \ 8 | --patience 10 --lr_policy constant --max_epochs 1000 9 | 10 | # faceforensics DF 11 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 12 | --name gp2-faceforensics-df --save_epoch_freq 200 \ 13 | --real_im_path dataset/faces/faceforensics_aligned/Deepfakes/original \ 14 | --fake_im_path dataset/faces/faceforensics_aligned/Deepfakes/manipulated \ 15 | --suffix baseline_xception_full \ 16 | --which_model_netD xception --model basic_discriminator \ 17 | --patience 10 --lr_policy constant --max_epochs 1000 18 | 19 | # faceforensics NT 20 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 21 | --name gp3-faceforensics-nt --save_epoch_freq 200 \ 22 | --real_im_path dataset/faces/faceforensics_aligned/NeuralTextures/original \ 23 | --fake_im_path dataset/faces/faceforensics_aligned/NeuralTextures/manipulated \ 24 | --suffix baseline_xception_full \ 25 | --which_model_netD xception --model basic_discriminator \ 26 | --patience 10 --lr_policy constant --max_epochs 1000 27 | 28 | # faceforensics FS 29 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 30 | --name gp4-faceforensics-fs --save_epoch_freq 200 \ 31 | --real_im_path dataset/faces/faceforensics_aligned/FaceSwap/original \ 32 | --fake_im_path dataset/faces/faceforensics_aligned/FaceSwap/manipulated \ 33 | --suffix baseline_xception_full \ 34 | --which_model_netD xception --model basic_discriminator \ 35 | --patience 10 --lr_policy constant --max_epochs 1000 36 | 37 | # faceforensics F2F 38 | python train.py --gpu_ids 0 --seed 0 --loadSize 299 --fineSize 299 \ 39 | --name gp5-faceforensics-f2f --save_epoch_freq 200 \ 40 | --real_im_path dataset/faces/faceforensics_aligned/Face2Face/original \ 41 | --fake_im_path dataset/faces/faceforensics_aligned/Face2Face/manipulated \ 42 | --suffix baseline_xception_full \ 43 | --which_model_netD xception --model basic_discriminator \ 44 | --patience 10 --lr_policy constant --max_epochs 1000 45 | 46 | -------------------------------------------------------------------------------- /scripts/03_train_resnet_block1.sh: -------------------------------------------------------------------------------- 1 | # pretrained pgan with inversions 2 | python train.py --gpu_ids 0 --seed 0 --loadSize 224 --fineSize 224 \ 3 | --name gp1-gan-winversion --save_epoch_freq 200 \ 4 | --real_im_path dataset/faces/celebahq/inverted_and_unpaired_real \ 5 | --fake_im_path dataset/faces/celebahq/inverted_and_unpaired_fake \ 6 | --suffix baseline_resnet18_layer1 \ 7 | --which_model_netD resnet18_layer1 --model patch_discriminator \ 8 | --patience 10 --lr_policy constant --max_epochs 1000 9 | 10 | # faceforensics DF 11 | python train.py --gpu_ids 0 --seed 0 --loadSize 224 --fineSize 224 \ 12 | --name gp2-faceforensics-df --save_epoch_freq 200 \ 13 | --real_im_path dataset/faces/faceforensics_aligned/Deepfakes/original \ 14 | --fake_im_path dataset/faces/faceforensics_aligned/Deepfakes/manipulated \ 15 | --suffix baseline_resnet18_layer1 \ 16 | --which_model_netD resnet18_layer1 --model patch_discriminator \ 17 | --patience 10 --lr_policy constant --max_epochs 1000 18 | 19 | # faceforensics NT 20 | python train.py --gpu_ids 0 --seed 0 --loadSize 224 --fineSize 224 \ 21 | --name gp3-faceforensics-nt --save_epoch_freq 200 \ 22 | --real_im_path dataset/faces/faceforensics_aligned/NeuralTextures/original \ 23 | --fake_im_path dataset/faces/faceforensics_aligned/NeuralTextures/manipulated \ 24 | --suffix baseline_resnet18_layer1 \ 25 | --which_model_netD resnet18_layer1 --model patch_discriminator \ 26 | --patience 10 --lr_policy constant --max_epochs 1000 27 | 28 | # faceforensics FS 29 | python train.py --gpu_ids 0 --seed 0 --loadSize 224 --fineSize 224 \ 30 | --name gp4-faceforensics-fs --save_epoch_freq 200 \ 31 | --real_im_path dataset/faces/faceforensics_aligned/FaceSwap/original \ 32 | --fake_im_path dataset/faces/faceforensics_aligned/FaceSwap/manipulated \ 33 | --suffix baseline_resnet18_layer1 \ 34 | --which_model_netD resnet18_layer1 --model patch_discriminator \ 35 | --patience 10 --lr_policy constant --max_epochs 1000 36 | 37 | # faceforensics F2F 38 | python train.py --gpu_ids 0 --seed 0 --loadSize 224 --fineSize 224 \ 39 | --name gp5-faceforensics-f2f --save_epoch_freq 200 \ 40 | --real_im_path dataset/faces/faceforensics_aligned/Face2Face/original \ 41 | --fake_im_path dataset/faces/faceforensics_aligned/Face2Face/manipulated \ 42 | --suffix baseline_resnet18_layer1 \ 43 | --which_model_netD resnet18_layer1 --model patch_discriminator \ 44 | --patience 10 --lr_policy constant --max_epochs 1000 45 | 46 | -------------------------------------------------------------------------------- /scripts/04_eval_checkpoint.sh: -------------------------------------------------------------------------------- 1 | # example: python test_runs.py checkpoints/temp-inverted-and-unpaired_seed0_xception_block5_constant_p10/ gen_models val 2 | 3 | # model arch and dataset 4 | for expt in checkpoints/gp1-* checkpoints/gp1[a-d]-* 5 | do 6 | for part in test # val 7 | do 8 | cmd="python test_runs.py $expt gen_models $part" 9 | echo $cmd 10 | eval $cmd 11 | done 12 | done 13 | 14 | # # faceforensics -- this will only run if the faceforensics 15 | # # dataset is processed according to 16 | # # scripts/00_data_processing_faceforensics_aligned_frames.sh 17 | # for expt in checkpoints/gp[2-5]-* 18 | # do 19 | # for part in val test 20 | # do 21 | # cmd="python test_runs.py $expt faceforensics $part" 22 | # echo $cmd 23 | # eval $cmd 24 | # done 25 | # done 26 | 27 | -------------------------------------------------------------------------------- /scripts/04_eval_patches_faceforensics_DF.sh: -------------------------------------------------------------------------------- 1 | # script to run top patches experiments 2 | # model: trained on deepfake 3 | partition=test 4 | 5 | # test on DF 6 | ckpt=gp2-faceforensics-df_seed0_xception_block3_constant_p10 7 | name=DF 8 | python patches.py --which_epoch bestval --gpu_ids 0 \ 9 | --topn 10000 --unique --partition $partition \ 10 | --dataset_name $name \ 11 | --train_config checkpoints/$ckpt/opt.yml \ 12 | --real_im_path dataset/faces/faceforensics_aligned/Deepfakes/original/$partition/ \ 13 | --fake_im_path dataset/faces/faceforensics_aligned/Deepfakes/manipulated/$partition/ 14 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000/ 15 | 16 | # test on NT 17 | ckpt=gp2-faceforensics-df_baseline_resnet18_layer1 18 | name=NT 19 | python patches.py --which_epoch bestval --gpu_ids 0 \ 20 | --topn 10000 --unique --partition $partition \ 21 | --dataset_name $name \ 22 | --train_config checkpoints/$ckpt/opt.yml \ 23 | --real_im_path dataset/faces/faceforensics_aligned/NeuralTextures/original/$partition/ \ 24 | --fake_im_path dataset/faces/faceforensics_aligned/NeuralTextures/manipulated/$partition/ 25 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000 26 | 27 | # test on F2F 28 | ckpt=gp2-faceforensics-df_baseline_resnet18_layer1 29 | name=F2F 30 | python patches.py --which_epoch bestval --gpu_ids 0 \ 31 | --topn 10000 --unique --partition $partition \ 32 | --dataset_name $name \ 33 | --train_config checkpoints/$ckpt/opt.yml \ 34 | --real_im_path dataset/faces/faceforensics_aligned/Face2Face/original/$partition/ \ 35 | --fake_im_path dataset/faces/faceforensics_aligned/Face2Face/manipulated/$partition/ 36 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000 37 | 38 | 39 | # test on FS 40 | ckpt=gp2-faceforensics-df_seed0_xception_block4_constant_p10 41 | name=FS 42 | python patches.py --which_epoch bestval --gpu_ids 0 \ 43 | --topn 10000 --unique --partition $partition \ 44 | --dataset_name $name \ 45 | --train_config checkpoints/$ckpt/opt.yml \ 46 | --real_im_path dataset/faces/faceforensics_aligned/FaceSwap/original/$partition/ \ 47 | --fake_im_path dataset/faces/faceforensics_aligned/FaceSwap/manipulated/$partition/ 48 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000 49 | -------------------------------------------------------------------------------- /scripts/04_eval_patches_faceforensics_F2F.sh: -------------------------------------------------------------------------------- 1 | # script to run top patches experiments 2 | # model: trained on face2face 3 | partition=test 4 | 5 | # test on DF 6 | ckpt=gp5-faceforensics-f2f_baseline_resnet18_layer1 7 | name=DF 8 | python patches.py --which_epoch bestval --gpu_ids 0 \ 9 | --topn 10000 --unique --partition $partition \ 10 | --dataset_name $name \ 11 | --train_config checkpoints/$ckpt/opt.yml \ 12 | --real_im_path dataset/faces/faceforensics_aligned/Deepfakes/original/$partition/ \ 13 | --fake_im_path dataset/faces/faceforensics_aligned/Deepfakes/manipulated/$partition/ 14 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000/ 15 | 16 | # test on NT 17 | ckpt=gp5-faceforensics-f2f_seed0_xception_block1_constant_p50 18 | name=NT 19 | python patches.py --which_epoch bestval --gpu_ids 0 \ 20 | --topn 10000 --unique --partition $partition \ 21 | --dataset_name $name \ 22 | --train_config checkpoints/$ckpt/opt.yml \ 23 | --real_im_path dataset/faces/faceforensics_aligned/NeuralTextures/original/$partition/ \ 24 | --fake_im_path dataset/faces/faceforensics_aligned/NeuralTextures/manipulated/$partition/ 25 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000 26 | 27 | # test on F2F 28 | ckpt=gp5-faceforensics-f2f_baseline_resnet18_layer1 29 | name=F2F 30 | python patches.py --which_epoch bestval --gpu_ids 0 \ 31 | --topn 10000 --unique --partition $partition \ 32 | --dataset_name $name \ 33 | --train_config checkpoints/$ckpt/opt.yml \ 34 | --real_im_path dataset/faces/faceforensics_aligned/Face2Face/original/$partition/ \ 35 | --fake_im_path dataset/faces/faceforensics_aligned/Face2Face/manipulated/$partition/ 36 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000 37 | 38 | 39 | # test on FS 40 | ckpt=gp5-faceforensics-f2f_seed0_xception_block2_constant_p20 41 | name=FS 42 | python patches.py --which_epoch bestval --gpu_ids 0 \ 43 | --topn 10000 --unique --partition $partition \ 44 | --dataset_name $name \ 45 | --train_config checkpoints/$ckpt/opt.yml \ 46 | --real_im_path dataset/faces/faceforensics_aligned/FaceSwap/original/$partition/ \ 47 | --fake_im_path dataset/faces/faceforensics_aligned/FaceSwap/manipulated/$partition/ 48 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000 49 | -------------------------------------------------------------------------------- /scripts/04_eval_patches_gen_models.sh: -------------------------------------------------------------------------------- 1 | # script to run top patches experiments 2 | 3 | partition=test 4 | 5 | # pgan pretrain 6 | ckpt=gp1-gan-winversion_seed0_xception_block2_constant_p20 7 | name=celebahq-pgan-pretrained 8 | python patches.py --which_epoch bestval --gpu_ids 0 \ 9 | --topn 10000 --unique --partition $partition \ 10 | --train_config checkpoints/$ckpt/opt.yml \ 11 | --dataset_name $name \ 12 | --real_im_path dataset/faces/celebahq/real-tfr-1024-resized128/$partition/ \ 13 | --fake_im_path dataset/faces/celebahq/pgan-pretrained-128-png/$partition/ 14 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000/ 15 | 16 | # sgan pretrain 17 | ckpt=gp1-gan-winversion_seed0_xception_block3_constant_p10 18 | name=celebahq-sgan-pretrained 19 | python patches.py --which_epoch bestval --gpu_ids 0 \ 20 | --topn 10000 --unique --partition $partition \ 21 | --train_config checkpoints/$ckpt/opt.yml \ 22 | --dataset_name $name \ 23 | --real_im_path dataset/faces/celebahq/real-tfr-1024-resized128/$partition \ 24 | --fake_im_path dataset/faces/celebahq/sgan-pretrained-128-png/$partition 25 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000/ 26 | 27 | # glow pretrain 28 | ckpt=gp1d-gan-samplesonly_seed0_xception_block1_constant_p50 29 | name=celebahq-glow-pretrained 30 | python patches.py --which_epoch bestval --gpu_ids 0 \ 31 | --topn 10000 --unique --partition $partition \ 32 | --train_config checkpoints/$ckpt/opt.yml \ 33 | --dataset_name $name \ 34 | --real_im_path dataset/faces/celebahq/real-tfr-1024-resized128/$partition \ 35 | --fake_im_path dataset/faces/celebahq/glow-pretrained-128-png/$partition 36 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000/ 37 | 38 | # gmm model 39 | ckpt=gp1-gan-winversion_seed0_xception_block2_constant_p20 40 | name=celeba-gmm 41 | python patches.py --which_epoch bestval --gpu_ids 0 \ 42 | --topn 10000 --unique --partition $partition \ 43 | --train_config checkpoints/$ckpt/opt.yml \ 44 | --dataset_name $name \ 45 | --real_im_path dataset/faces/celeba/mfa-real/$partition \ 46 | --fake_im_path dataset/faces/celeba/mfa-defaults/$partition 47 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000/ 48 | 49 | # ffhq pgan 50 | ckpt=gp1-gan-winversion_seed0_xception_block2_constant_p20 51 | name=ffhq-pgan 52 | python patches.py --which_epoch bestval --gpu_ids 0 \ 53 | --topn 10000 --unique --partition $partition \ 54 | --train_config checkpoints/$ckpt/opt.yml \ 55 | --dataset_name $name \ 56 | --real_im_path dataset/faces/ffhq/real-tfr-1024-resized128/$partition \ 57 | --fake_im_path dataset/faces/ffhq/pgan-9k-128-png/$partition 58 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000/ 59 | 60 | # ffhq sgan 61 | ckpt=gp1-gan-winversion_seed0_xception_block4_constant_p10 62 | name=ffhq-sgan 63 | python patches.py --which_epoch bestval --gpu_ids 0 \ 64 | --topn 10000 --unique --partition $partition \ 65 | --train_config checkpoints/$ckpt/opt.yml \ 66 | --dataset_name $name \ 67 | --real_im_path dataset/faces/ffhq/real-tfr-1024-resized128/$partition \ 68 | --fake_im_path dataset/faces/ffhq/sgan-pretrained-128-png/$partition 69 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000/ 70 | 71 | # ffhq sgan2 72 | ckpt=gp1-gan-winversion_seed0_xception_block3_constant_p10 73 | name=ffhq-sgan2 74 | python patches.py --which_epoch bestval --gpu_ids 0 \ 75 | --topn 10000 --unique --partition $partition \ 76 | --train_config checkpoints/$ckpt/opt.yml \ 77 | --dataset_name $name \ 78 | --real_im_path dataset/faces/ffhq/real-tfr-1024-resized128/$partition \ 79 | --fake_im_path dataset/faces/ffhq/sgan2-pretrained-128-png/$partition 80 | python segmenter.py results/$ckpt/$partition/epoch_bestval/$name/patches_top10000/ 81 | -------------------------------------------------------------------------------- /scripts/04_eval_visualize_faceforensics_DF.sh: -------------------------------------------------------------------------------- 1 | partition=test 2 | 3 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 4 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 5 | --dataset_name DF \ 6 | --real_im_path dataset/faces/faceforensics_aligned/Deepfakes/original/$partition \ 7 | --fake_im_path dataset/faces/faceforensics_aligned/Deepfakes/manipulated/$partition \ 8 | --train_config checkpoints/gp2-faceforensics-df_seed0_xception_block3_constant_p10/opt.yml 9 | 10 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 11 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 12 | --dataset_name NT \ 13 | --real_im_path dataset/faces/faceforensics_aligned/NeuralTextures/original/$partition \ 14 | --fake_im_path dataset/faces/faceforensics_aligned/NeuralTextures/manipulated/$partition \ 15 | --train_config checkpoints/gp2-faceforensics-df_baseline_resnet18_layer1/opt.yml 16 | 17 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 18 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 19 | --dataset_name F2F \ 20 | --real_im_path dataset/faces/faceforensics_aligned/Face2Face/original/$partition \ 21 | --fake_im_path dataset/faces/faceforensics_aligned/Face2Face/manipulated/$partition \ 22 | --train_config checkpoints/gp2-faceforensics-df_baseline_resnet18_layer1/opt.yml 23 | 24 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 25 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 26 | --dataset_name FS \ 27 | --real_im_path dataset/faces/faceforensics_aligned/FaceSwap/original/$partition \ 28 | --fake_im_path dataset/faces/faceforensics_aligned/FaceSwap/manipulated/$partition \ 29 | --train_config checkpoints/gp2-faceforensics-df_seed0_xception_block4_constant_p10/opt.yml 30 | -------------------------------------------------------------------------------- /scripts/04_eval_visualize_faceforensics_F2F.sh: -------------------------------------------------------------------------------- 1 | partition=test 2 | 3 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 4 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 5 | --dataset_name DF \ 6 | --real_im_path dataset/faces/faceforensics_aligned/Deepfakes/original/$partition \ 7 | --fake_im_path dataset/faces/faceforensics_aligned/Deepfakes/manipulated/$partition \ 8 | --train_config checkpoints/gp5-faceforensics-f2f_baseline_resnet18_layer1/opt.yml 9 | 10 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 11 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 12 | --dataset_name NT \ 13 | --real_im_path dataset/faces/faceforensics_aligned/NeuralTextures/original/$partition \ 14 | --fake_im_path dataset/faces/faceforensics_aligned/NeuralTextures/manipulated/$partition \ 15 | --train_config checkpoints/gp5-faceforensics-f2f_seed0_xception_block1_constant_p50/opt.yml 16 | 17 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 18 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 19 | --dataset_name F2F \ 20 | --real_im_path dataset/faces/faceforensics_aligned/Face2Face/original/$partition \ 21 | --fake_im_path dataset/faces/faceforensics_aligned/Face2Face/manipulated/$partition \ 22 | --train_config checkpoints/gp5-faceforensics-f2f_baseline_resnet18_layer1/opt.yml 23 | 24 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 25 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 26 | --dataset_name FS \ 27 | --real_im_path dataset/faces/faceforensics_aligned/FaceSwap/original/$partition \ 28 | --fake_im_path dataset/faces/faceforensics_aligned/FaceSwap/manipulated/$partition \ 29 | --train_config checkpoints/gp5-faceforensics-f2f_seed0_xception_block2_constant_p20/opt.yml 30 | -------------------------------------------------------------------------------- /scripts/04_eval_visualize_gen_models.sh: -------------------------------------------------------------------------------- 1 | partition=test 2 | 3 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 4 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 5 | --dataset_name celebahq-pgan-pretrained \ 6 | --real_im_path dataset/faces/celebahq/real-tfr-1024-resized128/$partition \ 7 | --fake_im_path dataset/faces/celebahq/pgan-pretrained-128-png/$partition \ 8 | --train_config checkpoints/gp1-gan-winversion_seed0_xception_block2_constant_p20/opt.yml 9 | 10 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 11 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 12 | --dataset_name celebahq-sgan-pretrained \ 13 | --real_im_path dataset/faces/celebahq/real-tfr-1024-resized128/$partition \ 14 | --fake_im_path dataset/faces/celebahq/sgan-pretrained-128-png/$partition \ 15 | --train_config checkpoints/gp1-gan-winversion_seed0_xception_block3_constant_p10/opt.yml 16 | 17 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 18 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 19 | --dataset_name celebahq-glow-pretrained \ 20 | --real_im_path dataset/faces/celebahq/real-tfr-1024-resized128/$partition \ 21 | --fake_im_path dataset/faces/celebahq/glow-pretrained-128-png/$partition \ 22 | --train_config checkpoints/gp1d-gan-samplesonly_seed0_xception_block1_constant_p50/opt.yml 23 | 24 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 25 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 26 | --dataset_name celeba-gmm \ 27 | --real_im_path dataset/faces/celeba/mfa-real/$partition \ 28 | --fake_im_path dataset/faces/celeba/mfa-defaults/$partition \ 29 | --train_config checkpoints/gp1-gan-winversion_seed0_xception_block2_constant_p20/opt.yml 30 | 31 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 32 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 33 | --dataset_name ffhq-pgan \ 34 | --real_im_path dataset/faces/ffhq/real-tfr-1024-resized128/$partition \ 35 | --fake_im_path dataset/faces/ffhq/pgan-9k-128-png/$partition \ 36 | --train_config checkpoints/gp1-gan-winversion_seed0_xception_block2_constant_p20/opt.yml 37 | 38 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 39 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 40 | --dataset_name ffhq-sgan \ 41 | --real_im_path dataset/faces/ffhq/real-tfr-1024-resized128/$partition \ 42 | --fake_im_path dataset/faces/ffhq/sgan-pretrained-128-png/$partition \ 43 | --train_config checkpoints/gp1-gan-winversion_seed0_xception_block4_constant_p10/opt.yml 44 | 45 | python test.py --which_epoch bestval --gpu_ids 0 --partition $partition \ 46 | --visualize --average_mode after_softmax --topn 100 --force_redo \ 47 | --dataset_name ffhq-sgan2 \ 48 | --real_im_path dataset/faces/ffhq/real-tfr-1024-resized128/$partition \ 49 | --fake_im_path dataset/faces/ffhq/sgan2-pretrained-128-png/$partition \ 50 | --train_config checkpoints/gp1-gan-winversion_seed0_xception_block3_constant_p10/opt.yml 51 | -------------------------------------------------------------------------------- /segmenter.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.transforms as transforms 3 | from PIL import Image 4 | import numpy as np 5 | import os 6 | from utils import rfutil, imutil, pidfile 7 | from tqdm import tqdm 8 | import cv2 9 | from collections import Counter, defaultdict, namedtuple 10 | import matplotlib.pyplot as plt 11 | import argparse 12 | import oyaml as yaml 13 | import sys 14 | sys.path.append('resources/face_parsing_pytorch/') 15 | from model import BiSeNet 16 | import random 17 | 18 | # grouped cluster assignments 19 | cluster_assn = { 20 | 0: 'background', 21 | 1: 'skin', 22 | 2: 'brows', 23 | 3: 'brows', 24 | 4: 'eye', 25 | 5: 'eye', 26 | 6: 'eye', 27 | 7: 'ear', 28 | 8: 'ear', 29 | 9: 'ear', 30 | 10: 'nose', 31 | 11: 'mouth', 32 | 12: 'mouth', 33 | 13: 'mouth', 34 | 14: 'neck', 35 | 15: 'neck', 36 | 16: 'clothes', 37 | 17: 'hair', 38 | 18: 'hat', 39 | } 40 | 41 | def cluster(args, outpath): 42 | 43 | # network setup 44 | n_classes = 19 45 | net = BiSeNet(n_classes=n_classes).cuda() 46 | save_pth = 'resources/face_parsing_pytorch/res/cp/79999_iter.pth' 47 | net.load_state_dict(torch.load(save_pth)) 48 | net.eval() 49 | to_tensor = transforms.Compose([ 50 | transforms.Resize((512, 512), Image.BILINEAR), 51 | transforms.ToTensor(), 52 | transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), 53 | ]) 54 | 55 | subsets = ['reals_easiest', 'fakes_easiest'] 56 | PatchInfo = namedtuple('PatchInfo', ['patch', 'pos', 'file', 'value']) 57 | 58 | for subset in subsets: 59 | print(subset) 60 | path = args.path 61 | cluster_dir = os.path.join(outpath, subset + '_clusters') 62 | os.makedirs(cluster_dir, exist_ok =True) 63 | with open(os.path.join(path, subset+'_files.txt')) as f: 64 | files = [line.strip() for line in f] 65 | patches = np.load(os.path.join(path, subset + '.npz')) 66 | which_model_netD = patches['which_model_netD'] 67 | fineSize = patches['finesize'] 68 | rfs = rfutil.find_rf_patches(which_model_netD, fineSize) 69 | 70 | clusters = defaultdict(list) 71 | clusters_baseline = Counter() # a counter for segmentations of random patches 72 | 73 | # assign each patch to a cluster based on segmentation 74 | for index, (patch, pos, value, file) in tqdm(enumerate( 75 | zip(patches['patch'], patches['pos'], patches['value'], files)), total=len(files)): 76 | image = Image.open(file).convert('RGB') 77 | with torch.no_grad(): 78 | tensor = to_tensor(image)[None].cuda() 79 | out = net(tensor)[0] 80 | parsing = out[0].cpu().numpy().argmax(0) 81 | interp = cv2.resize(parsing, (fineSize, fineSize), interpolation=cv2.INTER_NEAREST)[None] 82 | seg_patch = rfutil.get_patch_from_img(interp, rfs[(pos[0], pos[1])], patches['rf'], pad_value=-1) 83 | # how many pixels of each segmentation class in the patch 84 | counter_patch = Counter(seg_patch[0].ravel()) 85 | # how many pixels of each segmentation class in the full img 86 | counter_full = Counter(interp[0].ravel()) 87 | # normalize: omit padding value from normalization 88 | counter_norm = {k: counter_patch[k] / counter_full[k] for k in counter_patch.keys() if k != -1} 89 | cluster_id = max(counter_norm, key=counter_norm.get) 90 | cluster_label = cluster_assn[cluster_id] 91 | clusters[cluster_label].append(PatchInfo(patch, pos, file, value)) 92 | 93 | # pick a random patch for baseline 94 | random_patch = random.choice(list(rfs.values())) 95 | seg_patch = rfutil.get_patch_from_img(interp, random_patch, patches['rf'], pad_value=-1) 96 | counter_patch = Counter(seg_patch[0].ravel()) 97 | counter_full = Counter(interp[0].ravel()) 98 | # omit padding value from normalization 99 | counter_norm = {k: counter_patch[k] / counter_full[k] for k in counter_patch.keys() if k != -1} 100 | cluster_id = max(counter_norm, key=counter_norm.get) 101 | cluster_label = cluster_assn[cluster_id] 102 | clusters_baseline[cluster_label] += 1 103 | 104 | # plot each cluster in a grid 105 | counts, labels = [], [] 106 | infos = [] 107 | for index, (k,v) in enumerate(sorted(clusters.items(), key=lambda item: len(item[1]))[::-1]): 108 | line = '%d: %s, %d patches' % (index, k, len(v)) 109 | print(line) 110 | infos.append(line) 111 | counts.append(len(v)) 112 | labels.append(k) 113 | cluster = np.asarray([patchinfo.patch for patchinfo in v]) 114 | files = [patchinfo.file for patchinfo in clusters[k]] 115 | normalized = (cluster[-225:] * 0.5) + 0.5 # at most 15x15 grid 116 | grid = imutil.imgrid(np.uint8(normalized * 255), pad=0, cols=int(np.ceil(np.sqrt(normalized.shape[0])))) 117 | grid_im = Image.fromarray(grid) 118 | grid_im.save(os.path.join(cluster_dir, 'cluster_%d.png' % index)) 119 | np.savez(os.path.join(cluster_dir, 'cluster_%d.npz' % index), 120 | patch=cluster, rf=patches['rf'], 121 | finesize=patches['finesize'], 122 | outsize=patches['outsize'], 123 | which_model_netD=patches['which_model_netD'], 124 | pos=np.array([patchinfo.pos for patchinfo in v]), 125 | value=np.array([patchinfo.value for patchinfo in v])) 126 | with open(os.path.join(cluster_dir, 'cluster_%d.txt' % index), 'w') as f: 127 | [f.write('%s\n' % file) for file in files] 128 | 129 | # histogram 130 | f, ax = plt.subplots(1, 1, figsize=(6, 4)) 131 | ax.bar(range(1, len(labels) + 1), counts) 132 | ax.set_xticks(range(1, len(labels) + 1)) 133 | ax.set_xticklabels(labels, rotation='vertical') 134 | ax.set_ylabel('count') 135 | f.savefig(os.path.join(cluster_dir, 'histogram.pdf'), 136 | bbox_inches='tight') 137 | 138 | # write counts to file 139 | with open(os.path.join(cluster_dir, 'counts.txt'), 'w') as f: 140 | [f.write('%s\n' % line) for line in infos] 141 | 142 | # write random patch baseline to file 143 | infos = [] 144 | for index, (k,v) in enumerate(sorted(clusters_baseline.items(), key=lambda item: item[1])[::-1]): 145 | infos.append('%d: %s, %d patches' % (index, k, v)) 146 | with open(os.path.join(cluster_dir, 'baseline.txt'), 'w') as f: 147 | [f.write('%s\n' % line) for line in infos] 148 | 149 | 150 | if __name__ == '__main__': 151 | parser = argparse.ArgumentParser(description='Cluster patches using face segmentation.') 152 | parser.add_argument('path', type=str, help='path to precomputed top clusters') 153 | args = parser.parse_args() 154 | outpath = os.path.join(args.path, 'clusters') 155 | os.makedirs(outpath, exist_ok =True) 156 | pidfile.exit_if_job_done(outpath,redo=True) 157 | cluster(args, outpath) 158 | pidfile.mark_job_done(outpath) 159 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | from options.test_options import TestOptions 2 | from models import create_model 3 | import numpy as np 4 | import os 5 | import torch 6 | from utils import pidfile, util, imutil, pbar 7 | from utils import util 8 | from utils import imutil 9 | from torch.utils.data import DataLoader 10 | from data.unpaired_dataset import UnpairedDataset 11 | from sklearn import metrics 12 | import matplotlib.pyplot as plt 13 | from PIL import Image 14 | from IPython import embed 15 | 16 | torch.backends.cudnn.benchmark = True 17 | 18 | def run_eval(opt, output_dir): 19 | model = create_model(opt) 20 | model.setup(opt) 21 | model.eval() 22 | 23 | fake_label = opt.fake_class_id 24 | real_label = 1 - fake_label 25 | 26 | # values to track 27 | paths = [] 28 | prediction_voted = [] 29 | prediction_avg_after_softmax = [] 30 | prediction_avg_before_softmax = [] 31 | prediction_raw = [] 32 | labels = [] 33 | 34 | for data_path, label in zip([opt.real_im_path, opt.fake_im_path], 35 | [real_label, fake_label]): 36 | dset = UnpairedDataset(opt, data_path, is_val=True) 37 | dl = DataLoader(dset, batch_size=opt.batch_size, shuffle=False, 38 | num_workers=opt.nThreads, pin_memory=False) 39 | 40 | for i, data in enumerate(dl): 41 | # set model inputs 42 | ims = data['img'].to(opt.gpu_ids[0]) 43 | pred_labels = (torch.ones(ims.shape[0], dtype=torch.long) 44 | * label).to(opt.gpu_ids[0]) 45 | inputs = dict(ims=ims, labels=pred_labels) 46 | 47 | # forward pass 48 | model.reset() 49 | model.set_input(inputs) 50 | model.test(True) 51 | predictions = model.get_predictions() 52 | 53 | # update counts 54 | labels.append(pred_labels.cpu().numpy()) 55 | prediction_voted.append(predictions.vote) 56 | prediction_avg_before_softmax.append(predictions.before_softmax) 57 | prediction_avg_after_softmax.append(predictions.after_softmax) 58 | prediction_raw.append(predictions.raw) 59 | paths.extend(data['path']) 60 | 61 | # compute and save metrics 62 | if opt.model == 'patch_discriminator': 63 | # save precision, recall, AP metrics on voted predictions 64 | compute_metrics(np.concatenate(prediction_voted), 65 | np.concatenate(labels), 66 | os.path.join(output_dir, 'metrics_voted')) 67 | 68 | # save precision, recall, AP metrics, avg before softmax 69 | compute_metrics(np.concatenate(prediction_avg_before_softmax), 70 | np.concatenate(labels), 71 | os.path.join(output_dir, 'metrics_avg_before_softmax')) 72 | 73 | # save precision, recall, AP metrics, avg after softmax 74 | compute_metrics(np.concatenate(prediction_avg_after_softmax), 75 | np.concatenate(labels), 76 | os.path.join(output_dir, 'metrics_avg_after_softmax')) 77 | 78 | # save precision, recall, AP metrics, on raw patches 79 | # this can be slow, so will not plot AP curve 80 | patch_preds = np.concatenate(prediction_raw, axis=0) # N2HW 81 | patch_preds = patch_preds.transpose(0, 2, 3, 1) # NHW2 82 | n, h, w, c = patch_preds.shape 83 | patch_labels = np.concatenate(labels, axis=0)[:, None, None] 84 | patch_labels = np.tile(patch_labels, (1, h, w)) 85 | patch_preds = patch_preds.reshape(-1, 2) 86 | patch_labels = patch_labels.reshape(-1) 87 | compute_metrics(patch_preds, patch_labels, 88 | os.path.join(output_dir, 'metrics_patch'), 89 | plot=False) 90 | 91 | if opt.visualize: 92 | pred_output = { 93 | 'vote': np.concatenate(prediction_voted), 94 | 'before_softmax': np.concatenate(prediction_avg_before_softmax), 95 | 'after_softmax': np.concatenate(prediction_avg_after_softmax) 96 | }[opt.average_mode] 97 | vis_dir = os.path.join(output_dir, 'vis') 98 | transform = dset.transform 99 | model.visualize(pred_output, paths, np.concatenate(labels), 100 | transform, fake_label, 101 | os.path.join(vis_dir, 'fakes'), 102 | opt.topn) 103 | model.visualize(pred_output, paths, np.concatenate(labels), 104 | transform, real_label, 105 | os.path.join(vis_dir, 'reals'), 106 | opt.topn) 107 | else: 108 | # save precision, recall, AP metrics, on non-patch-based model 109 | compute_metrics(np.concatenate(prediction_raw), 110 | np.concatenate(labels), 111 | os.path.join(output_dir, 'metrics')) 112 | 113 | 114 | def compute_metrics(predictions, labels, save_path, threshold=0.5, plot=True): 115 | # save precision, recall, AP metrics on voted predictions 116 | print("Computing metrics for %s" % save_path) 117 | assert(len(np.unique(labels)) == 2) 118 | assert(np.ndim(predictions) == 2) 119 | assert(predictions.shape[1] == 2) 120 | assert(len(labels) == predictions.shape[0]) 121 | # predictions should be Nx2 np array 122 | # labels should be (N,) array with 0,1 values 123 | ap = metrics.average_precision_score(labels, predictions[:, 1]) 124 | precision, recall, thresholds = metrics.precision_recall_curve( 125 | labels, predictions[:, 1]) 126 | print("ap: %0.6f" % ap) 127 | acc = metrics.accuracy_score(labels, np.argmax(predictions, axis=1)) 128 | np.savez(save_path + '.npz', ap=ap, precision=precision, 129 | recall=recall, thresholds=thresholds, acc=acc, n=len(labels)) 130 | if plot: 131 | f, ax = plt.subplots(1, 1) 132 | ax.plot(recall, precision) 133 | ax.set_xlabel('recall') 134 | ax.set_ylabel('precision') 135 | ax.set_xlim([0, 1]) 136 | ax.set_ylim([0, 1]) 137 | f.savefig(save_path + '.pdf') 138 | 139 | if __name__ == '__main__': 140 | opt = TestOptions().parse() 141 | print("Evaluating model: %s epoch %s" % (opt.name, opt.which_epoch)) 142 | print("On dataset (real): %s" % (opt.real_im_path)) 143 | print("And dataset (fake): %s" % (opt.fake_im_path)) 144 | expdir = opt.name 145 | dataset_name = opt.dataset_name 146 | output_dir = os.path.join(opt.results_dir, expdir, opt.partition, 147 | 'epoch_%s' % opt.which_epoch, dataset_name) 148 | print(output_dir) 149 | os.makedirs(output_dir, exist_ok=True) 150 | 151 | # check if checkpoint is out of date (e.g. if model is still training) 152 | redo = opt.force_redo 153 | ckpt_path = os.path.join(opt.checkpoints_dir, opt.name, '%s_net_D.pth' 154 | % opt.which_epoch) 155 | timestamp_path = os.path.join(output_dir, 'timestamp_%s_net_D.txt' 156 | % opt.which_epoch) 157 | if util.check_timestamp(ckpt_path, timestamp_path): 158 | redo = True 159 | util.update_timestamp(ckpt_path, timestamp_path) 160 | pidfile.exit_if_job_done(output_dir, redo=redo) 161 | run_eval(opt, output_dir) 162 | pidfile.mark_job_done(output_dir) 163 | 164 | -------------------------------------------------------------------------------- /test_runs.py: -------------------------------------------------------------------------------- 1 | import time 2 | import os 3 | import sys 4 | import subprocess 5 | import shlex 6 | import glob 7 | import torch 8 | import argparse 9 | 10 | # wrapper to run test experiments 11 | 12 | # argparse model checkpoint 13 | parser = argparse.ArgumentParser('Model Test Pipeline') 14 | parser.add_argument('checkpoint_dir', help='directory of experiment checkpoints') 15 | parser.add_argument('dataset_type', help='gen_models, faceforensics, etc') 16 | parser.add_argument('partition', help='which partition to run [val|test]') 17 | args = parser.parse_args() 18 | checkpoint_dir = args.checkpoint_dir 19 | checkpoints = glob.glob(os.path.join(checkpoint_dir, '*_net_D.pth')) 20 | 21 | def get_dataset_paths(dataroot, datasets, partition): 22 | fake_datasets = [os.path.join(dataroot, dataset[0], partition) 23 | for dataset in datasets] 24 | real_datasets = [os.path.join(dataroot, dataset[1], partition) 25 | for dataset in datasets] 26 | dataset_names = [dataset[2] for dataset in datasets] 27 | return fake_datasets, real_datasets, dataset_names 28 | 29 | # datasets 30 | if args.dataset_type == 'gen_models': 31 | dataroot = 'dataset/faces/' 32 | partition = args.partition 33 | datasets = [ 34 | ('celebahq/pgan-pretrained-128-png', 35 | 'celebahq/real-tfr-1024-resized128', 'celebahq-pgan-pretrained'), 36 | ('celebahq/sgan-pretrained-128-png', 37 | 'celebahq/real-tfr-1024-resized128', 'celebahq-sgan-pretrained'), 38 | ('celebahq/glow-pretrained-128-png', 39 | 'celebahq/real-tfr-1024-resized128', 'celebahq-glow-pretrained'), 40 | ('celeba/mfa-defaults', 'celeba/mfa-real', 'celeba-gmm'), 41 | ('ffhq/pgan-9k-128-png', 'ffhq/real-tfr-1024-resized128', 'ffhq-pgan'), 42 | ('ffhq/sgan-pretrained-128-png', 'ffhq/real-tfr-1024-resized128', 43 | 'ffhq-sgan'), 44 | ('ffhq/sgan2-pretrained-128-png', 'ffhq/real-tfr-1024-resized128', 45 | 'ffhq-sgan2'), 46 | ] 47 | fake_datasets, real_datasets, dataset_names = get_dataset_paths( 48 | dataroot, datasets, partition) 49 | elif args.dataset_type == 'faceforensics': 50 | dataroot = 'dataset/faces/' 51 | partition = args.partition 52 | datasets = [ 53 | ('faceforensics_aligned/NeuralTextures/manipulated', 54 | 'faceforensics_aligned/NeuralTextures/original', 'NT'), 55 | ('faceforensics_aligned/Deepfakes/manipulated', 56 | 'faceforensics_aligned/Deepfakes/original', 'DF'), 57 | ('faceforensics_aligned/Face2Face/manipulated', 58 | 'faceforensics_aligned/Face2Face/original', 'F2F'), 59 | ('faceforensics_aligned/FaceSwap/manipulated', 60 | 'faceforensics_aligned/FaceSwap/original', 'FS'), 61 | ] 62 | fake_datasets, real_datasets, dataset_names = get_dataset_paths( 63 | dataroot, datasets, partition) 64 | else: 65 | raise NotImplementedError 66 | 67 | # print the datasets to test on 68 | print(real_datasets) 69 | print(fake_datasets) 70 | print(dataset_names) 71 | 72 | for checkpoint in checkpoints: 73 | for fake, real, name in zip(fake_datasets, real_datasets, dataset_names): 74 | which_epoch = os.path.basename(checkpoint).split('_')[0] 75 | if which_epoch != 'bestval': 76 | # only runs using the bestval checkpoint 77 | continue 78 | test_command = ('python test.py --train_config %s' % 79 | (os.path.join(checkpoint_dir, 'opt.yml'))) 80 | test_command += ' --which_epoch %s' % which_epoch 81 | test_command += ' --gpu_ids 0' 82 | test_command += ' --real_im_path %s' % real 83 | test_command += ' --fake_im_path %s' % fake 84 | test_command += ' --partition %s' % args.partition 85 | test_command += ' --dataset_name %s' % name 86 | 87 | print(test_command) 88 | os.system(test_command) 89 | 90 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import time 2 | from options.train_options import TrainOptions 3 | from models import create_model 4 | import numpy as np 5 | from utils.visualizer import Visualizer 6 | import logging 7 | import os 8 | from collections import OrderedDict 9 | from IPython import embed 10 | import torch 11 | import torchvision 12 | import torchvision.transforms as transforms 13 | import pdb 14 | from torch.utils.data import DataLoader 15 | from data.paired_dataset import PairedDataset 16 | from utils import pidfile, util 17 | import utils.logging 18 | 19 | def train(opt): 20 | torch.manual_seed(opt.seed) 21 | 22 | # load the train dataset 23 | dset = PairedDataset(opt, os.path.join(opt.real_im_path, 'train'), 24 | os.path.join(opt.fake_im_path, 'train')) 25 | # halves batch size since each batch returns both real and fake ims 26 | dl = DataLoader(dset, batch_size=opt.batch_size // 2, 27 | num_workers=opt.nThreads, pin_memory=False, 28 | shuffle=True) 29 | 30 | # setup class labeling 31 | assert(opt.fake_class_id in [0, 1]) 32 | fake_label = opt.fake_class_id 33 | real_label = 1 - fake_label 34 | logging.info("real label = %d" % real_label) 35 | logging.info("fake label = %d" % fake_label) 36 | dataset_size = 2 * len(dset) 37 | logging.info('# total images = %d' % dataset_size) 38 | logging.info('# total batches = %d' % len(dl)) 39 | 40 | # setup model and visualizer 41 | model = create_model(opt) 42 | epoch, best_val_metric, best_val_ep = model.setup(opt) 43 | 44 | visualizer_losses = model.loss_names + [n + '_val' for n in model.loss_names] 45 | visualizer = Visualizer(opt, visualizer_losses, model.visual_names) 46 | total_batches = epoch * len(dl) 47 | t_data = 0 48 | 49 | now = time.strftime("%c") 50 | logging.info('================ Training Loss (%s) ================\n' % now) 51 | 52 | while True: 53 | epoch_start_time = time.time() 54 | iter_data_time = time.time() 55 | epoch_iter = 0 56 | 57 | for i, ims in enumerate(dl): 58 | ims_real = ims['original'].to(opt.gpu_ids[0]) 59 | ims_fake = ims['manipulated'].to(opt.gpu_ids[0]) 60 | labels_real = real_label * torch.ones(ims_real.shape[0], dtype=torch.long).to(opt.gpu_ids[0]) 61 | labels_fake = fake_label * torch.ones(ims_fake.shape[0], dtype=torch.long).to(opt.gpu_ids[0]) 62 | 63 | batch_im = torch.cat((ims_real, ims_fake), axis=0) 64 | batch_label = torch.cat((labels_real, labels_fake), axis=0) 65 | batch_data = dict(ims=batch_im, labels=batch_label) 66 | 67 | iter_start_time = time.time() 68 | if total_batches % opt.print_freq == 0: 69 | # time to load data 70 | t_data = iter_start_time - iter_data_time 71 | 72 | total_batches += 1 73 | epoch_iter += 1 74 | model.reset() 75 | model.set_input(batch_data) 76 | model.optimize_parameters() 77 | 78 | if epoch_iter % opt.print_freq == 0: 79 | losses = model.get_current_losses() 80 | t = time.time() - iter_start_time 81 | visualizer.print_current_losses( 82 | epoch, float(epoch_iter)/len(dl), total_batches, 83 | losses, t, t_data) 84 | visualizer.plot_current_losses(total_batches, losses) 85 | 86 | if epoch_iter % opt.display_freq == 0: 87 | visualizer.display_current_results(model.get_current_visuals(), 88 | total_batches) 89 | 90 | if epoch_iter % opt.save_latest_freq == 0: 91 | logging.info('saving the latest model (epoch %d, total_batches %d)' % 92 | (epoch, total_batches)) 93 | model.save_networks('latest', epoch, best_val_metric, 94 | best_val_ep) 95 | 96 | model.reset() 97 | iter_data_time = time.time() 98 | 99 | # do validation loop at end of each epoch 100 | model.eval() 101 | val_start_time = time.time() 102 | val_losses = validate(model, opt) 103 | visualizer.plot_current_losses(epoch, val_losses) 104 | logging.info("Printing validation losses:") 105 | visualizer.print_current_losses( 106 | epoch, 0.0, total_batches, val_losses, 107 | time.time()-val_start_time, 0.0) 108 | model.train() 109 | model.reset() 110 | assert(model.net_D.training) 111 | 112 | # update best model and determine stopping conditions 113 | if val_losses[model.val_metric + '_val'] > best_val_metric: 114 | logging.info("Updating best val mode at ep %d" % epoch) 115 | logging.info("The previous values: ep %d, val %0.2f" % 116 | (best_val_ep, best_val_metric)) 117 | best_val_ep = epoch 118 | best_val_metric = val_losses[model.val_metric + '_val'] 119 | logging.info("The updated values: ep %d, val %0.2f" % 120 | (best_val_ep, best_val_metric)) 121 | model.save_networks('bestval', epoch, best_val_metric, best_val_ep) 122 | with open(os.path.join(model.save_dir, 'bestval_ep.txt'), 'a') as f: 123 | f.write('ep: %d %s: %f\n' % (epoch, model.val_metric + '_val', 124 | best_val_metric)) 125 | elif epoch > (best_val_ep + 5*opt.patience): 126 | logging.info("Current epoch %d, last updated val at ep %d" % 127 | (epoch, best_val_ep)) 128 | logging.info("Stopping training...") 129 | break 130 | elif best_val_metric == 1: 131 | logging.info("Reached perfect val accuracy metric") 132 | logging.info("Stopping training...") 133 | break 134 | elif opt.max_epochs and epoch > opt.max_epochs: 135 | logging.info("Reached max epoch count") 136 | logging.info("Stopping training...") 137 | break 138 | 139 | logging.info("Best val ep: %d" % best_val_ep) 140 | logging.info("Best val metric: %0.2f" % best_val_metric) 141 | 142 | # save final plots at end of each epoch 143 | visualizer.save_final_plots() 144 | 145 | if epoch % opt.save_epoch_freq == 0 and epoch > 0: 146 | logging.info('saving the model at the end of epoch %d, total batches %d' % (epoch, total_batches)) 147 | model.save_networks('latest', epoch, best_val_metric, 148 | best_val_ep) 149 | model.save_networks(epoch, epoch, best_val_metric, best_val_ep) 150 | 151 | logging.info('End of epoch %d \t Time Taken: %d sec' % 152 | (epoch, time.time() - epoch_start_time)) 153 | model.update_learning_rate(metric=val_losses[model.val_metric + '_val']) 154 | epoch += 1 155 | 156 | # save model at the end of training 157 | visualizer.save_final_plots() 158 | model.save_networks('latest', epoch, best_val_metric, 159 | best_val_ep) 160 | model.save_networks(epoch, epoch, best_val_metric, best_val_ep) 161 | logging.info("Finished Training") 162 | 163 | def validate(model, opt): 164 | # --- start evaluation loop --- 165 | logging.info('Starting evaluation loop ...') 166 | model.reset() 167 | assert(not model.net_D.training) 168 | val_dset = PairedDataset(opt, os.path.join(opt.real_im_path, 'val'), 169 | os.path.join(opt.fake_im_path, 'val'), 170 | is_val=True) 171 | val_dl = DataLoader(val_dset, batch_size=opt.batch_size // 2, 172 | num_workers=opt.nThreads, pin_memory=False, 173 | shuffle=False) 174 | val_losses = OrderedDict([(k + '_val', util.AverageMeter()) 175 | for k in model.loss_names]) 176 | fake_label = opt.fake_class_id 177 | real_label = 1 - fake_label 178 | val_start_time = time.time() 179 | for i, ims in enumerate(val_dl): 180 | ims_real = ims['original'].to(opt.gpu_ids[0]) 181 | ims_fake = ims['manipulated'].to(opt.gpu_ids[0]) 182 | labels_real = real_label * torch.ones(ims_real.shape[0], dtype=torch.long).to(opt.gpu_ids[0]) 183 | labels_fake = fake_label * torch.ones(ims_fake.shape[0], dtype=torch.long).to(opt.gpu_ids[0]) 184 | 185 | inputs = dict(ims=torch.cat((ims_real, ims_fake), axis=0), 186 | labels=torch.cat((labels_real, labels_fake), axis=0)) 187 | 188 | # forward pass 189 | model.reset() 190 | model.set_input(inputs) 191 | model.test(True) 192 | losses = model.get_current_losses() 193 | 194 | # update val losses 195 | for k, v in losses.items(): 196 | val_losses[k + '_val'].update(v, n=len(inputs['labels'])) 197 | 198 | # get average val losses 199 | for k, v in val_losses.items(): 200 | val_losses[k] = v.avg 201 | 202 | return val_losses 203 | 204 | 205 | if __name__ == '__main__': 206 | options = TrainOptions(print_opt=False) 207 | opt = options.parse() 208 | 209 | # lock active experiment directory and write out options 210 | os.makedirs(os.path.join(opt.checkpoints_dir, opt.name), exist_ok=True) 211 | pidfile.exit_if_job_done(os.path.join(opt.checkpoints_dir, opt.name)) 212 | options.print_options(opt) 213 | 214 | # configure logging file 215 | logging_file = os.path.join(opt.checkpoints_dir, opt.name, 'log.txt') 216 | utils.logging.configure(logging_file, append=False) 217 | 218 | # run train loop 219 | train(opt) 220 | 221 | # mark done and release lock 222 | pidfile.mark_job_done(os.path.join(opt.checkpoints_dir, opt.name)) 223 | -------------------------------------------------------------------------------- /utils/imutil.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from PIL import Image 4 | import cv2 5 | from scipy.ndimage.filters import gaussian_filter 6 | 7 | def save_image(image_numpy, image_path): 8 | image_pil = Image.fromarray(image_numpy) 9 | image_pil.save(image_path) 10 | 11 | # Arrange list of images in a grid with padding 12 | # adapted from: https://colab.research.google.com/github/tensorflow/hub/blob/master/examples/colab/biggan_generation_with_tf_hub.ipynb 13 | def imgrid(imarray_np, cols=5, pad=1): 14 | if imarray_np.dtype != np.uint8: 15 | raise ValueError('imgrid input imarray_np must be uint8') 16 | if imarray_np.shape[1] in [1, 3]: 17 | # reorder channel dimension 18 | imarray_np = np.transpose(imarray_np, (0, 2, 3, 1)) 19 | pad = int(pad) 20 | assert pad >= 0 21 | cols = int(cols) 22 | assert cols >= 1 23 | N, H, W, C = imarray_np.shape 24 | rows = int(np.ceil(N / float(cols))) 25 | batch_pad = rows * cols - N 26 | assert batch_pad >= 0 27 | post_pad = [batch_pad, pad, pad, 0] 28 | pad_arg = [[0, p] for p in post_pad] 29 | imarray_np = np.pad(imarray_np, pad_arg, 'constant', constant_values=255) 30 | H += pad 31 | W += pad 32 | grid = (imarray_np 33 | .reshape(rows, cols, H, W, C) 34 | .transpose(0, 2, 1, 3, 4) 35 | .reshape(rows*H, cols*W, C)) 36 | if pad: 37 | grid = grid[:-pad, :-pad] 38 | return grid 39 | 40 | def normalize_heatmap(heatmap): 41 | assert(np.ndim(heatmap) == 2) 42 | heatmap = heatmap - np.min(heatmap) 43 | heatmap = heatmap / (np.max(heatmap) +1e-6) # a bit of tolerance in div 44 | return heatmap 45 | 46 | def colorize_heatmap(heatmap, normalize=False): 47 | if normalize: 48 | heatmap = normalize_heatmap(heatmap) 49 | heatmap = np.uint8(255*heatmap) 50 | colorized = cv2.applyColorMap(heatmap, cv2.COLORMAP_BONE) #cv2.COLORMAP_JET) # COLORMAP_BONE 51 | colorized = colorized[:, :, ::-1] 52 | return colorized 53 | 54 | def overlay_heatmap(image, heatmap, normalize=False): 55 | assert(np.ndim(image) == 3) 56 | assert(np.ndim(heatmap) == 2) 57 | assert(image.shape[-1] in [1, 3]) # HWC 58 | if normalize: 59 | heatmap = normalize_heatmap(heatmap) 60 | (h, w) = image.shape[0:2] 61 | overlay = np.uint8(255*heatmap) 62 | overlay = cv2.resize(overlay, (h, w)) 63 | heatmap = cv2.applyColorMap(overlay, cv2.COLORMAP_JET) 64 | heatmap = heatmap[:, :, ::-1] 65 | result = heatmap * 0.5 + image * 0.5 66 | return np.uint8(result) 67 | 68 | def overlay_blur(image, heatmap, normalize=False, blur_sigma=3, 69 | add_threshold=False, add_contour=False, 70 | threshold=0.5, direction='below', color=[255, 255, 0]): 71 | heatmap_blurred = gaussian_filter(heatmap, sigma=blur_sigma) 72 | if normalize: 73 | heatmap_blurred = normalize_heatmap(heatmap_blurred) 74 | overlay = overlay_heatmap(image, heatmap_blurred) # already normalized 75 | if not add_threshold and not add_contour: 76 | return overlay 77 | mask = cv2.resize(heatmap_blurred, (image.shape[1], image.shape[0])) >= threshold 78 | mask = np.expand_dims(mask, axis=-1) # HW1 79 | if direction == 'above': # plot overlay red above threshold 80 | threshold_im = (overlay * np.uint8(mask) + 81 | (1 - np.uint8(mask)) * image) 82 | else: # plot overlay blue below threshold 83 | threshold_im = (overlay * (1 - np.uint8(mask)) + 84 | np.uint8(mask) * image) 85 | if not add_contour: 86 | return threshold_im 87 | border = border_from_mask(np.squeeze(mask)) 88 | contour_y, contour_x = np.where(border) 89 | threshold_im[contour_y, contour_x, :] = color 90 | return threshold_im 91 | 92 | def border_from_mask(a): 93 | out = np.zeros_like(a) 94 | h = (a[:-1,:] != a[1:,:]) 95 | v = (a[:,:-1] != a[:,1:]) 96 | d = (a[:-1,:-1] != a[1:,1:]) 97 | u = (a[1:,:-1] != a[:-1,1:]) 98 | out[:-1,:-1] |= d 99 | out[1:,1:] |= d 100 | out[1:,:-1] |= u 101 | out[:-1,1:] |= u 102 | out[:-1,:] |= h 103 | out[1:,:] |= h 104 | out[:,:-1] |= v 105 | out[:,1:] |= v 106 | out &= ~a 107 | return out 108 | 109 | -------------------------------------------------------------------------------- /utils/lightbox.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 12 | 15 | 19 | 22 | 26 | 33 | 34 | 35 |
36 |

Images in {{ directory }}

37 |
38 | Filter: 39 |
40 |
41 |
{{ r }}
42 | 43 |
44 |
45 | 46 | 75 | 76 | -------------------------------------------------------------------------------- /utils/logging.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import logging 4 | import tqdm 5 | import contextlib 6 | 7 | 8 | __all__ = ['configure', 'disable'] 9 | 10 | 11 | class TqdmLoggingHandler(logging.Handler): 12 | def __init__(self, level=logging.NOTSET): 13 | super(self.__class__, self).__init__(level) 14 | 15 | def emit(self, record): 16 | try: 17 | msg = self.format(record) 18 | tqdm.tqdm.write(msg) 19 | self.flush() 20 | except (KeyboardInterrupt, SystemExit): 21 | raise 22 | except: # noqa E722 23 | self.handleError(record) 24 | 25 | 26 | class MultiLineFormatter(logging.Formatter): 27 | 28 | def __init__(self, fmt=None, datefmt=None, style='%'): 29 | assert style == '%' 30 | super(MultiLineFormatter, self).__init__(fmt, datefmt, style) 31 | self.multiline_fmt = fmt 32 | 33 | def format(self, record): 34 | r""" 35 | This is mostly the same as logging.Formatter.format except for the splitlines() thing. 36 | This is done so (copied the code) to not make logging a bottleneck. It's not lots of code 37 | after all, and it's pretty straightforward. 38 | """ 39 | record.message = record.getMessage() 40 | if self.usesTime(): 41 | record.asctime = self.formatTime(record, self.datefmt) 42 | if '\n' in record.message: 43 | splitted = record.message.splitlines() 44 | output = self._fmt % dict(record.__dict__, message=splitted.pop(0)) 45 | output += ' \n' + '\n'.join( 46 | self.multiline_fmt % dict(record.__dict__, message=line) 47 | for line in splitted 48 | ) 49 | else: 50 | output = self._fmt % record.__dict__ 51 | 52 | if record.exc_info: 53 | # Cache the traceback text to avoid converting it multiple times 54 | # (it's constant anyway) 55 | if not record.exc_text: 56 | record.exc_text = self.formatException(record.exc_info) 57 | if record.exc_text: 58 | output += ' \n' 59 | try: 60 | output += '\n'.join( 61 | self.multiline_fmt % dict(record.__dict__, message=line) 62 | for index, line in enumerate(record.exc_text.splitlines()) 63 | ) 64 | except UnicodeError: 65 | output += '\n'.join( 66 | self.multiline_fmt % dict(record.__dict__, message=line) 67 | for index, line 68 | in enumerate(record.exc_text.decode(sys.getfilesystemencoding(), 'replace').splitlines()) 69 | ) 70 | return output 71 | 72 | 73 | # this should replace `sys.excepthook` 74 | # Shamelessly taken from https://stackoverflow.com/a/16993115 75 | def handle_exception(exc_type, exc_value, exc_traceback): 76 | if issubclass(exc_type, KeyboardInterrupt): 77 | sys.__excepthook__(exc_type, exc_value, exc_traceback) 78 | return 79 | 80 | logging.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback)) 81 | 82 | 83 | def configure(logging_file, log_level=logging.INFO, level_prefix='', prefix='', 84 | write_to_stdout=True, append=True): 85 | logging.getLogger().setLevel(logging.INFO) # set to info first to allow outputing in this function 86 | 87 | sys.excepthook = handle_exception # automatically log uncaught errors 88 | 89 | handlers = [] 90 | 91 | if write_to_stdout: 92 | handlers.append(TqdmLoggingHandler()) 93 | 94 | delayed_logging = [] # log after we set the handlers with the nice formatter 95 | 96 | if logging_file is not None: 97 | delayed_logging.append((logging.info, 'Logging to {}'.format(logging_file))) 98 | if append: 99 | if os.path.isfile(logging_file): 100 | delayed_logging.append((logging.warning, "Log file already exists, will append")) 101 | handlers.append(logging.FileHandler(logging_file)) 102 | else: 103 | delayed_logging.append((logging.warning, "Creating {} with mode write".format(logging_file))) 104 | handlers.append(logging.FileHandler(logging_file, mode='w')) 105 | 106 | 107 | 108 | formatter = MultiLineFormatter("{}%(asctime)s [{}%(levelname)-5s] %(message)s".format(prefix, level_prefix), 109 | "%Y-%m-%d %H:%M:%S") 110 | logger = logging.getLogger() 111 | logger.handlers = [] 112 | for h in handlers: 113 | h.setFormatter(formatter) 114 | logger.addHandler(h) 115 | 116 | logger.setLevel(log_level) 117 | 118 | # flush cached message 119 | for fn, msg in delayed_logging: 120 | fn(msg) 121 | 122 | return logger 123 | 124 | @contextlib.contextmanager 125 | def disable(level): 126 | # disables any level leq to :attr:`level` 127 | prev_level = logging.getLogger().getEffectiveLevel() 128 | logging.disable(level) 129 | yield 130 | logging.disable(prev_level) 131 | -------------------------------------------------------------------------------- /utils/options.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import oyaml as yaml 3 | import sys 4 | import time 5 | import os 6 | from collections import OrderedDict 7 | from . import util 8 | 9 | class Options(): 10 | def __init__(self): 11 | self.parser = parser = argparse.ArgumentParser() 12 | self.parser.add_argument('config_file', nargs='?', 13 | type=argparse.FileType(mode='r')) 14 | self.parser.add_argument('--overwrite_config', action='store_true', 15 | help="overwrite config files if they exist") 16 | 17 | def print_options(self, opt): 18 | opt_dict = OrderedDict() 19 | message = '' 20 | message += '----------------- Options ---------------\n' 21 | # top level options 22 | for k, v in sorted(vars(opt).items()): 23 | if type(v) == argparse.Namespace: 24 | grouped_k.append((k, v)) 25 | continue 26 | comment = '' 27 | default = self.parser.get_default(k) 28 | if v != default: 29 | comment = '\t[default: %s]' % str(default) 30 | message += '{:>25}: {:<30}{}\n'.format(str(k), str(v), comment) 31 | opt_dict[k] = v 32 | message += '----------------- End -------------------' 33 | print(message) 34 | 35 | # make experiment directory 36 | if hasattr(opt, 'checkpoints_dir') and hasattr(opt, 'name'): 37 | if opt.name != '': 38 | expr_dir = os.path.join(opt.checkpoints_dir, opt.name) 39 | else: 40 | expr_dir = os.path.join(opt.checkpoints_dir) 41 | else: 42 | expr_dir ='./' 43 | os.makedirs(expr_dir, exist_ok=True) 44 | 45 | # save to the disk 46 | file_name = os.path.join(expr_dir, 'opt.txt') 47 | if not opt.overwrite_config: 48 | assert(not os.path.isfile(file_name)), 'config file exists, use --overwrite_config' 49 | with open(file_name, 'wt') as opt_file: 50 | opt_file.write(message) 51 | opt_file.write('\n') 52 | 53 | file_name = os.path.join(expr_dir, 'opt.yml') 54 | if not opt.overwrite_config: 55 | assert(not os.path.isfile(file_name)), 'config file exists, use --overwrite_config' 56 | with open(file_name, 'wt') as opt_file: 57 | opt_dict['overwrite_config'] = False # make it false for saving 58 | yaml.dump(opt_dict, opt_file, default_flow_style=False) 59 | 60 | def parse(self, print_opt=True): 61 | 62 | # parse options 63 | opt = self.parser.parse_args() 64 | 65 | # get arguments specified in config file 66 | if opt.config_file: 67 | data = yaml.load(opt.config_file) 68 | else: 69 | data = {} 70 | 71 | # determine which options were specified 72 | # explicitly with command line args 73 | option_strings = {} 74 | for action_group in self.parser._action_groups: 75 | for action in action_group._group_actions: 76 | for option in action.option_strings: 77 | option_strings[option] = action.dest 78 | specified_options = set([option_strings[x] for x in 79 | sys.argv if x in option_strings]) 80 | 81 | # make namespace 82 | # by default, take the result from argparse 83 | # unless was specified in config file and not in command line 84 | args = {} 85 | for group in self.parser._action_groups: 86 | assert(group.title in ['positional arguments', 87 | 'optional arguments']) 88 | group_dict={a.dest: data[a.dest] if a.dest in data 89 | and a.dest not in specified_options 90 | else getattr(opt, a.dest, None) 91 | for a in group._group_actions} 92 | args.update(group_dict) 93 | 94 | opt = argparse.Namespace(**args) 95 | 96 | delattr(opt, 'config_file') 97 | 98 | # write the configurations to disk 99 | if print_opt: 100 | self.print_options(opt) 101 | 102 | self.opt = opt 103 | return opt 104 | -------------------------------------------------------------------------------- /utils/pbar.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Utilities for showing progress bars, controlling default verbosity, etc. 3 | ''' 4 | 5 | # If the tqdm package is not available, then do not show progress bars; 6 | # just connect print_progress to print. 7 | import sys, types 8 | try: 9 | from tqdm import tqdm, tqdm_notebook 10 | except: 11 | tqdm = None 12 | 13 | default_verbosity = True 14 | next_description = None 15 | 16 | def verbose(verbose): 17 | ''' 18 | Sets default verbosity level. Set to True to see progress bars. 19 | ''' 20 | global default_verbosity 21 | default_verbosity = verbose 22 | 23 | def post(**kwargs): 24 | ''' 25 | When within a progress loop, pbar.post(k=str) will display 26 | the given k=str status on the right-hand-side of the progress 27 | status bar. If not within a visible progress bar, does nothing. 28 | ''' 29 | innermost = innermost_tqdm() 30 | if innermost: 31 | innermost.set_postfix(**kwargs) 32 | 33 | def desc(desc): 34 | ''' 35 | When within a progress loop, pbar.desc(str) changes the 36 | left-hand-side description of the loop toe the given description. 37 | ''' 38 | innermost = innermost_tqdm() 39 | if innermost: 40 | innermost.set_description(str(desc)) 41 | 42 | def descnext(desc): 43 | ''' 44 | Called before starting a progress loop, pbar.descnext(str) 45 | sets the description text that will be used in the following loop. 46 | ''' 47 | global next_description 48 | if not default_verbosity or tqdm is None: 49 | return 50 | next_description = desc 51 | 52 | def print(*args): 53 | ''' 54 | When within a progress loop, will print above the progress loop. 55 | ''' 56 | global next_description 57 | next_description = None 58 | if default_verbosity: 59 | msg = ' '.join(str(s) for s in args) 60 | if tqdm is None: 61 | print(msg) 62 | else: 63 | tqdm.write(msg) 64 | 65 | def tqdm_terminal(it, *args, **kwargs): 66 | ''' 67 | Some settings for tqdm that make it run better in resizable terminals. 68 | ''' 69 | return tqdm(it, *args, dynamic_ncols=True, ascii=True, 70 | leave=(not innermost_tqdm()), **kwargs) 71 | 72 | def in_notebook(): 73 | ''' 74 | True if running inside a Jupyter notebook. 75 | ''' 76 | # From https://stackoverflow.com/a/39662359/265298 77 | try: 78 | shell = get_ipython().__class__.__name__ 79 | if shell == 'ZMQInteractiveShell': 80 | return True # Jupyter notebook or qtconsole 81 | elif shell == 'TerminalInteractiveShell': 82 | return False # Terminal running IPython 83 | else: 84 | return False # Other type (?) 85 | except NameError: 86 | return False # Probably standard Python interpreter 87 | 88 | def innermost_tqdm(): 89 | ''' 90 | Returns the innermost active tqdm progress loop on the stack. 91 | ''' 92 | if hasattr(tqdm, '_instances') and len(tqdm._instances) > 0: 93 | return max(tqdm._instances, key=lambda x: x.pos) 94 | else: 95 | return None 96 | 97 | def __call__(x, *args, **kwargs): 98 | ''' 99 | Invokes a progress function that can wrap iterators to print 100 | progress messages, if verbose is True. 101 | 102 | If verbose is False or tqdm is unavailable, then a quiet 103 | non-printing identity function is used. 104 | 105 | verbose can also be set to a spefific progress function rather 106 | than True, and that function will be used. 107 | ''' 108 | global default_verbosity, next_description 109 | if not default_verbosity or tqdm is None: 110 | return x 111 | if default_verbosity == True: 112 | fn = tqdm_notebook if in_notebook() else tqdm_terminal 113 | else: 114 | fn = default_verbosity 115 | if next_description is not None: 116 | kwargs = dict(kwargs) 117 | kwargs['desc'] = next_description 118 | next_description = None 119 | return fn(x, *args, **kwargs) 120 | 121 | class CallableModule(types.ModuleType): 122 | def __init__(self): 123 | # or super().__init__(__name__) for Python 3 124 | types.ModuleType.__init__(self, __name__) 125 | self.__dict__.update(sys.modules[__name__].__dict__) 126 | def __call__(self, x, *args, **kwargs): 127 | return __call__(x, *args, **kwargs) 128 | 129 | sys.modules[__name__] = CallableModule() 130 | 131 | -------------------------------------------------------------------------------- /utils/pidfile.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Utility for simple distribution of work on multiple processes, by 3 | making sure only one process is working on a job at once. 4 | ''' 5 | 6 | import os, errno, socket, atexit, time, sys 7 | 8 | def exit_if_job_done(directory, redo=False, force=False, verbose=True): 9 | if pidfile_taken(os.path.join(directory, 'lockfile.pid'), 10 | force=force, verbose=verbose): 11 | sys.exit(0) 12 | donefile = os.path.join(directory, 'done.txt') 13 | if os.path.isfile(donefile): 14 | with open(donefile) as f: 15 | msg = f.read() 16 | if redo or force: 17 | if verbose: 18 | print('Removing %s %s' % (donefile, msg)) 19 | os.remove(donefile) 20 | else: 21 | if verbose: 22 | print('%s %s' % (donefile, msg)) 23 | sys.exit(0) 24 | 25 | def mark_job_done(directory): 26 | with open(os.path.join(directory, 'done.txt'), 'w') as f: 27 | f.write('done by %d@%s %s at %s' % 28 | (os.getpid(), socket.gethostname(), 29 | os.getenv('STY', ''), 30 | time.strftime('%c'))) 31 | 32 | def pidfile_taken(path, verbose=False, force=False): 33 | ''' 34 | Usage. To grab an exclusive lock for the remaining duration of the 35 | current process (and exit if another process already has the lock), 36 | do this: 37 | 38 | if pidfile_taken('job_423/lockfile.pid', verbose=True): 39 | sys.exit(0) 40 | 41 | To do a batch of jobs, just run a script that does them all on 42 | each available machine, sharing a network filesystem. When each 43 | job grabs a lock, then this will automatically distribute the 44 | jobs so that each one is done just once on one machine. 45 | ''' 46 | 47 | # Try to create the file exclusively and write my pid into it. 48 | try: 49 | os.makedirs(os.path.dirname(path), exist_ok=True) 50 | fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR) 51 | except OSError as e: 52 | if e.errno == errno.EEXIST: 53 | # If we cannot because there was a race, yield the conflicter. 54 | conflicter = 'race' 55 | try: 56 | with open(path, 'r') as lockfile: 57 | conflicter = lockfile.read().strip() or 'empty' 58 | except: 59 | pass 60 | # Force is for manual one-time use, for deleting stale lockfiles. 61 | if force: 62 | if verbose: 63 | print('Removing %s from %s' % (path, conflicter)) 64 | os.remove(path) 65 | return pidfile_taken(path, verbose=verbose, force=False) 66 | if verbose: 67 | print('%s held by %s' % (path, conflicter)) 68 | return conflicter 69 | else: 70 | # Other problems get an exception. 71 | raise 72 | # Register to delete this file on exit. 73 | lockfile = os.fdopen(fd, 'r+') 74 | atexit.register(delete_pidfile, lockfile, path) 75 | # Write my pid into the open file. 76 | lockfile.write('%d@%s %s\n' % (os.getpid(), socket.gethostname(), 77 | os.getenv('STY', ''))) 78 | lockfile.flush() 79 | os.fsync(lockfile) 80 | # Return 'None' to say there was not a conflict. 81 | return None 82 | 83 | def delete_pidfile(lockfile, path): 84 | ''' 85 | Runs at exit after pidfile_taken succeeds. 86 | ''' 87 | if lockfile is not None: 88 | try: 89 | lockfile.close() 90 | except: 91 | pass 92 | try: 93 | os.unlink(path) 94 | except: 95 | pass 96 | -------------------------------------------------------------------------------- /utils/renormalize.py: -------------------------------------------------------------------------------- 1 | import numpy, torch, PIL 2 | from torchvision import transforms 3 | 4 | def as_tensor(data, source='zc', target='zc'): 5 | renorm = renormalizer(source=source, target=target) 6 | return renorm(data) 7 | 8 | def as_image(data, source='zc', target='byte'): 9 | assert len(data.shape) == 3 10 | renorm = renormalizer(source=source, target=target) 11 | return PIL.Image.fromarray(renorm(data). 12 | permute(1,2,0).cpu().numpy()) 13 | 14 | def renormalizer(source='zc', target='zc'): 15 | ''' 16 | Returns a function that imposes a standard normalization on 17 | the image data. The returned renormalizer operates on either 18 | 3d tensor (single image) or 4d tensor (image batch) data. 19 | The normalization target choices are: 20 | 21 | zc (default) - zero centered [-1..1] 22 | pt - pytorch [0..1] 23 | imagenet - zero mean, unit stdev imagenet stats (approx [-2.1...2.6]) 24 | byte - as from an image file, [0..255] 25 | 26 | If a source is provided (a dataset or transform), then, the renormalizer 27 | first reverses any normalization found in the data source before 28 | imposing the specified normalization. When no source is provided, 29 | the input data is assumed to be pytorch-normalized (range [0..1]). 30 | ''' 31 | if isinstance(source, str): 32 | oldoffset, oldscale = OFFSET_SCALE[source] 33 | else: 34 | normalizer = find_normalizer(source) 35 | oldoffset, oldscale = ( 36 | (normalizer.mean, normalizer.std) if normalizer is not None 37 | else OFFSET_SCALE['pt']) 38 | newoffset, newscale = (target if isinstance(target, tuple) 39 | else OFFSET_SCALE[target]) 40 | return Renormalizer(oldoffset, oldscale, newoffset, newscale, 41 | tobyte=(target == 'byte')) 42 | 43 | # The three commonly-seen image normalization schemes. 44 | OFFSET_SCALE=dict( 45 | pt=([0.0, 0.0, 0.0], [1.0, 1.0, 1.0]), 46 | zc=([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]), 47 | imagenet=([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), 48 | imagenet_meanonly=([0.485, 0.456, 0.406], 49 | [1.0/255, 1.0/255, 1.0/255]), 50 | places_meanonly=([0.475, 0.441, 0.408], 51 | [1.0/255, 1.0/255, 1.0/255]), 52 | byte=([0.0, 0.0, 0.0], [1.0/255, 1.0/255, 1.0/255])) 53 | 54 | NORMALIZER={k: transforms.Normalize(*OFFSET_SCALE[k]) for k in OFFSET_SCALE} 55 | 56 | def find_normalizer(source=None): 57 | ''' 58 | Crawl around the transforms attached to a dataset looking for a 59 | Normalize transform to return. 60 | ''' 61 | if source is None: 62 | return None 63 | if isinstance(source, (transforms.Normalize, Renormalizer)): 64 | return source 65 | t = getattr(source, 'transform', None) 66 | if t is not None: 67 | return find_normalizer(t) 68 | ts = getattr(source, 'transforms', None) 69 | if ts is not None: 70 | for t in reversed(ts): 71 | result = find_normalizer(t) 72 | if result is not None: 73 | return result 74 | return None 75 | 76 | class Renormalizer: 77 | def __init__(self, oldoffset, oldscale, newoffset, newscale, tobyte=False): 78 | self.mul = torch.from_numpy( 79 | numpy.array(oldscale) / numpy.array(newscale)) 80 | self.add = torch.from_numpy( 81 | (numpy.array(oldoffset) - numpy.array(newoffset)) 82 | / numpy.array(newscale)) 83 | self.tobyte = tobyte 84 | # Store these away to allow the data to be renormalized again 85 | self.mean = newoffset 86 | self.std = newscale 87 | 88 | def __call__(self, data): 89 | mul, add = [d.to(data.device, data.dtype) for d in [self.mul, self.add]] 90 | if data.ndimension() == 3: 91 | mul, add = [d[:, None, None] for d in [mul, add]] 92 | elif data.ndimension() == 4: 93 | mul, add = [d[None, :, None, None] for d in [mul, add]] 94 | result = data.mul(mul).add_(add) 95 | if self.tobyte: 96 | result = result.clamp(0, 255).byte() 97 | return result 98 | -------------------------------------------------------------------------------- /utils/rfutil.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from models.networks import networks 4 | import numpy as np 5 | import os 6 | from collections import OrderedDict 7 | from tqdm import tqdm 8 | 9 | def rf(output_size, ksize, stride): 10 | return (output_size -1) * stride + ksize 11 | 12 | def find_downsamples(model): 13 | downsamples = [] 14 | def find_downsamples_rec(model, layers): 15 | downsamples = (nn.Conv2d, nn.MaxPool2d, nn.AvgPool2d) 16 | for block in model.children(): 17 | if isinstance(block, downsamples): 18 | layers.append(block) 19 | layers = find_downsamples_rec(block, layers) 20 | return layers 21 | find_downsamples_rec(model, downsamples) 22 | return downsamples 23 | 24 | def find_downsamples_resnet(model): 25 | downsamples = ['conv', 'maxpool'] 26 | # skips the convs in the downsample block 27 | # which downsamples the input to add to residual features 28 | # it has smaller rf 29 | return [(n, l) for n, l in model.named_modules() if 30 | any([d in n for d in downsamples])] 31 | 32 | def find_downsamples_xceptionnet(model): 33 | downsamples = ['conv', 'maxpool'] 34 | ds_list = [] 35 | for n, l in model.named_modules(): 36 | if any([n.startswith(d) for d in downsamples]): 37 | ds_list.append((n, l)) 38 | elif n.endswith('rep'): 39 | print("adding: %s" % n) 40 | ds = find_downsamples(l) 41 | ds_list.extend([('%s_%s' % (n, i), d) for i, d in enumerate(ds)]) 42 | else: 43 | # skip duplicates counted in .rep modules 44 | # and the .skip modules 45 | print("skipping: %s" % n) 46 | return ds_list 47 | 48 | def find_rf_model(which_model_netD): 49 | model = networks.define_patch_D(which_model_netD, init_type=None) 50 | if 'resnet' in which_model_netD: 51 | ds = find_downsamples_resnet(model)[::-1] 52 | elif 'xception' in which_model_netD: 53 | ds = find_downsamples_xceptionnet(model)[::-1] 54 | else: 55 | raise NotImplementedError 56 | out = 1 57 | for name, layer in ds: 58 | k = (layer.kernel_size if isinstance(layer.kernel_size, int) else 59 | layer.kernel_size[0]) 60 | s = (layer.stride if isinstance(layer.stride, int) else 61 | layer.stride[0]) 62 | out = rf(out, k, s) 63 | print('%s, %d' % (name, out)) 64 | return out 65 | 66 | def find_rf_patches(which_model_netD, input_size): 67 | ''' given a patch model name and size of input image, 68 | calculate the receptive field of each coordinate of the output patch 69 | returns a dictionary mapping output (h,w) coordinate 70 | to input h1:h2, w1:w2 slices 71 | ''' 72 | # modified from https://github.com/rogertrullo/Receptive-Field-in-Pytorch/blob/master/Receptive_Field.ipynb 73 | 74 | # this can be kind of slow, so see if there are saved results first 75 | import pickle 76 | rf_cache = os.path.join('utils/rf_cache', '%s_rf_patches_%d.pkl' % 77 | (which_model_netD, input_size)) 78 | if os.path.isfile(rf_cache): 79 | RF = pickle.load(open(rf_cache, 'rb')) 80 | print("Found existing RF patches") 81 | return RF 82 | 83 | model = networks.define_patch_D(which_model_netD, init_type=None) 84 | 85 | # the following 2 steps are needed for this method to work 86 | # (1) replace maxpool layers with avgpool layers 87 | if hasattr(model, 'maxpool'): # resnet and resnet patch variations 88 | layer = getattr(model, 'maxpool') 89 | model.maxpool = nn.AvgPool2d(layer.kernel_size, layer.stride, 90 | layer.padding) 91 | elif hasattr(model, 'model'): # widenet variations 92 | if isinstance(model.model[3], torch.nn.MaxPool2d): 93 | layer = model.model[3] 94 | model.model[3] = nn.AvgPool2d(layer.kernel_size, layer.stride, 95 | layer.padding) 96 | elif hasattr(model, 'block1'): # xceptionet variations 97 | for b in range(1, 13): 98 | if not hasattr(model, 'block%d' % b): 99 | break # model was truncated here 100 | layer = getattr(model, 'block%d' % b) 101 | if isinstance(layer.rep[-1], torch.nn.MaxPool2d): 102 | print('here: block%d' % b) 103 | maxpool = layer.rep[-1] 104 | layer.rep[-1] = nn.AvgPool2d(maxpool.kernel_size, 105 | maxpool.stride, 106 | maxpool.padding) 107 | # (2) put in eval mode 108 | model.eval() 109 | 110 | # makes the gradients a bit more consistent 111 | for n,m in model.named_modules(): 112 | if isinstance(m, torch.nn.Conv2d): 113 | m.weight = torch.nn.Parameter(torch.ones_like(m.weight)) 114 | 115 | 116 | img_ = torch.ones((1, 3, input_size, input_size), requires_grad=True) 117 | print("Input shape: %s" % str(img_.shape)) 118 | out_cnn=model(img_) 119 | out_shape=out_cnn.size() 120 | print("Output shape: %s" % str(out_shape)) 121 | RF = OrderedDict() 122 | for h in tqdm(range(out_shape[2])): 123 | for w in range(out_shape[3]): 124 | grad=torch.zeros(out_cnn.size()) 125 | l_tmp = [0, 0, h, w] 126 | l_tmp = [int(x) for x in l_tmp] 127 | grad[tuple(l_tmp)]=1 128 | out_cnn.backward(gradient=grad, retain_graph=True) 129 | grad_np=img_.grad[0,0].data.numpy() 130 | idx_nonzeros=np.where(grad_np!=0) 131 | RF[(h, w)] = (slice(np.min(idx_nonzeros[0]), 132 | np.max(idx_nonzeros[0])+1), 133 | slice(np.min(idx_nonzeros[1]), 134 | np.max(idx_nonzeros[1])+1)) 135 | img_.grad.data.zero_() # zero the gradient 136 | 137 | # save the result to cache 138 | os.makedirs('utils/rf_cache', exist_ok=True) 139 | pickle.dump(RF, open(rf_cache, 'wb')) 140 | 141 | return RF 142 | 143 | def get_patch_from_img(img, indices, rf, pad_value=0): 144 | ''' given an image and indices to extract a patch from, 145 | extracts a patch of size rf by padding the border 146 | if necessary 147 | ''' 148 | 149 | slice_h, slice_w = indices 150 | assert(np.ndim(img) == 3) # no batch dim 151 | assert(img.shape[0] in [1, 3]) # channels first 152 | 153 | padded = np.ones((img.shape[0], rf, rf)) * pad_value 154 | patch = img[:, slice_h, slice_w] 155 | if patch.shape == padded.shape: 156 | return patch 157 | offset_h = max(0, rf - slice_h.stop) 158 | offset_w = max(0, rf - slice_w.stop) 159 | slice_h_offset = slice(offset_h, slice_h.stop - slice_h.start + offset_h) 160 | slice_w_offset = slice(offset_w, slice_w.stop - slice_w.start + offset_w) 161 | padded[:, slice_h_offset, slice_w_offset] = patch 162 | return padded 163 | 164 | def find_rf_numerical(which_model_netD, input_size): 165 | ''' computes the receptive field numerically by finding where 166 | there are non-zero gradients in the input wrt to an output 167 | location, should give the same answer as find_rf_model if 168 | rf < input_size 169 | ''' 170 | model = networks.define_patch_D(which_model_netD, init_type=None) 171 | 172 | # the following 2 steps are needed for this method to work 173 | # (1) replace maxpool layers with avgpool layers 174 | if hasattr(model, 'maxpool'): # resnet and resnet patch variations 175 | layer = getattr(model, 'maxpool') 176 | model.maxpool = nn.AvgPool2d(layer.kernel_size, layer.stride, 177 | layer.padding) 178 | elif hasattr(model, 'model'): # widenet variations 179 | if isinstance(model.model[3], torch.nn.MaxPool2d): 180 | layer = model.model[3] 181 | model.model[3] = nn.AvgPool2d(layer.kernel_size, layer.stride, 182 | layer.padding) 183 | elif hasattr(model, 'block1'): # xceptionet variations 184 | for b in range(1, 13): 185 | if not hasattr(model, 'block%d' % b): 186 | break # model was truncated here 187 | layer = getattr(model, 'block%d' % b) 188 | if isinstance(layer.rep[-1], torch.nn.MaxPool2d): 189 | print('here: block%d' % b) 190 | maxpool = layer.rep[-1] 191 | layer.rep[-1] = nn.AvgPool2d(maxpool.kernel_size, 192 | maxpool.stride, 193 | maxpool.padding) 194 | # (2) put in eval mode 195 | model.eval() 196 | 197 | # makes the gradients a bit more consistent 198 | for n,m in model.named_modules(): 199 | if isinstance(m, torch.nn.Conv2d): 200 | m.weight = torch.nn.Parameter(torch.ones_like(m.weight)) 201 | 202 | img_ = torch.ones((1, 3, input_size, input_size), requires_grad=True) 203 | print("Input shape: %s" % str(img_.shape)) 204 | out_cnn=model(img_) 205 | out_shape=out_cnn.size() 206 | print("Output shape: %s" % str(out_shape)) 207 | assert(len(out_shape) == 4) 208 | grad=torch.zeros(out_cnn.size()) 209 | l_tmp = [0, 0, out_shape[2] // 2, out_shape[3] // 2] 210 | l_tmp = [int(x) for x in l_tmp] 211 | grad[tuple(l_tmp)]=1 212 | out_cnn.backward(gradient=grad, retain_graph=True) 213 | grad_np=img_.grad[0,0].data.numpy() 214 | idx_nonzeros=np.where(grad_np!=0) 215 | print('RF (h) = %d' % (np.max(idx_nonzeros[0])+1 - 216 | np.min(idx_nonzeros[0]))) 217 | print('RF (w) = %d' % (np.max(idx_nonzeros[1])+1 - 218 | np.min(idx_nonzeros[1]))) 219 | img_.grad.data.zero_() # zero the gradient 220 | -------------------------------------------------------------------------------- /utils/show.py: -------------------------------------------------------------------------------- 1 | # show.py 2 | # 3 | # An abbreviated way to output simple HTML layout of text and images 4 | # into a python notebook. 5 | # 6 | # - show a PIL image to show an inline HTML . 7 | # - show an array of items to vertically stack them, centered in a block. 8 | # - show an array of arrays to horizontally lay them out as inline blocks. 9 | # - show an array of tuples to create a table. 10 | 11 | import PIL, base64, io, IPython, types, sys 12 | import html as html_module 13 | from IPython.display import display 14 | 15 | g_buffer = None 16 | 17 | def blocks(obj, space=''): 18 | return IPython.display.HTML(space.join(blocks_tags(obj))) 19 | 20 | def rows(obj, space=''): 21 | return IPython.display.HTML(space.join(rows_tags(obj))) 22 | 23 | def rows_tags(obj): 24 | if isinstance(obj, dict): 25 | obj = obj.items() 26 | results = [] 27 | results.append('') 28 | for row in obj: 29 | results.append('') 30 | for item in row: 31 | results.append('') 35 | results.append('') 36 | results.append('
') 33 | results.extend(blocks_tags(item)) 34 | results.append('
') 37 | return results 38 | 39 | def blocks_tags(obj): 40 | results = [] 41 | if isinstance(obj, PIL.Image.Image): 42 | results.append(pil_to_html(obj)) 43 | elif isinstance(obj, (str, int, float)): 44 | results.append('
') 45 | results.append(html_module.escape(str(obj))) 46 | results.append('
') 47 | elif isinstance(obj, IPython.display.HTML): 48 | results.append(obj.data) 49 | elif isinstance(obj, dict): 50 | results.extend(blocks_tags([(k, v) for k, v in obj.items()])) 51 | elif hasattr(obj, '__iter__'): 52 | blockstart, blockend, tstart, tend, rstart, rend, cstart, cend = [ 53 | '
', 55 | '
', 56 | '', 57 | '
', 58 | '', 59 | '', 60 | '', 61 | '', 62 | ] 63 | needs_end = False 64 | table_mode = False 65 | for i, line in enumerate(obj): 66 | if i == 0: 67 | needs_end = True 68 | if isinstance(line, tuple): 69 | table_mode = True 70 | results.append(tstart) 71 | else: 72 | results.append(blockstart) 73 | if table_mode: 74 | results.append(rstart) 75 | if not isinstance(line, str) and hasattr(line, '__iter__'): 76 | for cell in line: 77 | results.append(cstart) 78 | results.extend(blocks_tags(cell)) 79 | results.append(cend) 80 | else: 81 | results.append(cstart) 82 | results.extend(blocks_tags(line)) 83 | results.append(cend) 84 | results.append(rend) 85 | else: 86 | results.extend(blocks_tags(line)) 87 | if needs_end: 88 | results.append(table_mode and tend or blockend) 89 | return results 90 | 91 | def pil_to_b64(img, format='png'): 92 | buffered = io.BytesIO() 93 | img.save(buffered, format=format) 94 | return base64.b64encode(buffered.getvalue()).decode('utf-8') 95 | 96 | def pil_to_url(img, format='png'): 97 | return 'data:image/%s;base64,%s' % (format, pil_to_b64(img, format)) 98 | 99 | def pil_to_html(img, margin=1): 100 | mattr = ' style="margin:%dpx"' % margin 101 | return '' % (pil_to_url(img), mattr) 102 | 103 | def a(x, cols=None): 104 | global g_buffer 105 | if g_buffer is None: 106 | g_buffer = [] 107 | g_buffer.append(x) 108 | if cols is not None and len(g_buffer) >= cols: 109 | flush() 110 | 111 | def reset(): 112 | global g_buffer 113 | g_buffer = [] 114 | 115 | def flush(*args, **kwargs): 116 | global g_buffer 117 | if g_buffer is not None: 118 | x = g_buffer 119 | g_buffer = None 120 | display(blocks(x, *args, **kwargs)) 121 | 122 | def show(x=None, *args, **kwargs): 123 | flush(*args, **kwargs) 124 | if x is not None: 125 | display(blocks(x, *args, **kwargs)) 126 | 127 | class CallableModule(types.ModuleType): 128 | def __init__(self): 129 | # or super().__init__(__name__) for Python 3 130 | types.ModuleType.__init__(self, __name__) 131 | self.__dict__.update(sys.modules[__name__].__dict__) 132 | def __call__(self, x=None, *args, **kwargs): 133 | show(x, *args, **kwargs) 134 | 135 | sys.modules[__name__] = CallableModule() 136 | -------------------------------------------------------------------------------- /utils/tensorboard_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import numpy as np 4 | import time 5 | from tensorboardX import SummaryWriter 6 | import torchvision.utils as vutils 7 | import torch 8 | 9 | class LinePlotter(object): 10 | def __init__(self, writer, tag): 11 | self.writer = writer 12 | self.tag = tag 13 | 14 | def plot(self, x, data, walltime=None): 15 | # x is a scalar 16 | # data is a scalar 17 | if not hasattr(self, 'plot_data'): 18 | self.plot_data = {'X': [], 'Y': []} 19 | self.plot_data['X'].append(x) 20 | self.plot_data['Y'].append(data) 21 | self.writer.add_scalar(self.tag, data, x, walltime) 22 | 23 | def save_final_plot(self, save_dir): 24 | # save_dir = writer.logdir 25 | save_path = os.path.join(save_dir, '{}'.format(self.tag.replace('/', '_'))) 26 | os.makedirs(save_path, exist_ok=True) 27 | if hasattr(self, 'plot_data'): 28 | save_data = dict(X=np.array(self.plot_data['X']), 29 | Y=np.array(self.plot_data['Y']) 30 | ) 31 | np.savez(save_path + '.npz', **save_data) 32 | logging.info('Saved to {}'.format(save_path)) 33 | 34 | class ImageGridPlotter(object): 35 | def __init__(self, writer, ncols, grid=False): 36 | self.ncols = ncols 37 | self.writer = writer 38 | self.grid = grid 39 | 40 | def plot(self, visuals, niter=0): 41 | ncols = self.ncols 42 | ncols = min(ncols, len(visuals)) 43 | if self.grid: 44 | images = [] 45 | labels = '|' 46 | idx = 0 47 | for label, im in visuals.items(): 48 | images.append(im[0]) 49 | labels += label + '|' 50 | idx += 1 51 | if idx % ncols == 0 and idx > 0: 52 | labels += '||' 53 | blank_image = torch.ones_like(images[0]) 54 | while idx % ncols != 0: 55 | images.append(blank_image) 56 | idx += 1 57 | labels += ' |' 58 | self.writer.add_text('Visuals Labels', labels, niter) 59 | x = vutils.make_grid(images, normalize=True, nrow=ncols) # scale_each=True) 60 | self.writer.add_image('Visuals', x, niter) 61 | else: 62 | for label, im in visuals.items(): 63 | x = vutils.make_grid([im[0]], normalize=True) 64 | self.writer.add_image(label, x, niter) 65 | -------------------------------------------------------------------------------- /utils/util.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | from collections import OrderedDict 4 | import torch 5 | import torch.nn as nn 6 | from . import pbar 7 | import numpy as np 8 | from models import networks 9 | from datetime import datetime 10 | 11 | def remove_prefix(s, prefix): 12 | if s.startswith(prefix): 13 | s = s[len(prefix):] 14 | return s 15 | 16 | def get_subset_dict(in_dict,keys): 17 | if(len(keys)): 18 | subset = OrderedDict() 19 | for key in keys: 20 | subset[key] = in_dict[key] 21 | else: 22 | subset = in_dict 23 | return subset 24 | 25 | def datestring(): 26 | return time.strftime(r"%Y-%m-%d %H:%M:%S") 27 | 28 | def format_str_one(v, float_prec=6, int_pad=1): 29 | if isinstance(v, torch.Tensor) and v.numel() == 1: 30 | v = v.item() 31 | if isinstance(v, float): 32 | return ('{:.' + str(float_prec) + 'f}').format(v) 33 | if isinstance(v, int) and int_pad: 34 | return ('{:0' + str(int_pad) + 'd}').format(v) 35 | return str(v) 36 | 37 | def format_str(*args, format_opts={}, **kwargs): 38 | ss = [format_str_one(arg, **format_opts) for arg in args] 39 | for k, v in kwargs.items(): 40 | ss.append('{}: {}'.format(k, format_str_one(v, **format_opts))) 41 | return '\t'.join(ss) 42 | 43 | def complete_device(device): 44 | if not torch.cuda.is_available(): 45 | return torch.device('cpu') 46 | if type(device) == str: 47 | device = torch.device(device) 48 | if device.type == 'cuda' and device.index is None: 49 | return torch.device(device.type, torch.cuda.current_device()) 50 | return device 51 | 52 | def check_timestamp(checkpoint_path, timestamp_path): 53 | ''' returns True if checkpoint_path timestamp is different 54 | from timestamp path or timestamp_path doesn't exist''' 55 | if not os.path.isfile(timestamp_path): 56 | print("No timestamp found") 57 | return True 58 | newtime = os.path.getmtime(checkpoint_path) 59 | newtime = datetime.fromtimestamp(newtime).strftime('%Y-%m-%d %H:%M:%S') 60 | with open(timestamp_path) as f: 61 | oldtime = f.readlines()[0].strip() 62 | if oldtime != newtime: 63 | print("Timestamp out of date") 64 | return True 65 | print("Timestamp is correct") 66 | return False 67 | 68 | def update_timestamp(checkpoint_path, timestamp_path): 69 | ''' write the last modified date of checkpoint_path to the 70 | the file timestamp_path ''' 71 | newtime = os.path.getmtime(checkpoint_path) 72 | newtime = datetime.fromtimestamp(newtime).strftime('%Y-%m-%d %H:%M:%S') 73 | with open(timestamp_path, 'w') as f: 74 | f.write('%s' % newtime) 75 | 76 | class AverageMeter(object): 77 | """Computes and stores the average and current value""" 78 | def __init__(self): 79 | self.reset() 80 | 81 | def reset(self): 82 | self.val = 0 83 | self.avg = 0 84 | self.sum = 0 85 | self.count = 0 86 | 87 | def update(self, val, n=1): 88 | self.val = val 89 | self.sum += val * n 90 | self.count += n 91 | self.avg = self.sum / self.count 92 | -------------------------------------------------------------------------------- /utils/visualizer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | import logging 4 | from tensorboardX import SummaryWriter 5 | 6 | class Visualizer(): 7 | def __init__(self, opt, loss_names, visual_names=None): 8 | from . import tensorboard_utils as tb_utils 9 | self.name = opt.name 10 | self.opt = opt 11 | self.visual_names = visual_names 12 | # check that tensorboard history does not exist 13 | tb_path = os.path.join('runs', self.name) 14 | if os.path.isdir(tb_path): 15 | logging.info('Found existing tensorboard history at %s' % tb_path) 16 | if not opt.overwrite_config: 17 | logging.info('Use --overwrite_config to write to existing tensorboard history') 18 | exit(0) 19 | self.writer = SummaryWriter(logdir=tb_path) 20 | self.plotters = [] 21 | for name in loss_names: 22 | setattr(self, name + '_plotter', tb_utils.LinePlotter( 23 | self.writer, name.replace('_', '/', 1))) 24 | self.plotters.append(getattr(self, name + '_plotter')) 25 | self.imgrid = tb_utils.ImageGridPlotter( 26 | self.writer, ncols=5, grid=True) 27 | 28 | # |visuals|: dictionary of image tensors to display 29 | def display_current_results(self, visuals, epoch): 30 | # show images in the browser 31 | self.imgrid.plot(visuals, epoch) 32 | 33 | # losses: dictionary of error labels and values 34 | def plot_current_losses(self, niter, losses): 35 | for k, v in losses.items(): 36 | plotter = getattr(self, k + '_plotter') 37 | plotter.plot(niter, v) 38 | 39 | # losses: same format as |losses| of plot_current_losses 40 | def print_current_losses(self, epoch, iters, total_steps, losses, t, t_data, prefix=''): 41 | message = '(epoch: %d, iters: %.3f, time: %.3f, data: %.3f) ' % (epoch, iters, t, t_data) 42 | message += prefix 43 | message += ' ' 44 | for k, v in losses.items(): 45 | message += '%s: %.3f, ' % (k, v) 46 | 47 | logging.info('%s' % message) 48 | logging.info('Total batches: %0.2f k\n' % (total_steps / 1000)) 49 | 50 | def save_final_plots(self): 51 | save_dir = os.path.join(self.opt.checkpoints_dir, self.opt.name, 'visualize') 52 | for plotter in self.plotters: 53 | plotter.save_final_plot(save_dir) 54 | --------------------------------------------------------------------------------