├── Leaveout ├── run.sh ├── __pycache__ │ ├── model.cpython-36.pyc │ └── reader.cpython-36.pyc ├── reader │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── reader_rt.cpython-36.pyc │ │ ├── reader_diap.cpython-36.pyc │ │ ├── reader_mpii.cpython-36.pyc │ │ └── reader_gaze360.cpython-36.pyc │ ├── reader_mpii.py │ ├── reader_diap.py │ └── reader_rt.py ├── config │ ├── config_rt.yaml │ ├── config_mpii.yaml │ └── config_diap.yaml ├── test.py ├── train.py └── model.py ├── benchmarkA.png ├── benchmarkB.png ├── Traintest ├── __pycache__ │ ├── model.cpython-36.pyc │ └── reader.cpython-36.pyc ├── reader │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── reader_diap.cpython-36.pyc │ │ ├── reader_mpii.cpython-36.pyc │ │ └── reader_gaze360.cpython-36.pyc │ ├── reader_gaze360.py │ └── reader_diap.py ├── config │ └── config_gaze360.yaml ├── train.py ├── test.py ├── valid.py └── model.py └── README.md /Leaveout/run.sh: -------------------------------------------------------------------------------- 1 | for((i=0;i<3;i++)) 2 | do 3 | python $1 $2 $i 4 | done 5 | 6 | -------------------------------------------------------------------------------- /benchmarkA.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/benchmarkA.png -------------------------------------------------------------------------------- /benchmarkB.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/benchmarkB.png -------------------------------------------------------------------------------- /Leaveout/__pycache__/model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Leaveout/__pycache__/model.cpython-36.pyc -------------------------------------------------------------------------------- /Leaveout/__pycache__/reader.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Leaveout/__pycache__/reader.cpython-36.pyc -------------------------------------------------------------------------------- /Traintest/__pycache__/model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Traintest/__pycache__/model.cpython-36.pyc -------------------------------------------------------------------------------- /Traintest/__pycache__/reader.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Traintest/__pycache__/reader.cpython-36.pyc -------------------------------------------------------------------------------- /Leaveout/reader/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Leaveout/reader/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /Leaveout/reader/__pycache__/reader_rt.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Leaveout/reader/__pycache__/reader_rt.cpython-36.pyc -------------------------------------------------------------------------------- /Traintest/reader/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Traintest/reader/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /Leaveout/reader/__pycache__/reader_diap.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Leaveout/reader/__pycache__/reader_diap.cpython-36.pyc -------------------------------------------------------------------------------- /Leaveout/reader/__pycache__/reader_mpii.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Leaveout/reader/__pycache__/reader_mpii.cpython-36.pyc -------------------------------------------------------------------------------- /Traintest/reader/__pycache__/reader_diap.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Traintest/reader/__pycache__/reader_diap.cpython-36.pyc -------------------------------------------------------------------------------- /Traintest/reader/__pycache__/reader_mpii.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Traintest/reader/__pycache__/reader_mpii.cpython-36.pyc -------------------------------------------------------------------------------- /Leaveout/reader/__pycache__/reader_gaze360.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Leaveout/reader/__pycache__/reader_gaze360.cpython-36.pyc -------------------------------------------------------------------------------- /Traintest/reader/__pycache__/reader_gaze360.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yihuacheng/Dilated-Net/HEAD/Traintest/reader/__pycache__/reader_gaze360.cpython-36.pyc -------------------------------------------------------------------------------- /Leaveout/config/config_rt.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | train: 3 | params: 4 | batch_size: 64 5 | epoch: 100 6 | lr: 0.001 7 | decay: 0.1 8 | decay_step: 8000 9 | loss: MSELoss 10 | save: 11 | save_path: "/home/cyh/GazeBenchmark/exp/Implementation/Dilated-Net-rt" 12 | model_name: dilated 13 | step: 10 14 | data: 15 | image: "/home/cyh/GazeDataset20200519/Original/Rt-Gene" 16 | label: "/home/cyh/GazeDataset20200519/FaceBased/RT-Gene/Label/train" 17 | pretrains: "None" 18 | test: 19 | load: 20 | begin_step: 10 21 | end_step: 100 22 | steps: 10 23 | load_path: "/home/cyh/GazeBenchmark/exp/Implementation/Dilated-Net-rt" 24 | model_name: dilated 25 | data: 26 | image: "/home/cyh/GazeDataset20200519/Original/Rt-Gene" 27 | label: "/home/cyh/GazeDataset20200519/FaceBased/RT-Gene/Label-glasses/train" 28 | reader: reader_rt 29 | -------------------------------------------------------------------------------- /Leaveout/config/config_mpii.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | train: 3 | params: 4 | batch_size: 64 5 | epoch: 100 6 | lr: 0.001 7 | decay: 0.1 8 | decay_step: 8000 9 | loss: MSELoss 10 | save: 11 | save_path: "/home/cyh/GazeBenchmark/exp/Implementation/Dilated-Net" 12 | model_name: DilatedNet 13 | step: 20 14 | data: 15 | image: "/home/cyh/GazeDataset20200519/FaceBased/MPIIFaceGaze/Image" 16 | label: "/home/cyh/GazeDataset20200519/FaceBased/MPIIFaceGaze/Label" 17 | pretrains: "None" 18 | test: 19 | load: 20 | begin_step: 20 21 | end_step: 20 22 | steps: 20 23 | load_path: "/home/cyh/GazeBenchmark/exp/Implementation/Dilated-Net" 24 | model_name: DilatedNet 25 | data: 26 | image: "/home/cyh/GazeDataset20200519/FaceBased/MPIIFaceGaze/Image" 27 | label: "/home/cyh/GazeDataset20200519/FaceBased/MPIIFaceGaze/Label" 28 | reader: reader_mpii 29 | -------------------------------------------------------------------------------- /Leaveout/config/config_diap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | train: 3 | params: 4 | batch_size: 64 5 | epoch: 100 6 | lr: 0.001 7 | decay: 0.1 8 | decay_step: 8000 9 | loss: MSELoss 10 | save: 11 | save_path: "/home/cyh/GazeBenchmark/exp/Implementation/Dilated-Net-diapnoscale" 12 | model_name: DilatedNet 13 | step: 20 14 | data: 15 | image: "/home/cyh/GazeDataset20200519/FaceBased/EyeDiap/Image" 16 | label: "/home/cyh/GazeDataset20200519/FaceBased/EyeDiap/ClusterLabel" 17 | pretrains: "None" 18 | test: 19 | load: 20 | begin_step: 20 21 | end_step: 100 22 | steps: 20 23 | load_path: "/home/cyh/GazeBenchmark/exp/Implementation/Dilated-Net-diapnoscale" 24 | model_name: DilatedNet 25 | data: 26 | image: "/home/cyh/GazeDataset20200519/FaceBased/EyeDiap/Image" 27 | label: "/home/cyh/GazeDataset20200519/FaceBased/EyeDiap/ClusterLabel" 28 | reader: reader_diap 29 | -------------------------------------------------------------------------------- /Traintest/config/config_gaze360.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | train: 3 | params: 4 | batch_size: 64 5 | epoch: 100 6 | lr: 0.001 7 | decay: 0.1 8 | decay_step: 8000 9 | loss: MSELoss 10 | save: 11 | save_path: "/home/cyh/GazeBenchmark/exp/Implementation/Dilated-Net-gaze360" 12 | model_name: DilatedNet 13 | step: 20 14 | data: 15 | image: "/home/cyh/GazeDataset20200519/FaceBased/Gaze360/Image" 16 | label: "/home/cyh/GazeDataset20200519/FaceBased/Gaze360/Label/train.label" 17 | pretrains: "None" 18 | test: 19 | load: 20 | begin_step: 20 21 | end_step: 100 22 | steps: 20 23 | load_path: "/home/cyh/GazeBenchmark/exp/Implementation/Dilated-Net-gaze360" 24 | model_name: DilatedNet 25 | data: 26 | image: "/home/cyh/GazeDataset20200519/FaceBased/Gaze360/Image" 27 | label: "/home/cyh/GazeDataset20200519/FaceBased/Gaze360/Label/val.label" 28 | 29 | reader: reader_gaze360 30 | -------------------------------------------------------------------------------- /Traintest/reader/reader_gaze360.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import os 4 | from torch.utils.data import Dataset, DataLoader 5 | import torch 6 | 7 | def gazeto2d(gaze): 8 | yaw = np.arctan2(-gaze[0], -gaze[2]) 9 | pitch = np.arcsin(-gaze[1]) 10 | return np.array([yaw, pitch]) 11 | 12 | class loader(Dataset): 13 | def __init__(self, path, root, header=True): 14 | self.lines = [] 15 | if isinstance(path, list): 16 | for i in path: 17 | with open(i) as f: 18 | line = f.readlines() 19 | if header: line.pop(0) 20 | self.lines.extend(line) 21 | else: 22 | with open(path) as f: 23 | self.lines = f.readlines() 24 | if header: self.lines.pop(0) 25 | 26 | self.root = root 27 | 28 | def __len__(self): 29 | return len(self.lines) 30 | 31 | def __getitem__(self, idx): 32 | line = self.lines[idx] 33 | line = line.strip().split(" ") 34 | 35 | face = line[0] 36 | lefteye = line[1] 37 | righteye = line[2] 38 | name = line[3] 39 | gaze2d = line[5] 40 | 41 | label = np.array(gaze2d.split(",")).astype("float") 42 | label = torch.from_numpy(label).type(torch.FloatTensor) 43 | 44 | rimg = cv2.imread(os.path.join(self.root, righteye)) 45 | rimg = cv2.resize(rimg, (96, 64))/255 46 | rimg = rimg.transpose(2, 0, 1) 47 | 48 | limg = cv2.imread(os.path.join(self.root, lefteye)) 49 | limg = cv2.resize(limg, (96, 64))/255 50 | limg = limg.transpose(2, 0, 1) 51 | 52 | fimg = cv2.imread(os.path.join(self.root, face)) 53 | fimg = cv2.resize(fimg, (96, 96))/255 54 | fimg = fimg.transpose(2, 0, 1) 55 | 56 | img = {"left":torch.from_numpy(limg).type(torch.FloatTensor), 57 | "right":torch.from_numpy(rimg).type(torch.FloatTensor), 58 | "face":torch.from_numpy(fimg).type(torch.FloatTensor), 59 | "name":name} 60 | 61 | return img, label 62 | 63 | def txtload(labelpath, imagepath, batch_size, shuffle=True, num_workers=0, header=True): 64 | dataset = loader(labelpath, imagepath, header) 65 | print(f"[Read Data]: Total num: {len(dataset)}") 66 | print(f"[Read Data]: Label path: {labelpath}") 67 | load = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) 68 | return load 69 | 70 | 71 | if __name__ == "__main__": 72 | path = './p00.label' 73 | d = loader(path) 74 | print(len(d)) 75 | (data, label) = d.__getitem__(0) 76 | 77 | -------------------------------------------------------------------------------- /Leaveout/reader/reader_mpii.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import os 4 | from torch.utils.data import Dataset, DataLoader 5 | import torch 6 | 7 | def gazeto2d(gaze): 8 | yaw = np.arctan2(-gaze[0], -gaze[2]) 9 | pitch = np.arcsin(-gaze[1]) 10 | return np.array([yaw, pitch]) 11 | 12 | class loader(Dataset): 13 | def __init__(self, path, root, header=True): 14 | self.lines = [] 15 | if isinstance(path, list): 16 | for i in path: 17 | with open(i) as f: 18 | line = f.readlines() 19 | if header: line.pop(0) 20 | self.lines.extend(line) 21 | else: 22 | with open(path) as f: 23 | self.lines = f.readlines() 24 | if header: self.lines.pop(0) 25 | 26 | self.root = root 27 | 28 | def __len__(self): 29 | return len(self.lines) 30 | 31 | def __getitem__(self, idx): 32 | line = self.lines[idx] 33 | line = line.strip().split(" ") 34 | 35 | name = line[3] 36 | gaze2d = line[7] 37 | head2d = line[8] 38 | face = line[0] 39 | lefteye = line[1] 40 | righteye = line[2] 41 | 42 | label = np.array(gaze2d.split(",")).astype("float") 43 | label = torch.from_numpy(label).type(torch.FloatTensor) 44 | 45 | rimg = cv2.imread(os.path.join(self.root, righteye)) 46 | rimg = cv2.resize(rimg, (96, 64))/255 47 | rimg = rimg.transpose(2, 0, 1) 48 | 49 | limg = cv2.imread(os.path.join(self.root, lefteye)) 50 | limg = cv2.resize(limg, (96, 64))/255 51 | limg = limg.transpose(2, 0, 1) 52 | 53 | fimg = cv2.imread(os.path.join(self.root, face)) 54 | fimg = cv2.resize(fimg, (96, 96))/255 55 | fimg = fimg.transpose(2, 0, 1) 56 | 57 | img = {"left":torch.from_numpy(limg).type(torch.FloatTensor), 58 | "right":torch.from_numpy(rimg).type(torch.FloatTensor), 59 | "face":torch.from_numpy(fimg).type(torch.FloatTensor), 60 | "name":name} 61 | 62 | return img, label 63 | 64 | def txtload(labelpath, imagepath, batch_size, shuffle=True, num_workers=0, header=True): 65 | dataset = loader(labelpath, imagepath, header) 66 | print(f"[Read Data]: Total num: {len(dataset)}") 67 | print(f"[Read Data]: Label path: {labelpath}") 68 | load = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) 69 | return load 70 | 71 | 72 | if __name__ == "__main__": 73 | path = './p00.label' 74 | d = loader(path) 75 | print(len(d)) 76 | (data, label) = d.__getitem__(0) 77 | 78 | -------------------------------------------------------------------------------- /Leaveout/reader/reader_diap.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import os 4 | from torch.utils.data import Dataset, DataLoader 5 | import torch 6 | 7 | def gazeto2d(gaze): 8 | yaw = np.arctan2(-gaze[0], -gaze[2]) 9 | pitch = np.arcsin(-gaze[1]) 10 | return np.array([yaw, pitch]) 11 | 12 | class loader(Dataset): 13 | def __init__(self, path, root, header=True): 14 | self.lines = [] 15 | if isinstance(path, list): 16 | for i in path: 17 | with open(i) as f: 18 | line = f.readlines() 19 | if header: line.pop(0) 20 | self.lines.extend(line) 21 | else: 22 | with open(path) as f: 23 | self.lines = f.readlines() 24 | if header: self.lines.pop(0) 25 | 26 | self.root = root 27 | 28 | def __len__(self): 29 | return len(self.lines) 30 | 31 | def __getitem__(self, idx): 32 | line = self.lines[idx] 33 | line = line.strip().split(" ") 34 | 35 | gaze2d = line[6] 36 | head2d = line[7] 37 | 38 | face = line[0] 39 | lefteye = line[1] 40 | righteye = line[2] 41 | name = line[3] 42 | 43 | label = np.array(gaze2d.split(",")).astype("float") 44 | label = torch.from_numpy(label).type(torch.FloatTensor) 45 | 46 | rimg = cv2.imread(os.path.join(self.root, righteye)) 47 | rimg = cv2.resize(rimg, (96, 64))/255 48 | rimg = rimg.transpose(2, 0, 1) 49 | 50 | limg = cv2.imread(os.path.join(self.root, lefteye)) 51 | limg = cv2.resize(limg, (96, 64))/255 52 | limg = limg.transpose(2, 0, 1) 53 | 54 | fimg = cv2.imread(os.path.join(self.root, face)) 55 | fimg = cv2.resize(fimg, (96, 96))/255 56 | fimg = fimg.transpose(2, 0, 1) 57 | 58 | img = {"left":torch.from_numpy(limg).type(torch.FloatTensor), 59 | "right":torch.from_numpy(rimg).type(torch.FloatTensor), 60 | "face":torch.from_numpy(fimg).type(torch.FloatTensor), 61 | "name":name} 62 | 63 | return img, label 64 | 65 | def txtload(labelpath, imagepath, batch_size, shuffle=True, num_workers=0, header=True): 66 | dataset = loader(labelpath, imagepath, header) 67 | print(f"[Read Data]: Total num: {len(dataset)}") 68 | print(f"[Read Data]: Label path: {labelpath}") 69 | load = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) 70 | return load 71 | 72 | 73 | if __name__ == "__main__": 74 | path = './p00.label' 75 | d = loader(path) 76 | print(len(d)) 77 | (data, label) = d.__getitem__(0) 78 | 79 | -------------------------------------------------------------------------------- /Traintest/reader/reader_diap.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import os 4 | from torch.utils.data import Dataset, DataLoader 5 | import torch 6 | 7 | def gazeto2d(gaze): 8 | yaw = np.arctan2(-gaze[0], -gaze[2]) 9 | pitch = np.arcsin(-gaze[1]) 10 | return np.array([yaw, pitch]) 11 | 12 | class loader(Dataset): 13 | def __init__(self, path, root, header=True): 14 | self.lines = [] 15 | if isinstance(path, list): 16 | for i in path: 17 | with open(i) as f: 18 | line = f.readlines() 19 | if header: line.pop(0) 20 | self.lines.extend(line) 21 | else: 22 | with open(path) as f: 23 | self.lines = f.readlines() 24 | if header: self.lines.pop(0) 25 | 26 | self.root = root 27 | 28 | def __len__(self): 29 | return len(self.lines) 30 | 31 | def __getitem__(self, idx): 32 | line = self.lines[idx] 33 | line = line.strip().split(" ") 34 | 35 | gaze2d = line[6] 36 | head2d = line[7] 37 | 38 | face = line[0] 39 | lefteye = line[1] 40 | righteye = line[2] 41 | name = line[3] 42 | 43 | label = np.array(gaze2d.split(",")).astype("float") 44 | label = torch.from_numpy(label).type(torch.FloatTensor) 45 | 46 | rimg = cv2.imread(os.path.join(self.root, righteye)) 47 | rimg = cv2.resize(rimg, (96, 64))/255 48 | rimg = rimg.transpose(2, 0, 1) 49 | 50 | limg = cv2.imread(os.path.join(self.root, lefteye)) 51 | limg = cv2.resize(limg, (96, 64))/255 52 | limg = limg.transpose(2, 0, 1) 53 | 54 | fimg = cv2.imread(os.path.join(self.root, face)) 55 | fimg = cv2.resize(fimg, (96, 96))/255 56 | fimg = fimg.transpose(2, 0, 1) 57 | 58 | img = {"left":torch.from_numpy(limg).type(torch.FloatTensor), 59 | "right":torch.from_numpy(rimg).type(torch.FloatTensor), 60 | "face":torch.from_numpy(fimg).type(torch.FloatTensor), 61 | "name":name} 62 | 63 | return img, label 64 | 65 | def txtload(labelpath, imagepath, batch_size, shuffle=True, num_workers=0, header=True): 66 | dataset = loader(labelpath, imagepath, header) 67 | print(f"[Read Data]: Total num: {len(dataset)}") 68 | print(f"[Read Data]: Label path: {labelpath}") 69 | load = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) 70 | return load 71 | 72 | 73 | if __name__ == "__main__": 74 | path = './p00.label' 75 | d = loader(path) 76 | print(len(d)) 77 | (data, label) = d.__getitem__(0) 78 | 79 | -------------------------------------------------------------------------------- /Leaveout/reader/reader_rt.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import os 4 | from torch.utils.data import Dataset, DataLoader 5 | import torch 6 | from torchvision import transforms 7 | 8 | def gazeto2d(gaze): 9 | yaw = np.arctan2(-gaze[0], -gaze[2]) 10 | pitch = np.arcsin(-gaze[1]) 11 | return np.array([yaw, pitch]) 12 | 13 | class loader(Dataset): 14 | def __init__(self, path, root, header=True): 15 | self.lines = [] 16 | if isinstance(path, list): 17 | for i in path: 18 | with open(i) as f: 19 | line = f.readlines() 20 | if header: line.pop(0) 21 | self.lines.extend(line) 22 | else: 23 | with open(path) as f: 24 | self.lines = f.readlines() 25 | if header: self.lines.pop(0) 26 | 27 | self.root = root 28 | self.transform = transforms.ToTensor() 29 | 30 | def __len__(self): 31 | return len(self.lines) 32 | 33 | def __getitem__(self, idx): 34 | line = self.lines[idx] 35 | line = line.strip().split(" ") 36 | 37 | name = line[0] 38 | gaze2d = line[6] 39 | head2d = line[7] 40 | face = line[0] 41 | lefteye = line[1] 42 | righteye = line[2] 43 | 44 | label = np.array(gaze2d.split(",")).astype("float") 45 | label = torch.from_numpy(label).type(torch.FloatTensor) 46 | 47 | headpose = np.array(head2d.split(",")).astype("float") 48 | headpose = torch.from_numpy(headpose).type(torch.FloatTensor) 49 | 50 | fimg = cv2.imread(os.path.join(self.root, face)) 51 | fimg = cv2.resize(fimg, (96, 96)) 52 | fimg = self.transform(fimg) 53 | 54 | 55 | rimg = cv2.imread(os.path.join(self.root, righteye)) 56 | rimg = cv2.resize(rimg, (96, 64)) 57 | rimg = self.transform(rimg) 58 | 59 | limg = cv2.imread(os.path.join(self.root, lefteye)) 60 | limg = cv2.resize(limg, (96, 64)) 61 | limg = self.transform(limg) 62 | 63 | 64 | img = {"left": limg, 65 | "right": rimg, 66 | "face": fimg, 67 | "head_pose":headpose, 68 | "name":name} 69 | 70 | return img, label 71 | 72 | def txtload(labelpath, imagepath, batch_size, shuffle=True, num_workers=0, header=True): 73 | dataset = loader(labelpath, imagepath, header) 74 | print(f"[Read Data]: Total num: {len(dataset)}") 75 | print(f"[Read Data]: Label path: {labelpath}") 76 | load = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) 77 | return load 78 | 79 | 80 | if __name__ == "__main__": 81 | path = './p00.label' 82 | d = loader(path) 83 | print(len(d)) 84 | (data, label) = d.__getitem__(0) 85 | 86 | -------------------------------------------------------------------------------- /Traintest/train.py: -------------------------------------------------------------------------------- 1 | import model 2 | import numpy as np 3 | import importlib 4 | import torch 5 | import torch.nn as nn 6 | import torch.optim as optim 7 | import time 8 | import sys 9 | import os 10 | import copy 11 | import yaml 12 | 13 | if __name__ == "__main__": 14 | config = yaml.load(open(f"{sys.argv[1]}"), Loader=yaml.FullLoader) 15 | readername = config["reader"] 16 | dataloader = importlib.import_module("reader." + readername) 17 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 18 | 19 | config = config["train"] 20 | imagepath = config["data"]["image"] 21 | labelpath = config["data"]["label"] 22 | modelname = config["save"]["model_name"] 23 | 24 | 25 | # i represents the i-th folder used as the test set. 26 | savepath = os.path.join(config["save"]["save_path"], f"checkpoint/") 27 | if not os.path.exists(savepath): 28 | os.makedirs(savepath) 29 | 30 | print("Read data") 31 | dataset = dataloader.txtload(labelpath, imagepath, config["params"]["batch_size"], shuffle=True, num_workers=4, header=True) 32 | 33 | print("Model building") 34 | net = model.DilatedNet() 35 | net.train() 36 | net.to(device) 37 | 38 | print("optimizer building") 39 | lossfunc = config["params"]["loss"] 40 | loss_op = getattr(nn, lossfunc)().cuda() 41 | base_lr = config["params"]["lr"] 42 | 43 | decaysteps = config["params"]["decay_step"] 44 | decayratio = config["params"]["decay"] 45 | 46 | optimizer = optim.Adam(net.parameters(),lr=base_lr, betas=(0.9,0.95)) 47 | scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=decaysteps, gamma=decayratio) 48 | 49 | print("Traning") 50 | length = len(dataset) 51 | total = length * config["params"]["epoch"] 52 | cur = 0 53 | timebegin = time.time() 54 | with open(os.path.join(savepath, "train_log"), 'w') as outfile: 55 | for epoch in range(1, config["params"]["epoch"]+1): 56 | for i, (data, label) in enumerate(dataset): 57 | 58 | # Acquire data 59 | data["left"] = data["left"].to(device) 60 | data['right'] = data['right'].to(device) 61 | data['face'] = data['face'].to(device) 62 | label = label.to(device) 63 | 64 | # forward 65 | gaze = net(data) 66 | 67 | # loss calculation 68 | loss = loss_op(gaze, label) 69 | optimizer.zero_grad() 70 | 71 | # backward 72 | loss.backward() 73 | optimizer.step() 74 | scheduler.step() 75 | cur += 1 76 | 77 | # print logs 78 | if i % 20 == 0: 79 | timeend = time.time() 80 | resttime = (timeend - timebegin)/cur * (total-cur)/3600 81 | log = f"[{epoch}/{config['params']['epoch']}]: [{i}/{length}] loss:{loss} lr:{base_lr}, rest time:{resttime:.2f}h" 82 | print(log) 83 | outfile.write(log + "\n") 84 | sys.stdout.flush() 85 | outfile.flush() 86 | 87 | if epoch % config["save"]["step"] == 0: 88 | torch.save(net.state_dict(), os.path.join(savepath, f"Iter_{epoch}_{modelname}.pt")) 89 | 90 | -------------------------------------------------------------------------------- /Traintest/test.py: -------------------------------------------------------------------------------- 1 | import model 2 | import importlib 3 | import numpy as np 4 | import cv2 5 | import torch 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | import sys 9 | import yaml 10 | import os 11 | import copy 12 | 13 | def gazeto3d(gaze): 14 | gaze_gt = np.zeros([3]) 15 | gaze_gt[0] = -np.cos(gaze[1]) * np.sin(gaze[0]) 16 | gaze_gt[1] = -np.sin(gaze[1]) 17 | gaze_gt[2] = -np.cos(gaze[1]) * np.cos(gaze[0]) 18 | return gaze_gt 19 | 20 | def angular(gaze, label): 21 | total = np.sum(gaze * label) 22 | return np.arccos(min(total/(np.linalg.norm(gaze)* np.linalg.norm(label)), 0.9999999))*180/np.pi 23 | 24 | if __name__ == "__main__": 25 | config = yaml.load(open(sys.argv[1]), Loader = yaml.FullLoader) 26 | readername = config["reader"] 27 | dataloader = importlib.import_module("reader." + readername) 28 | 29 | config = config["test"] 30 | imagepath = config["data"]["image"] 31 | labelpath = config["data"]["label"] 32 | modelname = config["load"]["model_name"] 33 | 34 | loadpath = os.path.join(config["load"]["load_path"]) 35 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 36 | 37 | 38 | savepath = os.path.join(loadpath, f"checkpoint") 39 | 40 | if not os.path.exists(os.path.join(loadpath, f"evaluation")): 41 | os.makedirs(os.path.join(loadpath, f"evaluation")) 42 | 43 | print("Read data") 44 | dataset = dataloader.txtload(labelpath, imagepath, 32, num_workers=4, header=True) 45 | 46 | begin = config["load"]["begin_step"] 47 | end = config["load"]["end_step"] 48 | step = config["load"]["steps"] 49 | 50 | for saveiter in range(begin, end+step, step): 51 | print("Model building") 52 | net = model.DilatedNet() 53 | statedict = torch.load(os.path.join(savepath, f"Iter_{saveiter}_{modelname}.pt")) 54 | 55 | net.to(device) 56 | net.load_state_dict(statedict) 57 | net.eval() 58 | 59 | print(f"Test {saveiter}") 60 | length = len(dataset) 61 | accs = 0 62 | count = 0 63 | with torch.no_grad(): 64 | with open(os.path.join(loadpath, f"evaluation/{saveiter}.log"), 'w') as outfile: 65 | outfile.write("name results gts\n") 66 | for j, (data, label) in enumerate(dataset): 67 | 68 | fimg = data["face"].to(device) 69 | limg = data["left"].to(device) 70 | rimg = data["right"].to(device) 71 | names = data["name"] 72 | 73 | img = {"left":limg, "right":rimg, "face":fimg} 74 | gts = label.to(device) 75 | 76 | gazes = net(img) 77 | for k, gaze in enumerate(gazes): 78 | gaze = gaze.cpu().detach().numpy() 79 | count += 1 80 | accs += angular(gazeto3d(gaze), gazeto3d(gts.cpu().numpy()[k])) 81 | 82 | name = [names[k]] 83 | gaze = [str(u) for u in gaze] 84 | gt = [str(u) for u in gts.cpu().numpy()[k]] 85 | log = name + [",".join(gaze)] + [",".join(gt)] 86 | outfile.write(" ".join(log) + "\n") 87 | 88 | loger = f"[{saveiter}] Total Num: {count}, avg: {accs/count}" 89 | outfile.write(loger) 90 | print(loger) 91 | 92 | -------------------------------------------------------------------------------- /Traintest/valid.py: -------------------------------------------------------------------------------- 1 | import model 2 | import importlib 3 | import numpy as np 4 | import cv2 5 | import torch 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | import sys 9 | import yaml 10 | import os 11 | import copy 12 | 13 | def gazeto3d(gaze): 14 | gaze_gt = np.zeros([3]) 15 | gaze_gt[0] = -np.cos(gaze[1]) * np.sin(gaze[0]) 16 | gaze_gt[1] = -np.sin(gaze[1]) 17 | gaze_gt[2] = -np.cos(gaze[1]) * np.cos(gaze[0]) 18 | return gaze_gt 19 | 20 | def angular(gaze, label): 21 | total = np.sum(gaze * label) 22 | return np.arccos(min(total/(np.linalg.norm(gaze)* np.linalg.norm(label)), 0.9999999))*180/np.pi 23 | 24 | if __name__ == "__main__": 25 | config = yaml.load(open(sys.argv[1]), Loader = yaml.FullLoader) 26 | readername = config["reader"] 27 | dataloader = importlib.import_module("reader." + readername) 28 | 29 | config = config["test"] 30 | imagepath = config["data"]["image"] 31 | labelpath = config["data"]["label"] 32 | modelname = config["load"]["model_name"] 33 | 34 | loadpath = os.path.join(config["load"]["load_path"]) 35 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 36 | 37 | 38 | savepath = os.path.join(loadpath, f"checkpoint") 39 | 40 | if not os.path.exists(os.path.join(loadpath, f"validation")): 41 | os.makedirs(os.path.join(loadpath, f"validation")) 42 | 43 | print("Read data") 44 | dataset = dataloader.txtload(labelpath, imagepath, 32, num_workers=4, header=True) 45 | 46 | begin = config["load"]["begin_step"] 47 | end = config["load"]["end_step"] 48 | step = config["load"]["steps"] 49 | 50 | for saveiter in range(begin, end+step, step): 51 | print("Model building") 52 | net = model.DilatedNet() 53 | statedict = torch.load(os.path.join(savepath, f"Iter_{saveiter}_{modelname}.pt")) 54 | 55 | net.to(device) 56 | net.load_state_dict(statedict) 57 | net.eval() 58 | 59 | print(f"Test {saveiter}") 60 | length = len(dataset) 61 | accs = 0 62 | count = 0 63 | with torch.no_grad(): 64 | with open(os.path.join(loadpath, f"validation/{saveiter}.log"), 'w') as outfile: 65 | outfile.write("name results gts\n") 66 | for j, (data, label) in enumerate(dataset): 67 | 68 | fimg = data["face"].to(device) 69 | limg = data["left"].to(device) 70 | rimg = data["right"].to(device) 71 | names = data["name"] 72 | 73 | img = {"left":limg, "right":rimg, "face":fimg} 74 | gts = label.to(device) 75 | 76 | gazes = net(img) 77 | for k, gaze in enumerate(gazes): 78 | gaze = gaze.cpu().detach().numpy() 79 | count += 1 80 | accs += angular(gazeto3d(gaze), gazeto3d(gts.cpu().numpy()[k])) 81 | 82 | name = [names[k]] 83 | gaze = [str(u) for u in gaze] 84 | gt = [str(u) for u in gts.cpu().numpy()[k]] 85 | log = name + [",".join(gaze)] + [",".join(gt)] 86 | outfile.write(" ".join(log) + "\n") 87 | 88 | loger = f"[{saveiter}] Total Num: {count}, avg: {accs/count}" 89 | outfile.write(loger) 90 | print(loger) 91 | 92 | -------------------------------------------------------------------------------- /Leaveout/test.py: -------------------------------------------------------------------------------- 1 | import model 2 | import importlib 3 | import numpy as np 4 | import cv2 5 | import torch 6 | import torch.nn as nn 7 | import torch.optim as optim 8 | import sys 9 | import yaml 10 | import os 11 | import copy 12 | 13 | def gazeto3d(gaze): 14 | gaze_gt = np.zeros([3]) 15 | gaze_gt[0] = -np.cos(gaze[1]) * np.sin(gaze[0]) 16 | gaze_gt[1] = -np.sin(gaze[1]) 17 | gaze_gt[2] = -np.cos(gaze[1]) * np.cos(gaze[0]) 18 | return gaze_gt 19 | 20 | def angular(gaze, label): 21 | total = np.sum(gaze * label) 22 | return np.arccos(min(total/(np.linalg.norm(gaze)* np.linalg.norm(label)), 0.9999999))*180/np.pi 23 | 24 | if __name__ == "__main__": 25 | config = yaml.load(open(sys.argv[1]), Loader = yaml.FullLoader) 26 | readername = config["reader"] 27 | dataloader = importlib.import_module("reader." + readername) 28 | config = config["test"] 29 | imagepath = config["data"]["image"] 30 | labelpath = config["data"]["label"] 31 | modelname = config["load"]["model_name"] 32 | 33 | loadpath = os.path.join(config["load"]["load_path"]) 34 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 35 | 36 | folder = os.listdir(labelpath) 37 | i = int(sys.argv[2]) 38 | 39 | if i in range(15): 40 | tests = folder[i] 41 | print(f"Test Set: {tests}") 42 | 43 | savepath = os.path.join(loadpath, f"checkpoint/{tests}") 44 | 45 | if not os.path.exists(os.path.join(loadpath, f"evaluation/{tests}")): 46 | os.makedirs(os.path.join(loadpath, f"evaluation/{tests}")) 47 | 48 | print("Read data") 49 | dataset = dataloader.txtload(os.path.join(labelpath, tests), imagepath, 32, shuffle=False, num_workers=4, header=True) 50 | 51 | begin = config["load"]["begin_step"] 52 | end = config["load"]["end_step"] 53 | step = config["load"]["steps"] 54 | 55 | for saveiter in range(begin, end+step, step): 56 | print("Model building") 57 | net = model.DilatedNet() 58 | statedict = torch.load(os.path.join(savepath, f"Iter_{saveiter}_{modelname}.pt")) 59 | 60 | net.to(device) 61 | net.load_state_dict(statedict) 62 | net.eval() 63 | 64 | print(f"Test {saveiter}") 65 | length = len(dataset) 66 | accs = 0 67 | count = 0 68 | with torch.no_grad(): 69 | with open(os.path.join(loadpath, f"evaluation/{tests}/{saveiter}.log"), 'w') as outfile: 70 | outfile.write("name results gts\n") 71 | for j, (data, label) in enumerate(dataset): 72 | 73 | fimg = data["face"].to(device) 74 | limg = data["left"].to(device) 75 | rimg = data["right"].to(device) 76 | names = data["name"] 77 | 78 | img = {"left":limg, "right":rimg, "face":fimg} 79 | gts = label.to(device) 80 | 81 | gazes = net(img) 82 | for k, gaze in enumerate(gazes): 83 | gaze = gaze.cpu().detach().numpy() 84 | count += 1 85 | accs += angular(gazeto3d(gaze), gazeto3d(gts.cpu().numpy()[k])) 86 | 87 | name = [names[k]] 88 | gaze = [str(u) for u in gaze] 89 | gt = [str(u) for u in gts.cpu().numpy()[k]] 90 | log = name + [",".join(gaze)] + [",".join(gt)] 91 | outfile.write(" ".join(log) + "\n") 92 | 93 | loger = f"[{saveiter}] Total Num: {count}, avg: {accs/count}" 94 | outfile.write(loger) 95 | print(loger) 96 | 97 | -------------------------------------------------------------------------------- /Leaveout/train.py: -------------------------------------------------------------------------------- 1 | import model 2 | import numpy as np 3 | import importlib 4 | import torch 5 | import torch.nn as nn 6 | import torch.optim as optim 7 | import time 8 | import sys 9 | import os 10 | import copy 11 | import yaml 12 | 13 | if __name__ == "__main__": 14 | config = yaml.load(open(f"{sys.argv[1]}"), Loader=yaml.FullLoader) 15 | readername = config["reader"] 16 | dataloader = importlib.import_module("reader." + readername) 17 | 18 | config = config["train"] 19 | imagepath = config["data"]["image"] 20 | labelpath = config["data"]["label"] 21 | modelname = config["save"]["model_name"] 22 | 23 | folder = os.listdir(labelpath) 24 | folder.sort() 25 | 26 | # i represents the i-th folder used as the test set. 27 | i = int(sys.argv[2]) 28 | 29 | if i in list(range(15)): 30 | trains = copy.deepcopy(folder) 31 | tests = trains.pop(i) 32 | print(f"Train Set:{trains}") 33 | print(f"Test Set:{tests}") 34 | 35 | trainlabelpath = [os.path.join(labelpath, j) for j in trains] 36 | 37 | savepath = os.path.join(config["save"]["save_path"], f"checkpoint/{tests}") 38 | if not os.path.exists(savepath): 39 | os.makedirs(savepath) 40 | 41 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 42 | 43 | print("Read data") 44 | dataset = dataloader.txtload(trainlabelpath, imagepath, config["params"]["batch_size"], shuffle=True, num_workers=4, header=True) 45 | 46 | print("Model building") 47 | net = model.DilatedNet() 48 | net.train() 49 | net.to(device) 50 | 51 | print("optimizer building") 52 | lossfunc = config["params"]["loss"] 53 | loss_op = getattr(nn, lossfunc)().cuda() 54 | base_lr = config["params"]["lr"] 55 | 56 | decaysteps = config["params"]["decay_step"] 57 | decayratio = config["params"]["decay"] 58 | 59 | optimizer = optim.Adam(net.parameters(),lr=base_lr, betas=(0.9,0.95)) 60 | scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=decaysteps, gamma=decayratio) 61 | 62 | print("Traning") 63 | length = len(dataset) 64 | total = length * config["params"]["epoch"] 65 | cur = 0 66 | timebegin = time.time() 67 | with open(os.path.join(savepath, "train_log"), 'w') as outfile: 68 | for epoch in range(1, config["params"]["epoch"]+1): 69 | for i, (data, label) in enumerate(dataset): 70 | 71 | # Acquire data 72 | data["left"] = data["left"].to(device) 73 | data['right'] = data['right'].to(device) 74 | data['face'] = data['face'].to(device) 75 | label = label.to(device) 76 | 77 | # forward 78 | gaze = net(data) 79 | 80 | # loss calculation 81 | loss = loss_op(gaze, label) 82 | optimizer.zero_grad() 83 | 84 | # backward 85 | loss.backward() 86 | optimizer.step() 87 | scheduler.step() 88 | cur += 1 89 | 90 | # print logs 91 | if i % 20 == 0: 92 | timeend = time.time() 93 | resttime = (timeend - timebegin)/cur * (total-cur)/3600 94 | log = f"[{epoch}/{config['params']['epoch']}]: [{i}/{length}] loss:{loss} lr:{base_lr}, rest time:{resttime:.2f}h" 95 | print(log) 96 | outfile.write(log + "\n") 97 | sys.stdout.flush() 98 | outfile.flush() 99 | 100 | if epoch % config["save"]["step"] == 0: 101 | torch.save(net.state_dict(), os.path.join(savepath, f"Iter_{epoch}_{modelname}.pt")) 102 | 103 | -------------------------------------------------------------------------------- /Leaveout/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.models as models 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | import math 6 | 7 | class DilatedNet(nn.Module): 8 | def __init__(self): 9 | super(DilatedNet, self).__init__() 10 | # 36 * 60 11 | # 18 * 30 12 | # 9 * 15 13 | eyeconv = models.vgg16(pretrained=True).features 14 | # self.eyeStreamConv.load_state_dict(vgg16.features[0:9].state_dict()) 15 | # self.faceStreamPretrainedConv.load_state_dict(vgg16.features[0:15].state_dict()) 16 | 17 | # Eye Stream, is composed of Conv DConv and FC layers. 18 | self.eyeStreamConv = nn.Sequential( 19 | eyeconv[0], #nn.Conv2d(3, 64, 3, 1, 1), 20 | nn.BatchNorm2d(64), 21 | nn.ReLU(inplace=True), 22 | eyeconv[2], #nn.Conv2d(64, 64, 3, 1, 1), 23 | nn.BatchNorm2d(64), 24 | nn.ReLU(inplace=True), 25 | nn.MaxPool2d(2, 2), 26 | 27 | eyeconv[5], #nn.Conv2d(64, 128, 3, 1, 1), 28 | nn.BatchNorm2d(128), 29 | nn.ReLU(inplace=True), 30 | eyeconv[7], #nn.Conv2d(128, 128, 3, 1, 1), 31 | nn.BatchNorm2d(128), 32 | nn.ReLU(inplace=True), 33 | ) 34 | 35 | self.eyeStreamDConv = nn.Sequential( 36 | nn.Conv2d(128, 64, 1, 1), 37 | nn.BatchNorm2d(64), 38 | nn.ReLU(inplace=True), 39 | 40 | nn.Conv2d(64, 64, 3, 1, dilation=(2, 2)), 41 | nn.BatchNorm2d(64), 42 | nn.ReLU(inplace=True), 43 | 44 | nn.Conv2d(64, 64, 3, 1, dilation=(3, 3)), 45 | nn.BatchNorm2d(64), 46 | nn.ReLU(inplace=True), 47 | 48 | nn.Conv2d(64, 128, 3, 1, dilation=(4, 5)), 49 | nn.BatchNorm2d(128), 50 | nn.ReLU(inplace=True), 51 | 52 | nn.Conv2d(128, 128, 3, 1, dilation=(5, 11)), 53 | nn.BatchNorm2d(128), 54 | nn.ReLU(inplace=True), 55 | ) 56 | 57 | self.eyeStreamFC = nn.Sequential( 58 | nn.Linear(128*4*6, 256), 59 | nn.BatchNorm1d(256), 60 | nn.ReLU(inplace=True), 61 | nn.Dropout(0.5) 62 | ) 63 | 64 | # Face Stream, is composed of Conv and FC layers. 65 | faceconv = models.vgg16(pretrained=True).features 66 | self.faceStreamPretrainedConv = nn.Sequential( 67 | # 224*224 68 | faceconv[0], # nn.Conv2d(3, 64, 3, 1, 1), 69 | nn.BatchNorm2d(64), 70 | nn.ReLU(inplace=True), 71 | faceconv[2], # nn.Conv2d(64, 64, 3, 1, 1), 72 | nn.BatchNorm2d(64), 73 | nn.ReLU(inplace=True), 74 | nn.MaxPool2d(2, 2), 75 | 76 | # 112*112 77 | faceconv[5], # nn.Conv2d(64, 128, 3, 1, 1), 78 | nn.BatchNorm2d(128), 79 | nn.ReLU(inplace=True), 80 | faceconv[7], # nn.Conv2d(128, 128, 3, 1, 1), 81 | nn.BatchNorm2d(128), 82 | nn.ReLU(inplace=True), 83 | nn.MaxPool2d(2, 2), 84 | 85 | # 56*56 86 | faceconv[10], # nn.Conv2d(128, 256, 3, 1, 1), 87 | nn.BatchNorm2d(256), 88 | nn.ReLU(inplace=True), 89 | faceconv[12], # nn.Conv2d(256, 256, 3, 1, 1), 90 | nn.BatchNorm2d(256), 91 | nn.ReLU(inplace=True), 92 | faceconv[14], # nn.Conv2d(256, 256, 3, 1, 1), 93 | nn.BatchNorm2d(256), 94 | nn.ReLU(inplace=True), 95 | ) 96 | 97 | 98 | self.faceStreamConv = nn.Sequential( 99 | nn.Conv2d(256, 64, 1), 100 | nn.MaxPool2d(2,2), 101 | nn.BatchNorm2d(64), 102 | nn.ReLU(inplace=True), 103 | 104 | # 28*28 105 | nn.Conv2d(64, 64, 3, padding=1), 106 | nn.BatchNorm2d(64), 107 | nn.ReLU(inplace=True), 108 | 109 | nn.Conv2d(64, 64, 3, padding=1), 110 | nn.BatchNorm2d(64), 111 | nn.MaxPool2d(2,2), 112 | nn.ReLU(inplace=True), 113 | ) 114 | 115 | self.faceStreamFC = nn.Sequential( 116 | nn.Linear(64 * 6 * 6, 256), 117 | nn.BatchNorm1d(256), 118 | nn.ReLU(inplace=True), 119 | nn.Dropout(0.5), 120 | nn.Linear(256, 32), 121 | nn.BatchNorm1d(32), 122 | nn.ReLU(inplace=True), 123 | nn.Dropout(0.5) 124 | ) 125 | 126 | self.totalFC = nn.Sequential( 127 | nn.Linear(256+256+32, 256), 128 | nn.BatchNorm1d(256), 129 | nn.ReLU(inplace=True), 130 | nn.Dropout(0.5), 131 | nn.Linear(256, 2) 132 | ) 133 | 134 | def forward(self, x_in): 135 | # Get face feature 136 | faceFeatureMap = self.faceStreamPretrainedConv(x_in['face']) 137 | faceFeatureMap = self.faceStreamConv(faceFeatureMap) 138 | 139 | faceFeature = torch.flatten(faceFeatureMap, start_dim=1) 140 | faceFeature = self.faceStreamFC(faceFeature) 141 | 142 | # Get left feature 143 | leftEyeFeature = self.eyeStreamConv(x_in['left']) 144 | leftEyeFeature = self.eyeStreamDConv(leftEyeFeature) 145 | leftEyeFeature = torch.flatten(leftEyeFeature, start_dim=1) 146 | leftEyeFeature = self.eyeStreamFC(leftEyeFeature) 147 | 148 | # Get Right feature 149 | rightEyeFeature = self.eyeStreamConv(x_in['right']) 150 | rightEyeFeature = self.eyeStreamDConv(rightEyeFeature) 151 | rightEyeFeature = torch.flatten(rightEyeFeature, start_dim=1) 152 | rightEyeFeature = self.eyeStreamFC(rightEyeFeature) 153 | 154 | features = torch.cat((faceFeature, leftEyeFeature, rightEyeFeature), 1) 155 | 156 | gaze = self.totalFC(features) 157 | 158 | return gaze 159 | 160 | if __name__ == '__main__': 161 | m = DilatedNet() 162 | m.to("cuda") 163 | feature = {"face":torch.zeros(64, 3, 96, 96).to("cuda"), 164 | "left":torch.zeros(64, 3, 64,96).to("cuda"), 165 | "right":torch.zeros(64, 3, 64,96).to("cuda") 166 | } 167 | a = m(feature) 168 | print(m) 169 | 170 | -------------------------------------------------------------------------------- /Traintest/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.models as models 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | import math 6 | 7 | class DilatedNet(nn.Module): 8 | def __init__(self): 9 | super(DilatedNet, self).__init__() 10 | # 36 * 60 11 | # 18 * 30 12 | # 9 * 15 13 | eyeconv = models.vgg16(pretrained=True).features 14 | # self.eyeStreamConv.load_state_dict(vgg16.features[0:9].state_dict()) 15 | # self.faceStreamPretrainedConv.load_state_dict(vgg16.features[0:15].state_dict()) 16 | 17 | # Eye Stream, is composed of Conv DConv and FC layers. 18 | self.eyeStreamConv = nn.Sequential( 19 | eyeconv[0], #nn.Conv2d(3, 64, 3, 1, 1), 20 | nn.BatchNorm2d(64), 21 | nn.ReLU(inplace=True), 22 | eyeconv[2], #nn.Conv2d(64, 64, 3, 1, 1), 23 | nn.BatchNorm2d(64), 24 | nn.ReLU(inplace=True), 25 | nn.MaxPool2d(2, 2), 26 | 27 | eyeconv[5], #nn.Conv2d(64, 128, 3, 1, 1), 28 | nn.BatchNorm2d(128), 29 | nn.ReLU(inplace=True), 30 | eyeconv[7], #nn.Conv2d(128, 128, 3, 1, 1), 31 | nn.BatchNorm2d(128), 32 | nn.ReLU(inplace=True), 33 | ) 34 | 35 | self.eyeStreamDConv = nn.Sequential( 36 | nn.Conv2d(128, 64, 1, 1), 37 | nn.BatchNorm2d(64), 38 | nn.ReLU(inplace=True), 39 | 40 | nn.Conv2d(64, 64, 3, 1, dilation=(2, 2)), 41 | nn.BatchNorm2d(64), 42 | nn.ReLU(inplace=True), 43 | 44 | nn.Conv2d(64, 64, 3, 1, dilation=(3, 3)), 45 | nn.BatchNorm2d(64), 46 | nn.ReLU(inplace=True), 47 | 48 | nn.Conv2d(64, 128, 3, 1, dilation=(4, 5)), 49 | nn.BatchNorm2d(128), 50 | nn.ReLU(inplace=True), 51 | 52 | nn.Conv2d(128, 128, 3, 1, dilation=(5, 11)), 53 | nn.BatchNorm2d(128), 54 | nn.ReLU(inplace=True), 55 | ) 56 | 57 | self.eyeStreamFC = nn.Sequential( 58 | nn.Linear(128*4*6, 256), 59 | nn.BatchNorm1d(256), 60 | nn.ReLU(inplace=True), 61 | nn.Dropout(0.5) 62 | ) 63 | 64 | # Face Stream, is composed of Conv and FC layers. 65 | faceconv = models.vgg16(pretrained=True).features 66 | self.faceStreamPretrainedConv = nn.Sequential( 67 | # 224*224 68 | faceconv[0], # nn.Conv2d(3, 64, 3, 1, 1), 69 | nn.BatchNorm2d(64), 70 | nn.ReLU(inplace=True), 71 | faceconv[2], # nn.Conv2d(64, 64, 3, 1, 1), 72 | nn.BatchNorm2d(64), 73 | nn.ReLU(inplace=True), 74 | nn.MaxPool2d(2, 2), 75 | 76 | # 112*112 77 | faceconv[5], # nn.Conv2d(64, 128, 3, 1, 1), 78 | nn.BatchNorm2d(128), 79 | nn.ReLU(inplace=True), 80 | faceconv[7], # nn.Conv2d(128, 128, 3, 1, 1), 81 | nn.BatchNorm2d(128), 82 | nn.ReLU(inplace=True), 83 | nn.MaxPool2d(2, 2), 84 | 85 | # 56*56 86 | faceconv[10], # nn.Conv2d(128, 256, 3, 1, 1), 87 | nn.BatchNorm2d(256), 88 | nn.ReLU(inplace=True), 89 | faceconv[12], # nn.Conv2d(256, 256, 3, 1, 1), 90 | nn.BatchNorm2d(256), 91 | nn.ReLU(inplace=True), 92 | faceconv[14], # nn.Conv2d(256, 256, 3, 1, 1), 93 | nn.BatchNorm2d(256), 94 | nn.ReLU(inplace=True), 95 | ) 96 | 97 | 98 | self.faceStreamConv = nn.Sequential( 99 | nn.Conv2d(256, 64, 1), 100 | nn.MaxPool2d(2,2), 101 | nn.BatchNorm2d(64), 102 | nn.ReLU(inplace=True), 103 | 104 | # 28*28 105 | nn.Conv2d(64, 64, 3, padding=1), 106 | nn.BatchNorm2d(64), 107 | nn.ReLU(inplace=True), 108 | 109 | nn.Conv2d(64, 64, 3, padding=1), 110 | nn.BatchNorm2d(64), 111 | nn.MaxPool2d(2,2), 112 | nn.ReLU(inplace=True), 113 | ) 114 | 115 | self.faceStreamFC = nn.Sequential( 116 | nn.Linear(64 * 6 * 6, 256), 117 | nn.BatchNorm1d(256), 118 | nn.ReLU(inplace=True), 119 | nn.Dropout(0.5), 120 | nn.Linear(256, 32), 121 | nn.BatchNorm1d(32), 122 | nn.ReLU(inplace=True), 123 | nn.Dropout(0.5) 124 | ) 125 | 126 | self.totalFC = nn.Sequential( 127 | nn.Linear(256+256+32, 256), 128 | nn.BatchNorm1d(256), 129 | nn.ReLU(inplace=True), 130 | nn.Dropout(0.5), 131 | nn.Linear(256, 2) 132 | ) 133 | 134 | def forward(self, x_in): 135 | # Get face feature 136 | faceFeatureMap = self.faceStreamPretrainedConv(x_in['face']) 137 | faceFeatureMap = self.faceStreamConv(faceFeatureMap) 138 | 139 | faceFeature = torch.flatten(faceFeatureMap, start_dim=1) 140 | faceFeature = self.faceStreamFC(faceFeature) 141 | 142 | # Get left feature 143 | leftEyeFeature = self.eyeStreamConv(x_in['left']) 144 | leftEyeFeature = self.eyeStreamDConv(leftEyeFeature) 145 | leftEyeFeature = torch.flatten(leftEyeFeature, start_dim=1) 146 | leftEyeFeature = self.eyeStreamFC(leftEyeFeature) 147 | 148 | # Get Right feature 149 | rightEyeFeature = self.eyeStreamConv(x_in['right']) 150 | rightEyeFeature = self.eyeStreamDConv(rightEyeFeature) 151 | rightEyeFeature = torch.flatten(rightEyeFeature, start_dim=1) 152 | rightEyeFeature = self.eyeStreamFC(rightEyeFeature) 153 | 154 | features = torch.cat((faceFeature, leftEyeFeature, rightEyeFeature), 1) 155 | 156 | gaze = self.totalFC(features) 157 | 158 | return gaze 159 | 160 | if __name__ == '__main__': 161 | m = DilatedNet() 162 | m.to("cuda") 163 | feature = {"face":torch.zeros(64, 3, 96, 96).to("cuda"), 164 | "left":torch.zeros(64, 3, 64,96).to("cuda"), 165 | "right":torch.zeros(64, 3, 64,96).to("cuda") 166 | } 167 | a = m(feature) 168 | print(m) 169 | 170 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dilated-Net 2 | The Pytorch Implementation of "Appearance-Based Gaze Estimation Using Dilated-Convolutions". (updated in 2021/04/28) 3 | 4 | 5 | We build benchmarks for gaze estimation in our survey [**"Appearance-based Gaze Estimation With Deep Learning: A Review and Benchmark"**](https://arxiv.org/abs/2104.12668). 6 | This is the implemented code of the "Dilated-Net" method in our benchmark. Please refer our survey for more details. 7 | 8 | We recommend you to use **data processing codes** provided in *GazeHub*. 9 | You can direct run the method' code using the processed dataset. 10 | 11 | ## Links to gaze estimation codes. 12 | 13 | - A Coarse-to-fine Adaptive Network for Appearance-based Gaze Estimation, AAAI 2020 (Coming soon) 14 | - [Gaze360: Physically Unconstrained Gaze Estimation in the Wild](https://github.com/yihuacheng/Gaze360), ICCV 2019 15 | - [Appearance-Based Gaze Estimation Using Dilated-Convolutions](https://github.com/yihuacheng/Dilated-Net), ACCV 2019 16 | - [Appearance-Based Gaze Estimation via Evaluation-Guided Asymmetric Regression](https://github.com/yihuacheng/ARE-GazeEstimation), ECCV 2018 17 | - [RT-GENE: Real-Time Eye Gaze Estimation in Natural Environments](https://github.com/yihuacheng/RT-Gene), ECCV 2018 18 | - [MPIIGaze: Real-World Dataset and Deep Appearance-Based Gaze Estimation](https://github.com/yihuacheng/Gaze-Net), TPAMI 2017 19 | - [It’s written all over your face: Full-face appearance-based gaze estimation](https://github.com/yihuacheng/Full-face), CVPRW 2017 20 | - [Eye Tracking for Everyone](https://github.com/yihuacheng/Itracker), CVPR 2016 21 | - [Appearance-Based Gaze Estimation in the Wild](https://github.com/yihuacheng/Mnist), CVPR 2015 22 | 23 | ## Performance 24 | The method is evaluated in three tasks. Please refer our survey for more details. 25 | ![benchmarks](benchmarkA.png) 26 | ![benchmarks](benchmarkB.png) 27 | 28 | ## License 29 | The code is under the license of [CC BY-NC-SA 4.0 license](https://creativecommons.org/licenses/by-nc-sa/4.0/). 30 | 31 | ## Introduction 32 | We provide two similar projects for leave-one-person-out evaluation and evaluation of common training-test split. 33 | They have the same architecture but different started modes. 34 | 35 | Each project contains following files/folders. 36 | - `model.py`, the model code. 37 | - `train.py`, the entry for training. 38 | - `test.py`, the entry for testing. 39 | - `config/`, this folder contains the config of the experiment in each dataset. To run our code, **you should write your own** `config.yaml`. 40 | - `reader/`, the data loader code. You can use the provided reader or write your own reader. 41 | 42 | ## Getting Started 43 | ### Writing your own *config.yaml* 44 | 45 | Normally, for training, you should change 46 | 1. `train.save.save_path`, The model is saved in the `$save_path$/checkpoint/`. 47 | 2. `train.data.image`, This is the path of image, please use the provided data processing code in *GazeHub* 48 | 3. `train.data.label`, This is the path of label. 49 | 4. `reader`, This indicates the used reader. It is the filename in `reader` folder, e.g., *reader/reader_mpii.py* ==> `reader: reader_mpii`. 50 | 51 | For test, you should change 52 | 1. `test.load.load_path`, it is usually the same as `train.save.save_path`. The test result is saved in `$load_path$/evaluation/`. 53 | 2. `test.data.image`, it is usually the same as `train.data.image`. 54 | 3. `test.data.label`, it is usually the same as `train.data.label`. 55 | 56 | ### Training 57 | 58 | In the leaveout folder, you can run 59 | ``` 60 | python train.py config/config_mpii.yaml 0 61 | ``` 62 | This means the code will run with `config_mpii.yaml` and use the `0th` person as the test set. 63 | 64 | You also can run 65 | ``` 66 | bash run.sh train.py config/config_mpii.yaml 67 | ``` 68 | This means the code will perform leave-one-person-out training automatically. 69 | `run.sh` performs iteration, you can change the iteration times in `run.sh` for different datasets, e.g., set the iteration times as `4` for four-fold validation. 70 | 71 | In the traintest folder, you can run 72 | ``` 73 | python train.py config/config_mpii.yaml 74 | ``` 75 | 76 | ### Test 77 | In the leaveout folder, you can run 78 | ``` 79 | python test.py config/config_mpii.yaml 0 80 | ``` 81 | or 82 | ``` 83 | bash run.sh test.py config/config_mpii.yaml 84 | ``` 85 | 86 | In the traintest folder, you can run 87 | ``` 88 | python test.py config/config_mpii.yaml 89 | ``` 90 | 91 | ### Result 92 | After training or test, you can find the result from the `save_path` in `config_mpii.yaml`. 93 | 94 | 95 | ## Citation 96 | If you use our code, please cite: 97 | ``` 98 | @InProceedings{Chen_2019_ACCV, 99 | author="Chen, Zhaokang 100 | and Shi, Bertram E.", 101 | editor="Jawahar, C.V. 102 | and Li, Hongdong 103 | and Mori, Greg 104 | and Schindler, Konrad", 105 | title="Appearance-Based Gaze Estimation Using Dilated-Convolutions", 106 | booktitle="Computer Vision -- ACCV 2018", 107 | year="2019", 108 | publisher="Springer International Publishing", 109 | address="Cham", 110 | pages="309--324", 111 | isbn="978-3-030-20876-9" 112 | } 113 | 114 | @article{Cheng2021Survey, 115 | title={Appearance-based Gaze Estimation With Deep Learning: A Review and Benchmark}, 116 | author={Yihua Cheng and Haofei Wang and Yiwei Bao and Feng Lu}, 117 | journal={arXiv preprint arXiv:2104.12668}, 118 | year={2021} 119 | } 120 | ``` 121 | 122 | ## Contact 123 | Please email any questions or comments to yihua_c@buaa.edu.cn. 124 | 125 | 126 | ## Reference 127 | 128 | 1. MPIIGaze: Real-World Dataset and Deep Appearance-Based Gaze Estimation 129 | 2. EYEDIAP Database: Data Description and Gaze Tracking Evaluation Benchmarks 130 | 3. Learning-by-Synthesis for Appearance-based 3D Gaze Estimation 131 | 3. Gaze360: Physically Unconstrained Gaze Estimation in the Wild 132 | 5. ETH-XGaze: A Large Scale Dataset for Gaze Estimation under Extreme Head Pose and Gaze Variation 133 | 6. Appearance-Based Gaze Estimation in the Wild 134 | 7. Appearance-Based Gaze Estimation Using Dilated-Convolutions 135 | 8. RT-GENE: Real-Time Eye Gaze Estimation in Natural Environments 136 | 9. It’s written all over your face: Full-face appearance-based gaze estimation 137 | 10. A Coarse-to-fine Adaptive Network for Appearance-based Gaze Estimation 138 | 11. Eye Tracking for Everyone 139 | 12. Adaptive Feature Fusion Network for Gaze Tracking in Mobile Tablets 140 | 13. On-Device Few-Shot Personalization for Real-Time Gaze Estimation 141 | 14. A Generalized and Robust Method Towards Practical Gaze Estimation on Smart Phone 142 | --------------------------------------------------------------------------------