├── .gitignore ├── Chain-XOR ├── data_gen.py ├── nesy_eval.py ├── nesy_train.py ├── nn_utils.py ├── run.sh └── smt_solver.py ├── Conjunction ├── data_gen.py ├── nesy_eval.py ├── nesy_train.py ├── nn_utils.py ├── run.sh └── smt_solver.py ├── Kitti-Driving ├── astar.py ├── config.py ├── data │ └── readme.md ├── dataset.py ├── model.py ├── models │ ├── __init__.py │ ├── logic_layer.py │ ├── mingpt.py │ └── smoke │ │ ├── README.md │ │ ├── __init__.py │ │ ├── config │ │ ├── __init__.py │ │ ├── defaults.py │ │ └── paths_catalog.py │ │ ├── csrc │ │ ├── cpu │ │ │ ├── dcn_v2_cpu.cpp │ │ │ └── vision.h │ │ ├── cuda │ │ │ ├── dcn_v2_cuda.cu │ │ │ ├── dcn_v2_im2col_cuda.cu │ │ │ ├── dcn_v2_im2col_cuda.h │ │ │ ├── dcn_v2_psroi_pooling_cuda.cu │ │ │ └── vision.h │ │ ├── dcn_v2.h │ │ └── vision.cpp │ │ ├── data │ │ ├── __init__.py │ │ ├── build.py │ │ ├── collate_batch.py │ │ ├── datasets │ │ │ ├── __init__.py │ │ │ ├── concat_dataset.py │ │ │ ├── evaluation │ │ │ │ ├── __init__.py │ │ │ │ └── kitti │ │ │ │ │ ├── kitti_eval.py │ │ │ │ │ └── kitti_eval │ │ │ │ │ └── kitti_eval.exe │ │ │ ├── kitti.py │ │ │ ├── kitti_smoke_dataset.py │ │ │ └── nuscenes_smoke_dataset.py │ │ ├── samplers │ │ │ ├── __init__.py │ │ │ ├── distributed_sampler.py │ │ │ └── grouped_batch_sampler.py │ │ └── transforms │ │ │ ├── __init__.py │ │ │ ├── build.py │ │ │ └── transforms.py │ │ ├── engine │ │ ├── __init__.py │ │ ├── defaults.py │ │ ├── inference.py │ │ ├── launch.py │ │ ├── test_net.py │ │ └── trainer.py │ │ ├── layers │ │ ├── dcn_hdb.py │ │ ├── dcn_v2.py │ │ ├── deform_conv.py │ │ ├── focal_loss.py │ │ └── utils.py │ │ ├── modeling │ │ ├── backbone │ │ │ ├── __init__.py │ │ │ ├── backbone.py │ │ │ └── dla.py │ │ ├── detector │ │ │ ├── __init__.py │ │ │ ├── detectors.py │ │ │ └── keypoint_detector.py │ │ ├── heads │ │ │ ├── __init__.py │ │ │ ├── heads.py │ │ │ └── smoke_head │ │ │ │ ├── inference.py │ │ │ │ ├── loss.py │ │ │ │ ├── smoke_head.py │ │ │ │ └── smoke_predictor.py │ │ ├── heatmap_coder.py │ │ ├── make_layers.py │ │ ├── registry.py │ │ ├── smoke_coder.py │ │ └── utils.py │ │ ├── solver │ │ ├── __init__.py │ │ └── build.py │ │ ├── structures │ │ ├── image_list.py │ │ └── params_3d.py │ │ └── utils │ │ ├── check_point.py │ │ ├── collect_env.py │ │ ├── comm.py │ │ ├── envs.py │ │ ├── imports.py │ │ ├── logger.py │ │ ├── metric_logger.py │ │ ├── miscellaneous.py │ │ ├── model_serialization.py │ │ ├── model_zoo.py │ │ ├── plot.py │ │ ├── registry.py │ │ └── timer.py ├── nesy_eval_cnn.py ├── nesy_eval_gpt.py ├── nesy_train_cnn.py ├── nesy_train_gpt.py ├── nn_utils.py ├── run.sh ├── smt_solver.py ├── sup_eval.py ├── sup_train.py └── temp │ └── nesy_train_20230423.py ├── Nonograms ├── data │ ├── nonograms_15.csv │ └── nonograms_7.csv ├── dataset.py ├── gv_solver.py ├── nesy_eval.py ├── nesy_train.py ├── nn_utils.py ├── run.sh └── smt_solver.py ├── Nuscenes-Driving ├── astar.py ├── config.py ├── data │ └── readme.md ├── dataset.py ├── model.py ├── models │ ├── __init__.py │ ├── mingpt.py │ ├── smoke │ │ ├── README.md │ │ ├── __init__.py │ │ ├── config │ │ │ ├── __init__.py │ │ │ ├── defaults.py │ │ │ └── paths_catalog.py │ │ ├── csrc │ │ │ ├── cpu │ │ │ │ ├── dcn_v2_cpu.cpp │ │ │ │ └── vision.h │ │ │ ├── cuda │ │ │ │ ├── dcn_v2_cuda.cu │ │ │ │ ├── dcn_v2_im2col_cuda.cu │ │ │ │ ├── dcn_v2_im2col_cuda.h │ │ │ │ ├── dcn_v2_psroi_pooling_cuda.cu │ │ │ │ └── vision.h │ │ │ ├── dcn_v2.h │ │ │ └── vision.cpp │ │ ├── data │ │ │ ├── __init__.py │ │ │ ├── build.py │ │ │ ├── collate_batch.py │ │ │ ├── datasets │ │ │ │ ├── __init__.py │ │ │ │ ├── concat_dataset.py │ │ │ │ ├── evaluation │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── kitti │ │ │ │ │ │ ├── kitti_eval.py │ │ │ │ │ │ └── kitti_eval │ │ │ │ │ │ └── kitti_eval.exe │ │ │ │ ├── kitti.py │ │ │ │ ├── kitti_smoke_dataset.py │ │ │ │ └── nuscenes_smoke_dataset.py │ │ │ ├── samplers │ │ │ │ ├── __init__.py │ │ │ │ ├── distributed_sampler.py │ │ │ │ └── grouped_batch_sampler.py │ │ │ └── transforms │ │ │ │ ├── __init__.py │ │ │ │ ├── build.py │ │ │ │ └── transforms.py │ │ ├── engine │ │ │ ├── __init__.py │ │ │ ├── defaults.py │ │ │ ├── inference.py │ │ │ ├── launch.py │ │ │ ├── test_net.py │ │ │ └── trainer.py │ │ ├── layers │ │ │ ├── dcn_hdb.py │ │ │ ├── dcn_v2.py │ │ │ ├── deform_conv.py │ │ │ ├── focal_loss.py │ │ │ └── utils.py │ │ ├── modeling │ │ │ ├── backbone │ │ │ │ ├── __init__.py │ │ │ │ ├── backbone.py │ │ │ │ └── dla.py │ │ │ ├── detector │ │ │ │ ├── __init__.py │ │ │ │ ├── detectors.py │ │ │ │ └── keypoint_detector.py │ │ │ ├── heads │ │ │ │ ├── __init__.py │ │ │ │ ├── heads.py │ │ │ │ └── smoke_head │ │ │ │ │ ├── inference.py │ │ │ │ │ ├── loss.py │ │ │ │ │ ├── smoke_head.py │ │ │ │ │ └── smoke_predictor.py │ │ │ ├── heatmap_coder.py │ │ │ ├── make_layers.py │ │ │ ├── registry.py │ │ │ ├── smoke_coder.py │ │ │ └── utils.py │ │ ├── solver │ │ │ ├── __init__.py │ │ │ └── build.py │ │ ├── structures │ │ │ ├── image_list.py │ │ │ └── params_3d.py │ │ └── utils │ │ │ ├── check_point.py │ │ │ ├── collect_env.py │ │ │ ├── comm.py │ │ │ ├── envs.py │ │ │ ├── imports.py │ │ │ ├── logger.py │ │ │ ├── metric_logger.py │ │ │ ├── miscellaneous.py │ │ │ ├── model_serialization.py │ │ │ ├── model_zoo.py │ │ │ ├── plot.py │ │ │ ├── registry.py │ │ │ └── timer.py │ └── ste.py ├── nesy_eval_cnn.py ├── nesy_eval_gpt.py ├── nesy_train_cnn.py ├── nesy_train_gpt.py ├── nn_utils.py ├── run.sh ├── smt_solver.py ├── sup_eval.py └── sup_train.py ├── README.md ├── RRN-SudoKu ├── data_gen.py ├── dataset.py ├── models │ ├── __init__.py │ ├── gpt.py │ ├── logic_layer.py │ ├── mlp.py │ └── net.py ├── nesy_eval_cnn.py ├── nesy_eval_gpt.py ├── nesy_train_cnn.py ├── nesy_train_gpt.py ├── nn_utils.py ├── run.sh └── smt_solver.py ├── SATNet-SudoKu ├── data │ └── .DS_Store ├── dataset.py ├── models │ ├── __init__.py │ ├── gpt.py │ ├── logic_layer.py │ ├── mlp.py │ ├── net.py │ └── ste.py ├── nesy_eval_cnn.py ├── nesy_eval_gpt.py ├── nesy_train_cnn.py ├── nesy_train_gpt.py ├── nn_utils.py ├── run.sh └── smt_solver.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.t7 2 | *.pyc 3 | RRN-SudoKu/runs/* 4 | *.txt 5 | *.pt 6 | *.png 7 | *.json 8 | *.ipynb 9 | -------------------------------------------------------------------------------- /Chain-XOR/data_gen.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | 4 | def gen_data(seq_len, size, save_root, threshold=0.9, mode="xor"): 5 | # generate X 6 | X = torch.rand((size, seq_len)) 7 | X[X >= threshold] = 1 8 | X[X < threshold] = 0 9 | X = X.int() 10 | 11 | # generate Y 12 | Y = torch.sum(X, dim=1) 13 | if mode == "xor": 14 | Y = Y % 2 != 0 15 | elif mode == "disj": 16 | Y = Y != 0 17 | elif mode == "conj": 18 | Y = torch.ones_like(Y) * seq_len == Y 19 | Y = Y.int().unsqueeze(dim=1) 20 | 21 | # save data 22 | save_dir = os.path.join(save_root, str(seq_len)) 23 | if not os.path.exists(save_dir): 24 | os.makedirs(save_dir) 25 | 26 | torch.save(X, os.path.join(save_dir, "features.pt")) 27 | torch.save(Y, os.path.join(save_dir, "labels.pt")) 28 | 29 | return X, Y -------------------------------------------------------------------------------- /Chain-XOR/nesy_eval.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from nn_utils import * 3 | 4 | # Dataloader 5 | parser = argparse.ArgumentParser(description='PyTorch Chain-XOR') 6 | parser.add_argument('--device', default=0, type=int, help='Cuda device.') 7 | parser.add_argument('--len', default=20, type=int, help='the length of Chain-XOR') 8 | parser.add_argument('--exp_name', default='', type=str, help='Experiment name') 9 | opt = parser.parse_args() 10 | 11 | 12 | # init model 13 | X_train = torch.load('data/{}/X_train.pt'.format(opt.len)) 14 | y_train = torch.load('data/{}/y_train.pt'.format(opt.len)) 15 | X_test = torch.load('data/{}/X_test.pt'.format(opt.len)) 16 | y_test = torch.load('data/{}/y_test.pt'.format(opt.len)) 17 | 18 | # cuda 19 | torch.cuda.set_device(0) 20 | torch.set_default_tensor_type(torch.FloatTensor) 21 | 22 | # random seed 23 | random.seed(0) 24 | np.random.seed(0) 25 | torch.manual_seed(0) 26 | torch.cuda.manual_seed(0) 27 | 28 | # loading zero 29 | ckpt = './checkpoint/' + opt.exp_name + '__zero_0.t7' 30 | # init chektpoint 31 | static_dict = torch.load(ckpt) 32 | phi = static_dict['logic'] 33 | W0, b0, Z0 = phi 34 | m, n = W0.shape 35 | N, k = Z0.shape 36 | W0 = W0[:, 0:n-k].reshape(1,-1) 37 | 38 | # remove 0 39 | ind = (X_train == 0).all(dim=-1) 40 | W_zero = torch.ones(1,n).cuda() 41 | Z_zero = torch.zeros(1,k).cuda() 42 | b_zero = y_train[ind].mean().cuda().reshape(1,1) 43 | X_train, y_train = X_train[~ind], y_train[~ind] 44 | N = X_train.shape[0] 45 | 46 | # loading 47 | ckpt = './checkpoint/' + opt.exp_name + '__0.t7' 48 | # init chektpoint 49 | static_dict = torch.load(ckpt) 50 | phi = static_dict['logic'] 51 | Wtmp, btmp = bounding_box(phi, X_train, y_train) 52 | print(Wtmp, btmp) 53 | 54 | evaluate_batch(Wtmp, btmp, W0, b0, X_test, y_test) 55 | 56 | -------------------------------------------------------------------------------- /Chain-XOR/nesy_train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import torch 4 | import torch.nn.functional as F 5 | from nn_utils import * 6 | from scipy import linalg as splinalg 7 | import random 8 | import numpy as np 9 | import torch.nn as nn 10 | 11 | import sys 12 | sys.path.append("../") 13 | 14 | # Dataloader 15 | parser = argparse.ArgumentParser(description='PyTorch Chain-XOR') 16 | parser.add_argument('--device', default=0, type=int, help='Cuda device.') 17 | parser.add_argument('--seed', default=1, type=int, help='Random seed to use.') 18 | parser.add_argument('--lamda', default=0.1, type=float, help='the trust region penalty') 19 | parser.add_argument('--num_epochs', default=50000, type=int, help='the number of epochs') 20 | parser.add_argument('--num_iters', default=2000, type=int, help='the number of iters to increase t') 21 | parser.add_argument('--logic_lr', default=1e6, type=float, help='the step size of programming') 22 | parser.add_argument('--len', default=20, type=int, help='the length of Chain-XOR') 23 | parser.add_argument('--clauses', default=100, type=int, help='the number of clauses') 24 | parser.add_argument('--k', default=19, type=int, help='the number of aux variables') 25 | parser.add_argument('--update_b', default=0, type=int, help='the number of clauses') 26 | parser.add_argument('--data_split', default=0.9, type=float, help='the ratio of training/test set') 27 | parser.add_argument('--exp_name', default='', type=str, help='Experiment name') 28 | opt = parser.parse_args() 29 | 30 | # cuda 31 | torch.cuda.set_device(opt.device) 32 | torch.set_default_tensor_type(torch.FloatTensor) 33 | 34 | # random seed 35 | random.seed(opt.seed) 36 | np.random.seed(opt.seed) 37 | torch.manual_seed(opt.seed) 38 | torch.cuda.manual_seed(opt.seed) 39 | 40 | ##### dataset 41 | X = torch.load('data/{}/features.pt'.format(opt.len)) 42 | y = torch.load('data/{}/labels.pt'.format(opt.len)) 43 | 44 | #### split and preprocess 45 | ind = np.arange(X.shape[0]) 46 | np.random.shuffle(ind) 47 | N = int(opt.data_split*X.shape[0]) 48 | X_train, y_train = X[ind[0:N]].float().cuda(), y[ind[0:N]].float().cuda() 49 | X_test, y_test = X[ind[N:]].float().cuda(), y[ind[N:]].float().cuda() 50 | save_data(X_train, y_train, X_test, y_test, opt) 51 | phi0, X_train, y_train = preprocess(X_train, y_train, opt) 52 | 53 | #### training 54 | file_name = 'parity' + '_' + str(opt.len) + '_' + opt.exp_name + '_zero' 55 | save_logic(phi0, file_name) 56 | file_name = 'parity' + '_' + str(opt.len) + '_' + opt.exp_name 57 | phi = train(X_train, y_train, opt) 58 | save_logic(phi, file_name) 59 | 60 | 61 | -------------------------------------------------------------------------------- /Chain-XOR/run.sh: -------------------------------------------------------------------------------- 1 | # supervised training 2 | 3 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 0.0 --len 20 --k 19 --num_epochs 50000 4 | CUDA_VISIBLE_DEVICES=4,5 python3 nesy_eval.py --len 20 --exp_name 'parity_20' 5 | 6 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 0.0 --len 40 --k 39 --num_epochs 50000 7 | CUDA_VISIBLE_DEVICES=4,5 python3 nesy_eval.py --len 40 --exp_name 'parity_40' 8 | 9 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 0.0 --len 60 --k 59 --num_epochs 50000 10 | CUDA_VISIBLE_DEVICES=4,5 python3 nesy_eval.py --len 60 --exp_name 'parity_60' 11 | 12 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 0.0 --len 80 --k 79 --num_epochs 50000 13 | CUDA_VISIBLE_DEVICES=4,5 python3 nesy_eval.py --len 80 --exp_name 'parity_80' 14 | 15 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 0.0 --len 100 --k 99 --num_epochs 100000 16 | CUDA_VISIBLE_DEVICES=4,5 python3 nesy_eval.py --len 100 --exp_name 'parity_100' 17 | 18 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 0.0 --len 200 --k 199 --num_epochs 100000 19 | CUDA_VISIBLE_DEVICES=4,5 python3 nesy_eval.py --len 200 --exp_name 'parity_200' 20 | -------------------------------------------------------------------------------- /Chain-XOR/smt_solver.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from z3 import * 4 | import time 5 | 6 | size = 10 7 | 8 | def sat_solver(res, b): 9 | # check 10 | s = Solver() 11 | # set_option("parallel.enable", True) 12 | # s.set("timeout", 60000) 13 | X = Bool("X") 14 | 15 | # Default given: set the range of symbol 16 | for r, btmp in zip(res, b): 17 | eqn = [] 18 | for t in btmp: 19 | eqn.append(PbEq([(X,1)], t - r.item())) 20 | s.add(Or(eqn)) 21 | 22 | 23 | if s.check() == sat: 24 | res = [is_true(s.model()[X])] 25 | res = np.array(res)*1.0 26 | return True, res 27 | else: 28 | return False, None 29 | 30 | 31 | 32 | 33 | if __name__ == "__main__": 34 | W = torch.zeros(1, 729) 35 | bmin = torch.ones(1,1) 36 | bmax = torch.ones(1,1) 37 | W[:,0:9] = 1.0 38 | # init_check(W, b) 39 | maxsat_solver(W, bmin, bmax) 40 | # sat_solver(W, bmin, bmax) 41 | -------------------------------------------------------------------------------- /Conjunction/data_gen.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | 4 | def gen_data(seq_len, size, save_root, threshold=0.9, mode="xor"): 5 | # generate X 6 | X = torch.rand((size, seq_len)) 7 | X[X >= threshold] = 1 8 | X[X < threshold] = 0 9 | X = X.int() 10 | 11 | # generate Y 12 | Y = torch.sum(X, dim=1) 13 | if mode == "xor": 14 | Y = Y % 2 != 0 15 | elif mode == "disj": 16 | Y = Y != 0 17 | elif mode == "conj": 18 | Y = torch.ones_like(Y) * seq_len == Y 19 | Y = Y.int().unsqueeze(dim=1) 20 | print(Y.sum()) 21 | 22 | # save data 23 | save_dir = os.path.join(save_root, str(seq_len)) 24 | if not os.path.exists(save_dir): 25 | os.makedirs(save_dir) 26 | 27 | torch.save(X, os.path.join(save_dir, "features.pt")) 28 | torch.save(Y, os.path.join(save_dir, "labels.pt")) 29 | 30 | return X, Y 31 | 32 | 33 | if __name__ == "__main__": 34 | # seq_len = 20 35 | # size = 10000 36 | # save_root = './data/' 37 | # gen_data(seq_len, size, save_root, threshold=0.1, mode="conj") 38 | # seq_len = 40 39 | # size = 10000 40 | # save_root = './data/' 41 | # gen_data(seq_len, size, save_root, threshold=0.05, mode="conj") 42 | # seq_len = 60 43 | # size = 10000 44 | # save_root = './data/' 45 | # gen_data(seq_len, size, save_root, threshold=0.05, mode="conj") 46 | seq_len = 80 47 | size = 10000 48 | save_root = './data/' 49 | gen_data(seq_len, size, save_root, threshold=0.04, mode="conj") 50 | # seq_len = 100 51 | # size = 10000 52 | # save_root = './data/' 53 | # gen_data(seq_len, size, save_root, threshold=0.03, mode="conj") 54 | # seq_len = 200 55 | # size = 10000 56 | # save_root = './data/' 57 | # gen_data(seq_len, size, save_root, threshold=0.02, mode="conj") -------------------------------------------------------------------------------- /Conjunction/nesy_eval.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from nn_utils import * 3 | 4 | # Dataloader 5 | parser = argparse.ArgumentParser(description='PyTorch Chain-XOR') 6 | parser.add_argument('--device', default=0, type=int, help='Cuda device.') 7 | parser.add_argument('--len', default=20, type=int, help='the length of Chain-XOR') 8 | parser.add_argument('--exp_name', default='', type=str, help='Experiment name') 9 | opt = parser.parse_args() 10 | 11 | 12 | # init model 13 | X_train = torch.load('data/{}/X_train.pt'.format(opt.len)) 14 | y_train = torch.load('data/{}/y_train.pt'.format(opt.len)) 15 | X_test = torch.load('data/{}/X_test.pt'.format(opt.len)) 16 | y_test = torch.load('data/{}/y_test.pt'.format(opt.len)) 17 | 18 | # cuda 19 | torch.cuda.set_device(0) 20 | torch.set_default_tensor_type(torch.FloatTensor) 21 | 22 | # random seed 23 | random.seed(0) 24 | np.random.seed(0) 25 | torch.manual_seed(0) 26 | torch.cuda.manual_seed(0) 27 | 28 | # loading zero 29 | ckpt = './checkpoint/' + opt.exp_name + '__zero_0.t7' 30 | # init chektpoint 31 | static_dict = torch.load(ckpt) 32 | phi = static_dict['logic'] 33 | W0, b0, Z0 = phi 34 | m, n = W0.shape 35 | N, k = Z0.shape 36 | W0 = W0[:, 0:n-k].reshape(1,-1) 37 | 38 | # remove 0 39 | ind = (X_train == 0).all(dim=-1) 40 | W_zero = torch.ones(1,n).cuda() 41 | Z_zero = torch.zeros(1,k).cuda() 42 | b_zero = y_train[ind].mean().cuda().reshape(1,1) 43 | X_train, y_train = X_train[~ind], y_train[~ind] 44 | N = X_train.shape[0] 45 | 46 | # loading 47 | ckpt = './checkpoint/' + opt.exp_name + '__{}.0_0.t7'.format(opt.len) 48 | # init chektpoint 49 | static_dict = torch.load(ckpt) 50 | phi = static_dict['logic'] 51 | W1, b1 = bounding_box(phi, X_train, 1-y_train) 52 | print(W1.shape) 53 | # loading 54 | ckpt = './checkpoint/' + opt.exp_name + '__2.0_0.t7' 55 | # init chektpoint 56 | static_dict = torch.load(ckpt) 57 | phi = static_dict['logic'] 58 | W2, b2 = bounding_box(phi, X_train, 1-y_train) 59 | print(W2.shape) 60 | 61 | Wtmp = torch.cat([W1, W2], dim=0) 62 | btmp = b1 + b2 63 | print(Wtmp, btmp) 64 | 65 | evaluate_batch(Wtmp, btmp, W0, b0, X_test, y_test) 66 | 67 | -------------------------------------------------------------------------------- /Conjunction/nesy_train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import torch 4 | import torch.nn.functional as F 5 | from nn_utils import * 6 | from scipy import linalg as splinalg 7 | import random 8 | import numpy as np 9 | import torch.nn as nn 10 | 11 | import sys 12 | sys.path.append("../") 13 | 14 | # Dataloader 15 | parser = argparse.ArgumentParser(description='PyTorch Chain-XOR') 16 | parser.add_argument('--device', default=0, type=int, help='Cuda device.') 17 | parser.add_argument('--seed', default=1, type=int, help='Random seed to use.') 18 | parser.add_argument('--lamda', default=0.1, type=float, help='the trust region penalty') 19 | parser.add_argument('--num_epochs', default=50000, type=int, help='the number of epochs') 20 | parser.add_argument('--num_iters', default=2000, type=int, help='the number of iters to increase t') 21 | parser.add_argument('--logic_lr', default=1e6, type=float, help='the step size of programming') 22 | parser.add_argument('--len', default=20, type=int, help='the length of Chain-XOR') 23 | parser.add_argument('--clauses', default=100, type=int, help='the number of clauses') 24 | parser.add_argument('--k', default=19, type=int, help='the number of aux variables') 25 | parser.add_argument('--b', default=1, type=float, help='the pre-defined value of b') 26 | parser.add_argument('--update_b', default=0, type=int, help='the number of clauses') 27 | parser.add_argument('--data_split', default=0.9, type=float, help='the ratio of training/test set') 28 | parser.add_argument('--exp_name', default='', type=str, help='Experiment name') 29 | opt = parser.parse_args() 30 | 31 | # cuda 32 | torch.cuda.set_device(opt.device) 33 | torch.set_default_tensor_type(torch.FloatTensor) 34 | 35 | # random seed 36 | random.seed(opt.seed) 37 | np.random.seed(opt.seed) 38 | torch.manual_seed(opt.seed) 39 | torch.cuda.manual_seed(opt.seed) 40 | 41 | ##### dataset 42 | X = torch.load('data/{}/features.pt'.format(opt.len)) 43 | y = torch.load('data/{}/labels.pt'.format(opt.len)) 44 | 45 | #### split and preprocess 46 | ind = np.arange(X.shape[0]) 47 | np.random.shuffle(ind) 48 | N = int(opt.data_split*X.shape[0]) 49 | X_train, y_train = X[ind[0:N]].float().cuda(), y[ind[0:N]].float().cuda() 50 | X_test, y_test = X[ind[N:]].float().cuda(), y[ind[N:]].float().cuda() 51 | save_data(X_train, y_train, X_test, y_test, opt) 52 | phi0, X_train, y_train = preprocess(X_train, y_train, opt) 53 | 54 | #### training 55 | file_name = 'conj' + '_' + str(opt.len) + '_' + opt.exp_name + '_zero' 56 | save_logic(phi0, file_name) 57 | # learn clause of b = opt.len 58 | file_name = 'conj' + '_' + str(opt.len) + '_' + opt.exp_name + '_' + str(opt.b) 59 | phi = train(X_train, 1-y_train, opt) 60 | save_logic(phi, file_name) 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /Conjunction/run.sh: -------------------------------------------------------------------------------- 1 | # supervised training 2 | 3 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 0.0 --len 20 --k 19 --b 20 --clauses 1000 --num_epochs 200000 4 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 1000.0 --len 20 --k 19 --b 2 --clauses 1000 --num_epochs 500000 5 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_eval.py --len 20 --exp_name 'conj_20' 6 | 7 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 0.0 --len 40 --k 39 --b 40 --clauses 1000 --num_epochs 200000 8 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 1000.0 --len 40 --k 39 --b 2 --clauses 1000 --num_epochs 500000 9 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_eval.py --len 40 --exp_name 'conj_40' 10 | 11 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 0.0 --len 60 --k 59 --b 60 --clauses 1000 --num_epochs 200000 12 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 1000.0 --len 60 --k 59 --b 2 --clauses 1000 --num_epochs 500000 13 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_eval.py --len 60 --exp_name 'conj_60' 14 | 15 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 0.0 --len 80 --k 79 --b 80 --clauses 1000 --num_epochs 500000 16 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 1000.0 --len 80 --k 79 --b 2 --clauses 1500 --num_epochs 1000000 17 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_eval.py --len 80 --exp_name 'conj_80' 18 | 19 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 0.0 --len 100 --k 99 --b 100 --clauses 1000 --num_epochs 500000 20 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 1000.0 --len 100 --k 99 --b 2 --clauses 2000 --num_epochs 1000000 21 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_eval.py --len 100 --exp_name 'conj_100' 22 | 23 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 0.0 --len 200 --k 199 --b 200 --clauses 1000 --num_epochs 500000 24 | # CUDA_VISIBLE_DEVICES=4,5 python3 nesy_train.py --lamda 1000.0 --len 200 --k 199 --b 2 --clauses 3000 --num_epochs 1000000 25 | CUDA_VISIBLE_DEVICES=4,5 python3 nesy_eval.py --len 200 --exp_name 'conj_200' 26 | -------------------------------------------------------------------------------- /Conjunction/smt_solver.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from z3 import * 4 | import time 5 | 6 | size = 10 7 | 8 | def sat_solver(res, b): 9 | # check 10 | s = Solver() 11 | # set_option("parallel.enable", True) 12 | # s.set("timeout", 60000) 13 | X = Bool("X") 14 | 15 | # Default given: set the range of symbol 16 | for r, btmp in zip(res, b): 17 | eqn = [] 18 | for t in btmp: 19 | eqn.append(PbEq([(X,1)], t - r.item())) 20 | s.add(Or(eqn)) 21 | 22 | if s.check() == sat: 23 | res = [is_true(s.model()[X])] 24 | res = np.array(res)*1.0 25 | return True, res 26 | else: 27 | return False, None 28 | 29 | 30 | 31 | 32 | if __name__ == "__main__": 33 | W = torch.zeros(1, 729) 34 | bmin = torch.ones(1,1) 35 | bmax = torch.ones(1,1) 36 | W[:,0:9] = 1.0 37 | # init_check(W, b) 38 | maxsat_solver(W, bmin, bmax) 39 | # sat_solver(W, bmin, bmax) 40 | -------------------------------------------------------------------------------- /Kitti-Driving/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | class Config: 5 | def __init__(self, **kwargs): 6 | for k, v in kwargs.items(): 7 | setattr(self, k, v) 8 | 9 | def update(self, **kwargs): 10 | for k, v in kwargs.items(): 11 | setattr(self, k, v) 12 | 13 | 14 | def setup_cfg(): 15 | # init the default cfg 16 | cfg = Config( 17 | # log dir, default evolving with time 18 | LOG_DIR=os.path.join("./logs", "exp_" + time.strftime("%y-%m-%d--%H-%M-%S", time.localtime())), 19 | EXP_NAME="", 20 | # input shape, default as the resized shape with ratio width : height = 1 : 0.47 21 | INPUT_WIDTH=750, 22 | INPUT_HEIGHT=350, 23 | # output shape, choosing from (10,10), (50,50), (100,100) 24 | OUTPUT_WIDTH=10, 25 | OUTPUT_HEIGHT=10, 26 | # dataset, default as kitti dataset 27 | DATASET="kitti", 28 | DATASET_ROOT="../data/kitti", 29 | DETECT_CLASSES=("Car", "Pedestrian", "Cyclist"), 30 | MAX_OBJECTS=30, 31 | # data augmentation 32 | AUGMENT_BLUR=False, 33 | AUGMENT_JITTER=False, 34 | AUGMENT_FLIP=False, 35 | # data loader 36 | BATCH_SIZE=64, 37 | SHUFFLE=False, 38 | NUM_WORKERS=16, 39 | # model, default using resnet18 as backbone 40 | MODEL="Resnet", 41 | NUM_LAYERS=18, 42 | PRETRAINED=False, 43 | CKPT="", 44 | # train 45 | TRAINING=True, 46 | DEVICE="cuda:0", 47 | SEED=0, 48 | LEARNING_RATE=2.0e-4, 49 | OPTIMIZER="Adam", 50 | LOSS_FUNC="Focal", 51 | START_EPOCH=0, 52 | MAX_EPOCH=30, 53 | CKPT_PERIOD=1, 54 | EVAL_PERIOD=1, 55 | RESUME=False, 56 | # test 57 | PLANNER="Astar", 58 | THRESHOLD=0.15, 59 | METRICS=("accuracy", "precision", "recall", "f1", "collide", "dist", "length"), 60 | # focal loss hyper-params 61 | ALPHA=0.75, 62 | GAMMA=2, 63 | EPSILON=1e-5, 64 | ) 65 | 66 | return cfg 67 | -------------------------------------------------------------------------------- /Kitti-Driving/data/readme.md: -------------------------------------------------------------------------------- 1 | Save the data here. 2 | -------------------------------------------------------------------------------- /Kitti-Driving/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torchvision.models import ( 4 | resnet18, resnet34, resnet50, resnet101, 5 | ResNet18_Weights, ResNet34_Weights, ResNet50_Weights, ResNet101_Weights 6 | ) 7 | from config import setup_cfg 8 | from models.mingpt import GPTConfig, GPT 9 | 10 | 11 | def build_model_resnet(cfg, device, num_layers=18, pretrained=False): 12 | model_weights_dict = { 13 | 18: (resnet18, ResNet18_Weights.DEFAULT), 14 | 34: (resnet34, ResNet34_Weights.DEFAULT), 15 | 50: (resnet50, ResNet50_Weights.DEFAULT), 16 | 101: (resnet101, ResNet101_Weights.DEFAULT), 17 | } 18 | 19 | # construct basic resnet model with pretrained weights or not 20 | weights = model_weights_dict[num_layers][1] if pretrained else None 21 | basic_model = model_weights_dict[num_layers][0](weights=weights) 22 | 23 | class ResModel(nn.Module): 24 | def __init__(self, basic_model, width, height): 25 | super(ResModel, self).__init__() 26 | self.basic_model = basic_model 27 | self.width = width 28 | self.height = height 29 | 30 | in_channels = self.basic_model._modules['fc'].in_features 31 | self.basic_model.fc = nn.Linear(in_features=in_channels, 32 | out_features=self.width * self.height, 33 | bias=True) 34 | self.activation = nn.Sigmoid() 35 | 36 | self.basic_model.fc = nn.Sequential() 37 | 38 | # self.conv = nn.Sequential( 39 | # nn.Conv2d(in_channels=in_channels, 40 | # out_channels=out_channels, 41 | # kernel_size=1, stride=1, padding=1, bias=True), 42 | # # nn.BatchNorm2d(out_channels) 43 | # group_norm(out_channels) 44 | # ) 45 | # self.basic_model.layer4.add_module(name="planning_conv", module=self.conv) 46 | # self.basic_model.avgpool = nn.AdaptiveMaxPool2d(output_size=(1, 1)) 47 | 48 | self.fc = nn.Linear(in_features=in_channels, 49 | out_features=self.width * self.height, 50 | bias=True) 51 | 52 | self.activation = nn.Sigmoid() 53 | 54 | def forward(self, x): 55 | x = self.basic_model(x) 56 | x = self.fc(x) 57 | x = x.view(x.shape[0], self.width, self.height) 58 | 59 | return x 60 | 61 | 62 | model = ResModel(basic_model, cfg.OUTPUT_WIDTH, cfg.OUTPUT_HEIGHT) 63 | model.to(device) 64 | return model 65 | 66 | 67 | def build_model_rt(cfg, device): 68 | mconf = GPTConfig(block_size=cfg.OUTPUT_WIDTH * cfg.OUTPUT_HEIGHT, num_classes=1, 69 | n_layer=1, n_head=4, n_embd=128, n_recur=32, # default using: l1r32h4 70 | all_layers=True) 71 | model = GPT(mconf) 72 | 73 | model.to(device) 74 | return model 75 | 76 | 77 | if __name__ == "__main__": 78 | # test building the model 79 | cfg = setup_cfg() 80 | 81 | resnet_model = build_model_resnet(cfg, device=cfg.DEVICE) 82 | rt_model = build_model_rt(cfg, device=cfg.DEVICE) 83 | 84 | sample = torch.rand((16, 3, 350, 750), device=cfg.DEVICE) 85 | rt_target = torch.zeros((16, 2, 10, 10)).long().to("cuda:0") 86 | y = (torch.rand((16, 10, 10)) > 0.8).long().to("cuda:0") 87 | rt_target[:, 0, :, :] = y 88 | rt_target[:, 1, :, :] = -100 89 | 90 | t1 = rt_target[5] 91 | t11 = t1[0] 92 | t12 = t1[1] 93 | 94 | resnet_output = resnet_model(sample) 95 | rt_output, loss, _ = rt_model(sample, rt_target) 96 | 97 | print() 98 | -------------------------------------------------------------------------------- /Kitti-Driving/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lizn-zn/Nesy-Programming/1410e721c7be4782b50219f7e18a4e247638edb3/Kitti-Driving/models/__init__.py -------------------------------------------------------------------------------- /Kitti-Driving/models/logic_layer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import torch 3 | from torch import nn 4 | import torch.nn.functional as F 5 | 6 | class Logic(nn.Module): 7 | def __init__(self, input_dim, hidden=512, output_dim=1): 8 | super(Logic, self).__init__() 9 | # self.fc1 = nn.Linear(input_dim, hidden) 10 | # self.fc2 = nn.Linear(hidden, output_dim) 11 | self.fc = nn.Linear(input_dim, output_dim) 12 | 13 | def forward(self, x): 14 | N, M1, M2, num_classes = x.shape 15 | out = F.softmax(x, dim=-1) 16 | out = out.reshape(N, M1*M2*num_classes) 17 | # out = self.fc1(out) 18 | # out = torch.sigmoid(out) 19 | # out = self.fc2(out) 20 | out = self.fc(out) 21 | out = torch.sigmoid(out) 22 | return out 23 | 24 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/README.md: -------------------------------------------------------------------------------- 1 | 2 | # window 版 SMOKE 3 | 好用的话记得给个 star 哈 ... 4 | 5 | ![image](../../data/kitti-smoke/figures/result.gif) 6 | 7 | ## 添加功能 8 | 9 | - 使用了 pytorch 自带的 DConv，省去 linux 下编译 DConv 的 cuda 代码，可以直接在 window 下训练和测试 10 | - 提供了重新训练的模型：[百度云盘](https://pan.baidu.com/s/1GQdUfi6W9VbsCpwshMxlxA)，提取码： hdbb 11 | - 改变训练循环，并使用梯度累加机制 12 | - 增添了 finetune 和 resume 等功能 13 | - 提供了测试单张图像的例子 14 | 15 | 16 | ## 训练 17 | 18 | 1. 下载 KITTI 数据，并修改成一下结构，把图像列表放在 ImageSets 文件夹中，然后在 datasets 中创建 kitti 目录的软连接 19 | ``` 20 | kitti 21 | │──training 22 | │ ├──calib 23 | │ ├──label_2 24 | │ ├──image_2 25 | │ └──ImageSets 26 | └──testing 27 | ├──calib 28 | ├──image_2 29 | └──ImageSets 30 | ``` 31 | 32 | 2. 修改相关参数，比如 configs/smoke_gn_vector.yaml 和 smoke/config/defaults.py 33 | 34 | 3. 执行 tools/plain_train_net.py 即可 35 | 36 | ## 测试 37 | 38 | 1. 把 smoke/engine/defaults.py 中的 --eval-only 设成 True 39 | 40 | 3. 执行 tools/plain_train_net.py 即可 41 | 42 | 不过现在 KITTI 只有论文才能在线测试了，代码只能生成测试集的结果而已 43 | 44 | ## 测试单张图像 45 | 46 | 1. 指定 tools/detector.py 中 --ckpt 为训练好的模型，直接执行 detector.py 即可 47 | 48 | ![image](figures/result.png) 49 | 50 | 本人电脑比较弱鸡，batch 很小，就只训练了 50 epoch .... 51 | 52 | ## acknowledge 53 | 54 | https://github.com/lzccccc/SMOKE 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lizn-zn/Nesy-Programming/1410e721c7be4782b50219f7e18a4e247638edb3/Kitti-Driving/models/smoke/__init__.py -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/config/__init__.py: -------------------------------------------------------------------------------- 1 | from.defaults import _C as cfg -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/config/paths_catalog.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | class DatasetCatalog(): 5 | KITTI_DATA_DIR = "data/kitti-smoke/datasets" 6 | NUSCENES_DATA_DIR = "data/nuscenes-smoke" 7 | DATASETS = { 8 | "kitti_train": { 9 | "root": "kitti/training/", 10 | }, 11 | "kitti_test": { 12 | "root": "kitti/testing/", 13 | }, 14 | # kitti smoke(keypoints only) trainval / train / val test datasets root 15 | "kitti_smoke_trainval": { 16 | "root": "kitti/training/", 17 | }, 18 | "kitti_smoke_train": { 19 | "root": "kitti/training/", 20 | }, 21 | "kitti_smoke_val": { 22 | "root": "kitti/training/", 23 | }, 24 | "kitti_smoke_test": { 25 | "root": "kitti/training/", 26 | }, 27 | # nuscenes smoke(keypoints only) trainval / train / val test datasets root 28 | "nuscenes_smoke_trainval": { 29 | "root": "trainval_datasets/", 30 | }, 31 | "nuscenes_smoke_train": { 32 | "root": "trainval_datasets/", 33 | }, 34 | "nuscenes_smoke_val": { 35 | "root": "trainval_datasets/", 36 | }, 37 | "nuscenes_smoke_test": { 38 | "root": "trainval_datasets/", 39 | }, 40 | } 41 | 42 | @staticmethod 43 | def get(name): 44 | if "kitti_smoke" in name: 45 | data_dir = DatasetCatalog.KITTI_DATA_DIR 46 | attrs = DatasetCatalog.DATASETS[name] 47 | args = dict( 48 | root=os.path.join(os.getcwd(), data_dir, attrs["root"]), 49 | split= "trainval" if "trainval" in name else( 50 | "train" if "train" in name else ("val" if "val" in name else "test")), 51 | planning=True, # set param planning here 52 | ) 53 | return dict( 54 | factory="KittiSmokeDataset", 55 | args=args, 56 | ) 57 | elif "nuscenes_smoke" in name: 58 | data_dir = DatasetCatalog.NUSCENES_DATA_DIR 59 | attrs = DatasetCatalog.DATASETS[name] 60 | args = dict( 61 | root=os.path.join(os.getcwd(), data_dir, attrs["root"]), 62 | split="trainval" if "trainval" in name else ( 63 | "train" if "train" in name else ("val" if "val" in name else "test")), 64 | planning=True, # set param planning here 65 | ) 66 | return dict( 67 | factory="NuscensesSmokeDataset", 68 | args=args, 69 | ) 70 | elif "kitti" in name: 71 | data_dir = DatasetCatalog.KITTI_DATA_DIR 72 | attrs = DatasetCatalog.DATASETS[name] 73 | args = dict( 74 | root=os.path.join(os.getcwd(), data_dir, attrs["root"]), 75 | ) 76 | return dict( 77 | factory="KITTIDataset", 78 | args=args, 79 | ) 80 | raise RuntimeError("Dataset not available: {}".format(name)) 81 | 82 | 83 | class ModelCatalog(): 84 | IMAGENET_MODELS = { 85 | "DLA34": "http://dl.yf.io/dla/models/imagenet/dla34-ba72cf86.pth" 86 | } 87 | 88 | @staticmethod 89 | def get(name): 90 | if name.startswith("ImageNetPretrained"): 91 | return ModelCatalog.get_imagenet_pretrained(name) 92 | 93 | @staticmethod 94 | def get_imagenet_pretrained(name): 95 | name = name[len("ImageNetPretrained/"):] 96 | url = ModelCatalog.IMAGENET_MODELS[name] 97 | return url 98 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/csrc/cpu/dcn_v2_cpu.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | 6 | 7 | at::Tensor 8 | dcn_v2_cpu_forward(const at::Tensor &input, 9 | const at::Tensor &weight, 10 | const at::Tensor &bias, 11 | const at::Tensor &offset, 12 | const at::Tensor &mask, 13 | const int kernel_h, 14 | const int kernel_w, 15 | const int stride_h, 16 | const int stride_w, 17 | const int pad_h, 18 | const int pad_w, 19 | const int dilation_h, 20 | const int dilation_w, 21 | const int deformable_group) 22 | { 23 | AT_ERROR("Not implement on cpu"); 24 | } 25 | 26 | std::vector 27 | dcn_v2_cpu_backward(const at::Tensor &input, 28 | const at::Tensor &weight, 29 | const at::Tensor &bias, 30 | const at::Tensor &offset, 31 | const at::Tensor &mask, 32 | const at::Tensor &grad_output, 33 | int kernel_h, int kernel_w, 34 | int stride_h, int stride_w, 35 | int pad_h, int pad_w, 36 | int dilation_h, int dilation_w, 37 | int deformable_group) 38 | { 39 | AT_ERROR("Not implement on cpu"); 40 | } 41 | 42 | std::tuple 43 | dcn_v2_psroi_pooling_cpu_forward(const at::Tensor &input, 44 | const at::Tensor &bbox, 45 | const at::Tensor &trans, 46 | const int no_trans, 47 | const float spatial_scale, 48 | const int output_dim, 49 | const int group_size, 50 | const int pooled_size, 51 | const int part_size, 52 | const int sample_per_part, 53 | const float trans_std) 54 | { 55 | AT_ERROR("Not implement on cpu"); 56 | } 57 | 58 | std::tuple 59 | dcn_v2_psroi_pooling_cpu_backward(const at::Tensor &out_grad, 60 | const at::Tensor &input, 61 | const at::Tensor &bbox, 62 | const at::Tensor &trans, 63 | const at::Tensor &top_count, 64 | const int no_trans, 65 | const float spatial_scale, 66 | const int output_dim, 67 | const int group_size, 68 | const int pooled_size, 69 | const int part_size, 70 | const int sample_per_part, 71 | const float trans_std) 72 | { 73 | AT_ERROR("Not implement on cpu"); 74 | } -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/csrc/cpu/vision.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | at::Tensor 5 | dcn_v2_cpu_forward(const at::Tensor &input, 6 | const at::Tensor &weight, 7 | const at::Tensor &bias, 8 | const at::Tensor &offset, 9 | const at::Tensor &mask, 10 | const int kernel_h, 11 | const int kernel_w, 12 | const int stride_h, 13 | const int stride_w, 14 | const int pad_h, 15 | const int pad_w, 16 | const int dilation_h, 17 | const int dilation_w, 18 | const int deformable_group); 19 | 20 | std::vector 21 | dcn_v2_cpu_backward(const at::Tensor &input, 22 | const at::Tensor &weight, 23 | const at::Tensor &bias, 24 | const at::Tensor &offset, 25 | const at::Tensor &mask, 26 | const at::Tensor &grad_output, 27 | int kernel_h, int kernel_w, 28 | int stride_h, int stride_w, 29 | int pad_h, int pad_w, 30 | int dilation_h, int dilation_w, 31 | int deformable_group); 32 | 33 | 34 | std::tuple 35 | dcn_v2_psroi_pooling_cpu_forward(const at::Tensor &input, 36 | const at::Tensor &bbox, 37 | const at::Tensor &trans, 38 | const int no_trans, 39 | const float spatial_scale, 40 | const int output_dim, 41 | const int group_size, 42 | const int pooled_size, 43 | const int part_size, 44 | const int sample_per_part, 45 | const float trans_std); 46 | 47 | std::tuple 48 | dcn_v2_psroi_pooling_cpu_backward(const at::Tensor &out_grad, 49 | const at::Tensor &input, 50 | const at::Tensor &bbox, 51 | const at::Tensor &trans, 52 | const at::Tensor &top_count, 53 | const int no_trans, 54 | const float spatial_scale, 55 | const int output_dim, 56 | const int group_size, 57 | const int pooled_size, 58 | const int part_size, 59 | const int sample_per_part, 60 | const float trans_std); -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/csrc/cuda/vision.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | at::Tensor 5 | dcn_v2_cuda_forward(const at::Tensor &input, 6 | const at::Tensor &weight, 7 | const at::Tensor &bias, 8 | const at::Tensor &offset, 9 | const at::Tensor &mask, 10 | const int kernel_h, 11 | const int kernel_w, 12 | const int stride_h, 13 | const int stride_w, 14 | const int pad_h, 15 | const int pad_w, 16 | const int dilation_h, 17 | const int dilation_w, 18 | const int deformable_group); 19 | 20 | std::vector 21 | dcn_v2_cuda_backward(const at::Tensor &input, 22 | const at::Tensor &weight, 23 | const at::Tensor &bias, 24 | const at::Tensor &offset, 25 | const at::Tensor &mask, 26 | const at::Tensor &grad_output, 27 | int kernel_h, int kernel_w, 28 | int stride_h, int stride_w, 29 | int pad_h, int pad_w, 30 | int dilation_h, int dilation_w, 31 | int deformable_group); 32 | 33 | 34 | std::tuple 35 | dcn_v2_psroi_pooling_cuda_forward(const at::Tensor &input, 36 | const at::Tensor &bbox, 37 | const at::Tensor &trans, 38 | const int no_trans, 39 | const float spatial_scale, 40 | const int output_dim, 41 | const int group_size, 42 | const int pooled_size, 43 | const int part_size, 44 | const int sample_per_part, 45 | const float trans_std); 46 | 47 | std::tuple 48 | dcn_v2_psroi_pooling_cuda_backward(const at::Tensor &out_grad, 49 | const at::Tensor &input, 50 | const at::Tensor &bbox, 51 | const at::Tensor &trans, 52 | const at::Tensor &top_count, 53 | const int no_trans, 54 | const float spatial_scale, 55 | const int output_dim, 56 | const int group_size, 57 | const int pooled_size, 58 | const int part_size, 59 | const int sample_per_part, 60 | const float trans_std); -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/csrc/vision.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "dcn_v2.h" 3 | 4 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 5 | m.def("dcn_v2_forward", &dcn_v2_forward, "dcn_v2_forward"); 6 | m.def("dcn_v2_backward", &dcn_v2_backward, "dcn_v2_backward"); 7 | m.def("dcn_v2_psroi_pooling_forward", &dcn_v2_psroi_pooling_forward, "dcn_v2_psroi_pooling_forward"); 8 | m.def("dcn_v2_psroi_pooling_backward", &dcn_v2_psroi_pooling_backward, "dcn_v2_psroi_pooling_backward"); 9 | } 10 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/data/__init__.py: -------------------------------------------------------------------------------- 1 | from .build import make_data_loader 2 | from .build import build_test_loader -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/data/collate_batch.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | from ..structures.image_list import to_image_list 3 | 4 | 5 | class BatchCollator(object): 6 | """ 7 | From a list of samples from the dataset, 8 | returns the batched images and targets. 9 | This should be passed to the DataLoader 10 | """ 11 | 12 | def __init__(self, size_divisible=0): 13 | self.size_divisible = size_divisible 14 | 15 | def __call__(self, batch): 16 | transposed_batch = list(zip(*batch)) 17 | images = to_image_list(transposed_batch[0], self.size_divisible) 18 | targets = transposed_batch[1] 19 | img_ids = transposed_batch[2] 20 | return dict(images=images, 21 | targets=targets, 22 | img_ids=img_ids) 23 | 24 | 25 | class KeypointsOnlyBatchCollator(object): 26 | """ 27 | From a list of samples from the dataset, 28 | returns the batched images, img_infos, targets, grids and trajs 29 | This should be passed to the DataLoader 30 | """ 31 | 32 | def __init__(self, size_divisible=0): 33 | self.size_divisible = size_divisible 34 | 35 | def __call__(self, batch): 36 | transposed_batch = list(zip(*batch)) 37 | images = to_image_list(transposed_batch[0], self.size_divisible) 38 | img_infos = transposed_batch[1] 39 | targets = transposed_batch[2] 40 | grids = transposed_batch[3] 41 | trajs = transposed_batch[4] 42 | return dict(images=images, 43 | img_infos=img_infos, 44 | targets=targets, 45 | grids=grids, 46 | trajs=trajs) 47 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/data/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from .kitti import KITTIDataset 2 | from .concat_dataset import ConcatDataset 3 | from .kitti_smoke_dataset import KittiSmokeDataset 4 | from .nuscenes_smoke_dataset import NuscensesSmokeDataset 5 | 6 | __all__ = ["KITTIDataset", "KittiSmokeDataset", "NuscensesSmokeDataset"] 7 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/data/datasets/concat_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | import bisect 3 | 4 | from torch.utils.data.dataset import ConcatDataset as _ConcatDataset 5 | 6 | 7 | class ConcatDataset(_ConcatDataset): 8 | """ 9 | Same as torch.utils.data.dataset.ConcatDataset, but exposes an extra 10 | method for querying the sizes of the image 11 | """ 12 | 13 | def get_idxs(self, idx): 14 | dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx) 15 | if dataset_idx == 0: 16 | sample_idx = idx 17 | else: 18 | sample_idx = idx - self.cumulative_sizes[dataset_idx - 1] 19 | return dataset_idx, sample_idx 20 | 21 | def get_img_info(self, idx): 22 | dataset_idx, sample_idx = self.get_idxs(idx) 23 | return self.datasets[dataset_idx].get_img_info(sample_idx) 24 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/data/datasets/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | from ... import datasets 2 | 3 | from .kitti.kitti_eval import kitti_evaluation 4 | 5 | 6 | def evaluate(eval_type, dataset, predictions, output_folder): 7 | """evaluate dataset using different methods based on dataset type. 8 | Args: 9 | eval_type: 10 | dataset: Dataset object 11 | predictions(list[BoxList]): each item in the list represents the 12 | prediction results for one image. 13 | output_folder: output folder, to save evaluation files or results. 14 | **kwargs: other args. 15 | Returns: 16 | evaluation result 17 | """ 18 | args = dict( 19 | eval_type=eval_type, 20 | dataset=dataset, 21 | predictions=predictions, 22 | output_folder=output_folder, 23 | 24 | ) 25 | if isinstance(dataset, datasets.KITTIDataset): 26 | return kitti_evaluation(**args) 27 | else: 28 | dataset_name = dataset.__class__.__name__ 29 | raise NotImplementedError("Unsupported dataset type {}.".format(dataset_name)) 30 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/data/datasets/evaluation/kitti/kitti_eval.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import logging 4 | import subprocess 5 | 6 | from .....utils.miscellaneous import mkdir 7 | 8 | ID_TYPE_CONVERSION = { 9 | 0: 'Car', 10 | 1: 'Cyclist', 11 | 2: 'Pedestrian' 12 | } 13 | 14 | 15 | def kitti_evaluation( 16 | eval_type, 17 | dataset, 18 | predictions, 19 | output_folder, 20 | ): 21 | logger = logging.getLogger(__name__) 22 | if "detection" in eval_type: 23 | logger.info("performing kitti detection evaluation: ") 24 | do_kitti_detection_evaluation( 25 | dataset=dataset, 26 | predictions=predictions, 27 | output_folder=output_folder, 28 | logger=logger 29 | ) 30 | 31 | 32 | def do_kitti_detection_evaluation(dataset, 33 | predictions, 34 | output_folder, 35 | logger 36 | ): 37 | predict_folder = os.path.join(output_folder, 'data') # only recognize data 38 | mkdir(predict_folder) 39 | 40 | for image_id, prediction in predictions.items(): 41 | predict_txt = image_id + '.txt' 42 | predict_txt = os.path.join(predict_folder, predict_txt) 43 | 44 | generate_kitti_3d_detection(prediction, predict_txt) 45 | 46 | logger.info("Evaluate on KITTI dataset") 47 | output_dir = os.path.abspath(output_folder) 48 | os.chdir('./smoke/data/datasets/evaluation/kitti/kitti_eval') 49 | label_dir = getattr(dataset, 'label_dir') 50 | if not os.path.isfile('evaluate_object_3d_offline'): 51 | subprocess.Popen('g++ -O3 -DNDEBUG -o evaluate_object_3d_offline evaluate_object_3d_offline.cpp', shell=True) 52 | command = "./evaluate_object_3d_offline.exe {} {}".format(label_dir, output_dir) 53 | output = subprocess.check_output(command, shell=True, universal_newlines=True).strip() 54 | logger.info(output) 55 | os.chdir('./tools') 56 | 57 | 58 | def generate_kitti_3d_detection(prediction, predict_txt): 59 | with open(predict_txt, 'w', newline='') as f: 60 | w = csv.writer(f, delimiter=' ', lineterminator='\n') 61 | if len(prediction) == 0: 62 | w.writerow([]) 63 | else: 64 | for p in prediction: 65 | p = p.numpy() 66 | p = p.round(4) 67 | type = ID_TYPE_CONVERSION[int(p[0])] 68 | row = [type, 0, 0] + p[1:].tolist() 69 | w.writerow(row) 70 | 71 | check_last_line_break(predict_txt) 72 | 73 | 74 | def check_last_line_break(predict_txt): 75 | f = open(predict_txt, 'rb+') 76 | try: 77 | f.seek(-1, os.SEEK_END) 78 | except: 79 | pass 80 | else: 81 | if f.__next__() == b'\n': 82 | f.seek(-1, os.SEEK_END) 83 | f.truncate() 84 | f.close() 85 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/data/datasets/evaluation/kitti/kitti_eval/kitti_eval.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lizn-zn/Nesy-Programming/1410e721c7be4782b50219f7e18a4e247638edb3/Kitti-Driving/models/smoke/data/datasets/evaluation/kitti/kitti_eval/kitti_eval.exe -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/data/samplers/__init__.py: -------------------------------------------------------------------------------- 1 | from .grouped_batch_sampler import GroupedBatchSampler 2 | from .distributed_sampler import ( 3 | TrainingSampler, 4 | InferenceSampler, 5 | ) 6 | 7 | __all__ = ["GroupedBatchSampler", 8 | "TrainingSampler", 9 | "InferenceSampler",] 10 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/data/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | from .build import build_transforms -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/data/transforms/build.py: -------------------------------------------------------------------------------- 1 | from . import transforms as T 2 | 3 | 4 | def build_transforms(cfg, is_train=True): 5 | to_bgr = cfg.INPUT.TO_BGR 6 | 7 | normalize_transform = T.Normalize( 8 | mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD, to_bgr=to_bgr 9 | ) 10 | 11 | transform = T.Compose( 12 | [ 13 | T.ToTensor(), 14 | normalize_transform, 15 | ] 16 | ) 17 | return transform 18 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/data/transforms/transforms.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torchvision.transforms import functional as F 3 | 4 | 5 | class Compose(): 6 | def __init__(self, transforms): 7 | self.transforms = transforms 8 | 9 | def __call__(self, image, target): 10 | for t in self.transforms: 11 | image, target = t(image, target) 12 | return image, target 13 | 14 | 15 | class ToTensor(): 16 | def __call__(self, image, target): 17 | return F.to_tensor(image), target 18 | 19 | 20 | class Normalize(): 21 | def __init__(self, mean, std, to_bgr=True): 22 | self.mean = mean 23 | self.std = std 24 | self.to_bgr = to_bgr 25 | 26 | def __call__(self, image, target): 27 | if self.to_bgr: 28 | image = image[[2, 1, 0]] 29 | image = F.normalize(image, mean=self.mean, std=self.std) 30 | return image, target 31 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/engine/__init__.py: -------------------------------------------------------------------------------- 1 | from .defaults import * 2 | from .launch import * 3 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/engine/defaults.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import torch 5 | 6 | from ..utils import comm 7 | from ..utils.miscellaneous import mkdir 8 | from ..utils.logger import setup_logger 9 | from ..utils.collect_env import collect_env_info 10 | from ..utils.envs import seed_all_rng 11 | 12 | import platform 13 | 14 | __all__ = ["default_argument_parser", "default_setup"] 15 | 16 | 17 | def default_argument_parser(): 18 | parser = argparse.ArgumentParser(description="Detectron2 Training") 19 | parser.add_argument("--config-file", default="./data/kitti-smoke/configs/smoke_gn_vector.yaml", 20 | metavar="FILE", help="path to config file") 21 | parser.add_argument("--eval-only", action="store_true", help="perform evaluation only") 22 | parser.add_argument( 23 | "--ckpt", 24 | help="The path to the checkpoint for test, default is the latest checkpoint.", 25 | default=None, 26 | ) 27 | parser.add_argument("--num-gpus", type=int, default=1, help="number of gpus *per machine*") 28 | parser.add_argument("--num-machines", type=int, default=1) 29 | parser.add_argument( 30 | "--machine-rank", type=int, default=0, help="the rank of this machine (unique per machine)" 31 | ) 32 | 33 | # PyTorch still may leave orphan processes in multi-gpu training. 34 | # Therefore we use a deterministic way to obtain port, 35 | # so that users are aware of orphan processes by seeing the port occupied. 36 | plat = platform.system().lower() 37 | if plat == 'windows': 38 | port = 100 39 | else: 40 | port = 2 ** 15 + 2 ** 14 + hash(os.getuid()) % 2 ** 14 41 | parser.add_argument("--dist-url", default="tcp://127.0.0.1:{}".format(port)) 42 | parser.add_argument( 43 | "opts", 44 | help="Modify config options using the command-line", 45 | default=None, 46 | nargs=argparse.REMAINDER, 47 | ) 48 | return parser 49 | 50 | 51 | def default_setup(cfg, args): 52 | output_dir = cfg.OUTPUT_DIR 53 | if output_dir: 54 | mkdir(output_dir) 55 | 56 | rank = comm.get_rank() 57 | logger = setup_logger(output_dir, rank) 58 | logger.info("Using {} GPUs".format(args.num_gpus)) 59 | logger.info("Collecting environment info") 60 | logger.info("\n" + collect_env_info()) 61 | logger.info(args) 62 | 63 | logger.info("Loaded configuration file {}".format(args.config_file)) 64 | with open(args.config_file, "r") as cf: 65 | config_str = "\n" + cf.read() 66 | logger.info(config_str) 67 | logger.info("Running with config:\n{}".format(cfg)) 68 | 69 | # make sure each worker has a different, yet deterministic seed if specified 70 | seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank) 71 | 72 | # cudnn benchmark has large overhead. It shouldn't be used considering the small size of 73 | # typical validation set. 74 | if not (hasattr(args, "eval_only") and args.eval_only): 75 | torch.backends.cudnn.benchmark = cfg.CUDNN_BENCHMARK 76 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/engine/inference.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from tqdm import tqdm 3 | 4 | import torch 5 | 6 | from ..utils import comm 7 | from ..utils.timer import Timer, get_time_str 8 | from ..data.datasets.evaluation import evaluate 9 | 10 | 11 | def compute_on_dataset(model, data_loader, device, timer=None): 12 | model.eval() 13 | results_dict = {} 14 | cpu_device = torch.device("cpu") 15 | for _, batch in enumerate(tqdm(data_loader)): 16 | images, targets, image_ids = batch["images"], batch["targets"], batch["img_ids"] 17 | images = images.to(device) 18 | with torch.no_grad(): 19 | if timer: 20 | timer.tic() 21 | output = model(images, targets) 22 | if timer: 23 | torch.cuda.synchronize() 24 | timer.toc() 25 | output = output.to(cpu_device) 26 | results_dict.update( 27 | {img_id: output for img_id in image_ids} 28 | ) 29 | return results_dict 30 | 31 | 32 | def inference( 33 | model, 34 | data_loader, 35 | dataset_name, 36 | eval_types=("detections",), 37 | device="cuda", 38 | output_folder=None, 39 | 40 | ): 41 | device = torch.device(device) 42 | num_devices = comm.get_world_size() 43 | logger = logging.getLogger(__name__) 44 | dataset = data_loader.dataset 45 | logger.info("Start evaluation on {} dataset({} images).".format(dataset_name, len(dataset))) 46 | 47 | total_timer = Timer() 48 | inference_timer = Timer() 49 | total_timer.tic() 50 | predictions = compute_on_dataset(model, data_loader, device, inference_timer) 51 | comm.synchronize() 52 | 53 | total_time = total_timer.toc() 54 | total_time_str = get_time_str(total_time) 55 | logger.info( 56 | "Total run time: {} ({} s / img per device, on {} devices)".format( 57 | total_time_str, total_time * num_devices / len(dataset), num_devices 58 | ) 59 | ) 60 | total_infer_time = get_time_str(inference_timer.total_time) 61 | logger.info( 62 | "Model inference time: {} ({} s / img per device, on {} devices)".format( 63 | total_infer_time, 64 | inference_timer.total_time * num_devices / len(dataset), 65 | num_devices, 66 | ) 67 | ) 68 | if not comm.is_main_process(): 69 | return 70 | 71 | return evaluate(eval_type=eval_types, 72 | dataset=dataset, 73 | predictions=predictions, 74 | output_folder=output_folder, ) 75 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/engine/launch.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import torch 3 | import torch.distributed as dist 4 | import torch.multiprocessing as mp 5 | 6 | from ..utils import comm 7 | 8 | __all__ = ["launch"] 9 | 10 | 11 | def _find_free_port(): 12 | import socket 13 | 14 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 15 | # Binding to port 0 will cause the OS to find an available port for us 16 | sock.bind(("", 0)) 17 | port = sock.getsockname()[1] 18 | sock.close() 19 | # NOTE: there is still a chance the port could be taken by other processes. 20 | return port 21 | 22 | 23 | def launch(main_func, 24 | num_gpus_per_machine, 25 | num_machines=1, 26 | machine_rank=0, 27 | dist_url=None, 28 | args=()): 29 | """ 30 | Args: 31 | main_func: a function that will be called by `main_func(*args)` 32 | num_machines (int): the total number of machines 33 | machine_rank (int): the rank of this machine (one per machine) 34 | dist_url (str): url to connect to for distributed training, including protocol 35 | e.g. "tcp://127.0.0.1:8686". 36 | Can be set to auto to automatically select a free port on localhost 37 | args (tuple): arguments passed to main_func 38 | """ 39 | world_size = num_machines * num_gpus_per_machine 40 | if world_size > 1: 41 | # https://github.com/pytorch/pytorch/pull/14391 42 | # TODO prctl in spawned processes 43 | 44 | if dist_url == "auto": 45 | assert num_machines == 1, "dist_url=auto cannot work with distributed training." 46 | port = _find_free_port() 47 | dist_url = f"tcp://127.0.0.1:{port}" 48 | 49 | mp.spawn( 50 | _distributed_worker, 51 | nprocs=num_gpus_per_machine, 52 | args=(main_func, world_size, num_gpus_per_machine, machine_rank, dist_url, args), 53 | daemon=False, 54 | ) 55 | else: 56 | main_func(*args) 57 | 58 | 59 | def _distributed_worker( 60 | local_rank, main_func, world_size, num_gpus_per_machine, machine_rank, dist_url, args 61 | ): 62 | assert torch.cuda.is_available(), "cuda is not available. Please check your installation." 63 | global_rank = machine_rank * num_gpus_per_machine + local_rank 64 | try: 65 | dist.init_process_group( 66 | backend="NCCL", init_method=dist_url, world_size=world_size, rank=global_rank 67 | ) 68 | except Exception as e: 69 | logger = logging.getLogger(__name__) 70 | logger.error("Process group URL: {}".format(dist_url)) 71 | raise e 72 | # synchronize is needed here to prevent a possible timeout after calling init_process_group 73 | # See: https://github.com/facebookresearch/maskrcnn-benchmark/issues/172 74 | comm.synchronize() 75 | 76 | assert num_gpus_per_machine <= torch.cuda.device_count() 77 | torch.cuda.set_device(local_rank) 78 | 79 | # Setup the local process group (which contains ranks within the same machine) 80 | assert comm._LOCAL_PROCESS_GROUP is None 81 | num_machines = world_size // num_gpus_per_machine 82 | for i in range(num_machines): 83 | ranks_on_i = list(range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine)) 84 | pg = dist.new_group(ranks_on_i) 85 | if i == machine_rank: 86 | comm._LOCAL_PROCESS_GROUP = pg 87 | 88 | main_func(*args) 89 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/engine/test_net.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from ..data import build_test_loader 4 | from ..engine.inference import inference 5 | from ..utils import comm 6 | from ..utils.miscellaneous import mkdir 7 | 8 | 9 | def run_test(cfg, model): 10 | eval_types = ("detection",) 11 | output_folders = [None] * len(cfg.DATASETS.TEST) 12 | dataset_names = cfg.DATASETS.TEST 13 | if cfg.OUTPUT_DIR: 14 | for idx, dataset_name in enumerate(dataset_names): 15 | output_folder = os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name) 16 | mkdir(output_folder) 17 | output_folders[idx] = output_folder 18 | data_loaders_val = build_test_loader(cfg) 19 | for output_folder, dataset_name, data_loader_val in zip(output_folders, dataset_names, data_loaders_val): 20 | inference( 21 | model, 22 | data_loader_val, 23 | dataset_name=dataset_name, 24 | eval_types=eval_types, 25 | device=cfg.MODEL.DEVICE, 26 | output_folder=output_folder, 27 | ) 28 | comm.synchronize() 29 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/layers/deform_conv.py: -------------------------------------------------------------------------------- 1 | import torch.nn.functional as F 2 | from torch import nn 3 | 4 | # from smoke.layers.dcn_v2 import DCN 5 | from .dcn_hdb import DCN 6 | 7 | 8 | class DeformConv(nn.Module): 9 | def __init__(self, 10 | in_channel, 11 | out_channel, 12 | norm_func): 13 | super(DeformConv, self).__init__() 14 | 15 | self.norm = norm_func(out_channel) 16 | self.relu = nn.ReLU(inplace=True) 17 | self.deform_conv = DCN(in_channels=in_channel, 18 | out_channels=out_channel, 19 | kernel_size=(3, 3), 20 | stride=1, 21 | padding=1, 22 | dilation=1, 23 | deformable_groups=1) 24 | 25 | 26 | def forward(self, x): 27 | x = self.deform_conv(x) 28 | x = self.norm(x) 29 | x = self.relu(x) 30 | 31 | return x 32 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/layers/focal_loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | 4 | 5 | class FocalLoss(nn.Module): 6 | def __init__(self, alpha=2, beta=4): 7 | super(FocalLoss, self).__init__() 8 | self.alpha = alpha 9 | self.beta = beta 10 | 11 | def forward(self, prediction, target): 12 | positive_index = target.eq(1).float() 13 | negative_index = target.lt(1).float() 14 | 15 | negative_weights = torch.pow(1 - target, self.beta) 16 | loss = 0. 17 | 18 | positive_loss = torch.log(prediction) \ 19 | * torch.pow(1 - prediction, self.alpha) * positive_index 20 | negative_loss = torch.log(1 - prediction) \ 21 | * torch.pow(prediction, self.alpha) * negative_weights * negative_index 22 | 23 | num_positive = positive_index.float().sum() 24 | positive_loss = positive_loss.sum() 25 | negative_loss = negative_loss.sum() 26 | 27 | if num_positive == 0: 28 | loss -= negative_loss 29 | else: 30 | loss -= (positive_loss + negative_loss) / num_positive 31 | 32 | return loss 33 | 34 | 35 | class ClampedFocalLoss(nn.Module): 36 | def __init__(self, alpha=0.75, gamma=2, epsilon=0.0001): 37 | super(ClampedFocalLoss, self).__init__() 38 | self.alpha = alpha 39 | self.gamma = gamma 40 | self.epsilon = epsilon 41 | 42 | def forward(self, prediction, target): 43 | positive_index = target.eq(1).float() 44 | negative_index = target.lt(1).float() 45 | 46 | # negative_weights = torch.pow(1 - target, self.beta) 47 | loss = 0. 48 | 49 | # clamp the prediction to avoid log(0) 50 | prediction = torch.clamp(prediction, min=self.epsilon,max=1-self.epsilon) 51 | # 52 | positive_loss = -self.alpha * torch.pow(1-prediction, self.gamma) * torch.log(prediction) * positive_index 53 | negative_loss = -(1-self.alpha) * torch.pow(prediction, self.gamma) * torch.log(1-prediction) * negative_index 54 | 55 | # positive_loss = torch.log(prediction) \ 56 | # * torch.pow(1 - prediction, self.alpha) * positive_index 57 | # negative_loss = torch.log(1 - prediction) \ 58 | # * torch.pow(prediction, self.alpha) * negative_weights * negative_index 59 | # 60 | 61 | num_positive = positive_index.float().sum() 62 | positive_loss = positive_loss.sum() 63 | negative_loss = negative_loss.sum() 64 | 65 | # loss = positive_loss + negative_loss 66 | 67 | if num_positive == 0.: 68 | loss += negative_loss 69 | else: 70 | loss += (positive_loss + negative_loss) / num_positive 71 | 72 | return loss 73 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/layers/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.nn import functional as F 3 | 4 | 5 | def sigmoid_hm(hm_features): 6 | x = hm_features.sigmoid_() 7 | x = x.clamp(min=1e-4, max=1 - 1e-4) 8 | 9 | return x 10 | 11 | 12 | def nms_hm(heat_map, kernel=3): 13 | pad = (kernel - 1) // 2 14 | 15 | hmax = F.max_pool2d(heat_map, 16 | kernel_size=(kernel, kernel), 17 | stride=1, 18 | padding=pad) 19 | eq_index = (hmax == heat_map).float() 20 | 21 | return heat_map * eq_index 22 | 23 | 24 | def select_topk(heat_map, K=100): 25 | ''' 26 | Args: 27 | heat_map: heat_map in [N, C, H, W] 28 | K: top k samples to be selected 29 | score: detection threshold 30 | 31 | Returns: 32 | 33 | ''' 34 | batch, cls, height, width = heat_map.size() 35 | 36 | # First select topk scores in all classes and batchs 37 | # [N, C, H, W] -----> [N, C, H*W] 38 | heat_map = heat_map.view(batch, cls, -1) 39 | # Both in [N, C, K] 40 | topk_scores_all, topk_inds_all = torch.topk(heat_map, K) 41 | 42 | # topk_inds_all = topk_inds_all % (height * width) # todo: this seems redudant 43 | topk_ys = (topk_inds_all / width).float() 44 | topk_xs = (topk_inds_all % width).float() 45 | 46 | assert isinstance(topk_xs, torch.cuda.FloatTensor) 47 | assert isinstance(topk_ys, torch.cuda.FloatTensor) 48 | 49 | # Select topK examples across channel 50 | # [N, C, K] -----> [N, C*K] 51 | topk_scores_all = topk_scores_all.view(batch, -1) 52 | # Both in [N, K] 53 | topk_scores, topk_inds = torch.topk(topk_scores_all, K) 54 | topk_clses = (topk_inds / K).float() 55 | 56 | assert isinstance(topk_clses, torch.cuda.FloatTensor) 57 | 58 | # First expand it as 3 dimension 59 | topk_inds_all = _gather_feat(topk_inds_all.view(batch, -1, 1), topk_inds).view(batch, K) 60 | topk_ys = _gather_feat(topk_ys.view(batch, -1, 1), topk_inds).view(batch, K) 61 | topk_xs = _gather_feat(topk_xs.view(batch, -1, 1), topk_inds).view(batch, K) 62 | 63 | return topk_scores, topk_inds_all, topk_clses, topk_ys, topk_xs 64 | 65 | 66 | def _gather_feat(feat, ind): 67 | ''' 68 | Select specific indexs on featuremap 69 | Args: 70 | feat: all results in 3 dimensions 71 | ind: positive index 72 | 73 | Returns: 74 | 75 | ''' 76 | channel = feat.size(-1) 77 | ind = ind.unsqueeze(-1).expand(ind.size(0), ind.size(1), channel) 78 | feat = feat.gather(1, ind) 79 | 80 | return feat 81 | 82 | 83 | def select_point_of_interest(batch, index, feature_maps): 84 | ''' 85 | Select POI(point of interest) on feature map 86 | Args: 87 | batch: batch size 88 | index: in point format or index format 89 | feature_maps: regression feature map in [N, C, H, W] 90 | 91 | Returns: 92 | 93 | ''' 94 | w = feature_maps.shape[3] 95 | if len(index.shape) == 3: 96 | index = index[:, :, 1] * w + index[:, :, 0] 97 | index = index.view(batch, -1) 98 | # [N, C, H, W] -----> [N, H, W, C] 99 | feature_maps = feature_maps.permute(0, 2, 3, 1).contiguous() 100 | channel = feature_maps.shape[-1] 101 | # [N, H, W, C] -----> [N, H*W, C] 102 | feature_maps = feature_maps.view(batch, -1, channel) 103 | # expand index in channels 104 | index = index.unsqueeze(-1).repeat(1, 1, channel) 105 | # select specific features bases on POIs 106 | feature_maps = feature_maps.gather(1, index.long()) 107 | 108 | return feature_maps 109 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/modeling/backbone/__init__.py: -------------------------------------------------------------------------------- 1 | from .backbone import build_backbone -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/modeling/backbone/backbone.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | from torch import nn 4 | 5 | from .. import registry 6 | from . import dla 7 | 8 | @registry.BACKBONES.register("DLA-34-DCN") 9 | def build_dla_backbone(cfg): 10 | body = dla.DLA(cfg) 11 | model = nn.Sequential(OrderedDict([("body", body)])) 12 | model.out_channels = cfg.MODEL.BACKBONE.BACKBONE_OUT_CHANNELS 13 | return model 14 | 15 | 16 | def build_backbone(cfg): 17 | assert cfg.MODEL.BACKBONE.CONV_BODY in registry.BACKBONES, \ 18 | "cfg.MODEL.BACKBONE.CONV_BODY: {} are not registered in registry".format( 19 | cfg.MODEL.BACKBONE.CONV_BODY 20 | ) 21 | return registry.BACKBONES[cfg.MODEL.BACKBONE.CONV_BODY](cfg) 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/modeling/detector/__init__.py: -------------------------------------------------------------------------------- 1 | from .detectors import build_detection_model -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/modeling/detector/detectors.py: -------------------------------------------------------------------------------- 1 | from .keypoint_detector import KeypointDetector, KeypointsOnlyDetector, BinaryClassificationDetector 2 | 3 | 4 | def build_detection_model(cfg): 5 | if cfg.MODEL.SMOKE_ON: 6 | return KeypointDetector(cfg) 7 | elif cfg.MODEL.KEYPOINTS_ONLY: # build the Keypoints-Only Detector for our own initial experiments 8 | return KeypointsOnlyDetector(cfg) 9 | elif cfg.MODEL.BINARY_CLASSIFICATION: 10 | return BinaryClassificationDetector(cfg) -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/modeling/heads/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lizn-zn/Nesy-Programming/1410e721c7be4782b50219f7e18a4e247638edb3/Kitti-Driving/models/smoke/modeling/heads/__init__.py -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/modeling/heads/heads.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from .smoke_head.smoke_head import build_smoke_head, build_smoke_keypoints_only_head 4 | 5 | 6 | def build_heads(cfg, in_channels): 7 | if cfg.MODEL.SMOKE_ON: 8 | return build_smoke_head(cfg, in_channels) 9 | elif cfg.MODEL.KEYPOINTS_ONLY: # build SMOKEKeypointsOnlyHead used for KeyPointsOnlyDetector constructed by ourselves 10 | return build_smoke_keypoints_only_head(cfg, in_channels) 11 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/modeling/heatmap_coder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from skimage import transform as trans 3 | 4 | 5 | def get_transfrom_matrix(center_scale, output_size): 6 | center, scale = center_scale[0], center_scale[1] 7 | # todo: further add rot and shift here. 8 | src_w = scale[0] 9 | dst_w = output_size[0] 10 | dst_h = output_size[1] 11 | 12 | src_dir = np.array([0, src_w * -0.5]) 13 | dst_dir = np.array([0, dst_w * -0.5]) 14 | 15 | src = np.zeros((3, 2), dtype=np.float32) 16 | dst = np.zeros((3, 2), dtype=np.float32) 17 | src[0, :] = center 18 | src[1, :] = center + src_dir 19 | dst[0, :] = np.array([dst_w * 0.5, dst_h * 0.5]) 20 | dst[1, :] = np.array([dst_w * 0.5, dst_h * 0.5]) + dst_dir 21 | 22 | src[2, :] = get_3rd_point(src[0, :], src[1, :]) 23 | dst[2, :] = get_3rd_point(dst[0, :], dst[1, :]) 24 | 25 | get_matrix = trans.estimate_transform("affine", src, dst) 26 | matrix = get_matrix.params 27 | 28 | return matrix.astype(np.float32) 29 | 30 | 31 | def affine_transform(point, matrix): 32 | point_exd = np.array([point[0], point[1], 1.]) 33 | new_point = np.matmul(matrix, point_exd) 34 | 35 | return new_point[:2] 36 | 37 | 38 | def get_3rd_point(point_a, point_b): 39 | d = point_a - point_b 40 | point_c = point_b + np.array([-d[1], d[0]]) 41 | return point_c 42 | 43 | 44 | def gaussian_radius(h, w, thresh_min=0.7): 45 | a1 = 1 46 | b1 = h + w 47 | c1 = h * w * (1 - thresh_min) / (1 + thresh_min) 48 | sq1 = np.sqrt(b1 ** 2 - 4 * a1 * c1) 49 | r1 = (b1 - sq1) / (2 * a1) 50 | 51 | a2 = 4 52 | b2 = 2 * (h + w) 53 | c2 = (1 - thresh_min) * w * h 54 | sq2 = np.sqrt(b2 ** 2 - 4 * a2 * c2) 55 | r2 = (b2 - sq2) / (2 * a2) 56 | 57 | a3 = 4 * thresh_min 58 | b3 = -2 * thresh_min * (h + w) 59 | c3 = (thresh_min - 1) * w * h 60 | sq3 = np.sqrt(b3 ** 2 - 4 * a3 * c3) 61 | r3 = (b3 + sq3) / (2 * a3) 62 | 63 | return min(r1, r2, r3) 64 | 65 | 66 | def gaussian2D(shape, sigma=1): 67 | m, n = [(ss - 1.) / 2. for ss in shape] 68 | y, x = np.ogrid[-m:m + 1, -n:n + 1] 69 | 70 | h = np.exp(-(x * x + y * y) / (2 * sigma * sigma)) 71 | h[h < np.finfo(h.dtype).eps * h.max()] = 0 72 | return h 73 | 74 | 75 | def draw_umich_gaussian(heatmap, center, radius, k=1): 76 | diameter = 2 * radius + 1 77 | gaussian = gaussian2D((diameter, diameter), sigma=diameter / 6) 78 | 79 | x, y = int(center[0]), int(center[1]) 80 | 81 | height, width = heatmap.shape[0:2] 82 | 83 | left, right = min(x, radius), min(width - x, radius + 1) 84 | top, bottom = min(y, radius), min(height - y, radius + 1) 85 | 86 | masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] 87 | masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] 88 | if min(masked_gaussian.shape) > 0 and min(masked_heatmap.shape) > 0: 89 | np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) 90 | 91 | return heatmap 92 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/modeling/make_layers.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | from torch import nn 5 | 6 | from ..config import cfg 7 | 8 | 9 | def _make_conv_level(in_channels, out_channels, num_convs, norm_func, 10 | stride=1, dilation=1): 11 | """ 12 | make conv layers based on its number. 13 | """ 14 | modules = [] 15 | for i in range(num_convs): 16 | modules.extend([ 17 | nn.Conv2d(in_channels, out_channels, kernel_size=3, 18 | stride=stride if i == 0 else 1, 19 | padding=dilation, bias=False, dilation=dilation), 20 | norm_func(out_channels), 21 | nn.ReLU(inplace=True)]) 22 | in_channels = out_channels 23 | 24 | return nn.Sequential(*modules) 25 | 26 | 27 | def group_norm(out_channels): 28 | num_groups = cfg.MODEL.GROUP_NORM.NUM_GROUPS 29 | if out_channels % 32 == 0: 30 | return nn.GroupNorm(num_groups, out_channels) 31 | else: 32 | return nn.GroupNorm(num_groups // 2, out_channels) 33 | 34 | 35 | def _fill_up_weights(up): 36 | # todo: we can replace math here? 37 | w = up.weight.data 38 | f = math.ceil(w.size(2) / 2) 39 | c = (2 * f - 1 - f % 2) / (2. * f) 40 | for i in range(w.size(2)): 41 | for j in range(w.size(3)): 42 | w[0, 0, i, j] = \ 43 | (1 - math.fabs(i / f - c)) * (1 - math.fabs(j / f - c)) 44 | for c in range(1, w.size(0)): 45 | w[c, 0, :, :] = w[0, 0, :, :] 46 | 47 | 48 | def _fill_fc_weights(layers): 49 | for m in layers.modules(): 50 | if isinstance(m, nn.Conv2d): 51 | if m.bias is not None: 52 | nn.init.constant_(m.bias, 0) 53 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/modeling/registry.py: -------------------------------------------------------------------------------- 1 | from ..utils.registry import Registry 2 | 3 | BACKBONES = Registry() 4 | SMOKE_HEADS = Registry() 5 | SMOKE_PREDICTOR = Registry() 6 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/modeling/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def cat(tensors, dim=0): 4 | """ 5 | Efficient version of torch.cat that avoids a copy if there is only a single element in a list 6 | """ 7 | assert isinstance(tensors, (list, tuple)) 8 | if len(tensors) == 1: 9 | return tensors[0] 10 | return torch.cat(tensors, dim) 11 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/solver/__init__.py: -------------------------------------------------------------------------------- 1 | from .build import make_optimizer -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/solver/build.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def make_optimizer(cfg, model): 5 | params = [] 6 | for key, value in model.named_parameters(): 7 | if not value.requires_grad: 8 | continue 9 | lr = cfg.SOLVER.BASE_LR 10 | if "bias" in key: 11 | lr = cfg.SOLVER.BASE_LR * cfg.SOLVER.BIAS_LR_FACTOR 12 | params += [{"params": [value], "lr": lr}] 13 | 14 | optimizer = torch.optim.Adam(params, lr=lr) 15 | 16 | return optimizer 17 | 18 | 19 | def make_lr_scheduler(cfg, optimizer): 20 | return torch.optim.lr_scheduler.MultiStepLR( 21 | optimizer, 22 | cfg.SOLVER.STEPS 23 | ) 24 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/structures/image_list.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | 3 | import torch 4 | 5 | 6 | class ImageList(): 7 | """ 8 | Structure that holds a list of images (of possibly 9 | varying sizes) as a single tensor. 10 | This works by padding the images to the same size, 11 | and storing in a field the original sizes of each image 12 | """ 13 | 14 | def __init__(self, tensors, image_sizes): 15 | """ 16 | Arguments: 17 | tensors (tensor) 18 | image_sizes (list[tuple[int, int]]) 19 | """ 20 | self.tensors = tensors 21 | self.image_sizes = image_sizes 22 | 23 | def to(self, *args, **kwargs): 24 | cast_tensor = self.tensors.to(*args, **kwargs) 25 | return ImageList(cast_tensor, self.image_sizes) 26 | 27 | 28 | def to_image_list(tensors, size_divisible=0): 29 | """ 30 | tensors can be an ImageList, a torch.Tensor or 31 | an iterable of Tensors. It can't be a numpy array. 32 | When tensors is an iterable of Tensors, it pads 33 | the Tensors with zeros so that they have the same 34 | shape 35 | """ 36 | if isinstance(tensors, torch.Tensor) and size_divisible > 0: 37 | tensors = [tensors] 38 | 39 | if isinstance(tensors, ImageList): 40 | return tensors 41 | elif isinstance(tensors, torch.Tensor): 42 | # single tensor shape can be inferred 43 | if tensors.dim() == 3: 44 | tensors = tensors[None] 45 | assert tensors.dim() == 4 46 | image_sizes = [tensor.shape[-2:] for tensor in tensors] 47 | return ImageList(tensors, image_sizes) 48 | elif isinstance(tensors, (tuple, list)): 49 | max_size = tuple(max(s) for s in zip(*[img.shape for img in tensors])) 50 | 51 | # TODO Ideally, just remove this and let me model handle arbitrary 52 | # input sizs 53 | if size_divisible > 0: 54 | import math 55 | 56 | stride = size_divisible 57 | max_size = list(max_size) 58 | max_size[1] = int(math.ceil(max_size[1] / stride) * stride) 59 | max_size[2] = int(math.ceil(max_size[2] / stride) * stride) 60 | max_size = tuple(max_size) 61 | 62 | batch_shape = (len(tensors),) + max_size 63 | batched_imgs = tensors[0].new(*batch_shape).zero_() 64 | for img, pad_img in zip(tensors, batched_imgs): 65 | pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) 66 | 67 | image_sizes = [im.shape[-2:] for im in tensors] 68 | 69 | return ImageList(batched_imgs, image_sizes) 70 | else: 71 | raise TypeError("Unsupported type for to_image_list: {}".format(type(tensors))) -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/structures/params_3d.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class ParamsList(): 5 | """ 6 | This class represents labels of specific object. 7 | """ 8 | 9 | def __init__(self, image_size, is_train=True): 10 | self.size = image_size 11 | self.is_train = is_train 12 | self.extra_fields = {} 13 | 14 | def add_field(self, field, field_data): 15 | field_data = field_data if isinstance(field_data, torch.Tensor) else torch.as_tensor(field_data) 16 | self.extra_fields[field] = field_data 17 | 18 | def get_field(self, field): 19 | return self.extra_fields[field] 20 | 21 | def has_field(self, field): 22 | return field in self.extra_fields 23 | 24 | def fields(self): 25 | return list(self.extra_fields.keys()) 26 | 27 | def _copy_extra_fields(self, target): 28 | for k, v in target.extra_fields.items(): 29 | self.extra_fields[k] = v 30 | 31 | def to(self, device): 32 | target = ParamsList(self.size, self.is_train) 33 | for k, v in self.extra_fields.items(): 34 | if hasattr(v, "to"): 35 | v = v.to(device) 36 | target.add_field(k, v) 37 | return target 38 | 39 | def __len__(self): 40 | if self.is_train: 41 | reg_num = len(torch.nonzero(self.extra_fields["reg_mask"])) 42 | else: 43 | reg_num = 0 44 | return reg_num 45 | 46 | def __repr__(self): 47 | s = self.__class__.__name__ + "(" 48 | s += "regress_number={}, ".format(len(self)) 49 | s += "image_width={}, ".format(self.size[0]) 50 | s += "image_height={})".format(self.size[1]) 51 | return s 52 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/utils/collect_env.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | import PIL 3 | 4 | from torch.utils.collect_env import get_pretty_env_info 5 | 6 | 7 | def get_pil_version(): 8 | return "\n Pillow ({})".format(PIL.__version__) 9 | 10 | 11 | def collect_env_info(): 12 | env_str = get_pretty_env_info() 13 | env_str += get_pil_version() 14 | return env_str 15 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/utils/envs.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | import logging 4 | import random 5 | import numpy as np 6 | 7 | import torch 8 | 9 | 10 | def seed_all_rng(seed=None): 11 | """ 12 | Set the random seed for the RNG in torch, numpy and python. 13 | 14 | Args: 15 | seed (int): if None, will use a strong random seed. 16 | """ 17 | if seed is None: 18 | seed = ( 19 | os.getpid() 20 | + int(datetime.now().strftime("%S%f")) 21 | + int.from_bytes(os.urandom(2), "big") 22 | ) 23 | logger = logging.getLogger(__name__) 24 | logger.info("Using a generated random seed {}".format(seed)) 25 | np.random.seed(seed) 26 | torch.set_rng_state(torch.manual_seed(seed).get_state()) 27 | random.seed(seed) 28 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/utils/imports.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from importlib import util 3 | 4 | 5 | # from https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa 6 | def import_file(module_name, file_path, make_importable=False): 7 | spec = util.spec_from_file_location(module_name, file_path) 8 | module = util.module_from_spec(spec) 9 | spec.loader.exec_module(module) 10 | 11 | if make_importable: 12 | sys.modules[module_name] = module_name 13 | return module -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/utils/logger.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import logging 3 | import os 4 | import sys 5 | 6 | 7 | @functools.lru_cache() # so that calling setup_logger multiple times won't add many handlers 8 | def setup_logger(output_dir, distributed_rank=0, name="smoke", file_name="log.txt"): 9 | ''' 10 | Args: 11 | output_dir (str): a directory saves output log files 12 | name (str): name of the logger 13 | file_name (str): name of log file 14 | ''' 15 | logger = logging.getLogger(name) 16 | logger.setLevel(logging.DEBUG) 17 | # don't log results for the non-master process 18 | if distributed_rank > 0: 19 | return logger 20 | ch = logging.StreamHandler(stream=sys.stdout) 21 | ch.setLevel(logging.DEBUG) 22 | formatter = logging.Formatter("[%(asctime)s] %(name)s %(levelname)s: %(message)s") 23 | ch.setFormatter(formatter) 24 | logger.addHandler(ch) 25 | 26 | if output_dir: 27 | fh = logging.FileHandler(os.path.join(output_dir, file_name)) 28 | fh.setLevel(logging.DEBUG) 29 | fh.setFormatter(formatter) 30 | logger.addHandler(fh) 31 | 32 | return logger 33 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/utils/metric_logger.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | from collections import defaultdict 3 | from collections import deque 4 | 5 | import torch 6 | 7 | 8 | class SmoothedValue(): 9 | """Track a series of values and provide access to smoothed values over a 10 | window or the global series average. 11 | """ 12 | 13 | def __init__(self, window_size=3): 14 | self.window_size = window_size 15 | self.deque = deque(maxlen=window_size) 16 | self.series = [] 17 | self.total = 0.0 18 | self.count = 0 19 | 20 | def update(self, value): 21 | self.deque.append(value) 22 | self.series.append(value) 23 | self.count += 1 24 | self.total += value 25 | 26 | @property 27 | def median(self): 28 | d = torch.tensor(list(self.deque)) 29 | return d.median().item() 30 | 31 | @property 32 | def avg(self): 33 | d = torch.tensor(list(self.deque)) 34 | return d.mean().item() 35 | 36 | @property 37 | def global_avg(self): 38 | return self.total / self.count 39 | 40 | @property 41 | def reset(self): 42 | self.deque = deque(maxlen=self.window_size) 43 | self.series = [] 44 | self.total = 0.0 45 | self.count = 0 46 | 47 | 48 | class MetricLogger(): 49 | def __init__(self, delimiter="\t"): 50 | self.meters = defaultdict(SmoothedValue) 51 | self.delimiter = delimiter 52 | 53 | def update(self, **kwargs): 54 | for k, v in kwargs.items(): 55 | if isinstance(v, torch.Tensor): 56 | v = v.item() 57 | assert isinstance(v, (float, int)) 58 | self.meters[k].update(v) 59 | 60 | def reset(self, *argc): 61 | for k in argc: 62 | self.meters[k].reset 63 | 64 | def reset_all(self): 65 | for v in self.meters.values(): 66 | v.reset 67 | 68 | def __getattr__(self, attr): 69 | if attr in self.meters: 70 | return self.meters[attr] 71 | if attr in self.__dict__: 72 | return self.__dict__[attr] 73 | raise AttributeError("{} object has no attribute {}".format( 74 | type(self).__name__, attr)) 75 | 76 | def __str__(self): 77 | loss_str = [] 78 | for name, meter in self.meters.items(): 79 | loss_str.append( 80 | "{}: {:.4f} ({:.4f})".format(name, meter.median, meter.global_avg) 81 | ) 82 | return self.delimiter.join(loss_str) 83 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/utils/miscellaneous.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import os 3 | 4 | 5 | def mkdir(path): 6 | try: 7 | os.makedirs(path) 8 | except OSError as e: 9 | if e.errno != errno.EEXIST: 10 | raise -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/utils/model_serialization.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | from collections import OrderedDict 3 | import logging 4 | 5 | import torch 6 | 7 | 8 | def align_and_update_state_dicts(model_state_dict, loaded_state_dict): 9 | """ 10 | Strategy: suppose that the models that we will create will have prefixes appended 11 | to each of its keys, for example due to an extra level of nesting that the original 12 | pre-trained weights from ImageNet won't contain. For example, model.state_dict() 13 | might return backbone[0].body.res2.conv1.weight, while the pre-trained model contains 14 | res2.conv1.weight. We thus want to match both parameters together. 15 | For that, we look for each model weight, look among all loaded keys if there is one 16 | that is a suffix of the current weight name, and use it if that's the case. 17 | If multiple matches exist, take the one with longest size 18 | of the corresponding name. For example, for the same model as before, the pretrained 19 | weight file can contain both res2.conv1.weight, as well as conv1.weight. In this case, 20 | we want to match backbone[0].body.conv1.weight to conv1.weight, and 21 | backbone[0].body.res2.conv1.weight to res2.conv1.weight. 22 | """ 23 | current_keys = sorted(list(model_state_dict.keys())) 24 | loaded_keys = sorted(list(loaded_state_dict.keys())) 25 | # get a matrix of string matches, where each (i, j) entry correspond to the size of the 26 | # loaded_key string, if it matches 27 | match_matrix = [ 28 | len(j) if i.endswith(j) else 0 for i in current_keys for j in loaded_keys 29 | ] 30 | match_matrix = torch.as_tensor(match_matrix).view( 31 | len(current_keys), len(loaded_keys) 32 | ) 33 | max_match_size, idxs = match_matrix.max(1) 34 | # remove indices that correspond to no-match 35 | idxs[max_match_size == 0] = -1 36 | 37 | # used for logging 38 | max_size = max([len(key) for key in current_keys]) if current_keys else 1 39 | max_size_loaded = max([len(key) for key in loaded_keys]) if loaded_keys else 1 40 | log_str_template = "{: <{}} loaded from {: <{}} of shape {}" 41 | logger = logging.getLogger(__name__) 42 | for idx_new, idx_old in enumerate(idxs.tolist()): 43 | if idx_old == -1: 44 | continue 45 | key = current_keys[idx_new] 46 | key_old = loaded_keys[idx_old] 47 | model_state_dict[key] = loaded_state_dict[key_old] 48 | logger.info( 49 | log_str_template.format( 50 | key, 51 | max_size, 52 | key_old, 53 | max_size_loaded, 54 | tuple(loaded_state_dict[key_old].shape), 55 | ) 56 | ) 57 | 58 | 59 | def strip_prefix_if_present(state_dict, prefix): 60 | keys = sorted(state_dict.keys()) 61 | if not all(key.startswith(prefix) for key in keys): 62 | return state_dict 63 | stripped_state_dict = OrderedDict() 64 | for key, value in state_dict.items(): 65 | stripped_state_dict[key.replace(prefix, "")] = value 66 | return stripped_state_dict 67 | 68 | 69 | def load_state_dict(model, loaded_state_dict): 70 | model_state_dict = model.state_dict() 71 | # if the state_dict comes from a model that was wrapped in a 72 | # DataParallel or DistributedDataParallel during serialization, 73 | # remove the "module" prefix before performing the matching 74 | loaded_state_dict = strip_prefix_if_present(loaded_state_dict, prefix="module.") 75 | align_and_update_state_dicts(model_state_dict, loaded_state_dict) 76 | 77 | # use strict loading 78 | model.load_state_dict(model_state_dict) 79 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/utils/model_zoo.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | import os 3 | import sys 4 | 5 | from torch.hub import download_url_to_file 6 | from torch.hub import urlparse 7 | from torch.hub import HASH_REGEX 8 | 9 | from .comm import is_main_process 10 | from .comm import synchronize 11 | 12 | 13 | # very similar to https://github.com/pytorch/pytorch/blob/master/torch/utils/model_zoo.py 14 | # but with a few improvements and modifications 15 | def cache_url(url, model_dir=None, progress=True): 16 | r"""Loads the Torch serialized object at the given URL. 17 | If the object is already present in `model_dir`, it's deserialized and 18 | returned. The filename part of the URL should follow the naming convention 19 | ``filename-.ext`` where ```` is the first eight or more 20 | digits of the SHA256 hash of the contents of the file. The hash is used to 21 | ensure unique names and to verify the contents of the file. 22 | The default value of `model_dir` is ``$TORCH_HOME/models`` where 23 | ``$TORCH_HOME`` defaults to ``~/.torch``. The default directory can be 24 | overridden with the ``$TORCH_MODEL_ZOO`` environment variable. 25 | Args: 26 | url (string): URL of the object to download 27 | model_dir (string, optional): directory in which to save the object 28 | progress (bool, optional): whether or not to display a progress bar to stderr 29 | Example: 30 | >>> cached_file = maskrcnn_benchmark.utils.model_zoo.cache_url('https://s3.amazonaws.com/pytorch/models/resnet18-5c106cde.pth') 31 | """ 32 | if model_dir is None: 33 | torch_home = os.path.expanduser(os.getenv('TORCH_HOME', '~/.torch')) 34 | model_dir = os.getenv('TORCH_MODEL_ZOO', os.path.join(torch_home, 'models')) 35 | if not os.path.exists(model_dir): 36 | os.makedirs(model_dir) 37 | parts = urlparse(url) 38 | filename = os.path.basename(parts.path) 39 | if filename == "model_final.pkl": 40 | # workaround as pre-trained Caffe2 models from Detectron have all the same filename 41 | # so make the full path the filename by replacing / with _ 42 | filename = parts.path.replace("/", "_") 43 | cached_file = os.path.join(model_dir, filename) 44 | if not os.path.exists(cached_file) and is_main_process(): 45 | sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file)) 46 | hash_prefix = HASH_REGEX.search(filename) 47 | if hash_prefix is not None: 48 | hash_prefix = hash_prefix.group(1) 49 | # workaround: Caffe2 models don't have a hash, but follow the R-50 convention, 50 | # which matches the hash PyTorch uses. So we skip the hash matching 51 | # if the hash_prefix is less than 6 characters 52 | if len(hash_prefix) < 6: 53 | hash_prefix = None 54 | download_url_to_file(url, cached_file, hash_prefix, progress=progress) 55 | synchronize() 56 | return cached_file 57 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/utils/plot.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Dec 15 15:00:06 2022 4 | 5 | @author: hdb 6 | """ 7 | 8 | def plot_box(draw, box2d=[], box3d=[]): 9 | 10 | if len(box2d) > 0: 11 | num_obj = box2d.shape[0] 12 | for i in range(num_obj): 13 | 14 | x1, y1, x2, y2 = box2d[i].split(1, 0) 15 | draw.rectangle([x1,y1,x2,y2], outline=(0, 255, 0)) 16 | 17 | if len(box3d) > 0: 18 | box3d = box3d.permute(0, 2, 1) 19 | num_obj = box3d.shape[0] 20 | for i in range(num_obj): 21 | 22 | for j in range(box3d[i].shape[0]): 23 | x, y = box3d[i][j].split(1, 0) 24 | draw.ellipse((x-2,y-2, x+2,y+2), (255,0,0)) 25 | 26 | point_pair = [ 27 | [box3d[i][0].split(1, 0), box3d[i][1].split(1, 0)], 28 | [box3d[i][0].split(1, 0), box3d[i][5].split(1, 0)], 29 | [box3d[i][0].split(1, 0), box3d[i][7].split(1, 0)], 30 | [box3d[i][2].split(1, 0), box3d[i][1].split(1, 0)], 31 | [box3d[i][2].split(1, 0), box3d[i][3].split(1, 0)], 32 | [box3d[i][2].split(1, 0), box3d[i][7].split(1, 0)], 33 | [box3d[i][4].split(1, 0), box3d[i][1].split(1, 0)], 34 | [box3d[i][4].split(1, 0), box3d[i][3].split(1, 0)], 35 | [box3d[i][4].split(1, 0), box3d[i][5].split(1, 0)], 36 | [box3d[i][6].split(1, 0), box3d[i][3].split(1, 0)], 37 | [box3d[i][6].split(1, 0), box3d[i][5].split(1, 0)], 38 | [box3d[i][6].split(1, 0), box3d[i][7].split(1, 0)] 39 | ] 40 | 41 | for j in range(len(point_pair)): 42 | draw.line(point_pair[j], (0,255,0), width=2) -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/utils/registry.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | 3 | 4 | def _register_generic(module_dict, module_name, module): 5 | assert module_name not in module_dict 6 | module_dict[module_name] = module 7 | 8 | 9 | class Registry(dict): 10 | ''' 11 | A helper class for managing registering modules, it extends a dictionary 12 | and provides a register functions. 13 | 14 | Eg. creating a registry: 15 | some_registry = Registry({"default": default_module}) 16 | 17 | There're two ways of registering new modules: 18 | 1): normal way is just calling register function: 19 | def foo(): 20 | ... 21 | some_registry.register("foo_module", foo) 22 | 2): used as decorator when declaring the module: 23 | @some_registry.register("foo_module") 24 | @some_registry.register("foo_module_nickname") 25 | def foo(): 26 | ... 27 | 28 | Access of module is just like using a dictionary, eg: 29 | f = some_registry["foo_module"] 30 | ''' 31 | 32 | def __init__(self, *args, **kwargs): 33 | super(Registry, self).__init__(*args, **kwargs) 34 | 35 | def register(self, module_name, module=None): 36 | # used as function call 37 | if module is not None: 38 | _register_generic(self, module_name, module) 39 | return 40 | 41 | # used as decorator 42 | def register_fn(fn): 43 | _register_generic(self, module_name, fn) 44 | return fn 45 | 46 | return register_fn 47 | -------------------------------------------------------------------------------- /Kitti-Driving/models/smoke/utils/timer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | 3 | 4 | import time 5 | import datetime 6 | 7 | 8 | class Timer(object): 9 | def __init__(self): 10 | self.reset() 11 | 12 | @property 13 | def average_time(self): 14 | return self.total_time / self.calls if self.calls > 0 else 0.0 15 | 16 | def tic(self): 17 | # using time.time instead of time.clock because time time.clock 18 | # does not normalize for multithreading 19 | self.start_time = time.time() 20 | 21 | def toc(self, average=True): 22 | self.add(time.time() - self.start_time) 23 | if average: 24 | return self.average_time 25 | else: 26 | return self.diff 27 | 28 | def add(self, time_diff): 29 | self.diff = time_diff 30 | self.total_time += self.diff 31 | self.calls += 1 32 | 33 | def reset(self): 34 | self.total_time = 0.0 35 | self.calls = 0 36 | self.start_time = 0.0 37 | self.diff = 0.0 38 | 39 | def avg_time_str(self): 40 | time_str = str(datetime.timedelta(seconds=self.average_time)) 41 | return time_str 42 | 43 | 44 | def get_time_str(time_diff): 45 | time_str = str(datetime.timedelta(seconds=time_diff)) 46 | return time_str 47 | -------------------------------------------------------------------------------- /Kitti-Driving/run.sh: -------------------------------------------------------------------------------- 1 | # supervised training 2 | # CUDA_VISIBLE_DEVICES=4,5 python3 sup_train.py --seed 0 --alpha 0.75 --batch_size 32 3 | 4 | # nesy learning 5 | 6 | # CUDA_VISIBLE_DEVICES=0 python3 nesy_train_cnn.py --seed 0 --clauses 3000 --exp_name 'logic_clauses_3000' --iter 30 --num_epochs 300 7 | 8 | # CUDA_VISIBLE_DEVICES=0 python3 nesy_train_gpt.py --seed 0 --clauses 3000 --exp_name 'gpt_logic_clauses_3000' --iter 30 --num_epochs 300 9 | 10 | 11 | -------------------------------------------------------------------------------- /Nonograms/nesy_eval.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from nn_utils import * 3 | from dataset import * 4 | import argparse 5 | 6 | 7 | parser = argparse.ArgumentParser(description='PyTorch SudoKu Logic Training') 8 | parser.add_argument('--device', default=0, type=int, help='Cuda device.') 9 | parser.add_argument('--seed', default=1, type=int, help='Random seed to use.') 10 | parser.add_argument('--game_size', default=7, type=int, help='the size of nonograms') 11 | opt = parser.parse_args() 12 | 13 | n_train = 9000 14 | n_test = 1000 15 | 16 | if opt.game_size == 7: # for 7x7 board 17 | opt.max_hint_value = 6 18 | opt.max_num_per_hint = 4 19 | opt.n_embd = 128 # max_num_per_hint must be a factor of 2*emb_size 20 | elif opt.game_size == 15: # for 15x15 board 21 | opt.max_hint_value = 14 22 | opt.max_num_per_hint = 5 23 | opt.n_embd = 320 # max_num_per_hint must be a factdor of 2*emb_size 24 | else: 25 | raise Exception('not a valid game size') 26 | 27 | train_set = Nonogram_Dataset( 28 | data_path=f'./data/nonograms_{opt.game_size}.csv', 29 | split='train', 30 | board_dim=opt.game_size, 31 | max_num_per_hint=opt.max_num_per_hint, 32 | limit=(n_train + n_test), 33 | seed=opt.seed) 34 | test_set = Nonogram_Dataset( 35 | data_path=f'./data/nonograms_{opt.game_size}.csv', 36 | split='test', 37 | board_dim=opt.game_size, 38 | max_num_per_hint=opt.max_num_per_hint, 39 | limit=(n_train + n_test), 40 | seed=opt.seed) 41 | print(f'[nonogram-{opt.game_size}] use {len(train_set)} for training and {len(test_set)} for testing') 42 | X, Y = preprocess(train_set, length=opt.max_num_per_hint) 43 | 44 | # loading 45 | ckpt = './checkpoint/nonogram_{!s}__0_logic.t7'.format(opt.game_size) 46 | # init chektpoint 47 | static_dict = torch.load(ckpt) 48 | phi = static_dict['logic'] 49 | (W, b, Z) = phi 50 | Wtmp, btmp = bounding_box(phi, 1-X, Y) 51 | Wtmp = Wtmp.long() 52 | print(Wtmp.shape) 53 | 54 | evaluate_batch(Wtmp, btmp, test_set, opt) 55 | 56 | -------------------------------------------------------------------------------- /Nonograms/nesy_train.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import torch 4 | import torch.nn.functional as F 5 | from nn_utils import * 6 | from dataset import * 7 | from scipy import linalg as splinalg 8 | import random 9 | 10 | import sys 11 | import models 12 | sys.path.append("../") 13 | # from utils import * 14 | 15 | from joblib import Parallel, delayed 16 | from torch.utils.tensorboard import SummaryWriter 17 | 18 | # Dataloader 19 | parser = argparse.ArgumentParser(description='PyTorch Nonogram Logic Training') 20 | parser.add_argument('--device', default=0, type=int, help='Cuda device.') 21 | parser.add_argument('--seed', default=1, type=int, help='Random seed to use.') 22 | parser.add_argument('--lamda', default=0.1, type=float, help='the trust region penalty') 23 | parser.add_argument('--num_epochs', default=50000, type=int, help='the number of epochs') 24 | parser.add_argument('--num_iters', default=2000, type=int, help='the number of iters to increase t') 25 | parser.add_argument('--game_size', default=7, type=int, help='the size of nonograms') 26 | parser.add_argument('--k', default=2, type=int, help='the number of aux variables') 27 | parser.add_argument('--b', default=3, type=float, help='the pre-defined value of b') 28 | parser.add_argument('--logic_lr', default=1e6, type=float, help='the step size of programming') 29 | parser.add_argument('--clauses', default=1000, type=int, help='the number of clauses') 30 | parser.add_argument('--update_b', default=0, type=int, help='the number of clauses') 31 | parser.add_argument('--exp_name', default='', type=str, help='Experiment name') 32 | opt = parser.parse_args() 33 | 34 | # setting 35 | n_train = 9000 36 | n_test = 1000 37 | tol = 1e-3 38 | num_classes = 2; 39 | 40 | # cuda 41 | torch.cuda.set_device(opt.device) 42 | torch.set_default_tensor_type(torch.FloatTensor) 43 | 44 | # random seed 45 | random.seed(opt.seed) 46 | np.random.seed(opt.seed) 47 | torch.manual_seed(opt.seed) 48 | torch.cuda.manual_seed(opt.seed) 49 | 50 | ############# 51 | # Load data 52 | ############# 53 | if opt.game_size == 7: # for 7x7 board 54 | max_hint_value = 6 55 | max_num_per_hint = 4 56 | opt.n_embd = 128 # max_num_per_hint must be a factor of 2*emb_size 57 | elif opt.game_size == 15: # for 15x15 board 58 | max_hint_value = 14 59 | max_num_per_hint = 5 60 | opt.n_embd = 320 # max_num_per_hint must be a factdor of 2*emb_size 61 | else: 62 | raise Exception('not a valid game size') 63 | 64 | train_set = Nonogram_Dataset( 65 | data_path=f'./data/nonograms_{opt.game_size}.csv', 66 | split='train', 67 | board_dim=opt.game_size, 68 | max_num_per_hint=max_num_per_hint, 69 | limit=(n_train + n_test), 70 | seed=opt.seed) 71 | test_set = Nonogram_Dataset( 72 | data_path=f'./data/nonograms_{opt.game_size}.csv', 73 | split='test', 74 | board_dim=opt.game_size, 75 | max_num_per_hint=max_num_per_hint, 76 | limit=(n_train + n_test), 77 | seed=opt.seed) 78 | 79 | print(f'[nonogram-{opt.game_size}] use {len(train_set)} for training and {len(test_set)} for testing') 80 | X, Y = preprocess(train_set, length=max_num_per_hint) 81 | print('data size after preprocess: ', X.size[0]) 82 | 83 | #### training 84 | file_name = 'nonogram' + '_' + str(opt.game_size) + '_' + opt.exp_name 85 | phi = train(1-X, Y, opt) 86 | save_logic(phi, file_name) 87 | 88 | -------------------------------------------------------------------------------- /Nonograms/run.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | #### eval and transfer 4 | CUDA_VISIBLE_DEVICES=2 python3 nesy_train.py --game_size 7 --seed 0 --clauses 200 --lamda 1000.0 --num_epochs 500000 5 | # CUDA_VISIBLE_DEVICES=2 python3 nesy_train.py --game_size 15 --seed 0 --clauses 200 --lamda 1000.0 --num_epochs 500000 6 | 7 | CUDA_VISIBLE_DEVICES=2 python3 nesy_eval.py --game_size 7 8 | # CUDA_VISIBLE_DEVICES=2 python3 nesy_eval.py --game_size 15 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /Nuscenes-Driving/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | class Config: 5 | def __init__(self, **kwargs): 6 | for k, v in kwargs.items(): 7 | setattr(self, k, v) 8 | 9 | def update(self, **kwargs): 10 | for k, v in kwargs.items(): 11 | setattr(self, k, v) 12 | 13 | 14 | def setup_cfg(): 15 | # init the default cfg 16 | cfg = Config( 17 | # log dir, default evolving with time 18 | LOG_DIR=os.path.join("./logs", "exp_" + time.strftime("%y-%m-%d--%H-%M-%S", time.localtime())), 19 | EXP_NAME="", 20 | # input shape, default as the resized shape with ratio width : height = 1 : 0.47 21 | INPUT_WIDTH=750, 22 | INPUT_HEIGHT=350, 23 | # output shape, choosing from (10,10), (50,50), (100,100) 24 | OUTPUT_WIDTH=10, 25 | OUTPUT_HEIGHT=10, 26 | # dataset, default as kitti dataset 27 | DATASET="kitti", 28 | DATASET_ROOT="../data/kitti", 29 | DETECT_CLASSES=("Car", "Pedestrian", "Cyclist"), 30 | MAX_OBJECTS=30, 31 | # data augmentation 32 | AUGMENT_BLUR=False, 33 | AUGMENT_JITTER=False, 34 | AUGMENT_FLIP=False, 35 | # data loader 36 | BATCH_SIZE=64, 37 | SHUFFLE=False, 38 | NUM_WORKERS=16, 39 | # model, default using resnet18 as backbone 40 | MODEL="Resnet", 41 | NUM_LAYERS=18, 42 | PRETRAINED=False, 43 | CKPT="", 44 | # train 45 | TRAINING=True, 46 | DEVICE="cuda:0", 47 | SEED=0, 48 | LEARNING_RATE=2.0e-4, 49 | OPTIMIZER="Adam", 50 | LOSS_FUNC="Focal", 51 | START_EPOCH=0, 52 | MAX_EPOCH=30, 53 | CKPT_PERIOD=1, 54 | EVAL_PERIOD=1, 55 | RESUME=False, 56 | # test 57 | PLANNER="Astar", 58 | THRESHOLD=0.15, 59 | METRICS=("accuracy", "precision", "recall", "f1", "collide", "dist", "length"), 60 | # focal loss hyper-params 61 | ALPHA=0.75, 62 | GAMMA=2, 63 | EPSILON=1e-5, 64 | ) 65 | 66 | return cfg 67 | -------------------------------------------------------------------------------- /Nuscenes-Driving/data/readme.md: -------------------------------------------------------------------------------- 1 | Save the data here. 2 | -------------------------------------------------------------------------------- /Nuscenes-Driving/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torchvision.models import ( 4 | resnet18, resnet34, resnet50, resnet101, 5 | ResNet18_Weights, ResNet34_Weights, ResNet50_Weights, ResNet101_Weights 6 | ) 7 | from config import setup_cfg 8 | from models.mingpt import GPTConfig, GPT 9 | 10 | 11 | def build_model_resnet(cfg, device, num_layers=18, pretrained=False): 12 | model_weights_dict = { 13 | 18: (resnet18, ResNet18_Weights.DEFAULT), 14 | 34: (resnet34, ResNet34_Weights.DEFAULT), 15 | 50: (resnet50, ResNet50_Weights.DEFAULT), 16 | 101: (resnet101, ResNet101_Weights.DEFAULT), 17 | } 18 | 19 | # construct basic resnet model with pretrained weights or not 20 | weights = model_weights_dict[num_layers][1] if pretrained else None 21 | basic_model = model_weights_dict[num_layers][0](weights=weights) 22 | 23 | class ResModel(nn.Module): 24 | def __init__(self, basic_model, width, height): 25 | super(ResModel, self).__init__() 26 | self.basic_model = basic_model 27 | self.width = width 28 | self.height = height 29 | 30 | in_channels = self.basic_model._modules['fc'].in_features 31 | self.basic_model.fc = nn.Linear(in_features=in_channels, 32 | out_features=self.width * self.height, 33 | bias=True) 34 | self.activation = nn.Sigmoid() 35 | 36 | self.basic_model.fc = nn.Sequential() 37 | 38 | # self.conv = nn.Sequential( 39 | # nn.Conv2d(in_channels=in_channels, 40 | # out_channels=out_channels, 41 | # kernel_size=1, stride=1, padding=1, bias=True), 42 | # # nn.BatchNorm2d(out_channels) 43 | # group_norm(out_channels) 44 | # ) 45 | # self.basic_model.layer4.add_module(name="planning_conv", module=self.conv) 46 | # self.basic_model.avgpool = nn.AdaptiveMaxPool2d(output_size=(1, 1)) 47 | 48 | self.fc = nn.Linear(in_features=in_channels, 49 | out_features=self.width * self.height, 50 | bias=True) 51 | 52 | self.activation = nn.Sigmoid() 53 | 54 | def forward(self, x): 55 | x = self.basic_model(x) 56 | x = self.fc(x) 57 | x = x.view(x.shape[0], self.width, self.height) 58 | 59 | return x 60 | 61 | 62 | model = ResModel(basic_model, cfg.OUTPUT_WIDTH, cfg.OUTPUT_HEIGHT) 63 | model.to(device) 64 | return model 65 | 66 | 67 | def build_model_rt(cfg, device): 68 | mconf = GPTConfig(block_size=cfg.OUTPUT_WIDTH * cfg.OUTPUT_HEIGHT, num_classes=1, 69 | n_layer=1, n_head=4, n_embd=128, n_recur=32, # default using: l1r32h4 70 | all_layers=True) 71 | model = GPT(mconf) 72 | 73 | model.to(device) 74 | return model 75 | 76 | 77 | if __name__ == "__main__": 78 | # test building the model 79 | cfg = setup_cfg() 80 | 81 | resnet_model = build_model_resnet(cfg, device=cfg.DEVICE) 82 | rt_model = build_model_rt(cfg, device=cfg.DEVICE) 83 | 84 | sample = torch.rand((16, 3, 350, 750), device=cfg.DEVICE) 85 | rt_target = torch.zeros((16, 2, 10, 10)).long().to("cuda:0") 86 | y = (torch.rand((16, 10, 10)) > 0.8).long().to("cuda:0") 87 | rt_target[:, 0, :, :] = y 88 | rt_target[:, 1, :, :] = -100 89 | 90 | t1 = rt_target[5] 91 | t11 = t1[0] 92 | t12 = t1[1] 93 | 94 | resnet_output = resnet_model(sample) 95 | rt_output, loss, _ = rt_model(sample, rt_target) 96 | 97 | print() 98 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lizn-zn/Nesy-Programming/1410e721c7be4782b50219f7e18a4e247638edb3/Nuscenes-Driving/models/__init__.py -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/README.md: -------------------------------------------------------------------------------- 1 | 2 | # window 版 SMOKE 3 | 好用的话记得给个 star 哈 ... 4 | 5 | ![image](../../data/kitti-smoke/figures/result.gif) 6 | 7 | ## 添加功能 8 | 9 | - 使用了 pytorch 自带的 DConv，省去 linux 下编译 DConv 的 cuda 代码，可以直接在 window 下训练和测试 10 | - 提供了重新训练的模型：[百度云盘](https://pan.baidu.com/s/1GQdUfi6W9VbsCpwshMxlxA)，提取码： hdbb 11 | - 改变训练循环，并使用梯度累加机制 12 | - 增添了 finetune 和 resume 等功能 13 | - 提供了测试单张图像的例子 14 | 15 | 16 | ## 训练 17 | 18 | 1. 下载 KITTI 数据，并修改成一下结构，把图像列表放在 ImageSets 文件夹中，然后在 datasets 中创建 kitti 目录的软连接 19 | ``` 20 | kitti 21 | │──training 22 | │ ├──calib 23 | │ ├──label_2 24 | │ ├──image_2 25 | │ └──ImageSets 26 | └──testing 27 | ├──calib 28 | ├──image_2 29 | └──ImageSets 30 | ``` 31 | 32 | 2. 修改相关参数，比如 configs/smoke_gn_vector.yaml 和 smoke/config/defaults.py 33 | 34 | 3. 执行 tools/plain_train_net.py 即可 35 | 36 | ## 测试 37 | 38 | 1. 把 smoke/engine/defaults.py 中的 --eval-only 设成 True 39 | 40 | 3. 执行 tools/plain_train_net.py 即可 41 | 42 | 不过现在 KITTI 只有论文才能在线测试了，代码只能生成测试集的结果而已 43 | 44 | ## 测试单张图像 45 | 46 | 1. 指定 tools/detector.py 中 --ckpt 为训练好的模型，直接执行 detector.py 即可 47 | 48 | ![image](figures/result.png) 49 | 50 | 本人电脑比较弱鸡，batch 很小，就只训练了 50 epoch .... 51 | 52 | ## acknowledge 53 | 54 | https://github.com/lzccccc/SMOKE 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lizn-zn/Nesy-Programming/1410e721c7be4782b50219f7e18a4e247638edb3/Nuscenes-Driving/models/smoke/__init__.py -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/config/__init__.py: -------------------------------------------------------------------------------- 1 | from.defaults import _C as cfg -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/config/paths_catalog.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | class DatasetCatalog(): 5 | KITTI_DATA_DIR = "data/kitti-smoke/datasets" 6 | NUSCENES_DATA_DIR = "data/nuscenes-smoke" 7 | DATASETS = { 8 | "kitti_train": { 9 | "root": "kitti/training/", 10 | }, 11 | "kitti_test": { 12 | "root": "kitti/testing/", 13 | }, 14 | # kitti smoke(keypoints only) trainval / train / val test datasets root 15 | "kitti_smoke_trainval": { 16 | "root": "kitti/training/", 17 | }, 18 | "kitti_smoke_train": { 19 | "root": "kitti/training/", 20 | }, 21 | "kitti_smoke_val": { 22 | "root": "kitti/training/", 23 | }, 24 | "kitti_smoke_test": { 25 | "root": "kitti/training/", 26 | }, 27 | # nuscenes smoke(keypoints only) trainval / train / val test datasets root 28 | "nuscenes_smoke_trainval": { 29 | "root": "trainval_datasets/", 30 | }, 31 | "nuscenes_smoke_train": { 32 | "root": "trainval_datasets/", 33 | }, 34 | "nuscenes_smoke_val": { 35 | "root": "trainval_datasets/", 36 | }, 37 | "nuscenes_smoke_test": { 38 | "root": "trainval_datasets/", 39 | }, 40 | } 41 | 42 | @staticmethod 43 | def get(name): 44 | if "kitti_smoke" in name: 45 | data_dir = DatasetCatalog.KITTI_DATA_DIR 46 | attrs = DatasetCatalog.DATASETS[name] 47 | args = dict( 48 | root=os.path.join(os.getcwd(), data_dir, attrs["root"]), 49 | split= "trainval" if "trainval" in name else( 50 | "train" if "train" in name else ("val" if "val" in name else "test")), 51 | planning=True, # set param planning here 52 | ) 53 | return dict( 54 | factory="KittiSmokeDataset", 55 | args=args, 56 | ) 57 | elif "nuscenes_smoke" in name: 58 | data_dir = DatasetCatalog.NUSCENES_DATA_DIR 59 | attrs = DatasetCatalog.DATASETS[name] 60 | args = dict( 61 | root=os.path.join(os.getcwd(), data_dir, attrs["root"]), 62 | split="trainval" if "trainval" in name else ( 63 | "train" if "train" in name else ("val" if "val" in name else "test")), 64 | planning=True, # set param planning here 65 | ) 66 | return dict( 67 | factory="NuscensesSmokeDataset", 68 | args=args, 69 | ) 70 | elif "kitti" in name: 71 | data_dir = DatasetCatalog.KITTI_DATA_DIR 72 | attrs = DatasetCatalog.DATASETS[name] 73 | args = dict( 74 | root=os.path.join(os.getcwd(), data_dir, attrs["root"]), 75 | ) 76 | return dict( 77 | factory="KITTIDataset", 78 | args=args, 79 | ) 80 | raise RuntimeError("Dataset not available: {}".format(name)) 81 | 82 | 83 | class ModelCatalog(): 84 | IMAGENET_MODELS = { 85 | "DLA34": "http://dl.yf.io/dla/models/imagenet/dla34-ba72cf86.pth" 86 | } 87 | 88 | @staticmethod 89 | def get(name): 90 | if name.startswith("ImageNetPretrained"): 91 | return ModelCatalog.get_imagenet_pretrained(name) 92 | 93 | @staticmethod 94 | def get_imagenet_pretrained(name): 95 | name = name[len("ImageNetPretrained/"):] 96 | url = ModelCatalog.IMAGENET_MODELS[name] 97 | return url 98 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/csrc/cpu/dcn_v2_cpu.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | 6 | 7 | at::Tensor 8 | dcn_v2_cpu_forward(const at::Tensor &input, 9 | const at::Tensor &weight, 10 | const at::Tensor &bias, 11 | const at::Tensor &offset, 12 | const at::Tensor &mask, 13 | const int kernel_h, 14 | const int kernel_w, 15 | const int stride_h, 16 | const int stride_w, 17 | const int pad_h, 18 | const int pad_w, 19 | const int dilation_h, 20 | const int dilation_w, 21 | const int deformable_group) 22 | { 23 | AT_ERROR("Not implement on cpu"); 24 | } 25 | 26 | std::vector 27 | dcn_v2_cpu_backward(const at::Tensor &input, 28 | const at::Tensor &weight, 29 | const at::Tensor &bias, 30 | const at::Tensor &offset, 31 | const at::Tensor &mask, 32 | const at::Tensor &grad_output, 33 | int kernel_h, int kernel_w, 34 | int stride_h, int stride_w, 35 | int pad_h, int pad_w, 36 | int dilation_h, int dilation_w, 37 | int deformable_group) 38 | { 39 | AT_ERROR("Not implement on cpu"); 40 | } 41 | 42 | std::tuple 43 | dcn_v2_psroi_pooling_cpu_forward(const at::Tensor &input, 44 | const at::Tensor &bbox, 45 | const at::Tensor &trans, 46 | const int no_trans, 47 | const float spatial_scale, 48 | const int output_dim, 49 | const int group_size, 50 | const int pooled_size, 51 | const int part_size, 52 | const int sample_per_part, 53 | const float trans_std) 54 | { 55 | AT_ERROR("Not implement on cpu"); 56 | } 57 | 58 | std::tuple 59 | dcn_v2_psroi_pooling_cpu_backward(const at::Tensor &out_grad, 60 | const at::Tensor &input, 61 | const at::Tensor &bbox, 62 | const at::Tensor &trans, 63 | const at::Tensor &top_count, 64 | const int no_trans, 65 | const float spatial_scale, 66 | const int output_dim, 67 | const int group_size, 68 | const int pooled_size, 69 | const int part_size, 70 | const int sample_per_part, 71 | const float trans_std) 72 | { 73 | AT_ERROR("Not implement on cpu"); 74 | } -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/csrc/cpu/vision.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | at::Tensor 5 | dcn_v2_cpu_forward(const at::Tensor &input, 6 | const at::Tensor &weight, 7 | const at::Tensor &bias, 8 | const at::Tensor &offset, 9 | const at::Tensor &mask, 10 | const int kernel_h, 11 | const int kernel_w, 12 | const int stride_h, 13 | const int stride_w, 14 | const int pad_h, 15 | const int pad_w, 16 | const int dilation_h, 17 | const int dilation_w, 18 | const int deformable_group); 19 | 20 | std::vector 21 | dcn_v2_cpu_backward(const at::Tensor &input, 22 | const at::Tensor &weight, 23 | const at::Tensor &bias, 24 | const at::Tensor &offset, 25 | const at::Tensor &mask, 26 | const at::Tensor &grad_output, 27 | int kernel_h, int kernel_w, 28 | int stride_h, int stride_w, 29 | int pad_h, int pad_w, 30 | int dilation_h, int dilation_w, 31 | int deformable_group); 32 | 33 | 34 | std::tuple 35 | dcn_v2_psroi_pooling_cpu_forward(const at::Tensor &input, 36 | const at::Tensor &bbox, 37 | const at::Tensor &trans, 38 | const int no_trans, 39 | const float spatial_scale, 40 | const int output_dim, 41 | const int group_size, 42 | const int pooled_size, 43 | const int part_size, 44 | const int sample_per_part, 45 | const float trans_std); 46 | 47 | std::tuple 48 | dcn_v2_psroi_pooling_cpu_backward(const at::Tensor &out_grad, 49 | const at::Tensor &input, 50 | const at::Tensor &bbox, 51 | const at::Tensor &trans, 52 | const at::Tensor &top_count, 53 | const int no_trans, 54 | const float spatial_scale, 55 | const int output_dim, 56 | const int group_size, 57 | const int pooled_size, 58 | const int part_size, 59 | const int sample_per_part, 60 | const float trans_std); -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/csrc/cuda/vision.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | at::Tensor 5 | dcn_v2_cuda_forward(const at::Tensor &input, 6 | const at::Tensor &weight, 7 | const at::Tensor &bias, 8 | const at::Tensor &offset, 9 | const at::Tensor &mask, 10 | const int kernel_h, 11 | const int kernel_w, 12 | const int stride_h, 13 | const int stride_w, 14 | const int pad_h, 15 | const int pad_w, 16 | const int dilation_h, 17 | const int dilation_w, 18 | const int deformable_group); 19 | 20 | std::vector 21 | dcn_v2_cuda_backward(const at::Tensor &input, 22 | const at::Tensor &weight, 23 | const at::Tensor &bias, 24 | const at::Tensor &offset, 25 | const at::Tensor &mask, 26 | const at::Tensor &grad_output, 27 | int kernel_h, int kernel_w, 28 | int stride_h, int stride_w, 29 | int pad_h, int pad_w, 30 | int dilation_h, int dilation_w, 31 | int deformable_group); 32 | 33 | 34 | std::tuple 35 | dcn_v2_psroi_pooling_cuda_forward(const at::Tensor &input, 36 | const at::Tensor &bbox, 37 | const at::Tensor &trans, 38 | const int no_trans, 39 | const float spatial_scale, 40 | const int output_dim, 41 | const int group_size, 42 | const int pooled_size, 43 | const int part_size, 44 | const int sample_per_part, 45 | const float trans_std); 46 | 47 | std::tuple 48 | dcn_v2_psroi_pooling_cuda_backward(const at::Tensor &out_grad, 49 | const at::Tensor &input, 50 | const at::Tensor &bbox, 51 | const at::Tensor &trans, 52 | const at::Tensor &top_count, 53 | const int no_trans, 54 | const float spatial_scale, 55 | const int output_dim, 56 | const int group_size, 57 | const int pooled_size, 58 | const int part_size, 59 | const int sample_per_part, 60 | const float trans_std); -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/csrc/vision.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "dcn_v2.h" 3 | 4 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 5 | m.def("dcn_v2_forward", &dcn_v2_forward, "dcn_v2_forward"); 6 | m.def("dcn_v2_backward", &dcn_v2_backward, "dcn_v2_backward"); 7 | m.def("dcn_v2_psroi_pooling_forward", &dcn_v2_psroi_pooling_forward, "dcn_v2_psroi_pooling_forward"); 8 | m.def("dcn_v2_psroi_pooling_backward", &dcn_v2_psroi_pooling_backward, "dcn_v2_psroi_pooling_backward"); 9 | } 10 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/data/__init__.py: -------------------------------------------------------------------------------- 1 | from .build import make_data_loader 2 | from .build import build_test_loader -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/data/collate_batch.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | from ..structures.image_list import to_image_list 3 | 4 | 5 | class BatchCollator(object): 6 | """ 7 | From a list of samples from the dataset, 8 | returns the batched images and targets. 9 | This should be passed to the DataLoader 10 | """ 11 | 12 | def __init__(self, size_divisible=0): 13 | self.size_divisible = size_divisible 14 | 15 | def __call__(self, batch): 16 | transposed_batch = list(zip(*batch)) 17 | images = to_image_list(transposed_batch[0], self.size_divisible) 18 | targets = transposed_batch[1] 19 | img_ids = transposed_batch[2] 20 | return dict(images=images, 21 | targets=targets, 22 | img_ids=img_ids) 23 | 24 | 25 | class KeypointsOnlyBatchCollator(object): 26 | """ 27 | From a list of samples from the dataset, 28 | returns the batched images, img_infos, targets, grids and trajs 29 | This should be passed to the DataLoader 30 | """ 31 | 32 | def __init__(self, size_divisible=0): 33 | self.size_divisible = size_divisible 34 | 35 | def __call__(self, batch): 36 | transposed_batch = list(zip(*batch)) 37 | images = to_image_list(transposed_batch[0], self.size_divisible) 38 | img_infos = transposed_batch[1] 39 | targets = transposed_batch[2] 40 | grids = transposed_batch[3] 41 | trajs = transposed_batch[4] 42 | return dict(images=images, 43 | img_infos=img_infos, 44 | targets=targets, 45 | grids=grids, 46 | trajs=trajs) 47 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/data/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from .kitti import KITTIDataset 2 | from .concat_dataset import ConcatDataset 3 | from .kitti_smoke_dataset import KittiSmokeDataset 4 | from .nuscenes_smoke_dataset import NuscensesSmokeDataset 5 | 6 | __all__ = ["KITTIDataset", "KittiSmokeDataset", "NuscensesSmokeDataset"] 7 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/data/datasets/concat_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | import bisect 3 | 4 | from torch.utils.data.dataset import ConcatDataset as _ConcatDataset 5 | 6 | 7 | class ConcatDataset(_ConcatDataset): 8 | """ 9 | Same as torch.utils.data.dataset.ConcatDataset, but exposes an extra 10 | method for querying the sizes of the image 11 | """ 12 | 13 | def get_idxs(self, idx): 14 | dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx) 15 | if dataset_idx == 0: 16 | sample_idx = idx 17 | else: 18 | sample_idx = idx - self.cumulative_sizes[dataset_idx - 1] 19 | return dataset_idx, sample_idx 20 | 21 | def get_img_info(self, idx): 22 | dataset_idx, sample_idx = self.get_idxs(idx) 23 | return self.datasets[dataset_idx].get_img_info(sample_idx) 24 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/data/datasets/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | from ... import datasets 2 | 3 | from .kitti.kitti_eval import kitti_evaluation 4 | 5 | 6 | def evaluate(eval_type, dataset, predictions, output_folder): 7 | """evaluate dataset using different methods based on dataset type. 8 | Args: 9 | eval_type: 10 | dataset: Dataset object 11 | predictions(list[BoxList]): each item in the list represents the 12 | prediction results for one image. 13 | output_folder: output folder, to save evaluation files or results. 14 | **kwargs: other args. 15 | Returns: 16 | evaluation result 17 | """ 18 | args = dict( 19 | eval_type=eval_type, 20 | dataset=dataset, 21 | predictions=predictions, 22 | output_folder=output_folder, 23 | 24 | ) 25 | if isinstance(dataset, datasets.KITTIDataset): 26 | return kitti_evaluation(**args) 27 | else: 28 | dataset_name = dataset.__class__.__name__ 29 | raise NotImplementedError("Unsupported dataset type {}.".format(dataset_name)) 30 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/data/datasets/evaluation/kitti/kitti_eval.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import logging 4 | import subprocess 5 | 6 | from .....utils.miscellaneous import mkdir 7 | 8 | ID_TYPE_CONVERSION = { 9 | 0: 'Car', 10 | 1: 'Cyclist', 11 | 2: 'Pedestrian' 12 | } 13 | 14 | 15 | def kitti_evaluation( 16 | eval_type, 17 | dataset, 18 | predictions, 19 | output_folder, 20 | ): 21 | logger = logging.getLogger(__name__) 22 | if "detection" in eval_type: 23 | logger.info("performing kitti detection evaluation: ") 24 | do_kitti_detection_evaluation( 25 | dataset=dataset, 26 | predictions=predictions, 27 | output_folder=output_folder, 28 | logger=logger 29 | ) 30 | 31 | 32 | def do_kitti_detection_evaluation(dataset, 33 | predictions, 34 | output_folder, 35 | logger 36 | ): 37 | predict_folder = os.path.join(output_folder, 'data') # only recognize data 38 | mkdir(predict_folder) 39 | 40 | for image_id, prediction in predictions.items(): 41 | predict_txt = image_id + '.txt' 42 | predict_txt = os.path.join(predict_folder, predict_txt) 43 | 44 | generate_kitti_3d_detection(prediction, predict_txt) 45 | 46 | logger.info("Evaluate on KITTI dataset") 47 | output_dir = os.path.abspath(output_folder) 48 | os.chdir('./smoke/data/datasets/evaluation/kitti/kitti_eval') 49 | label_dir = getattr(dataset, 'label_dir') 50 | if not os.path.isfile('evaluate_object_3d_offline'): 51 | subprocess.Popen('g++ -O3 -DNDEBUG -o evaluate_object_3d_offline evaluate_object_3d_offline.cpp', shell=True) 52 | command = "./evaluate_object_3d_offline.exe {} {}".format(label_dir, output_dir) 53 | output = subprocess.check_output(command, shell=True, universal_newlines=True).strip() 54 | logger.info(output) 55 | os.chdir('./tools') 56 | 57 | 58 | def generate_kitti_3d_detection(prediction, predict_txt): 59 | with open(predict_txt, 'w', newline='') as f: 60 | w = csv.writer(f, delimiter=' ', lineterminator='\n') 61 | if len(prediction) == 0: 62 | w.writerow([]) 63 | else: 64 | for p in prediction: 65 | p = p.numpy() 66 | p = p.round(4) 67 | type = ID_TYPE_CONVERSION[int(p[0])] 68 | row = [type, 0, 0] + p[1:].tolist() 69 | w.writerow(row) 70 | 71 | check_last_line_break(predict_txt) 72 | 73 | 74 | def check_last_line_break(predict_txt): 75 | f = open(predict_txt, 'rb+') 76 | try: 77 | f.seek(-1, os.SEEK_END) 78 | except: 79 | pass 80 | else: 81 | if f.__next__() == b'\n': 82 | f.seek(-1, os.SEEK_END) 83 | f.truncate() 84 | f.close() 85 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/data/datasets/evaluation/kitti/kitti_eval/kitti_eval.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lizn-zn/Nesy-Programming/1410e721c7be4782b50219f7e18a4e247638edb3/Nuscenes-Driving/models/smoke/data/datasets/evaluation/kitti/kitti_eval/kitti_eval.exe -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/data/samplers/__init__.py: -------------------------------------------------------------------------------- 1 | from .grouped_batch_sampler import GroupedBatchSampler 2 | from .distributed_sampler import ( 3 | TrainingSampler, 4 | InferenceSampler, 5 | ) 6 | 7 | __all__ = ["GroupedBatchSampler", 8 | "TrainingSampler", 9 | "InferenceSampler",] 10 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/data/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | from .build import build_transforms -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/data/transforms/build.py: -------------------------------------------------------------------------------- 1 | from . import transforms as T 2 | 3 | 4 | def build_transforms(cfg, is_train=True): 5 | to_bgr = cfg.INPUT.TO_BGR 6 | 7 | normalize_transform = T.Normalize( 8 | mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD, to_bgr=to_bgr 9 | ) 10 | 11 | transform = T.Compose( 12 | [ 13 | T.ToTensor(), 14 | normalize_transform, 15 | ] 16 | ) 17 | return transform 18 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/data/transforms/transforms.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torchvision.transforms import functional as F 3 | 4 | 5 | class Compose(): 6 | def __init__(self, transforms): 7 | self.transforms = transforms 8 | 9 | def __call__(self, image, target): 10 | for t in self.transforms: 11 | image, target = t(image, target) 12 | return image, target 13 | 14 | 15 | class ToTensor(): 16 | def __call__(self, image, target): 17 | return F.to_tensor(image), target 18 | 19 | 20 | class Normalize(): 21 | def __init__(self, mean, std, to_bgr=True): 22 | self.mean = mean 23 | self.std = std 24 | self.to_bgr = to_bgr 25 | 26 | def __call__(self, image, target): 27 | if self.to_bgr: 28 | image = image[[2, 1, 0]] 29 | image = F.normalize(image, mean=self.mean, std=self.std) 30 | return image, target 31 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/engine/__init__.py: -------------------------------------------------------------------------------- 1 | from .defaults import * 2 | from .launch import * 3 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/engine/defaults.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import torch 5 | 6 | from ..utils import comm 7 | from ..utils.miscellaneous import mkdir 8 | from ..utils.logger import setup_logger 9 | from ..utils.collect_env import collect_env_info 10 | from ..utils.envs import seed_all_rng 11 | 12 | import platform 13 | 14 | __all__ = ["default_argument_parser", "default_setup"] 15 | 16 | 17 | def default_argument_parser(): 18 | parser = argparse.ArgumentParser(description="Detectron2 Training") 19 | parser.add_argument("--config-file", default="./data/kitti-smoke/configs/smoke_gn_vector.yaml", 20 | metavar="FILE", help="path to config file") 21 | parser.add_argument("--eval-only", action="store_true", help="perform evaluation only") 22 | parser.add_argument( 23 | "--ckpt", 24 | help="The path to the checkpoint for test, default is the latest checkpoint.", 25 | default=None, 26 | ) 27 | parser.add_argument("--num-gpus", type=int, default=1, help="number of gpus *per machine*") 28 | parser.add_argument("--num-machines", type=int, default=1) 29 | parser.add_argument( 30 | "--machine-rank", type=int, default=0, help="the rank of this machine (unique per machine)" 31 | ) 32 | 33 | # PyTorch still may leave orphan processes in multi-gpu training. 34 | # Therefore we use a deterministic way to obtain port, 35 | # so that users are aware of orphan processes by seeing the port occupied. 36 | plat = platform.system().lower() 37 | if plat == 'windows': 38 | port = 100 39 | else: 40 | port = 2 ** 15 + 2 ** 14 + hash(os.getuid()) % 2 ** 14 41 | parser.add_argument("--dist-url", default="tcp://127.0.0.1:{}".format(port)) 42 | parser.add_argument( 43 | "opts", 44 | help="Modify config options using the command-line", 45 | default=None, 46 | nargs=argparse.REMAINDER, 47 | ) 48 | return parser 49 | 50 | 51 | def default_setup(cfg, args): 52 | output_dir = cfg.OUTPUT_DIR 53 | if output_dir: 54 | mkdir(output_dir) 55 | 56 | rank = comm.get_rank() 57 | logger = setup_logger(output_dir, rank) 58 | logger.info("Using {} GPUs".format(args.num_gpus)) 59 | logger.info("Collecting environment info") 60 | logger.info("\n" + collect_env_info()) 61 | logger.info(args) 62 | 63 | logger.info("Loaded configuration file {}".format(args.config_file)) 64 | with open(args.config_file, "r") as cf: 65 | config_str = "\n" + cf.read() 66 | logger.info(config_str) 67 | logger.info("Running with config:\n{}".format(cfg)) 68 | 69 | # make sure each worker has a different, yet deterministic seed if specified 70 | seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank) 71 | 72 | # cudnn benchmark has large overhead. It shouldn't be used considering the small size of 73 | # typical validation set. 74 | if not (hasattr(args, "eval_only") and args.eval_only): 75 | torch.backends.cudnn.benchmark = cfg.CUDNN_BENCHMARK 76 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/engine/inference.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from tqdm import tqdm 3 | 4 | import torch 5 | 6 | from ..utils import comm 7 | from ..utils.timer import Timer, get_time_str 8 | from ..data.datasets.evaluation import evaluate 9 | 10 | 11 | def compute_on_dataset(model, data_loader, device, timer=None): 12 | model.eval() 13 | results_dict = {} 14 | cpu_device = torch.device("cpu") 15 | for _, batch in enumerate(tqdm(data_loader)): 16 | images, targets, image_ids = batch["images"], batch["targets"], batch["img_ids"] 17 | images = images.to(device) 18 | with torch.no_grad(): 19 | if timer: 20 | timer.tic() 21 | output = model(images, targets) 22 | if timer: 23 | torch.cuda.synchronize() 24 | timer.toc() 25 | output = output.to(cpu_device) 26 | results_dict.update( 27 | {img_id: output for img_id in image_ids} 28 | ) 29 | return results_dict 30 | 31 | 32 | def inference( 33 | model, 34 | data_loader, 35 | dataset_name, 36 | eval_types=("detections",), 37 | device="cuda", 38 | output_folder=None, 39 | 40 | ): 41 | device = torch.device(device) 42 | num_devices = comm.get_world_size() 43 | logger = logging.getLogger(__name__) 44 | dataset = data_loader.dataset 45 | logger.info("Start evaluation on {} dataset({} images).".format(dataset_name, len(dataset))) 46 | 47 | total_timer = Timer() 48 | inference_timer = Timer() 49 | total_timer.tic() 50 | predictions = compute_on_dataset(model, data_loader, device, inference_timer) 51 | comm.synchronize() 52 | 53 | total_time = total_timer.toc() 54 | total_time_str = get_time_str(total_time) 55 | logger.info( 56 | "Total run time: {} ({} s / img per device, on {} devices)".format( 57 | total_time_str, total_time * num_devices / len(dataset), num_devices 58 | ) 59 | ) 60 | total_infer_time = get_time_str(inference_timer.total_time) 61 | logger.info( 62 | "Model inference time: {} ({} s / img per device, on {} devices)".format( 63 | total_infer_time, 64 | inference_timer.total_time * num_devices / len(dataset), 65 | num_devices, 66 | ) 67 | ) 68 | if not comm.is_main_process(): 69 | return 70 | 71 | return evaluate(eval_type=eval_types, 72 | dataset=dataset, 73 | predictions=predictions, 74 | output_folder=output_folder, ) 75 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/engine/launch.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import torch 3 | import torch.distributed as dist 4 | import torch.multiprocessing as mp 5 | 6 | from ..utils import comm 7 | 8 | __all__ = ["launch"] 9 | 10 | 11 | def _find_free_port(): 12 | import socket 13 | 14 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 15 | # Binding to port 0 will cause the OS to find an available port for us 16 | sock.bind(("", 0)) 17 | port = sock.getsockname()[1] 18 | sock.close() 19 | # NOTE: there is still a chance the port could be taken by other processes. 20 | return port 21 | 22 | 23 | def launch(main_func, 24 | num_gpus_per_machine, 25 | num_machines=1, 26 | machine_rank=0, 27 | dist_url=None, 28 | args=()): 29 | """ 30 | Args: 31 | main_func: a function that will be called by `main_func(*args)` 32 | num_machines (int): the total number of machines 33 | machine_rank (int): the rank of this machine (one per machine) 34 | dist_url (str): url to connect to for distributed training, including protocol 35 | e.g. "tcp://127.0.0.1:8686". 36 | Can be set to auto to automatically select a free port on localhost 37 | args (tuple): arguments passed to main_func 38 | """ 39 | world_size = num_machines * num_gpus_per_machine 40 | if world_size > 1: 41 | # https://github.com/pytorch/pytorch/pull/14391 42 | # TODO prctl in spawned processes 43 | 44 | if dist_url == "auto": 45 | assert num_machines == 1, "dist_url=auto cannot work with distributed training." 46 | port = _find_free_port() 47 | dist_url = f"tcp://127.0.0.1:{port}" 48 | 49 | mp.spawn( 50 | _distributed_worker, 51 | nprocs=num_gpus_per_machine, 52 | args=(main_func, world_size, num_gpus_per_machine, machine_rank, dist_url, args), 53 | daemon=False, 54 | ) 55 | else: 56 | main_func(*args) 57 | 58 | 59 | def _distributed_worker( 60 | local_rank, main_func, world_size, num_gpus_per_machine, machine_rank, dist_url, args 61 | ): 62 | assert torch.cuda.is_available(), "cuda is not available. Please check your installation." 63 | global_rank = machine_rank * num_gpus_per_machine + local_rank 64 | try: 65 | dist.init_process_group( 66 | backend="NCCL", init_method=dist_url, world_size=world_size, rank=global_rank 67 | ) 68 | except Exception as e: 69 | logger = logging.getLogger(__name__) 70 | logger.error("Process group URL: {}".format(dist_url)) 71 | raise e 72 | # synchronize is needed here to prevent a possible timeout after calling init_process_group 73 | # See: https://github.com/facebookresearch/maskrcnn-benchmark/issues/172 74 | comm.synchronize() 75 | 76 | assert num_gpus_per_machine <= torch.cuda.device_count() 77 | torch.cuda.set_device(local_rank) 78 | 79 | # Setup the local process group (which contains ranks within the same machine) 80 | assert comm._LOCAL_PROCESS_GROUP is None 81 | num_machines = world_size // num_gpus_per_machine 82 | for i in range(num_machines): 83 | ranks_on_i = list(range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine)) 84 | pg = dist.new_group(ranks_on_i) 85 | if i == machine_rank: 86 | comm._LOCAL_PROCESS_GROUP = pg 87 | 88 | main_func(*args) 89 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/engine/test_net.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from ..data import build_test_loader 4 | from ..engine.inference import inference 5 | from ..utils import comm 6 | from ..utils.miscellaneous import mkdir 7 | 8 | 9 | def run_test(cfg, model): 10 | eval_types = ("detection",) 11 | output_folders = [None] * len(cfg.DATASETS.TEST) 12 | dataset_names = cfg.DATASETS.TEST 13 | if cfg.OUTPUT_DIR: 14 | for idx, dataset_name in enumerate(dataset_names): 15 | output_folder = os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name) 16 | mkdir(output_folder) 17 | output_folders[idx] = output_folder 18 | data_loaders_val = build_test_loader(cfg) 19 | for output_folder, dataset_name, data_loader_val in zip(output_folders, dataset_names, data_loaders_val): 20 | inference( 21 | model, 22 | data_loader_val, 23 | dataset_name=dataset_name, 24 | eval_types=eval_types, 25 | device=cfg.MODEL.DEVICE, 26 | output_folder=output_folder, 27 | ) 28 | comm.synchronize() 29 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/layers/deform_conv.py: -------------------------------------------------------------------------------- 1 | import torch.nn.functional as F 2 | from torch import nn 3 | 4 | # from smoke.layers.dcn_v2 import DCN 5 | from .dcn_hdb import DCN 6 | 7 | 8 | class DeformConv(nn.Module): 9 | def __init__(self, 10 | in_channel, 11 | out_channel, 12 | norm_func): 13 | super(DeformConv, self).__init__() 14 | 15 | self.norm = norm_func(out_channel) 16 | self.relu = nn.ReLU(inplace=True) 17 | self.deform_conv = DCN(in_channels=in_channel, 18 | out_channels=out_channel, 19 | kernel_size=(3, 3), 20 | stride=1, 21 | padding=1, 22 | dilation=1, 23 | deformable_groups=1) 24 | 25 | 26 | def forward(self, x): 27 | x = self.deform_conv(x) 28 | x = self.norm(x) 29 | x = self.relu(x) 30 | 31 | return x 32 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/layers/focal_loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | 4 | 5 | class FocalLoss(nn.Module): 6 | def __init__(self, alpha=2, beta=4): 7 | super(FocalLoss, self).__init__() 8 | self.alpha = alpha 9 | self.beta = beta 10 | 11 | def forward(self, prediction, target): 12 | positive_index = target.eq(1).float() 13 | negative_index = target.lt(1).float() 14 | 15 | negative_weights = torch.pow(1 - target, self.beta) 16 | loss = 0. 17 | 18 | positive_loss = torch.log(prediction) \ 19 | * torch.pow(1 - prediction, self.alpha) * positive_index 20 | negative_loss = torch.log(1 - prediction) \ 21 | * torch.pow(prediction, self.alpha) * negative_weights * negative_index 22 | 23 | num_positive = positive_index.float().sum() 24 | positive_loss = positive_loss.sum() 25 | negative_loss = negative_loss.sum() 26 | 27 | if num_positive == 0: 28 | loss -= negative_loss 29 | else: 30 | loss -= (positive_loss + negative_loss) / num_positive 31 | 32 | return loss 33 | 34 | 35 | class ClampedFocalLoss(nn.Module): 36 | def __init__(self, alpha=0.75, gamma=2, epsilon=0.0001): 37 | super(ClampedFocalLoss, self).__init__() 38 | self.alpha = alpha 39 | self.gamma = gamma 40 | self.epsilon = epsilon 41 | 42 | def forward(self, prediction, target): 43 | positive_index = target.eq(1).float() 44 | negative_index = target.lt(1).float() 45 | 46 | # negative_weights = torch.pow(1 - target, self.beta) 47 | loss = 0. 48 | 49 | # clamp the prediction to avoid log(0) 50 | prediction = torch.clamp(prediction, min=self.epsilon,max=1-self.epsilon) 51 | # 52 | positive_loss = -self.alpha * torch.pow(1-prediction, self.gamma) * torch.log(prediction) * positive_index 53 | negative_loss = -(1-self.alpha) * torch.pow(prediction, self.gamma) * torch.log(1-prediction) * negative_index 54 | 55 | # positive_loss = torch.log(prediction) \ 56 | # * torch.pow(1 - prediction, self.alpha) * positive_index 57 | # negative_loss = torch.log(1 - prediction) \ 58 | # * torch.pow(prediction, self.alpha) * negative_weights * negative_index 59 | # 60 | 61 | num_positive = positive_index.float().sum() 62 | positive_loss = positive_loss.sum() 63 | negative_loss = negative_loss.sum() 64 | 65 | # loss = positive_loss + negative_loss 66 | 67 | if num_positive == 0.: 68 | loss += negative_loss 69 | else: 70 | loss += (positive_loss + negative_loss) / num_positive 71 | 72 | return loss 73 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/layers/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.nn import functional as F 3 | 4 | 5 | def sigmoid_hm(hm_features): 6 | x = hm_features.sigmoid_() 7 | x = x.clamp(min=1e-4, max=1 - 1e-4) 8 | 9 | return x 10 | 11 | 12 | def nms_hm(heat_map, kernel=3): 13 | pad = (kernel - 1) // 2 14 | 15 | hmax = F.max_pool2d(heat_map, 16 | kernel_size=(kernel, kernel), 17 | stride=1, 18 | padding=pad) 19 | eq_index = (hmax == heat_map).float() 20 | 21 | return heat_map * eq_index 22 | 23 | 24 | def select_topk(heat_map, K=100): 25 | ''' 26 | Args: 27 | heat_map: heat_map in [N, C, H, W] 28 | K: top k samples to be selected 29 | score: detection threshold 30 | 31 | Returns: 32 | 33 | ''' 34 | batch, cls, height, width = heat_map.size() 35 | 36 | # First select topk scores in all classes and batchs 37 | # [N, C, H, W] -----> [N, C, H*W] 38 | heat_map = heat_map.view(batch, cls, -1) 39 | # Both in [N, C, K] 40 | topk_scores_all, topk_inds_all = torch.topk(heat_map, K) 41 | 42 | # topk_inds_all = topk_inds_all % (height * width) # todo: this seems redudant 43 | topk_ys = (topk_inds_all / width).float() 44 | topk_xs = (topk_inds_all % width).float() 45 | 46 | assert isinstance(topk_xs, torch.cuda.FloatTensor) 47 | assert isinstance(topk_ys, torch.cuda.FloatTensor) 48 | 49 | # Select topK examples across channel 50 | # [N, C, K] -----> [N, C*K] 51 | topk_scores_all = topk_scores_all.view(batch, -1) 52 | # Both in [N, K] 53 | topk_scores, topk_inds = torch.topk(topk_scores_all, K) 54 | topk_clses = (topk_inds / K).float() 55 | 56 | assert isinstance(topk_clses, torch.cuda.FloatTensor) 57 | 58 | # First expand it as 3 dimension 59 | topk_inds_all = _gather_feat(topk_inds_all.view(batch, -1, 1), topk_inds).view(batch, K) 60 | topk_ys = _gather_feat(topk_ys.view(batch, -1, 1), topk_inds).view(batch, K) 61 | topk_xs = _gather_feat(topk_xs.view(batch, -1, 1), topk_inds).view(batch, K) 62 | 63 | return topk_scores, topk_inds_all, topk_clses, topk_ys, topk_xs 64 | 65 | 66 | def _gather_feat(feat, ind): 67 | ''' 68 | Select specific indexs on featuremap 69 | Args: 70 | feat: all results in 3 dimensions 71 | ind: positive index 72 | 73 | Returns: 74 | 75 | ''' 76 | channel = feat.size(-1) 77 | ind = ind.unsqueeze(-1).expand(ind.size(0), ind.size(1), channel) 78 | feat = feat.gather(1, ind) 79 | 80 | return feat 81 | 82 | 83 | def select_point_of_interest(batch, index, feature_maps): 84 | ''' 85 | Select POI(point of interest) on feature map 86 | Args: 87 | batch: batch size 88 | index: in point format or index format 89 | feature_maps: regression feature map in [N, C, H, W] 90 | 91 | Returns: 92 | 93 | ''' 94 | w = feature_maps.shape[3] 95 | if len(index.shape) == 3: 96 | index = index[:, :, 1] * w + index[:, :, 0] 97 | index = index.view(batch, -1) 98 | # [N, C, H, W] -----> [N, H, W, C] 99 | feature_maps = feature_maps.permute(0, 2, 3, 1).contiguous() 100 | channel = feature_maps.shape[-1] 101 | # [N, H, W, C] -----> [N, H*W, C] 102 | feature_maps = feature_maps.view(batch, -1, channel) 103 | # expand index in channels 104 | index = index.unsqueeze(-1).repeat(1, 1, channel) 105 | # select specific features bases on POIs 106 | feature_maps = feature_maps.gather(1, index.long()) 107 | 108 | return feature_maps 109 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/modeling/backbone/__init__.py: -------------------------------------------------------------------------------- 1 | from .backbone import build_backbone -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/modeling/backbone/backbone.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | from torch import nn 4 | 5 | from .. import registry 6 | from . import dla 7 | 8 | @registry.BACKBONES.register("DLA-34-DCN") 9 | def build_dla_backbone(cfg): 10 | body = dla.DLA(cfg) 11 | model = nn.Sequential(OrderedDict([("body", body)])) 12 | model.out_channels = cfg.MODEL.BACKBONE.BACKBONE_OUT_CHANNELS 13 | return model 14 | 15 | 16 | def build_backbone(cfg): 17 | assert cfg.MODEL.BACKBONE.CONV_BODY in registry.BACKBONES, \ 18 | "cfg.MODEL.BACKBONE.CONV_BODY: {} are not registered in registry".format( 19 | cfg.MODEL.BACKBONE.CONV_BODY 20 | ) 21 | return registry.BACKBONES[cfg.MODEL.BACKBONE.CONV_BODY](cfg) 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/modeling/detector/__init__.py: -------------------------------------------------------------------------------- 1 | from .detectors import build_detection_model -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/modeling/detector/detectors.py: -------------------------------------------------------------------------------- 1 | from .keypoint_detector import KeypointDetector, KeypointsOnlyDetector, BinaryClassificationDetector 2 | 3 | 4 | def build_detection_model(cfg): 5 | if cfg.MODEL.SMOKE_ON: 6 | return KeypointDetector(cfg) 7 | elif cfg.MODEL.KEYPOINTS_ONLY: # build the Keypoints-Only Detector for our own initial experiments 8 | return KeypointsOnlyDetector(cfg) 9 | elif cfg.MODEL.BINARY_CLASSIFICATION: 10 | return BinaryClassificationDetector(cfg) -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/modeling/heads/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lizn-zn/Nesy-Programming/1410e721c7be4782b50219f7e18a4e247638edb3/Nuscenes-Driving/models/smoke/modeling/heads/__init__.py -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/modeling/heads/heads.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from .smoke_head.smoke_head import build_smoke_head, build_smoke_keypoints_only_head 4 | 5 | 6 | def build_heads(cfg, in_channels): 7 | if cfg.MODEL.SMOKE_ON: 8 | return build_smoke_head(cfg, in_channels) 9 | elif cfg.MODEL.KEYPOINTS_ONLY: # build SMOKEKeypointsOnlyHead used for KeyPointsOnlyDetector constructed by ourselves 10 | return build_smoke_keypoints_only_head(cfg, in_channels) 11 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/modeling/heatmap_coder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from skimage import transform as trans 3 | 4 | 5 | def get_transfrom_matrix(center_scale, output_size): 6 | center, scale = center_scale[0], center_scale[1] 7 | # todo: further add rot and shift here. 8 | src_w = scale[0] 9 | dst_w = output_size[0] 10 | dst_h = output_size[1] 11 | 12 | src_dir = np.array([0, src_w * -0.5]) 13 | dst_dir = np.array([0, dst_w * -0.5]) 14 | 15 | src = np.zeros((3, 2), dtype=np.float32) 16 | dst = np.zeros((3, 2), dtype=np.float32) 17 | src[0, :] = center 18 | src[1, :] = center + src_dir 19 | dst[0, :] = np.array([dst_w * 0.5, dst_h * 0.5]) 20 | dst[1, :] = np.array([dst_w * 0.5, dst_h * 0.5]) + dst_dir 21 | 22 | src[2, :] = get_3rd_point(src[0, :], src[1, :]) 23 | dst[2, :] = get_3rd_point(dst[0, :], dst[1, :]) 24 | 25 | get_matrix = trans.estimate_transform("affine", src, dst) 26 | matrix = get_matrix.params 27 | 28 | return matrix.astype(np.float32) 29 | 30 | 31 | def affine_transform(point, matrix): 32 | point_exd = np.array([point[0], point[1], 1.]) 33 | new_point = np.matmul(matrix, point_exd) 34 | 35 | return new_point[:2] 36 | 37 | 38 | def get_3rd_point(point_a, point_b): 39 | d = point_a - point_b 40 | point_c = point_b + np.array([-d[1], d[0]]) 41 | return point_c 42 | 43 | 44 | def gaussian_radius(h, w, thresh_min=0.7): 45 | a1 = 1 46 | b1 = h + w 47 | c1 = h * w * (1 - thresh_min) / (1 + thresh_min) 48 | sq1 = np.sqrt(b1 ** 2 - 4 * a1 * c1) 49 | r1 = (b1 - sq1) / (2 * a1) 50 | 51 | a2 = 4 52 | b2 = 2 * (h + w) 53 | c2 = (1 - thresh_min) * w * h 54 | sq2 = np.sqrt(b2 ** 2 - 4 * a2 * c2) 55 | r2 = (b2 - sq2) / (2 * a2) 56 | 57 | a3 = 4 * thresh_min 58 | b3 = -2 * thresh_min * (h + w) 59 | c3 = (thresh_min - 1) * w * h 60 | sq3 = np.sqrt(b3 ** 2 - 4 * a3 * c3) 61 | r3 = (b3 + sq3) / (2 * a3) 62 | 63 | return min(r1, r2, r3) 64 | 65 | 66 | def gaussian2D(shape, sigma=1): 67 | m, n = [(ss - 1.) / 2. for ss in shape] 68 | y, x = np.ogrid[-m:m + 1, -n:n + 1] 69 | 70 | h = np.exp(-(x * x + y * y) / (2 * sigma * sigma)) 71 | h[h < np.finfo(h.dtype).eps * h.max()] = 0 72 | return h 73 | 74 | 75 | def draw_umich_gaussian(heatmap, center, radius, k=1): 76 | diameter = 2 * radius + 1 77 | gaussian = gaussian2D((diameter, diameter), sigma=diameter / 6) 78 | 79 | x, y = int(center[0]), int(center[1]) 80 | 81 | height, width = heatmap.shape[0:2] 82 | 83 | left, right = min(x, radius), min(width - x, radius + 1) 84 | top, bottom = min(y, radius), min(height - y, radius + 1) 85 | 86 | masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] 87 | masked_gaussian = gaussian[radius - top:radius + bottom, radius - left:radius + right] 88 | if min(masked_gaussian.shape) > 0 and min(masked_heatmap.shape) > 0: 89 | np.maximum(masked_heatmap, masked_gaussian * k, out=masked_heatmap) 90 | 91 | return heatmap 92 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/modeling/make_layers.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | import torch 4 | from torch import nn 5 | 6 | from ..config import cfg 7 | 8 | 9 | def _make_conv_level(in_channels, out_channels, num_convs, norm_func, 10 | stride=1, dilation=1): 11 | """ 12 | make conv layers based on its number. 13 | """ 14 | modules = [] 15 | for i in range(num_convs): 16 | modules.extend([ 17 | nn.Conv2d(in_channels, out_channels, kernel_size=3, 18 | stride=stride if i == 0 else 1, 19 | padding=dilation, bias=False, dilation=dilation), 20 | norm_func(out_channels), 21 | nn.ReLU(inplace=True)]) 22 | in_channels = out_channels 23 | 24 | return nn.Sequential(*modules) 25 | 26 | 27 | def group_norm(out_channels): 28 | num_groups = cfg.MODEL.GROUP_NORM.NUM_GROUPS 29 | if out_channels % 32 == 0: 30 | return nn.GroupNorm(num_groups, out_channels) 31 | else: 32 | return nn.GroupNorm(num_groups // 2, out_channels) 33 | 34 | 35 | def _fill_up_weights(up): 36 | # todo: we can replace math here? 37 | w = up.weight.data 38 | f = math.ceil(w.size(2) / 2) 39 | c = (2 * f - 1 - f % 2) / (2. * f) 40 | for i in range(w.size(2)): 41 | for j in range(w.size(3)): 42 | w[0, 0, i, j] = \ 43 | (1 - math.fabs(i / f - c)) * (1 - math.fabs(j / f - c)) 44 | for c in range(1, w.size(0)): 45 | w[c, 0, :, :] = w[0, 0, :, :] 46 | 47 | 48 | def _fill_fc_weights(layers): 49 | for m in layers.modules(): 50 | if isinstance(m, nn.Conv2d): 51 | if m.bias is not None: 52 | nn.init.constant_(m.bias, 0) 53 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/modeling/registry.py: -------------------------------------------------------------------------------- 1 | from ..utils.registry import Registry 2 | 3 | BACKBONES = Registry() 4 | SMOKE_HEADS = Registry() 5 | SMOKE_PREDICTOR = Registry() 6 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/modeling/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def cat(tensors, dim=0): 4 | """ 5 | Efficient version of torch.cat that avoids a copy if there is only a single element in a list 6 | """ 7 | assert isinstance(tensors, (list, tuple)) 8 | if len(tensors) == 1: 9 | return tensors[0] 10 | return torch.cat(tensors, dim) 11 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/solver/__init__.py: -------------------------------------------------------------------------------- 1 | from .build import make_optimizer -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/solver/build.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def make_optimizer(cfg, model): 5 | params = [] 6 | for key, value in model.named_parameters(): 7 | if not value.requires_grad: 8 | continue 9 | lr = cfg.SOLVER.BASE_LR 10 | if "bias" in key: 11 | lr = cfg.SOLVER.BASE_LR * cfg.SOLVER.BIAS_LR_FACTOR 12 | params += [{"params": [value], "lr": lr}] 13 | 14 | optimizer = torch.optim.Adam(params, lr=lr) 15 | 16 | return optimizer 17 | 18 | 19 | def make_lr_scheduler(cfg, optimizer): 20 | return torch.optim.lr_scheduler.MultiStepLR( 21 | optimizer, 22 | cfg.SOLVER.STEPS 23 | ) 24 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/structures/image_list.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | 3 | import torch 4 | 5 | 6 | class ImageList(): 7 | """ 8 | Structure that holds a list of images (of possibly 9 | varying sizes) as a single tensor. 10 | This works by padding the images to the same size, 11 | and storing in a field the original sizes of each image 12 | """ 13 | 14 | def __init__(self, tensors, image_sizes): 15 | """ 16 | Arguments: 17 | tensors (tensor) 18 | image_sizes (list[tuple[int, int]]) 19 | """ 20 | self.tensors = tensors 21 | self.image_sizes = image_sizes 22 | 23 | def to(self, *args, **kwargs): 24 | cast_tensor = self.tensors.to(*args, **kwargs) 25 | return ImageList(cast_tensor, self.image_sizes) 26 | 27 | 28 | def to_image_list(tensors, size_divisible=0): 29 | """ 30 | tensors can be an ImageList, a torch.Tensor or 31 | an iterable of Tensors. It can't be a numpy array. 32 | When tensors is an iterable of Tensors, it pads 33 | the Tensors with zeros so that they have the same 34 | shape 35 | """ 36 | if isinstance(tensors, torch.Tensor) and size_divisible > 0: 37 | tensors = [tensors] 38 | 39 | if isinstance(tensors, ImageList): 40 | return tensors 41 | elif isinstance(tensors, torch.Tensor): 42 | # single tensor shape can be inferred 43 | if tensors.dim() == 3: 44 | tensors = tensors[None] 45 | assert tensors.dim() == 4 46 | image_sizes = [tensor.shape[-2:] for tensor in tensors] 47 | return ImageList(tensors, image_sizes) 48 | elif isinstance(tensors, (tuple, list)): 49 | max_size = tuple(max(s) for s in zip(*[img.shape for img in tensors])) 50 | 51 | # TODO Ideally, just remove this and let me model handle arbitrary 52 | # input sizs 53 | if size_divisible > 0: 54 | import math 55 | 56 | stride = size_divisible 57 | max_size = list(max_size) 58 | max_size[1] = int(math.ceil(max_size[1] / stride) * stride) 59 | max_size[2] = int(math.ceil(max_size[2] / stride) * stride) 60 | max_size = tuple(max_size) 61 | 62 | batch_shape = (len(tensors),) + max_size 63 | batched_imgs = tensors[0].new(*batch_shape).zero_() 64 | for img, pad_img in zip(tensors, batched_imgs): 65 | pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) 66 | 67 | image_sizes = [im.shape[-2:] for im in tensors] 68 | 69 | return ImageList(batched_imgs, image_sizes) 70 | else: 71 | raise TypeError("Unsupported type for to_image_list: {}".format(type(tensors))) -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/structures/params_3d.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class ParamsList(): 5 | """ 6 | This class represents labels of specific object. 7 | """ 8 | 9 | def __init__(self, image_size, is_train=True): 10 | self.size = image_size 11 | self.is_train = is_train 12 | self.extra_fields = {} 13 | 14 | def add_field(self, field, field_data): 15 | field_data = field_data if isinstance(field_data, torch.Tensor) else torch.as_tensor(field_data) 16 | self.extra_fields[field] = field_data 17 | 18 | def get_field(self, field): 19 | return self.extra_fields[field] 20 | 21 | def has_field(self, field): 22 | return field in self.extra_fields 23 | 24 | def fields(self): 25 | return list(self.extra_fields.keys()) 26 | 27 | def _copy_extra_fields(self, target): 28 | for k, v in target.extra_fields.items(): 29 | self.extra_fields[k] = v 30 | 31 | def to(self, device): 32 | target = ParamsList(self.size, self.is_train) 33 | for k, v in self.extra_fields.items(): 34 | if hasattr(v, "to"): 35 | v = v.to(device) 36 | target.add_field(k, v) 37 | return target 38 | 39 | def __len__(self): 40 | if self.is_train: 41 | reg_num = len(torch.nonzero(self.extra_fields["reg_mask"])) 42 | else: 43 | reg_num = 0 44 | return reg_num 45 | 46 | def __repr__(self): 47 | s = self.__class__.__name__ + "(" 48 | s += "regress_number={}, ".format(len(self)) 49 | s += "image_width={}, ".format(self.size[0]) 50 | s += "image_height={})".format(self.size[1]) 51 | return s 52 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/utils/collect_env.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | import PIL 3 | 4 | from torch.utils.collect_env import get_pretty_env_info 5 | 6 | 7 | def get_pil_version(): 8 | return "\n Pillow ({})".format(PIL.__version__) 9 | 10 | 11 | def collect_env_info(): 12 | env_str = get_pretty_env_info() 13 | env_str += get_pil_version() 14 | return env_str 15 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/utils/envs.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | import logging 4 | import random 5 | import numpy as np 6 | 7 | import torch 8 | 9 | 10 | def seed_all_rng(seed=None): 11 | """ 12 | Set the random seed for the RNG in torch, numpy and python. 13 | 14 | Args: 15 | seed (int): if None, will use a strong random seed. 16 | """ 17 | if seed is None: 18 | seed = ( 19 | os.getpid() 20 | + int(datetime.now().strftime("%S%f")) 21 | + int.from_bytes(os.urandom(2), "big") 22 | ) 23 | logger = logging.getLogger(__name__) 24 | logger.info("Using a generated random seed {}".format(seed)) 25 | np.random.seed(seed) 26 | torch.set_rng_state(torch.manual_seed(seed).get_state()) 27 | random.seed(seed) 28 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/utils/imports.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from importlib import util 3 | 4 | 5 | # from https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa 6 | def import_file(module_name, file_path, make_importable=False): 7 | spec = util.spec_from_file_location(module_name, file_path) 8 | module = util.module_from_spec(spec) 9 | spec.loader.exec_module(module) 10 | 11 | if make_importable: 12 | sys.modules[module_name] = module_name 13 | return module -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/utils/logger.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import logging 3 | import os 4 | import sys 5 | 6 | 7 | @functools.lru_cache() # so that calling setup_logger multiple times won't add many handlers 8 | def setup_logger(output_dir, distributed_rank=0, name="smoke", file_name="log.txt"): 9 | ''' 10 | Args: 11 | output_dir (str): a directory saves output log files 12 | name (str): name of the logger 13 | file_name (str): name of log file 14 | ''' 15 | logger = logging.getLogger(name) 16 | logger.setLevel(logging.DEBUG) 17 | # don't log results for the non-master process 18 | if distributed_rank > 0: 19 | return logger 20 | ch = logging.StreamHandler(stream=sys.stdout) 21 | ch.setLevel(logging.DEBUG) 22 | formatter = logging.Formatter("[%(asctime)s] %(name)s %(levelname)s: %(message)s") 23 | ch.setFormatter(formatter) 24 | logger.addHandler(ch) 25 | 26 | if output_dir: 27 | fh = logging.FileHandler(os.path.join(output_dir, file_name)) 28 | fh.setLevel(logging.DEBUG) 29 | fh.setFormatter(formatter) 30 | logger.addHandler(fh) 31 | 32 | return logger 33 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/utils/metric_logger.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | from collections import defaultdict 3 | from collections import deque 4 | 5 | import torch 6 | 7 | 8 | class SmoothedValue(): 9 | """Track a series of values and provide access to smoothed values over a 10 | window or the global series average. 11 | """ 12 | 13 | def __init__(self, window_size=3): 14 | self.window_size = window_size 15 | self.deque = deque(maxlen=window_size) 16 | self.series = [] 17 | self.total = 0.0 18 | self.count = 0 19 | 20 | def update(self, value): 21 | self.deque.append(value) 22 | self.series.append(value) 23 | self.count += 1 24 | self.total += value 25 | 26 | @property 27 | def median(self): 28 | d = torch.tensor(list(self.deque)) 29 | return d.median().item() 30 | 31 | @property 32 | def avg(self): 33 | d = torch.tensor(list(self.deque)) 34 | return d.mean().item() 35 | 36 | @property 37 | def global_avg(self): 38 | return self.total / self.count 39 | 40 | @property 41 | def reset(self): 42 | self.deque = deque(maxlen=self.window_size) 43 | self.series = [] 44 | self.total = 0.0 45 | self.count = 0 46 | 47 | 48 | class MetricLogger(): 49 | def __init__(self, delimiter="\t"): 50 | self.meters = defaultdict(SmoothedValue) 51 | self.delimiter = delimiter 52 | 53 | def update(self, **kwargs): 54 | for k, v in kwargs.items(): 55 | if isinstance(v, torch.Tensor): 56 | v = v.item() 57 | assert isinstance(v, (float, int)) 58 | self.meters[k].update(v) 59 | 60 | def reset(self, *argc): 61 | for k in argc: 62 | self.meters[k].reset 63 | 64 | def reset_all(self): 65 | for v in self.meters.values(): 66 | v.reset 67 | 68 | def __getattr__(self, attr): 69 | if attr in self.meters: 70 | return self.meters[attr] 71 | if attr in self.__dict__: 72 | return self.__dict__[attr] 73 | raise AttributeError("{} object has no attribute {}".format( 74 | type(self).__name__, attr)) 75 | 76 | def __str__(self): 77 | loss_str = [] 78 | for name, meter in self.meters.items(): 79 | loss_str.append( 80 | "{}: {:.4f} ({:.4f})".format(name, meter.median, meter.global_avg) 81 | ) 82 | return self.delimiter.join(loss_str) 83 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/utils/miscellaneous.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import os 3 | 4 | 5 | def mkdir(path): 6 | try: 7 | os.makedirs(path) 8 | except OSError as e: 9 | if e.errno != errno.EEXIST: 10 | raise -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/utils/model_serialization.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | from collections import OrderedDict 3 | import logging 4 | 5 | import torch 6 | 7 | 8 | def align_and_update_state_dicts(model_state_dict, loaded_state_dict): 9 | """ 10 | Strategy: suppose that the models that we will create will have prefixes appended 11 | to each of its keys, for example due to an extra level of nesting that the original 12 | pre-trained weights from ImageNet won't contain. For example, model.state_dict() 13 | might return backbone[0].body.res2.conv1.weight, while the pre-trained model contains 14 | res2.conv1.weight. We thus want to match both parameters together. 15 | For that, we look for each model weight, look among all loaded keys if there is one 16 | that is a suffix of the current weight name, and use it if that's the case. 17 | If multiple matches exist, take the one with longest size 18 | of the corresponding name. For example, for the same model as before, the pretrained 19 | weight file can contain both res2.conv1.weight, as well as conv1.weight. In this case, 20 | we want to match backbone[0].body.conv1.weight to conv1.weight, and 21 | backbone[0].body.res2.conv1.weight to res2.conv1.weight. 22 | """ 23 | current_keys = sorted(list(model_state_dict.keys())) 24 | loaded_keys = sorted(list(loaded_state_dict.keys())) 25 | # get a matrix of string matches, where each (i, j) entry correspond to the size of the 26 | # loaded_key string, if it matches 27 | match_matrix = [ 28 | len(j) if i.endswith(j) else 0 for i in current_keys for j in loaded_keys 29 | ] 30 | match_matrix = torch.as_tensor(match_matrix).view( 31 | len(current_keys), len(loaded_keys) 32 | ) 33 | max_match_size, idxs = match_matrix.max(1) 34 | # remove indices that correspond to no-match 35 | idxs[max_match_size == 0] = -1 36 | 37 | # used for logging 38 | max_size = max([len(key) for key in current_keys]) if current_keys else 1 39 | max_size_loaded = max([len(key) for key in loaded_keys]) if loaded_keys else 1 40 | log_str_template = "{: <{}} loaded from {: <{}} of shape {}" 41 | logger = logging.getLogger(__name__) 42 | for idx_new, idx_old in enumerate(idxs.tolist()): 43 | if idx_old == -1: 44 | continue 45 | key = current_keys[idx_new] 46 | key_old = loaded_keys[idx_old] 47 | model_state_dict[key] = loaded_state_dict[key_old] 48 | logger.info( 49 | log_str_template.format( 50 | key, 51 | max_size, 52 | key_old, 53 | max_size_loaded, 54 | tuple(loaded_state_dict[key_old].shape), 55 | ) 56 | ) 57 | 58 | 59 | def strip_prefix_if_present(state_dict, prefix): 60 | keys = sorted(state_dict.keys()) 61 | if not all(key.startswith(prefix) for key in keys): 62 | return state_dict 63 | stripped_state_dict = OrderedDict() 64 | for key, value in state_dict.items(): 65 | stripped_state_dict[key.replace(prefix, "")] = value 66 | return stripped_state_dict 67 | 68 | 69 | def load_state_dict(model, loaded_state_dict): 70 | model_state_dict = model.state_dict() 71 | # if the state_dict comes from a model that was wrapped in a 72 | # DataParallel or DistributedDataParallel during serialization, 73 | # remove the "module" prefix before performing the matching 74 | loaded_state_dict = strip_prefix_if_present(loaded_state_dict, prefix="module.") 75 | align_and_update_state_dicts(model_state_dict, loaded_state_dict) 76 | 77 | # use strict loading 78 | model.load_state_dict(model_state_dict) 79 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/utils/model_zoo.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | import os 3 | import sys 4 | 5 | from torch.hub import download_url_to_file 6 | from torch.hub import urlparse 7 | from torch.hub import HASH_REGEX 8 | 9 | from .comm import is_main_process 10 | from .comm import synchronize 11 | 12 | 13 | # very similar to https://github.com/pytorch/pytorch/blob/master/torch/utils/model_zoo.py 14 | # but with a few improvements and modifications 15 | def cache_url(url, model_dir=None, progress=True): 16 | r"""Loads the Torch serialized object at the given URL. 17 | If the object is already present in `model_dir`, it's deserialized and 18 | returned. The filename part of the URL should follow the naming convention 19 | ``filename-.ext`` where ```` is the first eight or more 20 | digits of the SHA256 hash of the contents of the file. The hash is used to 21 | ensure unique names and to verify the contents of the file. 22 | The default value of `model_dir` is ``$TORCH_HOME/models`` where 23 | ``$TORCH_HOME`` defaults to ``~/.torch``. The default directory can be 24 | overridden with the ``$TORCH_MODEL_ZOO`` environment variable. 25 | Args: 26 | url (string): URL of the object to download 27 | model_dir (string, optional): directory in which to save the object 28 | progress (bool, optional): whether or not to display a progress bar to stderr 29 | Example: 30 | >>> cached_file = maskrcnn_benchmark.utils.model_zoo.cache_url('https://s3.amazonaws.com/pytorch/models/resnet18-5c106cde.pth') 31 | """ 32 | if model_dir is None: 33 | torch_home = os.path.expanduser(os.getenv('TORCH_HOME', '~/.torch')) 34 | model_dir = os.getenv('TORCH_MODEL_ZOO', os.path.join(torch_home, 'models')) 35 | if not os.path.exists(model_dir): 36 | os.makedirs(model_dir) 37 | parts = urlparse(url) 38 | filename = os.path.basename(parts.path) 39 | if filename == "model_final.pkl": 40 | # workaround as pre-trained Caffe2 models from Detectron have all the same filename 41 | # so make the full path the filename by replacing / with _ 42 | filename = parts.path.replace("/", "_") 43 | cached_file = os.path.join(model_dir, filename) 44 | if not os.path.exists(cached_file) and is_main_process(): 45 | sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file)) 46 | hash_prefix = HASH_REGEX.search(filename) 47 | if hash_prefix is not None: 48 | hash_prefix = hash_prefix.group(1) 49 | # workaround: Caffe2 models don't have a hash, but follow the R-50 convention, 50 | # which matches the hash PyTorch uses. So we skip the hash matching 51 | # if the hash_prefix is less than 6 characters 52 | if len(hash_prefix) < 6: 53 | hash_prefix = None 54 | download_url_to_file(url, cached_file, hash_prefix, progress=progress) 55 | synchronize() 56 | return cached_file 57 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/utils/plot.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Dec 15 15:00:06 2022 4 | 5 | @author: hdb 6 | """ 7 | 8 | def plot_box(draw, box2d=[], box3d=[]): 9 | 10 | if len(box2d) > 0: 11 | num_obj = box2d.shape[0] 12 | for i in range(num_obj): 13 | 14 | x1, y1, x2, y2 = box2d[i].split(1, 0) 15 | draw.rectangle([x1,y1,x2,y2], outline=(0, 255, 0)) 16 | 17 | if len(box3d) > 0: 18 | box3d = box3d.permute(0, 2, 1) 19 | num_obj = box3d.shape[0] 20 | for i in range(num_obj): 21 | 22 | for j in range(box3d[i].shape[0]): 23 | x, y = box3d[i][j].split(1, 0) 24 | draw.ellipse((x-2,y-2, x+2,y+2), (255,0,0)) 25 | 26 | point_pair = [ 27 | [box3d[i][0].split(1, 0), box3d[i][1].split(1, 0)], 28 | [box3d[i][0].split(1, 0), box3d[i][5].split(1, 0)], 29 | [box3d[i][0].split(1, 0), box3d[i][7].split(1, 0)], 30 | [box3d[i][2].split(1, 0), box3d[i][1].split(1, 0)], 31 | [box3d[i][2].split(1, 0), box3d[i][3].split(1, 0)], 32 | [box3d[i][2].split(1, 0), box3d[i][7].split(1, 0)], 33 | [box3d[i][4].split(1, 0), box3d[i][1].split(1, 0)], 34 | [box3d[i][4].split(1, 0), box3d[i][3].split(1, 0)], 35 | [box3d[i][4].split(1, 0), box3d[i][5].split(1, 0)], 36 | [box3d[i][6].split(1, 0), box3d[i][3].split(1, 0)], 37 | [box3d[i][6].split(1, 0), box3d[i][5].split(1, 0)], 38 | [box3d[i][6].split(1, 0), box3d[i][7].split(1, 0)] 39 | ] 40 | 41 | for j in range(len(point_pair)): 42 | draw.line(point_pair[j], (0,255,0), width=2) -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/utils/registry.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | 3 | 4 | def _register_generic(module_dict, module_name, module): 5 | assert module_name not in module_dict 6 | module_dict[module_name] = module 7 | 8 | 9 | class Registry(dict): 10 | ''' 11 | A helper class for managing registering modules, it extends a dictionary 12 | and provides a register functions. 13 | 14 | Eg. creating a registry: 15 | some_registry = Registry({"default": default_module}) 16 | 17 | There're two ways of registering new modules: 18 | 1): normal way is just calling register function: 19 | def foo(): 20 | ... 21 | some_registry.register("foo_module", foo) 22 | 2): used as decorator when declaring the module: 23 | @some_registry.register("foo_module") 24 | @some_registry.register("foo_module_nickname") 25 | def foo(): 26 | ... 27 | 28 | Access of module is just like using a dictionary, eg: 29 | f = some_registry["foo_module"] 30 | ''' 31 | 32 | def __init__(self, *args, **kwargs): 33 | super(Registry, self).__init__(*args, **kwargs) 34 | 35 | def register(self, module_name, module=None): 36 | # used as function call 37 | if module is not None: 38 | _register_generic(self, module_name, module) 39 | return 40 | 41 | # used as decorator 42 | def register_fn(fn): 43 | _register_generic(self, module_name, fn) 44 | return fn 45 | 46 | return register_fn 47 | -------------------------------------------------------------------------------- /Nuscenes-Driving/models/smoke/utils/timer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 2 | 3 | 4 | import time 5 | import datetime 6 | 7 | 8 | class Timer(object): 9 | def __init__(self): 10 | self.reset() 11 | 12 | @property 13 | def average_time(self): 14 | return self.total_time / self.calls if self.calls > 0 else 0.0 15 | 16 | def tic(self): 17 | # using time.time instead of time.clock because time time.clock 18 | # does not normalize for multithreading 19 | self.start_time = time.time() 20 | 21 | def toc(self, average=True): 22 | self.add(time.time() - self.start_time) 23 | if average: 24 | return self.average_time 25 | else: 26 | return self.diff 27 | 28 | def add(self, time_diff): 29 | self.diff = time_diff 30 | self.total_time += self.diff 31 | self.calls += 1 32 | 33 | def reset(self): 34 | self.total_time = 0.0 35 | self.calls = 0 36 | self.start_time = 0.0 37 | self.diff = 0.0 38 | 39 | def avg_time_str(self): 40 | time_str = str(datetime.timedelta(seconds=self.average_time)) 41 | return time_str 42 | 43 | 44 | def get_time_str(time_diff): 45 | time_str = str(datetime.timedelta(seconds=time_diff)) 46 | return time_str 47 | -------------------------------------------------------------------------------- /Nuscenes-Driving/run.sh: -------------------------------------------------------------------------------- 1 | # supervised training 2 | # CUDA_VISIBLE_DEVICES=4,5 python3 sup_train.py --seed 0 --alpha 0.75 --batch_size 32 3 | 4 | # nesy learning 5 | 6 | # CUDA_VISIBLE_DEVICES=0 python3 nesy_train_cnn.py --seed 0 --clauses 3000 --exp_name 'logic_clauses_3000' --iter 30 --num_epochs 300 7 | 8 | CUDA_VISIBLE_DEVICES=0 python3 nesy_train_gpt.py --seed 0 --clauses 3000 --exp_name 'gpt_logic_clauses_3000' --iter 30 --num_epochs 300 9 | 10 | 11 | # CUDA_VISIBLE_DEVICES=0 python3 nesy_eval.py --seed 0 --conf_threshold 0.9995 --test_root './data' --file_name 'checkpoint/lenet_0_nesy_programming_0.t7' 12 | 13 | # CUDA_VISIBLE_DEVICES=0 python3 nesy_eval.py --seed 0 --conf_threshold 0.0 --test_root '../RRN-SudoKu/data' --file_name 'checkpoint/lenet_0_nesy_programming_0.t7' 14 | 15 | # CUDA_VISIBLE_DEVICES=0 python3 nesy_eval.py --seed 0 --conf_threshold 0.9995 --test_root './data' --file_name 'checkpoint/lenet_0_nesy_sgd_0.t7' -------------------------------------------------------------------------------- /Nuscenes-Driving/smt_solver.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from z3 import * 4 | import time 5 | 6 | size = 10 7 | 8 | def maxsat_solver(W, bmin, bmax, sxy, txy): 9 | # check 10 | m, n = W.shape 11 | s = Optimize() 12 | # set_option("parallel.enable", True) 13 | # s.set("timeout", 60000) 14 | X = [Bool("X_%s" % (i)) for i in range(n)] 15 | 16 | s.add(X[sxy] == True) 17 | s.add(X[txy] == True) 18 | 19 | # Default given: set the range of symbol 20 | for w, l, u in zip(W, bmin, bmax): 21 | l, u = l.item(), u.item() 22 | w = w.reshape(-1) 23 | U = torch.where(w != 0)[0] 24 | num = U.shape[0] 25 | # ignore meaningless logic 26 | if Sum([X[U[i]] for i in range(num)]) == 0.0: 27 | continue 28 | # unweighted 29 | s.add_soft(Sum([X[U[i]] for i in range(num)]) <= u, 1) 30 | s.add_soft(Sum([X[U[i]] for i in range(num)]) >= l, 1) 31 | 32 | # induce the point 33 | for i in range(n): 34 | s.add_soft(X[i] == True, 1) 35 | 36 | if s.check() == sat: 37 | res = [is_true(s.model()[X[i]]) for i in range(n)] 38 | res = np.array(res)*1.0 39 | return True, res 40 | else: 41 | return False, None 42 | 43 | def maxsat_solver_gpt(output, W, bmin, bmax, sxy, txy): 44 | # check 45 | m, n = W.shape 46 | s = Optimize() 47 | # set_option("parallel.enable", True) 48 | # s.set("timeout", 60000) 49 | X = [Bool("X_%s" % (i)) for i in range(n)] 50 | 51 | s.add(X[sxy] == True) 52 | s.add(X[txy] == True) 53 | 54 | for i in range(n): 55 | s.add_soft(X[i] == bool(output[i]), 1) 56 | 57 | # Default given: set the range of symbol 58 | for w, l, u in zip(W, bmin, bmax): 59 | l, u = l.item(), u.item() 60 | w = w.reshape(-1) 61 | U = torch.where(w != 0)[0] 62 | num = U.shape[0] 63 | # ignore meaningless logic 64 | if Sum([X[U[i]] for i in range(num)]) == 0.0: 65 | continue 66 | # unweighted 67 | s.add_soft(Sum([X[U[i]] for i in range(num)]) <= u, 1) 68 | s.add_soft(Sum([X[U[i]] for i in range(num)]) >= l, 1) 69 | # s.add(Sum([X[U[i]] for i in range(num)]) <= u) 70 | # s.add(Sum([X[U[i]] for i in range(num)]) >= l) 71 | 72 | 73 | if s.check() == sat: 74 | res = [is_true(s.model()[X[i]]) for i in range(n)] 75 | res = np.array(res)*1.0 76 | return True, res 77 | else: 78 | return False, None 79 | 80 | 81 | 82 | def sat_solver(W, bmin, bmax, sxy, txy): 83 | # check 84 | m, n = W.shape 85 | s = Solver() 86 | # set_option("parallel.enable", True) 87 | # s.set("timeout", 60000) 88 | X = [Bool("X_%s" % (i)) for i in range(n)] 89 | 90 | s.add(X[sxy] == True) 91 | s.add(X[txy] == True) 92 | 93 | # Default given: set the range of symbol 94 | for w, l, u in zip(W, bmin, bmax): 95 | l, u = int(l.item()), int(u.item()) 96 | w = w.reshape(-1) 97 | U = torch.where(w != 0)[0] 98 | num = U.shape[0] 99 | # ignore meaningless logic 100 | if Sum([X[U[i]] for i in range(num)]) == 0.0: 101 | continue 102 | # unweighted 103 | s.add(PbLe([(X[U[i]],1) for i in range(num)], u)) 104 | s.add(PbGe([(X[U[i]],1) for i in range(num)], l)) 105 | 106 | if s.check() == sat: 107 | res = [is_true(s.model()[X[i]]) for i in range(n)] 108 | res = np.array(res)*1.0 109 | return True, res 110 | else: 111 | return False, None 112 | 113 | 114 | 115 | if __name__ == "__main__": 116 | W = torch.zeros(1, 729) 117 | bmin = torch.ones(1,1) 118 | bmax = torch.ones(1,1) 119 | W[:,0:9] = 1.0 120 | # init_check(W, b) 121 | maxsat_solver(W, bmin, bmax) 122 | # sat_solver(W, bmin, bmax) 123 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nesy-Programming 2 | 3 | Code for the paper "Neuro-symbolic Learning Yielding Logical Constraints" 4 | 5 | ### Usage 6 | 7 | To reproduce the experimental results, Run the command sh run.sh in each task folder. 8 | -------------------------------------------------------------------------------- /RRN-SudoKu/dataset.py: -------------------------------------------------------------------------------- 1 | from nn_utils import * 2 | import os.path as osp 3 | import pickle 4 | import random 5 | import torch 6 | import copy 7 | 8 | class SudoKuDataset(Dataset): 9 | def __init__(self, root='./data', split='train', numSamples=None, randomSeed=None): 10 | super(SudoKuDataset, self).__init__() 11 | 12 | # shape of [num, 729] 13 | self.board_path = ('%s/%s/board.pt' %(root, split)) 14 | # shape of [num, 81, 1, 28, 28] 15 | # shape of [num, 81, 3, 32, 32] 16 | self.board_img_path = ('%s/%s/board_img.pt' %(root, split)) 17 | # shape of [num, 729] given cells is the same as board 18 | self.label_path = ('%s/%s/label.pt' %(root, split)) 19 | # shape of [num, 729] 20 | self.input_mask_path = ('%s/%s/input_mask.pt' %(root, split)) 21 | 22 | with open(self.board_path, 'rb') as f: 23 | self.board = pickle.load(f) 24 | 25 | with open(self.board_img_path, 'rb') as f: 26 | self.board_img = pickle.load(f) 27 | 28 | with open(self.label_path, 'rb') as f: 29 | self.label = pickle.load(f) 30 | 31 | with open(self.input_mask_path, 'rb') as f: 32 | self.input_mask = pickle.load(f) 33 | 34 | def __getitem__(self, index): 35 | sample = { 'input': None, 'label': None, 'symbol': None, 'index': None} 36 | sample['input'] = self.board_img[index] 37 | sample['label'] = self.board[index] 38 | sample['symbol'] = self.label[index] 39 | sample['mask'] = self.input_mask[index] 40 | sample['index'] = index 41 | return sample 42 | 43 | def __len__(self): 44 | return len(self.board_img) 45 | -------------------------------------------------------------------------------- /RRN-SudoKu/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .mlp import LeNet, CNN 2 | from .logic_layer import Logic 3 | from .net import CatNet 4 | from .gpt import GPT, GPTConfig -------------------------------------------------------------------------------- /RRN-SudoKu/models/logic_layer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import torch 3 | from torch import nn 4 | import torch.nn.functional as F 5 | 6 | class Logic(nn.Module): 7 | def __init__(self, input_dim, hidden=512, output_dim=1): 8 | super(Logic, self).__init__() 9 | # self.fc1 = nn.Linear(input_dim, hidden) 10 | # self.fc2 = nn.Linear(hidden, output_dim) 11 | self.fc = nn.Linear(input_dim, output_dim) 12 | 13 | def forward(self, x): 14 | N, M1, M2, num_classes = x.shape 15 | out = F.softmax(x, dim=-1) 16 | out = out.reshape(N, M1*M2*num_classes) 17 | # out = self.fc1(out) 18 | # out = torch.sigmoid(out) 19 | # out = self.fc2(out) 20 | out = self.fc(out) 21 | out = torch.sigmoid(out) 22 | return out 23 | 24 | -------------------------------------------------------------------------------- /RRN-SudoKu/models/net.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import torch 3 | from torch import nn 4 | from torch.nn import functional as F 5 | 6 | class CatNet(nn.Module): 7 | def __init__(self, module, classifier): 8 | super(CatNet, self).__init__() 9 | self.module = module 10 | self.classifier = classifier 11 | 12 | def forward(self, x): 13 | x = self.module(x) 14 | x = self.classifier(x) 15 | return x -------------------------------------------------------------------------------- /RRN-SudoKu/nesy_eval_cnn.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import torch 4 | import torch.nn.functional as F 5 | from torch.utils.tensorboard import SummaryWriter 6 | from nn_utils import * 7 | from dataset import * 8 | from scipy import linalg as splinalg 9 | 10 | import sys 11 | import models 12 | sys.path.append("../") 13 | 14 | 15 | # Dataloader 16 | parser = argparse.ArgumentParser(description='PyTorch SudoKu Logic Training') 17 | parser.add_argument('--derive', default=1, type=int, help='Cuda device.') 18 | parser.add_argument('--seed', default=1, type=int, help='Random seed to use.') 19 | parser.add_argument('--file_name', default='', type=str, help='Experiment name') 20 | parser.add_argument('--test_root', default='./data', type=str, help='root of data') 21 | parser.add_argument('--logic_threshold', default=0.99, type=float, help='Threshold for logic rule learning') 22 | parser.add_argument('--conf_threshold', default=0.0, type=float, help='Threshold for confidence') 23 | opt = parser.parse_args() 24 | 25 | # setting 26 | logic_threshold = opt.logic_threshold 27 | conf_threshold = opt.conf_threshold 28 | num_classes = 9; 29 | 30 | # cuda 31 | torch.cuda.set_device(0) 32 | torch.set_default_tensor_type(torch.FloatTensor) 33 | 34 | # random seed 35 | random.seed(opt.seed) 36 | np.random.seed(opt.seed) 37 | torch.manual_seed(opt.seed) 38 | torch.cuda.manual_seed(opt.seed) 39 | 40 | train_set = SudoKuDataset(split='train') 41 | test_set = SudoKuDataset(root=opt.test_root, split='test') 42 | 43 | 44 | def bounding_box(net, phi, dataset): 45 | 46 | dataloader = torch.utils.data.DataLoader(dataset, batch_size=64, 47 | shuffle=True, num_workers=0) 48 | 49 | W, b = phi 50 | m, n = W.shape 51 | N = len(dataset) 52 | Z = torch.rand(N, n).cuda() 53 | # filter 54 | net.eval() 55 | with torch.no_grad(): 56 | for batch_idx, sample in enumerate(dataloader): 57 | inputs, labels = sample['input'], sample['label'] 58 | masks, index = sample['mask'], sample['index'] 59 | symbols = sample['symbol'] 60 | 61 | num = inputs.size(0) 62 | Z_shape = (num, size**2*num_classes) 63 | 64 | # inference 65 | inputs = inputs.cuda() 66 | _, preds = torch.max(net(inputs), dim=-1) 67 | preds = F.one_hot(preds, num_classes=num_classes).reshape(Z_shape) 68 | mask_index, label_index = torch.where(masks == 1), torch.where(masks == 0) 69 | Zin = preds[mask_index] # mask the images with labels 70 | 71 | # define the symbol 72 | out = torch.zeros(Z_shape).cuda() 73 | y = symbols[label_index].cuda() 74 | out[mask_index] = Zin.float(); out[label_index] = y.float() 75 | out = out.reshape(num, n) 76 | Z[index,:] = out 77 | 78 | Wtmp = W.reshape(m,n).clone() 79 | Wtmp[Wtmp < 0.5] = 0.0 80 | Wtmp[Wtmp > 0.5] = 1.0 81 | btmp = (Wtmp@Z.T) 82 | btmp, _ = torch.sort(btmp, dim=-1) 83 | ind1, ind2 = int(N*(1-logic_threshold)), int(N*logic_threshold) 84 | bmin, bmax = btmp[:,ind1].reshape(-1, 1), btmp[:,ind2].reshape(-1, 1) 85 | # remove redundancy 86 | tmp = torch.hstack([Wtmp, b, bmin, bmax]) 87 | tmp = torch.unique(tmp, dim=0) 88 | Wtmp, b, bmin, bmax = tmp[:,0:-3], tmp[:,-3], tmp[:,-2], tmp[:,-1] 89 | b, bmax, bmin = b.reshape(-1, 1), bmax.reshape(-1,1), bmin.reshape(-1,1) 90 | return Wtmp, b, bmin, bmax 91 | 92 | 93 | 94 | if __name__ == "__main__": 95 | net = torch.load(opt.file_name)['net'] 96 | phi = torch.load(opt.file_name)['logic'] 97 | net.cuda() 98 | W, b, bmin, bmax = bounding_box(net, phi, train_set) 99 | res = evaluate_batch_cnn(net, W, b, bmin, bmax, test_set, conf_threshold) -------------------------------------------------------------------------------- /RRN-SudoKu/run.sh: -------------------------------------------------------------------------------- 1 | CUDA_VISIBLE_DEVICES=0 python3 nesy_train_cnn.py --seed 0 --clauses 2000 --exp_name cnn 2 | 3 | # CUDA_VISIBLE_DEVICES=1 python3 nesy_train_gpt.py --seed 0 --clauses 2000 --exp_name gpt 4 | 5 | 6 | #### eval and transfer 7 | 8 | # CUDA_VISIBLE_DEVICES=2 python3 nesy_eval_cnn.py --seed 0 --test_root './data/' --file_name 'checkpoint/cnn_model.t7' 9 | 10 | # CUDA_VISIBLE_DEVICES=2 python3 nesy_eval_gpt.py --seed 0 --test_root './data/' --file_name 'checkpoint/gpt_model.t7' 11 | 12 | # CUDA_VISIBLE_DEVICES=2 python3 nesy_eval_gpt.py --seed 0 --test_root '../SATNet-SudoKu/data' --file_name 'checkpoint/gpt_model.t7' 13 | 14 | # CUDA_VISIBLE_DEVICES=1 python3 nesy_eval_cnn.py --seed 0 --test_root '../SATNet-SudoKu/data' --file_name 'checkpoint/cnn_model.t7' 15 | -------------------------------------------------------------------------------- /RRN-SudoKu/smt_solver.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from z3 import * 4 | import time 5 | 6 | size = 9 7 | num_classes = 9 8 | 9 | def maxsat_solver(W, bmin, bmax): 10 | # check 11 | m, n = W.shape 12 | s = Optimize() 13 | # set_option("parallel.enable", True) 14 | s.set("timeout", 60000) 15 | X = [[Bool("X_%s_%s" % (i+1, j+1)) for j in range(num_classes)] for i in range(int(n/num_classes))] 16 | 17 | # # Default given: set the range of symbol 18 | for i in range(len(X)): 19 | s.add(Sum(X[i]) == 1) 20 | 21 | for w, l, u in zip(W, bmin, bmax): 22 | l, u = l.item(), u.item() 23 | w = w.reshape(-1, num_classes) 24 | U, V = torch.where(w != 0) 25 | num = U.shape[0] 26 | # ignore meaningless logic 27 | if Sum([X[U[i]][V[i]] for i in range(num)]) == 0.0: 28 | continue 29 | # unweighted 30 | s.add_soft(Sum([X[U[i]][V[i]] for i in range(num)]) <= u, 1) 31 | s.add_soft(Sum([X[U[i]][V[i]] for i in range(num)]) >= l, 1) 32 | 33 | if s.check() == sat: 34 | res = [[is_true(s.model()[X[i][j]]) for j in range(num_classes)] for i in range(int(n/num_classes))] 35 | res = np.array(res)*1.0 36 | return True, res 37 | else: 38 | return False, None 39 | 40 | 41 | def sat_solver(W, bmin, bmax): 42 | # check 43 | m, n = W.shape 44 | s = Solver() 45 | # set_option("parallel.enable", True) 46 | X = [[Bool("X_%s_%s" % (i+1, j+1)) for j in range(num_classes)] for i in range(int(n/num_classes))] 47 | 48 | # # Default given: set the range of symbol 49 | for i in range(len(X)): 50 | s.add(Sum(X[i]) == 1) 51 | 52 | for w, l, u in zip(W, bmin, bmax): 53 | l, u = int(l.item()), int(u.item()) 54 | w = w.reshape(-1, num_classes) 55 | U, V = torch.where(w != 0) 56 | num = U.shape[0] 57 | # ignore meaningless logic 58 | if Sum([X[U[i]][V[i]] for i in range(num)]) == 0.0: 59 | continue 60 | # s.add(Sum([X[U[i]][V[i]] for i in range(num)]) <= u) 61 | # s.add(Sum([X[U[i]][V[i]] for i in range(num)]) >= l) 62 | s.add(PbLe([(X[U[i]][V[i]],1) for i in range(num)], u)) 63 | s.add(PbGe([(X[U[i]][V[i]],1) for i in range(num)], l)) 64 | 65 | if s.check() == sat: 66 | res = [[is_true(s.model()[X[i][j]]) for j in range(num_classes)] for i in range(int(n/num_classes))] 67 | res = np.array(res)*1.0 68 | return True, res 69 | else: 70 | return False, None 71 | 72 | 73 | if __name__ == "__main__": 74 | W = torch.zeros(1, 729) 75 | bmin = torch.ones(1,1) 76 | bmax = torch.ones(1,1) 77 | W[:,0:9] = 1.0 78 | # init_check(W, b) 79 | maxsat_solver(W, bmin, bmax) 80 | # sat_solver(W, bmin, bmax) 81 | -------------------------------------------------------------------------------- /SATNet-SudoKu/data/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lizn-zn/Nesy-Programming/1410e721c7be4782b50219f7e18a4e247638edb3/SATNet-SudoKu/data/.DS_Store -------------------------------------------------------------------------------- /SATNet-SudoKu/dataset.py: -------------------------------------------------------------------------------- 1 | from nn_utils import * 2 | import os.path as osp 3 | import pickle 4 | import random 5 | import torch 6 | import copy 7 | 8 | class SudoKuDataset(Dataset): 9 | def __init__(self, root='./data', split='train', numSamples=None, randomSeed=None): 10 | super(SudoKuDataset, self).__init__() 11 | 12 | # shape of [num, 729] 13 | self.board_path = ('%s/%s/board.pt' %(root, split)) 14 | # shape of [num, 81, 1, 28, 28] 15 | # shape of [num, 81, 3, 32, 32] 16 | self.board_img_path = ('%s/%s/board_img.pt' %(root, split)) 17 | # shape of [num, 729] given cells is the same as board 18 | self.label_path = ('%s/%s/label.pt' %(root, split)) 19 | # shape of [num, 729] 20 | self.input_mask_path = ('%s/%s/input_mask.pt' %(root, split)) 21 | 22 | with open(self.board_path, 'rb') as f: 23 | self.board = pickle.load(f) 24 | 25 | with open(self.board_img_path, 'rb') as f: 26 | self.board_img = pickle.load(f) 27 | 28 | with open(self.label_path, 'rb') as f: 29 | self.label = pickle.load(f) 30 | 31 | with open(self.input_mask_path, 'rb') as f: 32 | self.input_mask = pickle.load(f) 33 | 34 | def __getitem__(self, index): 35 | sample = { 'input': None, 'label': None, 'symbol': None, 'index': None} 36 | sample['input'] = self.board_img[index] 37 | sample['label'] = self.board[index] 38 | sample['symbol'] = self.label[index] 39 | sample['mask'] = self.input_mask[index] 40 | sample['index'] = index 41 | return sample 42 | 43 | def __len__(self): 44 | return len(self.board_img) 45 | -------------------------------------------------------------------------------- /SATNet-SudoKu/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .mlp import LeNet, CNN, ModelM3, ModelM5, ModelM7 2 | from .logic_layer import Logic 3 | from .net import CatNet 4 | from .gpt import GPT, GPTConfig -------------------------------------------------------------------------------- /SATNet-SudoKu/models/logic_layer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import torch 3 | from torch import nn 4 | import torch.nn.functional as F 5 | 6 | class Logic(nn.Module): 7 | def __init__(self, input_dim, hidden=512, output_dim=1): 8 | super(Logic, self).__init__() 9 | # self.fc1 = nn.Linear(input_dim, hidden) 10 | # self.fc2 = nn.Linear(hidden, output_dim) 11 | self.fc = nn.Linear(input_dim, output_dim) 12 | 13 | def forward(self, x): 14 | N, M1, M2, num_classes = x.shape 15 | out = F.softmax(x, dim=-1) 16 | out = out.reshape(N, M1*M2*num_classes) 17 | # out = self.fc1(out) 18 | # out = torch.sigmoid(out) 19 | # out = self.fc2(out) 20 | out = self.fc(out) 21 | out = torch.sigmoid(out) 22 | return out 23 | 24 | -------------------------------------------------------------------------------- /SATNet-SudoKu/models/net.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import torch 3 | from torch import nn 4 | from torch.nn import functional as F 5 | 6 | class CatNet(nn.Module): 7 | def __init__(self, module, classifier): 8 | super(CatNet, self).__init__() 9 | self.module = module 10 | self.classifier = classifier 11 | 12 | def forward(self, x): 13 | x = self.module(x) 14 | x = self.classifier(x) 15 | return x -------------------------------------------------------------------------------- /SATNet-SudoKu/nesy_eval_cnn.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import torch 4 | import torch.nn.functional as F 5 | from torch.utils.tensorboard import SummaryWriter 6 | from nn_utils import * 7 | from dataset import * 8 | from scipy import linalg as splinalg 9 | 10 | import sys 11 | import models 12 | sys.path.append("../") 13 | 14 | 15 | # Dataloader 16 | parser = argparse.ArgumentParser(description='PyTorch SudoKu Logic Training') 17 | parser.add_argument('--derive', default=0, type=int, help='Cuda device.') 18 | parser.add_argument('--seed', default=1, type=int, help='Random seed to use.') 19 | parser.add_argument('--file_name', default='', type=str, help='Experiment name') 20 | parser.add_argument('--test_root', default='./data', type=str, help='root of data') 21 | parser.add_argument('--logic_threshold', default=0.99, type=float, help='Threshold for logic rule learning') 22 | parser.add_argument('--conf_threshold', default=0.0, type=float, help='Threshold for confidence') 23 | opt = parser.parse_args() 24 | 25 | # setting 26 | logic_threshold = opt.logic_threshold 27 | conf_threshold = opt.conf_threshold 28 | num_classes = 9; 29 | 30 | # cuda 31 | torch.cuda.set_device(0) 32 | torch.set_default_tensor_type(torch.FloatTensor) 33 | 34 | # random seed 35 | random.seed(opt.seed) 36 | np.random.seed(opt.seed) 37 | torch.manual_seed(opt.seed) 38 | torch.cuda.manual_seed(opt.seed) 39 | 40 | train_set = SudoKuDataset(split='train') 41 | test_set = SudoKuDataset(root=opt.test_root, split='test') 42 | 43 | 44 | def bounding_box(net, phi, dataset): 45 | 46 | dataloader = torch.utils.data.DataLoader(dataset, batch_size=64, 47 | shuffle=True, num_workers=0) 48 | 49 | W, b = phi 50 | m, n = W.shape 51 | N = len(dataset) 52 | Z = torch.rand(N, n).cuda() 53 | # filter 54 | net.eval() 55 | with torch.no_grad(): 56 | for batch_idx, sample in enumerate(dataloader): 57 | inputs, labels = sample['input'], sample['label'] 58 | masks, index = sample['mask'], sample['index'] 59 | symbols = sample['symbol'] 60 | 61 | num = inputs.size(0) 62 | Z_shape = (num, size**2*num_classes) 63 | 64 | # inference 65 | inputs = inputs.cuda() 66 | _, preds = torch.max(net(inputs), dim=-1) 67 | preds = F.one_hot(preds, num_classes=num_classes).reshape(Z_shape) 68 | mask_index, label_index = torch.where(masks == 1), torch.where(masks == 0) 69 | Zin = preds[mask_index] # mask the images with labels 70 | 71 | # define the symbol 72 | out = torch.zeros(Z_shape).cuda() 73 | y = symbols[label_index].cuda() 74 | out[mask_index] = Zin.float(); out[label_index] = y.float() 75 | out = out.reshape(num, n) 76 | Z[index,:] = out 77 | 78 | Wtmp = W.reshape(m,n).clone() 79 | Wtmp[Wtmp < 0.5] = 0.0 80 | Wtmp[Wtmp > 0.5] = 1.0 81 | btmp = (Wtmp@Z.T) 82 | btmp, _ = torch.sort(btmp, dim=-1) 83 | ind1, ind2 = int(N*(1-logic_threshold)), int(N*logic_threshold) 84 | bmin, bmax = btmp[:,ind1].reshape(-1, 1), btmp[:,ind2].reshape(-1, 1) 85 | # remove redundancy 86 | tmp = torch.hstack([Wtmp, b, bmin, bmax]) 87 | tmp = torch.unique(tmp, dim=0) 88 | Wtmp, b, bmin, bmax = tmp[:,0:-3], tmp[:,-3], tmp[:,-2], tmp[:,-1] 89 | b, bmax, bmin = b.reshape(-1, 1), bmax.reshape(-1,1), bmin.reshape(-1,1) 90 | return Wtmp, b, bmin, bmax 91 | 92 | 93 | 94 | if __name__ == "__main__": 95 | net = torch.load(opt.file_name)['net'] 96 | phi = torch.load(opt.file_name)['logic'] 97 | net.cuda() 98 | W, b, bmin, bmax = bounding_box(net, phi, train_set) 99 | res = evaluate_batch_cnn(net, W, b, bmin, bmax, test_set, conf_threshold) -------------------------------------------------------------------------------- /SATNet-SudoKu/nesy_eval_gpt.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import torch 4 | import torch.nn.functional as F 5 | from torch.utils.tensorboard import SummaryWriter 6 | from nn_utils import * 7 | from dataset import * 8 | from scipy import linalg as splinalg 9 | 10 | import sys 11 | import models 12 | sys.path.append("../") 13 | 14 | 15 | # Dataloader 16 | parser = argparse.ArgumentParser(description='PyTorch SudoKu Logic Training') 17 | parser.add_argument('--derive', default=1, type=int, help='Cuda device.') 18 | parser.add_argument('--seed', default=1, type=int, help='Random seed to use.') 19 | parser.add_argument('--file_name', default='', type=str, help='Experiment name') 20 | parser.add_argument('--test_root', default='./data', type=str, help='root of data') 21 | parser.add_argument('--logic_threshold', default=0.99, type=float, help='Threshold for logic rule learning') 22 | parser.add_argument('--conf_threshold', default=0.0, type=float, help='Threshold for confidence') 23 | opt = parser.parse_args() 24 | 25 | # setting 26 | logic_threshold = opt.logic_threshold 27 | conf_threshold = opt.conf_threshold 28 | num_classes = 9; 29 | 30 | # cuda 31 | torch.cuda.set_device(0) 32 | torch.set_default_tensor_type(torch.FloatTensor) 33 | 34 | # random seed 35 | random.seed(opt.seed) 36 | np.random.seed(opt.seed) 37 | torch.manual_seed(opt.seed) 38 | torch.cuda.manual_seed(opt.seed) 39 | 40 | train_set = SudoKuDataset(split='train') 41 | test_set = SudoKuDataset(root=opt.test_root, split='test') 42 | 43 | 44 | def bounding_box(net, phi, dataset): 45 | 46 | dataloader = torch.utils.data.DataLoader(dataset, batch_size=64, 47 | shuffle=True, num_workers=0) 48 | 49 | W, b = phi 50 | m, n = W.shape 51 | N = len(dataset) 52 | Z = torch.rand(N, n).cuda() 53 | # filter 54 | net.eval() 55 | with torch.no_grad(): 56 | for batch_idx, sample in enumerate(dataloader): 57 | inputs, labels = sample['input'], sample['label'] 58 | masks, index = sample['mask'], sample['index'] 59 | symbols = sample['symbol'] 60 | 61 | num = inputs.size(0) 62 | Z_shape = (num, size**2*num_classes) 63 | 64 | # inference 65 | inputs = inputs.cuda() 66 | _, preds = torch.max(net(inputs)[0], dim=-1) 67 | preds = F.one_hot(preds, num_classes=num_classes).reshape(Z_shape) 68 | mask_index, label_index = torch.where(masks == 1), torch.where(masks == 0) 69 | Zin = preds[mask_index] # mask the images with labels 70 | 71 | # define the symbol 72 | out = torch.zeros(Z_shape).cuda() 73 | y = symbols[label_index].cuda() 74 | out[mask_index] = Zin.float(); out[label_index] = y.float() 75 | out = out.reshape(num, n) 76 | Z[index,:] = out 77 | 78 | Wtmp = W.reshape(m,n).clone() 79 | Wtmp[Wtmp < 0.5] = 0.0 80 | Wtmp[Wtmp > 0.5] = 1.0 81 | btmp = (Wtmp@Z.T) 82 | btmp, _ = torch.sort(btmp, dim=-1) 83 | ind1, ind2 = int(N*(1-logic_threshold)), int(N*logic_threshold) 84 | bmin, bmax = btmp[:,ind1].reshape(-1, 1), btmp[:,ind2].reshape(-1, 1) 85 | # remove redundancy 86 | tmp = torch.hstack([Wtmp, b, bmin, bmax]) 87 | tmp = torch.unique(tmp, dim=0) 88 | Wtmp, b, bmin, bmax = tmp[:,0:-3], tmp[:,-3], tmp[:,-2], tmp[:,-1] 89 | b, bmax, bmin = b.reshape(-1, 1), bmax.reshape(-1,1), bmin.reshape(-1,1) 90 | return Wtmp, b, bmin, bmax 91 | 92 | 93 | 94 | if __name__ == "__main__": 95 | net = torch.load(opt.file_name)['net'] 96 | phi = torch.load(opt.file_name)['logic'] 97 | net.cuda() 98 | W, b, bmin, bmax = bounding_box(net, phi, train_set) 99 | res = evaluate_batch_gpt(net, W, b, bmin, bmax, test_set, conf_threshold) -------------------------------------------------------------------------------- /SATNet-SudoKu/run.sh: -------------------------------------------------------------------------------- 1 | # CUDA_VISIBLE_DEVICES=4 python3 nesy_train_cnn.py --seed 0 --clauses 2000 --exp_name cnn 2 | 3 | # CUDA_VISIBLE_DEVICES=1 python3 nesy_train_gpt.py --seed 0 --clauses 2000 --exp_name gpt 4 | 5 | 6 | 7 | #### eval and transfer 8 | 9 | # CUDA_VISIBLE_DEVICES=1 python3 nesy_eval_cnn.py --seed 0 --test_root './data' --file_name 'checkpoint/cnn_model.t7' 10 | 11 | # CUDA_VISIBLE_DEVICES=2 python3 nesy_eval_gpt.py --seed 0 --test_root './data/' --file_name 'checkpoint/gpt_model.t7' 12 | 13 | # CUDA_VISIBLE_DEVICES=1 python3 nesy_eval_cnn.py --seed 0 --test_root '../RRN-SudoKu/data' --file_name 'checkpoint/cnn_model.t7' 14 | 15 | # CUDA_VISIBLE_DEVICES=2 python3 nesy_eval_gpt.py --seed 0 --test_root '../RRN-SudoKu/data' --file_name 'checkpoint/gpt_model.t7' 16 | -------------------------------------------------------------------------------- /SATNet-SudoKu/smt_solver.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from z3 import * 4 | import time 5 | 6 | size = 9 7 | num_classes = 9 8 | 9 | def maxsat_solver(W, bmin, bmax): 10 | # check 11 | m, n = W.shape 12 | s = Optimize() 13 | # set_option("parallel.enable", True) 14 | s.set("timeout", 60000) 15 | X = [[Bool("X_%s_%s" % (i+1, j+1)) for j in range(num_classes)] for i in range(int(n/num_classes))] 16 | 17 | # # Default given: set the range of symbol 18 | for i in range(len(X)): 19 | s.add(Sum(X[i]) == 1) 20 | 21 | for w, l, u in zip(W, bmin, bmax): 22 | l, u = l.item(), u.item() 23 | w = w.reshape(-1, num_classes) 24 | U, V = torch.where(w != 0) 25 | num = U.shape[0] 26 | # ignore meaningless logic 27 | if Sum([X[U[i]][V[i]] for i in range(num)]) == 0.0: 28 | continue 29 | # unweighted 30 | s.add_soft(Sum([X[U[i]][V[i]] for i in range(num)]) <= u, 1) 31 | s.add_soft(Sum([X[U[i]][V[i]] for i in range(num)]) >= l, 1) 32 | 33 | if s.check() == sat: 34 | res = [[is_true(s.model()[X[i][j]]) for j in range(num_classes)] for i in range(int(n/num_classes))] 35 | res = np.array(res)*1.0 36 | return True, res 37 | else: 38 | return False, None 39 | 40 | 41 | def sat_solver(W, bmin, bmax): 42 | # check 43 | m, n = W.shape 44 | s = Solver() 45 | # set_option("parallel.enable", True) 46 | X = [[Bool("X_%s_%s" % (i+1, j+1)) for j in range(num_classes)] for i in range(int(n/num_classes))] 47 | 48 | # # Default given: set the range of symbol 49 | for i in range(len(X)): 50 | s.add(Sum(X[i]) == 1) 51 | 52 | for w, l, u in zip(W, bmin, bmax): 53 | l, u = int(l.item()), int(u.item()) 54 | w = w.reshape(-1, num_classes) 55 | U, V = torch.where(w != 0) 56 | num = U.shape[0] 57 | # ignore meaningless logic 58 | if Sum([X[U[i]][V[i]] for i in range(num)]) == 0.0: 59 | continue 60 | # s.add(Sum([X[U[i]][V[i]] for i in range(num)]) <= u) 61 | # s.add(Sum([X[U[i]][V[i]] for i in range(num)]) >= l) 62 | s.add(PbLe([(X[U[i]][V[i]],1) for i in range(num)], u)) 63 | s.add(PbGe([(X[U[i]][V[i]],1) for i in range(num)], l)) 64 | 65 | if s.check() == sat: 66 | res = [[is_true(s.model()[X[i][j]]) for j in range(num_classes)] for i in range(int(n/num_classes))] 67 | res = np.array(res)*1.0 68 | return True, res 69 | else: 70 | return False, None 71 | 72 | 73 | if __name__ == "__main__": 74 | W = torch.zeros(1, 729) 75 | bmin = torch.ones(1,1) 76 | bmax = torch.ones(1,1) 77 | W[:,0:9] = 1.0 78 | # init_check(W, b) 79 | maxsat_solver(W, bmin, bmax) 80 | # sat_solver(W, bmin, bmax) 81 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tqdm 2 | torchvision 3 | torchvision 4 | matplotlib 5 | tensorboard 6 | scipy 7 | joblib --------------------------------------------------------------------------------