├── champs-scalar-coupling ├── mpnn_model │ ├── __init__.py │ ├── lib │ │ ├── __init__.py │ │ ├── net │ │ │ ├── __init__.py │ │ │ └── rate.py │ │ ├── utility │ │ │ ├── __init__.py │ │ │ ├── file.py │ │ │ └── draw.py │ │ └── include.py │ ├── helpers.py │ ├── common_constants.py │ ├── common.py │ ├── parallel_process.py │ ├── RNN_attention.py │ ├── regression_head.py │ ├── GaussRank.py │ ├── build_predictions.py │ ├── callback.py │ ├── common_model.py │ ├── train_loss.py │ ├── radam.py │ ├── message_passing.py │ └── data_collate.py ├── build_data │ ├── lib │ │ ├── __init__.py │ │ ├── utility │ │ │ ├── __init__.py │ │ │ ├── file.py │ │ │ └── draw.py │ │ ├── include.py │ │ └── net │ │ │ └── rate.py │ ├── common.py │ ├── parallel_process.py │ └── create_parquet.py ├── README.md ├── README_old.md ├── experiments │ ├── .nfs0000000001371b7a000004c5 │ ├── MPNN_RNN_MAE_GAUSSRANK_SINGLE_TYPE.yaml │ ├── MPNN_RNN_MAE_WO_GAUSSRANK_SINGLE_TYPE.yaml │ ├── MPNN_EMBED_TYPE_LMAE_WO_GAUSSRANK.yaml │ ├── MPNN_RNN_EMBED_TYPE_LMAE_WO_GAUSSRANK.yaml │ ├── MPNN_PREDICT_TYPE_MLMAE_GAUSSRANK.yaml │ ├── MPNN_RNN_PREDICT_TYPE_LMAE_WO_GAUSSRANK.yaml │ ├── MPNN_EMBED_TYPE_LMAE_GAUSSRANK.yaml │ ├── MPNN_PREDICT_TYPE_LMAE_GAUSSRANK.yaml │ ├── MPNN_PREDICT_TYPE_LMAE_WO_GAUSSRANK.yaml │ ├── MPNN_RNN_EMBED_TYPE_LMAE_GAUSSRANK.yaml │ ├── MPNN_RNN_PREDICT_TYPE_LMAE_GAUSSRANK.yaml │ ├── MPNN_RNN_PREDICT_TYPE_MLMAE_GAUSSRANK.yaml │ ├── MPNN_PREDICT_TYPE_LMAE_GAUSSRANK_BOOTSTRAP.yaml │ ├── MPNN_PREDICT_TYPE_MLMAE_GAUSSRANK_BOOTSTRAP.yaml │ ├── MPNN_RNN_PREDICT_TYPE_LMAE_GAUSSRANK_BOOTSTRAP.yaml │ └── MPNN_RNN_PREDICT_TYPE_MLMAE_GAUSSRANK_BOOTSTRAP.yaml ├── tree.md └── save_pretrained_single_models.ipynb ├── .gitignore ├── pytorch ├── optimizers │ ├── README.md │ ├── __init__.py │ ├── setup.py │ ├── adam.py │ └── adamw.py ├── README.md └── batch_dataloader │ ├── batch_dataloader.py │ ├── README.md │ └── batch_dataset.py ├── RecSys2020Tutorial ├── images │ ├── te_oof.png │ ├── gaussrank.png │ ├── normalization.png │ └── dask_dag_cartoon.png ├── LICENSE ├── Dockerfile ├── README.md ├── data │ └── README.md └── 00_0_Initial.ipynb ├── RecSys2019 └── README.md ├── RecSys2020 └── README.md ├── WSDM2021 └── README.md └── README.md /champs-scalar-coupling/mpnn_model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /champs-scalar-coupling/build_data/lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/lib/net/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /champs-scalar-coupling/build_data/lib/utility/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/lib/utility/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /champs-scalar-coupling/README.md: -------------------------------------------------------------------------------- 1 | Placeholder for our 33rd place solution. 2 | -------------------------------------------------------------------------------- /champs-scalar-coupling/README_old.md: -------------------------------------------------------------------------------- 1 | Placeholder for our 33rd place solution. 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.parquet 3 | *.csv 4 | *.pkl 5 | __pycache__/ 6 | .ipynb_checkpoints/ 7 | -------------------------------------------------------------------------------- /pytorch/optimizers/README.md: -------------------------------------------------------------------------------- 1 | # pytorch-optimizers 2 | Numba accelerated PyTorch Optimizers 3 | -------------------------------------------------------------------------------- /RecSys2020Tutorial/images/te_oof.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rapidsai/deeplearning/HEAD/RecSys2020Tutorial/images/te_oof.png -------------------------------------------------------------------------------- /RecSys2020Tutorial/images/gaussrank.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rapidsai/deeplearning/HEAD/RecSys2020Tutorial/images/gaussrank.png -------------------------------------------------------------------------------- /RecSys2020Tutorial/images/normalization.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rapidsai/deeplearning/HEAD/RecSys2020Tutorial/images/normalization.png -------------------------------------------------------------------------------- /pytorch/optimizers/__init__.py: -------------------------------------------------------------------------------- 1 | from .adam import Adam 2 | from .adamw import AdamW 3 | from .radam import RAdam, PlainRAdam, FusedRAdam 4 | -------------------------------------------------------------------------------- /RecSys2020Tutorial/images/dask_dag_cartoon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rapidsai/deeplearning/HEAD/RecSys2020Tutorial/images/dask_dag_cartoon.png -------------------------------------------------------------------------------- /RecSys2019/README.md: -------------------------------------------------------------------------------- 1 | ## Accelerating Recommender Systems by 15x with RAPIDS (Source Code) 2 | 3 | This content was moved to a new [competition repository](https://github.com/NVIDIA-Merlin/competitions). 4 | -------------------------------------------------------------------------------- /RecSys2020/README.md: -------------------------------------------------------------------------------- 1 | ## GPU Accelerated Feature Engineering and Training for Recommender Systems (source) 2 | 3 | This content was moved to a new [competition repository](https://github.com/NVIDIA-Merlin/competitions). 4 | -------------------------------------------------------------------------------- /WSDM2021/README.md: -------------------------------------------------------------------------------- 1 | # Using Deep Learning to Win the Booking.com WSDM WebTour21 Challenge on Sequential Recommendations 2 | 3 | This content was moved to a new [competition repository](https://github.com/NVIDIA-Merlin/competitions). 4 | -------------------------------------------------------------------------------- /pytorch/README.md: -------------------------------------------------------------------------------- 1 | [PyTorch](https://pytorch.org/) is an open source machine learning framework designed to accelerate the path from research prototyping to production deployment. RAPIDS is an active contributor to PyTorch, developing preprocessing functionality and dataloading on the GPU, along with improvements to kernels and optimizers critical to deep learning on tabular data. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### RAPIDS.AI Deep Learning Repo 2 | This repository is the home of our efforts to integrate RAPIDS acceleration of dataframes on GPU into popular deep learning frameworks. The work can be broken down into three main sections: 3 | 4 | - Dataloaders and preprocessing functionality developed to help provide connectivity between RAPIDS cuDF dataframes and the different deep learning libraries available. 5 | - Improvements to optimizers through the fusion of GPU operations. 6 | - Examples of the use of each of the above in competitions or on real world datasets. 7 | 8 | Each deep learning library is contained within it's own subfolder, with the different dataloader options and examples contained within further subfolders. For now our focus is on PyTorch, however we expect to add other libraries in the future. 9 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/helpers.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter 3 | 4 | ############################################################################################################# 5 | # # 6 | # Load experiment configuration # 7 | # # 8 | ############################################################################################################# 9 | import yaml 10 | def load_cfg(yaml_filepath): 11 | """ 12 | Load a YAML configuration file. 13 | 14 | Parameters 15 | ---------- 16 | yaml_filepath : str 17 | 18 | Returns 19 | ------- 20 | cfg : dict 21 | """ 22 | # Read YAML experiment definition file 23 | with open(yaml_filepath, 'r') as stream: 24 | cfg = yaml.load(stream) 25 | return cfg 26 | -------------------------------------------------------------------------------- /pytorch/optimizers/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup, find_packages 4 | 5 | setup( 6 | name='pytorch_optimizers', 7 | version='0.0.1', 8 | 9 | description='Numba accelerated PyTorch Optimizers', 10 | 11 | # The project's main homepage. 12 | url='https://github.com/madsbk/pytorch-optimizers', 13 | 14 | # Author details 15 | author='Mads R. B. Kristensen', 16 | author_email='madsbk@gmail.com', 17 | 18 | # Choose your license 19 | license='Apache 2.0', 20 | 21 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 22 | classifiers=[ 23 | 'Development Status :: 3 - Alpha', 24 | 'License :: OSI Approved :: Apache License 2.0', 25 | 'Programming Language :: Python :: 3', 26 | ], 27 | 28 | # What does your project relate to? 29 | keywords='PyTorch', 30 | 31 | # You can just specify the packages manually here if your project is 32 | # simple. Or you can use find_packages(). 33 | packages=find_packages(exclude=['contrib', 'docs', 'tests']), 34 | 35 | install_requires=[ 36 | 'torch', 37 | ], 38 | ) 39 | 40 | -------------------------------------------------------------------------------- /RecSys2020Tutorial/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020, NVIDIA CORPORATION. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /RecSys2020Tutorial/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG dev=false 2 | FROM nvcr.io/nvidia/cuda:10.2-devel-ubuntu18.04 AS base 3 | 4 | # install python and cudf 5 | RUN apt-get update 6 | RUN apt-get -y install graphviz git 7 | 8 | ADD https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh /miniconda.sh 9 | RUN sh /miniconda.sh -b -p /conda && /conda/bin/conda update -n base conda && /conda/bin/conda create --name nvtabular -c rapidsai -c nvidia -c numba -c conda-forge -c defaults pip cudf=0.15 python=3.7 cudatoolkit=10.2 dask-cudf nodejs>=10.0.0 ipython jupyterlab 10 | 11 | ENV PATH=${PATH}:/conda/bin 12 | SHELL ["/bin/bash", "-c"] 13 | 14 | RUN source activate nvtabular && pip3 install matplotlib pydotplus sklearn torch dask_cuda graphviz xgboost 15 | RUN source activate nvtabular && pip3 install git+https://github.com/NVIDIA/NVTabular.git 16 | 17 | # Create working directory to add repo. 18 | WORKDIR /dli 19 | 20 | # Load contents into student working directory, excluding anything in .dockerignore 21 | ADD . . 22 | 23 | # Set the initial working directory for students. 24 | WORKDIR /dli/task 25 | 26 | # Jupyter listens on 8888. 27 | EXPOSE 8888 28 | 29 | ENTRYPOINT ["/dli/entrypoint.sh"] 30 | 31 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/lib/include.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | PROJECT_PATH = os.path.dirname(os.path.realpath(__file__).replace('/lib','')) 4 | IDENTIFIER = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') 5 | 6 | 7 | #numerical libs 8 | import math 9 | import numpy as np 10 | import random 11 | import PIL 12 | #import cv2 13 | import matplotlib 14 | print('matplotlib.get_backend : ', matplotlib.get_backend()) 15 | 16 | # torch libs 17 | import torch 18 | from torch.utils.data.dataset import Dataset 19 | from torch.utils.data import DataLoader 20 | from torch.utils.data.sampler import * 21 | 22 | import torch.nn as nn 23 | import torch.nn.functional as F 24 | import torch.optim as optim 25 | from torch.nn.parallel.data_parallel import data_parallel 26 | 27 | from torch.nn.utils.rnn import * 28 | 29 | 30 | # std libs 31 | import collections 32 | import copy 33 | import numbers 34 | import inspect 35 | import shutil 36 | from timeit import default_timer as timer 37 | import itertools 38 | from collections import OrderedDict 39 | from multiprocessing import Pool 40 | import multiprocessing as mp 41 | 42 | #from pprintpp import pprint, pformat 43 | import json 44 | import zipfile 45 | 46 | 47 | 48 | import csv 49 | import pandas as pd 50 | import pickle 51 | import glob 52 | import sys 53 | from distutils.dir_util import copy_tree 54 | import time 55 | 56 | 57 | import matplotlib.pyplot as plt 58 | from mpl_toolkits.mplot3d import Axes3D 59 | 60 | 61 | 62 | # constant # 63 | PI = np.pi 64 | INF = np.inf 65 | EPS = 1e-12 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /champs-scalar-coupling/build_data/lib/include.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | PROJECT_PATH = os.path.dirname(os.path.realpath(__file__).replace('/lib','')) 4 | IDENTIFIER = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') 5 | 6 | 7 | #numerical libs 8 | import math 9 | import numpy as np 10 | import random 11 | import PIL 12 | #import cv2 13 | import matplotlib 14 | #matplotlib.use('TkAgg') 15 | #matplotlib.use('WXAgg') 16 | #matplotlib.use('Qt4Agg') 17 | #matplotlib.use('Qt5Agg') #Qt4Agg 18 | print('matplotlib.get_backend : ', matplotlib.get_backend()) 19 | #print(matplotlib.__version__) 20 | 21 | 22 | # torch libs 23 | import torch 24 | from torch.utils.data.dataset import Dataset 25 | from torch.utils.data import DataLoader 26 | from torch.utils.data.sampler import * 27 | 28 | import torch.nn as nn 29 | import torch.nn.functional as F 30 | import torch.optim as optim 31 | from torch.nn.parallel.data_parallel import data_parallel 32 | 33 | from torch.nn.utils.rnn import * 34 | 35 | 36 | # std libs 37 | import collections 38 | import copy 39 | import numbers 40 | import inspect 41 | import shutil 42 | from timeit import default_timer as timer 43 | import itertools 44 | from collections import OrderedDict 45 | from multiprocessing import Pool 46 | import multiprocessing as mp 47 | 48 | #from pprintpp import pprint, pformat 49 | import json 50 | import zipfile 51 | 52 | 53 | 54 | import csv 55 | import pandas as pd 56 | import pickle 57 | import glob 58 | import sys 59 | from distutils.dir_util import copy_tree 60 | import time 61 | 62 | 63 | import matplotlib.pyplot as plt 64 | from mpl_toolkits.mplot3d import Axes3D 65 | 66 | 67 | 68 | # constant # 69 | PI = np.pi 70 | INF = np.inf 71 | EPS = 1e-12 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /RecSys2020Tutorial/README.md: -------------------------------------------------------------------------------- 1 | # RecSys2020 Tutorial: Feature Engineering for Recommender Systems 2 | 3 | by Chris Deotte (NVidia), Benedikt Schifferer (NVidia) and Even Oldridge (NVidia) 4 | 5 | ### Content 6 | 7 | The selection of features and proper preparation of data for deep learning or machine learning models plays a significant role in the performance of recommender systems. To address this we propose a tutorial highlighting best practices and optimization techniques for feature engineering and preprocessing of recommender system datasets. The tutorial will explore feature engineering using pandas and Dask, and will also cover acceleration on the GPU using open source libraries like RAPIDS and NVTabular. Proposed length is 180min. We’ve designed the tutorial as a combination of a lecture covering the mathematical and theoretical background and an interactive session based on jupyter notebooks. Participants will practice the discussed features by writing their own implementation in Python. NVIDIA will host the tutorial on their infrastructure, providing dataset, jupyter notebooks and GPUs. Participants will be able to easily attend the tutorial via their web browsers, avoiding any complicated setup. 8 | 9 | Beginner to intermediate users are the target audience, which should have prior knowledge in python programming using libraries, such as pandas and NumPy. In addition, they should have a basic understanding of recommender systems, decision trees and feed forward neural networks. 10 | 11 | ### Requirements 12 | 13 | * RAPIDS cuDF 0.15 14 | * NVTabular 0.2 15 | * PyTorch 16 | 17 | ### Structure 18 | 19 | * Notebooks contains theory and exercises 20 | * /solutions/ contains solutions for the exercises 21 | * /data/ is the path with the expected parquet files 22 | -------------------------------------------------------------------------------- /RecSys2020Tutorial/data/README.md: -------------------------------------------------------------------------------- 1 | # RecSys2020 Tutorial: Feature Engineering for Recommender Systems 2 | 3 | by Chris Deotte (Nvidia), Benedikt Schifferer (Nvidia) and Even Oldridge (Nvidia) 4 | 5 | ### Content 6 | 7 | The selection of features and proper preparation of data for deep learning or machine learning models plays a significant role in the performance of recommender systems. To address this we propose a tutorial highlighting best practices and optimization techniques for feature engineering and preprocessing of recommender system datasets. The tutorial will explore feature engineering using pandas and Dask, and will also cover acceleration on the GPU using open source libraries like RAPIDS and NVTabular. Proposed length is 180min. We’ve designed the tutorial as a combination of a lecture covering the mathematical and theoretical background and an interactive session based on jupyter notebooks. Participants will practice the discussed features by writing their own implementation in Python. NVIDIA will host the tutorial on their infrastructure, providing dataset, jupyter notebooks and GPUs. Participants will be able to easily attend the tutorial via their web browsers, avoiding any complicated setup. 8 | 9 | Beginner to intermediate users are the target audience, which should have prior knowledge in python programming using libraries, such as pandas and NumPy. In addition, they should have a basic understanding of recommender systems, decision trees and feed forward neural networks. 10 | 11 | ### Requirements 12 | 13 | * RAPIDS cuDF 0.15 14 | * NVTabular 0.2 15 | * PyTorch 16 | 17 | ### Structure 18 | 19 | * Notebooks contains theory and exercises 20 | * /solutions/ contains solutions for the exercises 21 | * /data/ is the path with the expected parquet files 22 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/common_constants.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | ### Helpers for normalization 5 | NUM_COUPLING_TYPE=8 6 | COUPLING_TYPE_STATS=[ 7 | #type #mean, std, min, max 8 | '1JHC', 94.9761528641869, 18.27722399839607, 66.6008, 204.8800, 9 | '2JHC', -0.2706244378832, 4.52360876732858, -36.2186, 42.8192, 10 | '3JHC', 3.6884695895355, 3.07090647005439, -18.5821, 76.0437, 11 | '1JHN', 47.4798844844683, 10.92204561670947, 24.3222, 80.4187, 12 | '2JHN', 3.1247536134185, 3.67345877025737, -2.6209, 17.7436, 13 | '3JHN', 0.9907298624944, 1.31538940138001, -3.1724, 10.9712, 14 | '2JHH', -10.2866051639817, 3.97960190019757, -35.1761, 11.8542, 15 | '3JHH', 4.7710233597359, 3.70498129755812, -3.0205, 17.4841, 16 | ] 17 | 18 | NUM_COUPLING_TYPE = len(COUPLING_TYPE_STATS)//5 19 | COUPLING_TYPE = [ COUPLING_TYPE_STATS[i*5 ] for i in range(NUM_COUPLING_TYPE)] 20 | REVERSE_COUPLING_TYPE = dict(zip(range(8), COUPLING_TYPE)) 21 | 22 | COUPLING_TYPE_MEAN = torch.tensor([COUPLING_TYPE_STATS[i*5+1] for i in range(NUM_COUPLING_TYPE)], dtype=torch.float32).cuda() 23 | COUPLING_TYPE_STD = torch.tensor([ COUPLING_TYPE_STATS[i*5+2] for i in range(NUM_COUPLING_TYPE)], dtype=torch.float32).cuda() 24 | 25 | COUPLING_MIN_ = [ COUPLING_TYPE_STATS[i*5+3 ] for i in range(NUM_COUPLING_TYPE)] 26 | COUPLING_MAX_ = [ COUPLING_TYPE_STATS[i*5+4 ] for i in range(NUM_COUPLING_TYPE)] 27 | 28 | NODE_MAX, EDGE_MAX = 32, 816 29 | 30 | COUPLING_MAX_DICT = {'1JHC': 20, '2JHC': 36, '3JHC': 66, '1JHN': 8, '2JHN': 12, '3JHN': 18, '3JHH': 36, '2JHH': 19 } 31 | 32 | #--- Set of Categorical modalities 33 | SYMBOL = ['H', 'C', 'N', 'O', 'F'] 34 | 35 | # model criterion 36 | model_dict = { '1JHC': 'lmae', '2JHC': 'lmae', '3JHC': 'lmae', '3JHH': 'lmae', 37 | '1JHN': 'mlmae' , '2JHN':'mlmae' , '3JHN':'mlmae', '2JHH':'mlmae'} 38 | -------------------------------------------------------------------------------- /champs-scalar-coupling/build_data/common.py: -------------------------------------------------------------------------------- 1 | from lib.include import * 2 | from lib.utility.draw import * 3 | from lib.utility.file import * 4 | from lib.net.rate import * 5 | 6 | 7 | #--------------------------------------------------------------------------------- 8 | COMMON_STRING ='@%s: \n' % os.path.basename(__file__) 9 | 10 | if 1: 11 | SEED = int(time.time()) #35202 #35202 #123 # 12 | random.seed(SEED) 13 | np.random.seed(SEED) 14 | torch.manual_seed(SEED) 15 | torch.cuda.manual_seed_all(SEED) 16 | COMMON_STRING += '\tset random seed\n' 17 | COMMON_STRING += '\t\tSEED = %d\n'%SEED 18 | 19 | torch.backends.cudnn.benchmark = True ##uses the inbuilt cudnn auto-tuner to find the fastest convolution algorithms. - 20 | torch.backends.cudnn.enabled = True 21 | torch.backends.cudnn.deterministic = True 22 | 23 | COMMON_STRING += '\tset cuda environment\n' 24 | COMMON_STRING += '\t\ttorch.__version__ = %s\n'%torch.__version__ 25 | COMMON_STRING += '\t\ttorch.version.cuda = %s\n'%torch.version.cuda 26 | COMMON_STRING += '\t\ttorch.backends.cudnn.version() = %s\n'%torch.backends.cudnn.version() 27 | try: 28 | COMMON_STRING += '\t\tos[\'CUDA_VISIBLE_DEVICES\'] = %s\n'%os.environ['CUDA_VISIBLE_DEVICES'] 29 | NUM_CUDA_DEVICES = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) 30 | except Exception: 31 | COMMON_STRING += '\t\tos[\'CUDA_VISIBLE_DEVICES\'] = None\n' 32 | NUM_CUDA_DEVICES = 1 33 | 34 | COMMON_STRING += '\t\ttorch.cuda.device_count() = %d\n'%torch.cuda.device_count() 35 | #print ('\t\ttorch.cuda.current_device() =', torch.cuda.current_device()) 36 | 37 | 38 | COMMON_STRING += '\n' 39 | 40 | #--------------------------------------------------------------------------------- 41 | ## useful : http://forums.fast.ai/t/model-visualization/12365/2 42 | 43 | 44 | if __name__ == '__main__': 45 | print (COMMON_STRING) -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/common.py: -------------------------------------------------------------------------------- 1 | from .lib.include import * 2 | from .lib.utility.draw import * 3 | from .lib.utility.file import * 4 | from .lib.net.rate import * 5 | 6 | 7 | #--------------------------------------------------------------------------------- 8 | COMMON_STRING ='@%s: \n' % os.path.basename(__file__) 9 | 10 | if 1: 11 | SEED = int(time.time()) #35202 #35202 #123 # 12 | random.seed(SEED) 13 | np.random.seed(SEED) 14 | torch.manual_seed(SEED) 15 | torch.cuda.manual_seed_all(SEED) 16 | COMMON_STRING += '\tset random seed\n' 17 | COMMON_STRING += '\t\tSEED = %d\n'%SEED 18 | 19 | torch.backends.cudnn.benchmark = True ##uses the inbuilt cudnn auto-tuner to find the fastest convolution algorithms. - 20 | torch.backends.cudnn.enabled = True 21 | torch.backends.cudnn.deterministic = True 22 | 23 | COMMON_STRING += '\tset cuda environment\n' 24 | COMMON_STRING += '\t\ttorch.__version__ = %s\n'%torch.__version__ 25 | COMMON_STRING += '\t\ttorch.version.cuda = %s\n'%torch.version.cuda 26 | COMMON_STRING += '\t\ttorch.backends.cudnn.version() = %s\n'%torch.backends.cudnn.version() 27 | try: 28 | COMMON_STRING += '\t\tos[\'CUDA_VISIBLE_DEVICES\'] = %s\n'%os.environ['CUDA_VISIBLE_DEVICES'] 29 | NUM_CUDA_DEVICES = len(os.environ['CUDA_VISIBLE_DEVICES'].split(',')) 30 | except Exception: 31 | COMMON_STRING += '\t\tos[\'CUDA_VISIBLE_DEVICES\'] = None\n' 32 | NUM_CUDA_DEVICES = 1 33 | 34 | COMMON_STRING += '\t\ttorch.cuda.device_count() = %d\n'%torch.cuda.device_count() 35 | #print ('\t\ttorch.cuda.current_device() =', torch.cuda.current_device()) 36 | 37 | 38 | COMMON_STRING += '\n' 39 | 40 | #--------------------------------------------------------------------------------- 41 | ## useful : http://forums.fast.ai/t/model-visualization/12365/2 42 | 43 | 44 | if __name__ == '__main__': 45 | print (COMMON_STRING) -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/.nfs0000000001371b7a000004c5: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output 4 | script_path: /rapids/notebooks/srabhi/champs-2019/CherKeng_solution/fastai_code/dataset.py 5 | graph_path: /rapids/notebooks/srabhi/champs-2019/input/structure/graph4 6 | normalize: False 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/CherKeng_solution/fastai_code/model.py 10 | num_target : 8 11 | mpnn: 12 | T_steps: 6 13 | node_encoder: 14 | encoding: label 15 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 16 | n_cont: 1 17 | node_dim: 7 18 | layers: [128, 128] 19 | activation: relu 20 | dropout: 0. 21 | 22 | 23 | edge_encoder: 24 | encoding: label 25 | emb_sz: [[5,3]] 26 | n_cont: 2 27 | node_dim: 128 28 | edge_dim: 3 29 | layers: [256, 256, 128] 30 | activation: relu 31 | dropout: 0. 32 | 33 | Set2Set: 34 | processing_step: 6 35 | num_layer: 1 36 | in_channel: 128 37 | batch_size: 64 38 | 39 | y_range: [-2.326753765513524, 2.3267537655135464] 40 | 41 | regression: 42 | num_output: 1 43 | node_dim: 128 44 | shared_layers: [1024, 512] 45 | activation: relu 46 | dropout: 0. 47 | branch_layers: [512, 128] 48 | num_target: 8 49 | predict_type: True 50 | 51 | Classif: 52 | y_range: [-2.326753765513524, 2.3267537655135464] 53 | 54 | 55 | train: 56 | train_shape: 4658147 57 | test_shape: 2505542 58 | batch_size: 64 59 | epochs: 180 60 | max_lr: 0.005 61 | loss_name: lmae 62 | callback_metric: LMAE 63 | pretrain: False 64 | predict_type: True 65 | model_name: mpnn_gauss_rank_predict_type_180epochs_ 66 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_RNN_MAE_GAUSSRANK_SINGLE_TYPE.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: True 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 1 11 | RNN : True 12 | mpnn: 13 | T_steps: 4 14 | 15 | node_encoder: 16 | encoding: label 17 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 18 | n_cont: 1 19 | node_dim: 7 20 | layers: [128, 128] 21 | activation: relu 22 | dropout: 0. 23 | 24 | edge_encoder: 25 | encoding: label 26 | emb_sz: [[5,3]] 27 | n_cont: 2 28 | node_dim: 128 29 | edge_dim: 3 30 | layers: [128, 64] 31 | activation: relu 32 | dropout: 0. 33 | 34 | Set2Set: 35 | processing_step: 4 36 | num_layer: 1 37 | in_channel: 128 38 | batch_size: 64 39 | 40 | regression: 41 | num_output: 1 42 | input_dim: 768 43 | shared_layers: [1024, 512, 128] 44 | activation: relu 45 | dropout: 0. 46 | num_target: 1 47 | predict_type: False 48 | 49 | node_seq: 50 | node_dim: 128 51 | hidden_size: 256 52 | num_layers: 1 53 | dropout: 0.05 54 | batch_first: True 55 | bidirectional: True 56 | rnn_model: 'LSTM' 57 | attention: True 58 | 59 | train: 60 | train_shape: 4658147 61 | test_shape: 2505542 62 | batch_size: 64 63 | epochs: 1 64 | max_lr: 0.001 65 | loss_name: lmaeo2ceha 66 | callback_metric: LMAE 67 | pretrain: False 68 | predict_type: False 69 | model_name: MPNN_RNN_SINGLE_TYPE_GAUSSRANK_ 70 | device: cuda 71 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_RNN_MAE_WO_GAUSSRANK_SINGLE_TYPE.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: False 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 1 11 | RNN : True 12 | mpnn: 13 | T_steps: 4 14 | 15 | node_encoder: 16 | encoding: label 17 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 18 | n_cont: 1 19 | node_dim: 7 20 | layers: [128, 128] 21 | activation: relu 22 | dropout: 0. 23 | 24 | edge_encoder: 25 | encoding: label 26 | emb_sz: [[5,3]] 27 | n_cont: 2 28 | node_dim: 128 29 | edge_dim: 3 30 | layers: [128, 64] 31 | activation: relu 32 | dropout: 0. 33 | 34 | Set2Set: 35 | processing_step: 4 36 | num_layer: 1 37 | in_channel: 128 38 | batch_size: 64 39 | 40 | regression: 41 | num_output: 1 42 | input_dim: 768 43 | shared_layers: [1024, 512, 128] 44 | activation: relu 45 | dropout: 0. 46 | num_target: 1 47 | predict_type: False 48 | 49 | node_seq: 50 | node_dim: 128 51 | hidden_size: 256 52 | num_layers: 1 53 | dropout: 0.05 54 | batch_first: True 55 | bidirectional: True 56 | rnn_model: 'LSTM' 57 | attention: True 58 | 59 | train: 60 | train_shape: 4658147 61 | test_shape: 2505542 62 | batch_size: 64 63 | epochs: 1 64 | max_lr: 0.001 65 | loss_name: lmaeo2ceha 66 | callback_metric: LMAE 67 | pretrain: False 68 | predict_type: False 69 | model_name: MPNN_RNN_SINGLE_TYPE_WO_GAUSSRANK_ 70 | device: cuda 71 | -------------------------------------------------------------------------------- /champs-scalar-coupling/tree.md: -------------------------------------------------------------------------------- 1 | . 2 | ├── build_data 3 | │   ├── __init__.py 4 | │   ├── lib 5 | │   │   ├── include.py 6 | │   │   ├── __init__.py 7 | │   │   ├── net 8 | │   │   │   ├── __init__.py 9 | │   │   │   ├── __pycache__ 10 | │   │   │   │   ├── __init__ 11 | │   │   │   └── rate.py 12 | │   │   └── utility 13 | │   │   ├── draw.py 14 | │   │   ├── file.py 15 | │   │   ├── __init__.py 16 | │   ├── common.py 17 | │   ├── data-cudf.py 18 | │   ├── data.py 19 | │   ├── parallel_process.py 20 | │   ├── atom_features.py 21 | │   ├── baseline_node_frame_from_csv.ipynb 22 | │   ├── build_baseline_dataframes.ipynb 23 | │   ├── build_preds_from_checkpoints.ipynb 24 | │   ├── build_stack_train_validation.ipynb 25 | │   ├── build_train_validation.ipynb 26 | │   ├── build_train_validation_rnn.ipynb 27 | │   ├── build_train_validation_rnn_per_type.ipynb 28 | │   ├── build_train_validation_rnn_scalar.ipynb 29 | ├── experiments 30 | │   ├── \*.yaml 31 | ├── merge_predictions_per_type.ipynb 32 | ├── models 33 | │   ├── \*.pth 34 | ├── mpnn_model 35 | │   ├── __init__.py 36 | │   ├── lib 37 | │   │   ├── include.py 38 | │   │   ├── __init__.py 39 | │   │   ├── net 40 | │   │   │   ├── __init__.py 41 | │   │   │   ├── __pycache__ 42 | │   │   │   │   ├── __init__ 43 | │   │   │   └── rate.py 44 | │   │   └── utility 45 | │   │   ├── draw.py 46 | │   │   ├── file.py 47 | │   │   ├── __init__.py 48 | │   ├── build_predictions.py 49 | │   ├── callback.py 50 | │   ├── common_constants.py 51 | │   ├── common_model.py 52 | │   ├── common.py 53 | │   ├── data_collate.py 54 | │   ├── data.py 55 | │   ├── dataset.py 56 | │   ├── GaussRank.py 57 | │   ├── helpers.py 58 | │   ├── message_passing.py 59 | │   ├── model.py 60 | │   ├── parallel_process.py 61 | │   ├── radam.py 62 | │   ├── regression_head.py 63 | │   ├── RNN_attention.py 64 | │   └── train_loss.py 65 | ├── pre_trained_models 66 | │   ├── \*.pth 67 | ├── scripts 68 | │   ├── bootsrap_train_mpnn_rnn.py 69 | │   ├── train_mpnn.py 70 | │   ├── train_mpnn_rnn.py 71 | │   └── train_type.py 72 | ├── train_MPNN_RNN.ipynb 73 | ├── train_MPNN_RNN_SINGLE_TYPE.ipynb 74 | ├── save_pretrained_single_models.ipynb 75 | ├── README.md 76 | └── tree.md -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_EMBED_TYPE_LMAE_WO_GAUSSRANK.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: False 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: False 12 | y_range: [-36.2186, 204.8800] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 512 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 1 50 | predict_type: False 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 90 67 | max_lr: 0.005 68 | loss_name: lmae_embed_type 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: False 72 | model_name: MPNN_RNN_EMBED_TYPE_WO_GAUSSRANK_LMAE 73 | device: cuda 74 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_RNN_EMBED_TYPE_LMAE_WO_GAUSSRANK.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: False 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: True 12 | y_range: [-36.2186, 204.8800] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 800 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 1 50 | predict_type: False 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 90 67 | max_lr: 0.005 68 | loss_name: lmae_embed_type 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: False 72 | model_name: MPNN_RNN_EMBED_TYPE_WO_GAUSSRANK_LMAE 73 | device: cuda 74 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_PREDICT_TYPE_MLMAE_GAUSSRANK.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: True 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: False 12 | y_range: [-2.326753765513524, 2.3267537655135464] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 768 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 8 50 | predict_type: True 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 1 67 | max_lr: 0.005 68 | loss_name: mlmaeo2ceha 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: True 72 | model_name: MPNN_GAUSSRANK_PREDICT_TYPE_MLMAE_ 73 | device: cuda 74 | 75 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_RNN_PREDICT_TYPE_LMAE_WO_GAUSSRANK.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: False 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: True 12 | y_range: [-36.2186, 204.8800] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 768 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 8 50 | predict_type: True 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 1 67 | max_lr: 0.005 68 | loss_name: lmaeo2ceha 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: True 72 | model_name: mpnn_gauss_rank_MLMAE_2CE_RNN_V3_type_seq_v3_ 73 | device: cuda 74 | 75 | 76 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_EMBED_TYPE_LMAE_GAUSSRANK.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: True 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: False 12 | y_range: [-2.326753765513524, 2.3267537655135464] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 512 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 1 50 | predict_type: False 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 90 67 | max_lr: 0.005 68 | loss_name: lmae_embed_type 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: False 72 | model_name: MPNN_EMBED_TYPE_GAUSSRANK_LMAE 73 | device: cuda 74 | 75 | 76 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_PREDICT_TYPE_LMAE_GAUSSRANK.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/CherKeng_solution/fastai_code/dataset.py 5 | normalize: False 6 | gaussrank: True 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/CherKeng_solution/fastai_code/model.py 10 | num_type: 8 11 | RNN: False 12 | y_range: [-2.326753765513524, 2.3267537655135464] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 768 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 8 50 | predict_type: True 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 1 67 | max_lr: 0.005 68 | loss_name: lmaeo2ceha 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: True 72 | model_name: MPNN_GAUSSRANK_PREDICT_TYPE_LMAE_ 73 | device: cuda 74 | 75 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_PREDICT_TYPE_LMAE_WO_GAUSSRANK.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: False 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: False 12 | y_range: [-2.326753765513524, 2.3267537655135464] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 768 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 8 50 | predict_type: True 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 1 67 | max_lr: 0.005 68 | loss_name: lmaeo2ceha 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: True 72 | model_name: MPNN_WO_GAUSSRANK_PREDICT_TYPE_LMAE_ 73 | device: cuda 74 | 75 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_RNN_EMBED_TYPE_LMAE_GAUSSRANK.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: True 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: True 12 | y_range: [-2.326753765513524, 2.3267537655135464] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 800 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 1 50 | predict_type: False 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 90 67 | max_lr: 0.005 68 | loss_name: lmae_embed_type 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: False 72 | model_name: MPNN_RNN_EMBED_TYPE_GAUSSRANK_LMAE 73 | device: cuda 74 | 75 | 76 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_RNN_PREDICT_TYPE_LMAE_GAUSSRANK.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: True 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: True 12 | y_range: [-2.326753765513524, 2.3267537655135464] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 768 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 8 50 | predict_type: True 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 1 67 | max_lr: 0.005 68 | loss_name: lmaeo2ceha 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: True 72 | model_name: MPNN_RNN_PREDICT_TYPE_LMAE_GAUSSRANK 73 | device: cuda 74 | 75 | 76 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_RNN_PREDICT_TYPE_MLMAE_GAUSSRANK.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: True 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: True 12 | y_range: [-2.326753765513524, 2.3267537655135464] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 768 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 8 50 | predict_type: True 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 1 67 | max_lr: 0.005 68 | loss_name: mlmaeo2ceha 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: True 72 | model_name: MPNN_RNN_PREDICT_TYPE_MLMAE_GAUSSRANK 73 | device: cuda 74 | 75 | 76 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_PREDICT_TYPE_LMAE_GAUSSRANK_BOOTSTRAP.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: True 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: False 12 | y_range: [-2.326753765513524, 2.3267537655135464] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 768 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 8 50 | predict_type: True 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 1 67 | max_lr: 0.005 68 | loss_name: lmaeo2ceha 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: True 72 | model_name: MPNN_GAUSSRANK_PREDICT_TYPE_LMAE_BOOTSTRAP_ 73 | device: cuda 74 | 75 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_PREDICT_TYPE_MLMAE_GAUSSRANK_BOOTSTRAP.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: True 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: False 12 | y_range: [-2.326753765513524, 2.3267537655135464] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 768 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 8 50 | predict_type: True 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 1 67 | max_lr: 0.005 68 | loss_name: mlmaeo2ceha 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: True 72 | model_name: MPNN_GAUSSRANK_PREDICT_TYPE_MLMAE_BOOTSTRAP_ 73 | device: cuda 74 | 75 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_RNN_PREDICT_TYPE_LMAE_GAUSSRANK_BOOTSTRAP.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: True 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: True 12 | y_range: [-2.326753765513524, 2.3267537655135464] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 768 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 8 50 | predict_type: True 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 1 67 | max_lr: 0.005 68 | loss_name: lmaeo2ceha 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: True 72 | model_name: MPNN_RNN_GAUSSRANK_PREDICT_TYPE_LMAE_BOOTSTRAP_ 73 | device: cuda 74 | 75 | -------------------------------------------------------------------------------- /champs-scalar-coupling/experiments/MPNN_RNN_PREDICT_TYPE_MLMAE_GAUSSRANK_BOOTSTRAP.yaml: -------------------------------------------------------------------------------- 1 | dataset: 2 | input_path: /rapids/notebooks/srabhi/champs-2019/input 3 | output_path: /rapids/notebooks/srabhi/champs-2019/output/ 4 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/dataset.py 5 | normalize: False 6 | gaussrank: True 7 | 8 | model: 9 | script_path: /rapids/notebooks/srabhi/champs-2019/final_solution/mpnn_model/model.py 10 | num_type: 8 11 | RNN: True 12 | y_range: [-2.326753765513524, 2.3267537655135464] 13 | mpnn: 14 | T_steps: 4 15 | 16 | node_encoder: 17 | encoding: label 18 | emb_sz: [[6,3], [3,3], [3, 3], [3,3], [5,3], [8, 4]] 19 | n_cont: 1 20 | node_dim: 7 21 | layers: [128, 128] 22 | activation: relu 23 | dropout: 0. 24 | 25 | edge_encoder: 26 | encoding: label 27 | emb_sz: [[5,3]] 28 | n_cont: 2 29 | node_dim: 128 30 | edge_dim: 3 31 | layers: [256, 128] 32 | activation: relu 33 | dropout: 0. 34 | 35 | Set2Set: 36 | processing_step: 4 37 | num_layer: 1 38 | in_channel: 128 39 | batch_size: 64 40 | 41 | 42 | regression: 43 | num_output: 1 44 | input_dim: 768 45 | shared_layers: [1024, 512, 128] 46 | activation: relu 47 | dropout: 0. 48 | branch_layers: [512, 128] 49 | num_target: 8 50 | predict_type: True 51 | 52 | node_seq: 53 | node_dim: 128 54 | hidden_size: 256 55 | num_layers: 1 56 | dropout: 0.05 57 | batch_first: True 58 | bidirectional: True 59 | rnn_model: 'LSTM' 60 | attention: True 61 | 62 | train: 63 | train_shape: 4658147 64 | test_shape: 2505542 65 | batch_size: 64 66 | epochs: 1 67 | max_lr: 0.005 68 | loss_name: mlmaeo2ceha 69 | callback_metric: LMAE 70 | pretrain: False 71 | predict_type: True 72 | model_name: MPNN_RNN_GAUSSRANK_PREDICT_TYPE_MLMAE_BOOTSTRAP_ 73 | device: cuda 74 | 75 | -------------------------------------------------------------------------------- /champs-scalar-coupling/build_data/parallel_process.py: -------------------------------------------------------------------------------- 1 | from tqdm import tqdm 2 | from concurrent.futures import ProcessPoolExecutor, as_completed 3 | """ 4 | Credit to http://danshiebler.com 5 | """ 6 | def parallel_process(array, function, n_jobs=16, use_kwargs=False, front_num=3): 7 | """ 8 | A parallel version of the map function with a progress bar. 9 | 10 | Args: 11 | array (array-like): An array to iterate over. 12 | function (function): A python function to apply to the elements of array 13 | n_jobs (int, default=16): The number of cores to use 14 | use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of 15 | keyword arguments to function 16 | front_num (int, default=3): The number of iterations to run serially before kicking off the parallel job. 17 | Useful for catching bugs 18 | Returns: 19 | [function(array[0]), function(array[1]), ...] 20 | """ 21 | #We run the first few iterations serially to catch bugs 22 | if front_num > 0: 23 | front = [function(**a) if use_kwargs else function(a) for a in array[:front_num]] 24 | #If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging. 25 | if n_jobs==1: 26 | return front + [function(**a) if use_kwargs else function(a) for a in tqdm(array[front_num:])] 27 | #Assemble the workers 28 | with ProcessPoolExecutor(max_workers=n_jobs) as pool: 29 | #Pass the elements of array into function 30 | if use_kwargs: 31 | futures = [pool.submit(function, **a) for a in array[front_num:]] 32 | else: 33 | futures = [pool.submit(function, a) for a in array[front_num:]] 34 | kwargs = { 35 | 'total': len(futures), 36 | 'unit': 'it', 37 | 'unit_scale': True, 38 | 'leave': True 39 | } 40 | #Print out the progress as tasks complete 41 | for f in tqdm(as_completed(futures), **kwargs): 42 | pass 43 | out = [] 44 | #Get the results from the futures. 45 | for i, future in tqdm(enumerate(futures)): 46 | try: 47 | out.append(future.result()) 48 | except Exception as e: 49 | out.append(e) 50 | return front + out -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/parallel_process.py: -------------------------------------------------------------------------------- 1 | from tqdm import tqdm 2 | from concurrent.futures import ProcessPoolExecutor, as_completed 3 | """ 4 | Credit to http://danshiebler.com 5 | """ 6 | def parallel_process(array, function, n_jobs=16, use_kwargs=False, front_num=3): 7 | """ 8 | A parallel version of the map function with a progress bar. 9 | 10 | Args: 11 | array (array-like): An array to iterate over. 12 | function (function): A python function to apply to the elements of array 13 | n_jobs (int, default=16): The number of cores to use 14 | use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of 15 | keyword arguments to function 16 | front_num (int, default=3): The number of iterations to run serially before kicking off the parallel job. 17 | Useful for catching bugs 18 | Returns: 19 | [function(array[0]), function(array[1]), ...] 20 | """ 21 | #We run the first few iterations serially to catch bugs 22 | if front_num > 0: 23 | front = [function(**a) if use_kwargs else function(a) for a in array[:front_num]] 24 | #If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging. 25 | if n_jobs==1: 26 | return front + [function(**a) if use_kwargs else function(a) for a in tqdm(array[front_num:])] 27 | #Assemble the workers 28 | with ProcessPoolExecutor(max_workers=n_jobs) as pool: 29 | #Pass the elements of array into function 30 | if use_kwargs: 31 | futures = [pool.submit(function, **a) for a in array[front_num:]] 32 | else: 33 | futures = [pool.submit(function, a) for a in array[front_num:]] 34 | kwargs = { 35 | 'total': len(futures), 36 | 'unit': 'it', 37 | 'unit_scale': True, 38 | 'leave': True 39 | } 40 | #Print out the progress as tasks complete 41 | for f in tqdm(as_completed(futures), **kwargs): 42 | pass 43 | out = [] 44 | #Get the results from the futures. 45 | for i, future in tqdm(enumerate(futures)): 46 | try: 47 | out.append(future.result()) 48 | except Exception as e: 49 | out.append(e) 50 | return front + out -------------------------------------------------------------------------------- /pytorch/batch_dataloader/batch_dataloader.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import _utils 3 | 4 | class BatchDataLoader(object): 5 | """Batch Data loader. Takes in a batch dataset and returns iterators that return whole batches of data. 6 | Arguments: 7 | batchdataset (BatchDataset): dataset from which to load the data. 8 | shuffle (bool, optional): set to ``True`` to have the data reshuffled 9 | at every epoch (default: ``False``). 10 | pin_memory (bool, optional): If ``True``, the data loader will copy tensors 11 | into CUDA pinned memory before returning them. 12 | drop_last (bool, optional): set to ``True`` to drop the last incomplete batch, 13 | if the dataset size is not divisible by the batch size. If ``False`` and 14 | the size of dataset is not divisible by the batch size, then the last batch 15 | will be smaller. (default: ``False``) 16 | """ 17 | 18 | def __init__(self, batchdataset, shuffle=False, 19 | pin_memory=False, drop_last=False): 20 | self.batchdataset = batchdataset 21 | self.batch_size = batchdataset.batch_size 22 | 23 | self.shuffle = shuffle 24 | self.pin_memory = pin_memory 25 | self.drop_last = drop_last 26 | 27 | 28 | def __iter__(self): 29 | return _BatchDataLoaderIter(self) 30 | 31 | def __len__(self): 32 | if self.drop_last and self.batchdataset.num_samples%self.batch_size != 0: 33 | return len(self.batchdataset)-1 34 | else: 35 | return len(self.batchdataset) 36 | 37 | 38 | class _BatchDataLoaderIter(object): 39 | """Iterates once over the BatchDataLoader's batchdataset, shuffling if requested""" 40 | def __init__(self, loader): 41 | self.batchdataset = loader.batchdataset 42 | self.batch_size = loader.batch_size 43 | self.pin_memory = loader.pin_memory and torch.cuda.is_available() 44 | self.drop_last = loader.drop_last 45 | 46 | if loader.shuffle: 47 | self.batchdataset.shuffle() 48 | 49 | self.idx = 0 50 | 51 | def __len__(self): 52 | if self.drop_last and self.batchdataset.num_samples%self.batch_size != 0: 53 | return len(self.batchdataset)-1 54 | else: 55 | return len(self.batchdataset) 56 | 57 | 58 | def __next__(self): 59 | if self.idx >= len(self): 60 | raise StopIteration 61 | batch = self.batchdataset[self.idx] 62 | # Note Pinning memory was ~10% _slower_ for the test examples I explored 63 | if self.pin_memory: 64 | batch = _utils.pin_memory.pin_memory_batch(batch) 65 | self.idx = self.idx+1 66 | return batch 67 | 68 | next = __next__ # Python 2 compatibility 69 | 70 | def __iter__(self): 71 | return self 72 | -------------------------------------------------------------------------------- /pytorch/batch_dataloader/README.md: -------------------------------------------------------------------------------- 1 | ### Pytorch Batch Dataloader 2 | ## 🚀 Feature 3 | A dataloader and dataset that operate at the batch level, rather than the item level, pulling batches from contiguous blocks of memory and avoiding random access patterns in the dataloader. 4 | 5 | ## Motivation 6 | Loading data item by item and coallating into a batch is very inefficient, particularly in the case of tabular or text data where the items are small. This is compounded further when you want to use large batch sizes. By pre shuffling the data each epoch (when required) we can grab each batch as a single read from contiguous memory. This much faster and scales better with batch size, removing the necessity of multiprocessing, which adds complexity in the form of bus errors when not enough shared memory is available (https://github.com/pytorch/pytorch/issues/5040), CUDA init issues when forking (https://github.com/pytorch/pytorch/issues/4377), etc. This forking issue was one of my original motivations as it solves the issue of using the dataloader in conjunction with RAPIDS or any other code that calls CUDA before the dataloader workers are forked. It should also solve the issue on windows with the speed of dataloaders, at least for tabular and text data, (https://github.com/pytorch/pytorch/issues/12831) as spawning is not necessary. 7 | 8 | Using the proposed method results in better GPU utilization, and better throughput when training in the tests on tabular data that I've run. With no multiprocessing I've measured a 5-15% improvement* in throughput over an 8 worker vanilla dataloader (more were tried but it maxed out at 8). I've also been able to increase batch sizes for tabular data into the 800K+ range with no loss of accuracy and get a 2x performance improvement over the best multiprocessor dataloader I could run without running into bus error issues that cropped up with large batch sizes. 9 | 10 | *depends on tensor and batch size 11 | 12 | ## Pitch 13 | 14 | I've created source for a batch dataloader and batch dataset modelled after their vanilla counterparts and would love to see it integrated into the PyTorch repo. Usage is similar, and I've tried to stick to the pytorch variable naming and formatting. 15 | 16 | Code can be found here: https://github.com/rapidsai/dataloaders/tree/main/pytorch/batch_dataloader 17 | 18 | It should hopefully be ready to go; I've tested it with both base pytorch and with ignite, but more eyes on it would definitely be beneficial, particularly in use cases beyond tabular like text or small images. It should be applicable to anyone who isn't doing large images or a lot of image augmentation. It's undergone an internal (NVidia) review of @ptrblck who was immensely helpful in refining it and @ngimel who reviewed the codebase and had helpful suggestions regarding memory pinning. 19 | 20 | I'm happy to work with the team to create test cases similar to those for dataset and dataloader and would love feedback on it. 21 | 22 | ## Alternatives 23 | 24 | One possible solution to the CUDA Init before fork issue is to spawn, however as seen in windows this is significantly slower and I had trouble getting it working. 25 | 26 | ## Additional context 27 | 28 | I'm also working on versions of this that work with larger than CPU memory datasets and on a version that works in GPU memory doing a 0-copy transform of a rapids cudf dataframe via dlpack. 29 | -------------------------------------------------------------------------------- /pytorch/batch_dataloader/batch_dataset.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | class BatchDataset(object): 4 | """An abstract class representing a Batch Dataset. 5 | All other datasets should subclass this. All subclasses should override 6 | ``__len__``, which provides the size of the dataset, ``__getitem__``, 7 | supporting integer indexing of batches in range from 0 to len(self)//batchsize exclusive, 8 | and ``shuffle`` which randomly shuffles the data, generally called per epoch. 9 | Batch datasets are meant to be iterated over in order rather than randomly accessed 10 | so the randomization has to happen first. 11 | """ 12 | 13 | def __getitem__(self, index): 14 | raise NotImplementedError 15 | 16 | def __len__(self): 17 | raise NotImplementedError 18 | 19 | def __add__(self): 20 | raise NotImplementedError 21 | 22 | def shuffle(self): 23 | raise NotImplementedError 24 | 25 | 26 | 27 | class TensorBatchDataset(BatchDataset): 28 | """Batch Dataset wrapping Tensors. 29 | Arguments: 30 | *tensors (Tensor): tensors that have the same size of the first dimension. 31 | batch_size: The size of the batch to return 32 | pin_memory (bool, optional): If ``True``, the dataset will be pinned memory for faster copy to GPU. 33 | I saw no performance improvement to doing so but results may vary. 34 | """ 35 | def __init__(self, tensors, batch_size=1, pin_memory=False): 36 | assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors) 37 | self.tensors = tensors 38 | self.batch_size=batch_size 39 | 40 | self.num_samples = tensors[0].size(0) 41 | 42 | if pin_memory: 43 | for tensor in self.tensors: 44 | tensor.pin_memory() 45 | 46 | def __len__(self): 47 | if self.num_samples%self.batch_size == 0: 48 | return self.num_samples // self.batch_size 49 | else: 50 | return self.num_samples // self.batch_size + 1 51 | 52 | def __getitem__(self, item): 53 | idx = item*self.batch_size 54 | #Need to handle odd sized batches if data isn't divisible by batchsize 55 | if idx < self.num_samples and (idx + self.batch_size < self.num_samples or self.num_samples%self.batch_size == 0): 56 | return [tensor[idx:idx+self.batch_size] for tensor in self.tensors] 57 | elif idx < self.num_samples and idx + self.batch_size> self.num_samples : 58 | return [tensor[idx:] for tensor in self.tensors] 59 | else: 60 | raise IndexError 61 | 62 | def __add__(self, tensors): 63 | assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors) 64 | assert len(self.tensors) == len(tensors) 65 | assert all(self_tensor[0].shape == tensor[0].shape for self_tensor, tensor in zip(self.tensors, tensors)) 66 | 67 | num_add_samples = tensors[0].size(0) 68 | self.num_samples = self.num_samples + num_add_samples 69 | self.tensors = [torch.cat((self_tensor, tensor)) for self_tensor, tensor in zip(self.tensors, tensors)] 70 | 71 | def shuffle(self): 72 | idx = torch.randperm(self.num_samples, dtype=torch.int64) 73 | self.tensors = [tensor[idx] for tensor in self.tensors] 74 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/RNN_attention.py: -------------------------------------------------------------------------------- 1 | from mpnn_model.common import * 2 | import torch 3 | import torch.nn as nn 4 | 5 | ############################################################################################################# 6 | # # 7 | # Nodes sequence : Attention-Bidirectional # 8 | # # 9 | ############################################################################################################# 10 | 11 | class BI_RNN_Nodes(torch.nn.Module): 12 | def attention_neuralnet(self, rnn_out, state): 13 | 14 | """ 15 | #### credit to : https://github.com/wabyking/TextClassificationBenchmark 16 | """ 17 | merged_state = torch.cat([s for s in state],1) # merge the hidden states of the two directions 18 | merged_state = merged_state.squeeze(0).unsqueeze(2) 19 | 20 | # (batch, seq_len, cell_size) * (batch, cell_size, 1) = (batch, seq_len, 1) 21 | weights = torch.bmm(rnn_out, merged_state) 22 | weights = torch.nn.functional.softmax(weights.squeeze(2)).unsqueeze(2) 23 | 24 | # (batch, cell_size, seq_len) * (batch, seq_len, 1) = (batch, cell_size, 1) 25 | return torch.bmm(torch.transpose(rnn_out, 1, 2), weights).squeeze(2), weights 26 | 27 | 28 | def __init__(self, 29 | node_dim, 30 | hidden_size, 31 | num_layers, 32 | dropout, 33 | batch_first, 34 | bidirectional, 35 | rnn_model='LSTM', 36 | attention=True): 37 | 38 | 39 | super(BI_RNN_Nodes, self).__init__() 40 | 41 | self.type_encoder = nn.Embedding(16, 32, padding_idx=0) 42 | 43 | self.atomic_encoder = nn.Embedding(16, 32, padding_idx=0) 44 | 45 | 46 | self.attention = attention 47 | if rnn_model == 'LSTM': 48 | self.rnn = nn.LSTM(input_size= node_dim + 64, hidden_size=hidden_size, 49 | num_layers=num_layers, dropout=dropout, 50 | batch_first=batch_first, bidirectional=bidirectional) 51 | else: 52 | raise LookupError('only support LSTM ') 53 | 54 | def forward(self, x_nodes, x_coupling_type, x_atomic): 55 | ''' 56 | x_nodes [batch_size x 4 x node_dim] : sequence of nodes embeddings of the coupling's shortest path 57 | x_coupling_type [batch_size x 4 x 1]: sequence of in-coming bond type 58 | X_atomic [batch_size x 4 x 1]: sequence of node's atomic number 59 | ''' 60 | 61 | x_type = self.type_encoder(x_coupling_type+1).squeeze() # +1 to encode padded/missing values to 0 62 | x_atomic = self.atomic_encoder(x_atomic+1).squeeze() 63 | 64 | x = torch.cat([x_nodes, x_type, x_atomic], dim=2) 65 | 66 | rnn_out, (final_hidden_state, final_cell_state) = self.rnn(x, None) 67 | 68 | if self.attention: 69 | last_tensor_item, weights = self.attention_neuralnet(rnn_out, final_hidden_state) 70 | else: 71 | # use mean instead of weighted attention 72 | last_tensor = rnn_out[row_indices, :, :] 73 | last_tensor_item = torch.mean(last_tensor, dim=1) 74 | 75 | return last_tensor_item 76 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/regression_head.py: -------------------------------------------------------------------------------- 1 | from mpnn_model.common import * 2 | from mpnn_model.common_model import * 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | 8 | __all__ = ['get_regression_module'] 9 | 10 | ############################################################################################################# 11 | # # 12 | # Output models # 13 | # # 14 | ############################################################################################################# 15 | 16 | def get_regression_module(num_output=1, 17 | input_dim=128, 18 | shared_layers=[1024, 512], 19 | activation='relu', 20 | dropout= 0., 21 | branch_layers=[], 22 | num_target=8, 23 | predict_type =False): 24 | ''' 25 | Regression module 26 | Args: 27 | num_output: [1, 5]: Whether to predict only scalar coupling or scalar coupling + the 4 contributions 28 | input_dim: the dimension of regression's head input: 29 | Combination of Graph representation, nodes' reprensentation of the coupling edge, nodes sequence hidden states. 30 | shared_layers: the dimension of the fully connected network shared between all the possible model's outputs 31 | activation: 32 | dropout: probability dropout for regresson regularization 33 | branch_layers: the fully connected branch network to predict each contribution value 34 | num_target: Whether to predict all the coupling type or fine-tune a single model per type 35 | predict_type: For num_output =1, whether to jointly predict the bond type or to embed the categorical variable "bond type" 36 | 37 | Outputs: 4 branches 38 | dense_layer: shared branch that learns a dense representation from the concatenation of a combination of 39 | Graph representation, nodes' reprensentation of the coupling edge, . 40 | classify: if predict_type==True, Classification branch that computes the logits of the 8 classes of coupling type 41 | predict: if num_output==1, Regression branch that computes scalar coupling constant vector: 8 values (per type) 42 | predicition_layers: if num_output==5, 8 regression branches (one for each coupling types) that computes 43 | the scalar coupling constant and the contribution components. 44 | ''' 45 | 46 | predicition_layers = [] 47 | classify =[] 48 | predict = [] 49 | dense_layer = LinearBn(input_dim, shared_layers[0], act=activation) 50 | 51 | ### Whether to predict only scalar coupling or scalar coupling + the 4 contributions 52 | if num_output==1: 53 | predict = nn.Sequential( 54 | MlpBn(shared_layers[0], dimensions=shared_layers[1:], activation=activation, dropout=dropout), 55 | nn.Linear(shared_layers[-1], num_target) 56 | ) 57 | ### Whether to jointly predict the bond type or to embed the categorical variable "bond type" 58 | if predict_type: 59 | classify = nn.Sequential( 60 | LinearBn( 1024, 512), 61 | nn.ReLU(inplace=True), 62 | nn.Linear(512, num_target),) 63 | 64 | elif num_output == 5: 65 | model = nn.Sequential( 66 | MlpBn(shared_layers[0], 67 | dimensions=branch_layers, 68 | activation=activation, 69 | dropout=dropout), 70 | nn.Linear(branch_layers[-1], num_output)) 71 | predicition_layers = nn.ModuleList([model for i in range(num_target)]) 72 | 73 | return dense_layer, classify, predict, predicition_layers 74 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/GaussRank.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.special import erfinv 3 | from bisect import bisect_left 4 | import pandas as pd 5 | class GaussRankMap(): 6 | 7 | def __init__(self, training_maps=[], coupling_order=[]): 8 | self.epsilon = 0.001 9 | self.lower = -1 + self.epsilon 10 | self.upper = 1 - self.epsilon 11 | self.range = self.upper - self.lower 12 | 13 | self.training_maps = training_maps 14 | self.coupling_order = coupling_order 15 | 16 | def fit_training(self, df, reset=False): 17 | if self.training_maps and reset == True: 18 | self.training_maps = [] 19 | self.coupling_order = [] 20 | elif self.training_maps: 21 | print('GaussRank Mapping already exists. To overide set reset=True.') 22 | return 23 | 24 | tf = None 25 | 26 | for coupling_type in df['type'].unique(): 27 | self.coupling_order.append(coupling_type) 28 | X = df[df['type']==coupling_type]['scalar_coupling_constant'] 29 | i = np.argsort(X, axis=0) 30 | j = np.argsort(i, axis=0) 31 | 32 | assert (j.min() == 0).all() 33 | assert (j.max() == len(j) - 1).all() 34 | 35 | j_range = len(j) - 1 36 | self.divider = j_range / self.range 37 | 38 | transformed = j / self.divider 39 | transformed = transformed - self.upper 40 | transformed = erfinv(transformed) 41 | 42 | #print(coupling_type, len(X), len(transformed)) 43 | 44 | if tf is None: 45 | tf = transformed.copy(deep=True) 46 | 47 | else: 48 | tf = tf.append(transformed.copy(deep=True)) 49 | 50 | 51 | training_map = pd.concat([X, transformed], axis=1) 52 | training_map.columns=['sc','sct'] 53 | training_map.sort_values(['sc'], ascending=[1], inplace=True) 54 | training_map.reset_index(inplace=True, drop=True) 55 | 56 | self.training_maps.append(training_map) 57 | return tf 58 | 59 | def convert_df(self, df, from_coupling=True): 60 | #coupling_idx = self.coupling_order.index(coupling_type) 61 | if from_coupling==True: 62 | column = 'sc' 63 | target = 'sct' 64 | df_column = 'scalar_coupling_constant' 65 | 66 | else: 67 | column = 'sct' 68 | target = 'sc' 69 | df_column = 'prediction' 70 | 71 | output = None 72 | # Do all of the sorts per coupling type in a single operation 73 | for coupling_type in df['type'].unique(): 74 | training_map = self.training_maps[self.coupling_order.index(coupling_type)] 75 | #training_map = cudf.DataFrame.from_pandas(self.training_maps[self.coupling_order.index(coupling_type)]) 76 | pos = training_map[column].searchsorted(df[df['type']==coupling_type][df_column], side='left') 77 | pos[pos>=len(training_map)] = len(training_map)-1 78 | pos[pos-1<=0] = 0 79 | 80 | x1 = training_map[column].iloc[pos].reset_index(drop=True) 81 | x2 = training_map[column].iloc[pos-1].reset_index(drop=True) # larger of the two 82 | y1 = training_map[target].iloc[pos].reset_index(drop=True) 83 | y2 = training_map[target].iloc[pos-1].reset_index(drop=True) 84 | z = df[df['type']==coupling_type].reset_index(drop=False)[['index',df_column]] 85 | 86 | relative = z['index'],(z[df_column]-x2) / (x1-x2) 87 | if output is None: 88 | output = pd.DataFrame(list(zip(relative[0],((1-relative[1])*y2 + (relative[1]*y1))))) 89 | else: 90 | output = output.append(pd.DataFrame(list(zip(relative[0],((1-relative[1])*y2 + (relative[1]*y1)))))) 91 | 92 | output.columns = ['index',target] 93 | output = output.set_index('index', drop=True) 94 | # output = output.sort_index() 95 | # < min or > max 96 | return output #pd.DataFrame(list(zip(relative[0],((1-relative[1])*y2 + (relative[1]*y1))))) -------------------------------------------------------------------------------- /champs-scalar-coupling/build_data/lib/utility/file.py: -------------------------------------------------------------------------------- 1 | from lib.include import * 2 | import builtins 3 | import re 4 | 5 | class Struct(object): 6 | def __init__(self, is_copy=False, **kwargs): 7 | self.add(is_copy, **kwargs) 8 | 9 | def add(self, is_copy=False, **kwargs): 10 | #self.__dict__.update(kwargs) 11 | 12 | if is_copy == False: 13 | for key, value in kwargs.items(): 14 | setattr(self, key, value) 15 | else: 16 | for key, value in kwargs.items(): 17 | try: 18 | setattr(self, key, copy.deepcopy(value)) 19 | #setattr(self, key, value.copy()) 20 | except Exception: 21 | setattr(self, key, value) 22 | 23 | def __str__(self): 24 | return str(self.__dict__.keys()) 25 | 26 | 27 | 28 | # log ------------------------------------ 29 | def remove_comments(lines, token='#'): 30 | """ Generator. Strips comments and whitespace from input lines. 31 | """ 32 | 33 | l = [] 34 | for line in lines: 35 | s = line.split(token, 1)[0].strip() 36 | if s != '': 37 | l.append(s) 38 | return l 39 | 40 | 41 | def open(file, mode=None, encoding=None): 42 | if mode == None: mode = 'r' 43 | 44 | if '/' in file: 45 | if 'w' or 'a' in mode: 46 | dir = os.path.dirname(file) 47 | if not os.path.isdir(dir): os.makedirs(dir) 48 | 49 | f = builtins.open(file, mode=mode, encoding=encoding) 50 | return f 51 | 52 | 53 | def remove(file): 54 | if os.path.exists(file): os.remove(file) 55 | 56 | 57 | def empty(dir): 58 | if os.path.isdir(dir): 59 | shutil.rmtree(dir, ignore_errors=True) 60 | else: 61 | os.makedirs(dir) 62 | 63 | 64 | # http://stackoverflow.com/questions/34950201/pycharm-print-end-r-statement-not-working 65 | class Logger(object): 66 | def __init__(self): 67 | self.terminal = sys.stdout #stdout 68 | self.file = None 69 | 70 | def open(self, file, mode=None): 71 | if mode is None: mode ='w' 72 | self.file = open(file, mode) 73 | 74 | def write(self, message, is_terminal=1, is_file=1 ): 75 | if '\r' in message: is_file=0 76 | 77 | if is_terminal == 1: 78 | self.terminal.write(message) 79 | self.terminal.flush() 80 | #time.sleep(1) 81 | 82 | if is_file == 1: 83 | self.file.write(message) 84 | self.file.flush() 85 | 86 | def flush(self): 87 | # this flush method is needed for python 3 compatibility. 88 | # this handles the flush command by doing nothing. 89 | # you might want to specify some extra behavior here. 90 | pass 91 | 92 | # io ------------------------------------ 93 | def write_list_to_file(list_file, strings): 94 | with open(list_file, 'w') as f: 95 | for s in strings: 96 | f.write('%s\n'%str(s)) 97 | pass 98 | 99 | 100 | def read_list_from_file(list_file, comment='#'): 101 | with open(list_file) as f: 102 | lines = f.readlines() 103 | 104 | strings=[] 105 | for line in lines: 106 | if comment is not None: 107 | s = line.split(comment, 1)[0].strip() 108 | else: 109 | s = line.strip() 110 | 111 | if s != '': 112 | strings.append(s) 113 | 114 | 115 | return strings 116 | 117 | 118 | 119 | def read_pickle_from_file(pickle_file): 120 | with open(pickle_file,'rb') as f: 121 | x = pickle.load(f) 122 | return x 123 | 124 | def write_pickle_to_file(pickle_file, x): 125 | with open(pickle_file, 'wb') as f: 126 | pickle.dump(x, f, pickle.HIGHEST_PROTOCOL) 127 | 128 | 129 | 130 | # backup ------------------------------------ 131 | 132 | #https://stackoverflow.com/questions/1855095/how-to-create-a-zip-archive-of-a-directory 133 | def backup_project_as_zip(project_dir, zip_file): 134 | assert(os.path.isdir(project_dir)) 135 | assert(os.path.isdir(os.path.dirname(zip_file))) 136 | shutil.make_archive(zip_file.replace('.zip',''), 'zip', project_dir) 137 | pass 138 | 139 | 140 | # etc ------------------------------------ 141 | def time_to_str(t, mode='min'): 142 | if mode=='min': 143 | t = int(t)/60 144 | hr = t//60 145 | min = t%60 146 | return '%2d hr %02d min'%(hr,min) 147 | 148 | elif mode=='sec': 149 | t = int(t) 150 | min = t//60 151 | sec = t%60 152 | return '%2d min %02d sec'%(min,sec) 153 | 154 | 155 | else: 156 | raise NotImplementedError 157 | 158 | 159 | def np_float32_to_uint8(x, scale=255): 160 | return (x*scale).astype(np.uint8) 161 | 162 | def np_uint8_to_float32(x, scale=255): 163 | return (x/scale).astype(np.float32) 164 | 165 | 166 | def int_tuple(x): 167 | return tuple( [int(round(xx)) for xx in x] ) 168 | 169 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/lib/utility/file.py: -------------------------------------------------------------------------------- 1 | from mpnn_model.lib.include import * 2 | import builtins 3 | import re 4 | 5 | class Struct(object): 6 | def __init__(self, is_copy=False, **kwargs): 7 | self.add(is_copy, **kwargs) 8 | 9 | def add(self, is_copy=False, **kwargs): 10 | #self.__dict__.update(kwargs) 11 | 12 | if is_copy == False: 13 | for key, value in kwargs.items(): 14 | setattr(self, key, value) 15 | else: 16 | for key, value in kwargs.items(): 17 | try: 18 | setattr(self, key, copy.deepcopy(value)) 19 | #setattr(self, key, value.copy()) 20 | except Exception: 21 | setattr(self, key, value) 22 | 23 | def __str__(self): 24 | return str(self.__dict__.keys()) 25 | 26 | 27 | 28 | # log ------------------------------------ 29 | def remove_comments(lines, token='#'): 30 | """ Generator. Strips comments and whitespace from input lines. 31 | """ 32 | 33 | l = [] 34 | for line in lines: 35 | s = line.split(token, 1)[0].strip() 36 | if s != '': 37 | l.append(s) 38 | return l 39 | 40 | 41 | def open(file, mode=None, encoding=None): 42 | if mode == None: mode = 'r' 43 | 44 | if '/' in file: 45 | if 'w' or 'a' in mode: 46 | dir = os.path.dirname(file) 47 | if not os.path.isdir(dir): os.makedirs(dir) 48 | 49 | f = builtins.open(file, mode=mode, encoding=encoding) 50 | return f 51 | 52 | 53 | def remove(file): 54 | if os.path.exists(file): os.remove(file) 55 | 56 | 57 | def empty(dir): 58 | if os.path.isdir(dir): 59 | shutil.rmtree(dir, ignore_errors=True) 60 | else: 61 | os.makedirs(dir) 62 | 63 | 64 | # http://stackoverflow.com/questions/34950201/pycharm-print-end-r-statement-not-working 65 | class Logger(object): 66 | def __init__(self): 67 | self.terminal = sys.stdout #stdout 68 | self.file = None 69 | 70 | def open(self, file, mode=None): 71 | if mode is None: mode ='w' 72 | self.file = open(file, mode) 73 | 74 | def write(self, message, is_terminal=1, is_file=1 ): 75 | if '\r' in message: is_file=0 76 | 77 | if is_terminal == 1: 78 | self.terminal.write(message) 79 | self.terminal.flush() 80 | #time.sleep(1) 81 | 82 | if is_file == 1: 83 | self.file.write(message) 84 | self.file.flush() 85 | 86 | def flush(self): 87 | # this flush method is needed for python 3 compatibility. 88 | # this handles the flush command by doing nothing. 89 | # you might want to specify some extra behavior here. 90 | pass 91 | 92 | # io ------------------------------------ 93 | def write_list_to_file(list_file, strings): 94 | with open(list_file, 'w') as f: 95 | for s in strings: 96 | f.write('%s\n'%str(s)) 97 | pass 98 | 99 | 100 | def read_list_from_file(list_file, comment='#'): 101 | with open(list_file) as f: 102 | lines = f.readlines() 103 | 104 | strings=[] 105 | for line in lines: 106 | if comment is not None: 107 | s = line.split(comment, 1)[0].strip() 108 | else: 109 | s = line.strip() 110 | 111 | if s != '': 112 | strings.append(s) 113 | 114 | 115 | return strings 116 | 117 | 118 | 119 | def read_pickle_from_file(pickle_file): 120 | with open(pickle_file,'rb') as f: 121 | x = pickle.load(f) 122 | return x 123 | 124 | def write_pickle_to_file(pickle_file, x): 125 | with open(pickle_file, 'wb') as f: 126 | pickle.dump(x, f, pickle.HIGHEST_PROTOCOL) 127 | 128 | 129 | 130 | # backup ------------------------------------ 131 | 132 | #https://stackoverflow.com/questions/1855095/how-to-create-a-zip-archive-of-a-directory 133 | def backup_project_as_zip(project_dir, zip_file): 134 | assert(os.path.isdir(project_dir)) 135 | assert(os.path.isdir(os.path.dirname(zip_file))) 136 | shutil.make_archive(zip_file.replace('.zip',''), 'zip', project_dir) 137 | pass 138 | 139 | 140 | # etc ------------------------------------ 141 | def time_to_str(t, mode='min'): 142 | if mode=='min': 143 | t = int(t)/60 144 | hr = t//60 145 | min = t%60 146 | return '%2d hr %02d min'%(hr,min) 147 | 148 | elif mode=='sec': 149 | t = int(t) 150 | min = t//60 151 | sec = t%60 152 | return '%2d min %02d sec'%(min,sec) 153 | 154 | 155 | else: 156 | raise NotImplementedError 157 | 158 | 159 | def np_float32_to_uint8(x, scale=255): 160 | return (x*scale).astype(np.uint8) 161 | 162 | def np_uint8_to_float32(x, scale=255): 163 | return (x/scale).astype(np.float32) 164 | 165 | 166 | def int_tuple(x): 167 | return tuple( [int(round(xx)) for xx in x] ) 168 | 169 | -------------------------------------------------------------------------------- /champs-scalar-coupling/build_data/lib/utility/draw.py: -------------------------------------------------------------------------------- 1 | import os 2 | #qt bug ??? 3 | os.environ['QT_XKB_CONFIG_ROOT']='/usr/share/X11/xkb/' 4 | 5 | from lib.include import * 6 | import matplotlib.cm 7 | 8 | 9 | # draw ----------------------------------- 10 | def image_show(name, image, resize=1): 11 | H,W = image.shape[0:2] 12 | cv2.namedWindow(name, cv2.WINDOW_GUI_NORMAL) #WINDOW_NORMAL 13 | #cv2.namedWindow(name, cv2.WINDOW_GUI_EXPANDED) #WINDOW_GUI_EXPANDED 14 | cv2.imshow(name, image.astype(np.uint8)) 15 | cv2.resizeWindow(name, round(resize*W), round(resize*H)) 16 | 17 | 18 | def image_show_norm(name, image, max=None, min=None, resize=1): 19 | if max is None: max=image.max() 20 | if min is None: min=image.min() 21 | 22 | H,W = image.shape[0:2] 23 | cv2.namedWindow(name, cv2.WINDOW_GUI_NORMAL) #WINDOW_NORMAL 24 | cv2.imshow(name, ((image-min)/(max-min)*255).astype(np.uint8)) 25 | cv2.resizeWindow(name, round(resize*W), round(resize*H)) 26 | 27 | def draw_shadow_text(img, text, pt, fontScale, color, thickness, color1=None, thickness1=None): 28 | if color1 is None: color1=(0,0,0) 29 | if thickness1 is None: thickness1 = thickness+2 30 | 31 | font = cv2.FONT_HERSHEY_SIMPLEX 32 | cv2.putText(img, text, pt, font, fontScale, color1, thickness1, cv2.LINE_AA) 33 | cv2.putText(img, text, pt, font, fontScale, color, thickness, cv2.LINE_AA) 34 | 35 | def to_color_image(image, max=None): 36 | if max is None: max=image.max() 37 | 38 | image = (image/max*255).astype(np.uint8) 39 | image = cv2.cvtColor(image,cv2.COLOR_GRAY2BGR) 40 | return image 41 | 42 | 43 | 44 | ##http://stackoverflow.com/questions/26690932/opencv-rectangle-with-dotted-or-dashed-lines 45 | def draw_dotted_line(image, pt1, pt2, color, thickness=1, gap=20): 46 | 47 | dist =((pt1[0]-pt2[0])**2+(pt1[1]-pt2[1])**2)**.5 48 | pts= [] 49 | for i in np.arange(0,dist,gap): 50 | r=i/dist 51 | x=int((pt1[0]*(1-r)+pt2[0]*r)+.5) 52 | y=int((pt1[1]*(1-r)+pt2[1]*r)+.5) 53 | p = (x,y) 54 | pts.append(p) 55 | 56 | if gap==1: 57 | for p in pts: 58 | cv2.circle(image,p,thickness,color,-1,cv2.LINE_AA) 59 | else: 60 | def pairwise(iterable): 61 | "s -> (s0, s1), (s2, s3), (s4, s5), ..." 62 | a = iter(iterable) 63 | return zip(a, a) 64 | 65 | for p, q in pairwise(pts): 66 | cv2.line(image,p, q, color,thickness,cv2.LINE_AA) 67 | 68 | 69 | def draw_dotted_poly(image, pts, color, thickness=1, gap=20): 70 | s=pts[0] 71 | e=pts[0] 72 | pts.append(pts.pop(0)) 73 | for p in pts: 74 | s=e 75 | e=p 76 | draw_dotted_line(image,s,e,color,thickness,gap) 77 | 78 | 79 | def draw_dotted_rect(image, pt1, pt2, color, thickness=1, gap=3): 80 | pts = [pt1,(pt2[0],pt1[1]),pt2,(pt1[0],pt2[1])] 81 | draw_dotted_poly(image, pts, color, thickness, gap) 82 | 83 | def draw_screen_rect(image, pt1, pt2, color, alpha=0.5): 84 | x1, y1 = pt1 85 | x2, y2 = pt2 86 | image[y1:y2,x1:x2,:] = (1-alpha)*image[y1:y2,x1:x2,:] + (alpha)*np.array(color, np.uint8) 87 | 88 | 89 | 90 | # def draw_mask(image, mask, color=(255,255,255), α=1, β=0.25, λ=0., threshold=32 ): 91 | # # image * α + mask * β + λ 92 | # 93 | # if threshold is None: 94 | # mask = mask/255 95 | # else: 96 | # mask = clean_mask(mask,threshold,1) 97 | # 98 | # mask = np.dstack((color[0]*mask,color[1]*mask,color[2]*mask)).astype(np.uint8) 99 | # image[...] = cv2.addWeighted(image, α, mask, β, λ) 100 | # 101 | 102 | 103 | # def draw_contour(image, mask, color=(0,255,0), thickness=1, threshold=127): 104 | # ret, thresh = cv2.threshold(mask,threshold,255,0) 105 | # ret = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) 106 | # hierarchy = ret[0] 107 | # contours = ret[1] 108 | # #image[...]=image 109 | # cv2.drawContours(image, contours, -1, color, thickness, cv2.LINE_AA) 110 | # ## drawContours(image, contours, contourIdx, color, thickness=None, lineType=None, hierarchy=None, maxLevel=None, offset=None): # real signature unknown; restored from __doc__ 111 | # 112 | # 113 | 114 | 115 | def to_color(s, color=None): 116 | 117 | if type(color) in [str] or color is None: 118 | #https://matplotlib.org/xkcd/examples/color/colormaps_reference.html 119 | 120 | if color is None: color='cool' 121 | color = matplotlib.get_cmap(color)(s) 122 | b = int(255*color[2]) 123 | g = int(255*color[1]) 124 | r = int(255*color[0]) 125 | 126 | elif type(color) in [list,tuple]: 127 | b = int(s*color[0]) 128 | g = int(s*color[1]) 129 | r = int(s*color[2]) 130 | 131 | return b,g,r 132 | 133 | 134 | 135 | # main ################################################################# 136 | if __name__ == '__main__': 137 | print( '%s: calling main function ... ' % os.path.basename(__file__)) 138 | 139 | 140 | image = np.zeros((50,50,3), np.uint8) 141 | cv2.rectangle(image, (0,0),(49,49), (0,0,255),1) #inclusive 142 | 143 | image[8,8]=[255,255,255] 144 | 145 | image_show('image',image,10) 146 | cv2.waitKey(0) 147 | 148 | 149 | print('\nsucess!') -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/lib/utility/draw.py: -------------------------------------------------------------------------------- 1 | import os 2 | #qt bug ??? 3 | os.environ['QT_XKB_CONFIG_ROOT']='/usr/share/X11/xkb/' 4 | 5 | from mpnn_model.lib.include import * 6 | import matplotlib.cm 7 | 8 | 9 | # draw ----------------------------------- 10 | def image_show(name, image, resize=1): 11 | H,W = image.shape[0:2] 12 | cv2.namedWindow(name, cv2.WINDOW_GUI_NORMAL) #WINDOW_NORMAL 13 | #cv2.namedWindow(name, cv2.WINDOW_GUI_EXPANDED) #WINDOW_GUI_EXPANDED 14 | cv2.imshow(name, image.astype(np.uint8)) 15 | cv2.resizeWindow(name, round(resize*W), round(resize*H)) 16 | 17 | 18 | def image_show_norm(name, image, max=None, min=None, resize=1): 19 | if max is None: max=image.max() 20 | if min is None: min=image.min() 21 | 22 | H,W = image.shape[0:2] 23 | cv2.namedWindow(name, cv2.WINDOW_GUI_NORMAL) #WINDOW_NORMAL 24 | cv2.imshow(name, ((image-min)/(max-min)*255).astype(np.uint8)) 25 | cv2.resizeWindow(name, round(resize*W), round(resize*H)) 26 | 27 | def draw_shadow_text(img, text, pt, fontScale, color, thickness, color1=None, thickness1=None): 28 | if color1 is None: color1=(0,0,0) 29 | if thickness1 is None: thickness1 = thickness+2 30 | 31 | font = cv2.FONT_HERSHEY_SIMPLEX 32 | cv2.putText(img, text, pt, font, fontScale, color1, thickness1, cv2.LINE_AA) 33 | cv2.putText(img, text, pt, font, fontScale, color, thickness, cv2.LINE_AA) 34 | 35 | def to_color_image(image, max=None): 36 | if max is None: max=image.max() 37 | 38 | image = (image/max*255).astype(np.uint8) 39 | image = cv2.cvtColor(image,cv2.COLOR_GRAY2BGR) 40 | return image 41 | 42 | 43 | 44 | ##http://stackoverflow.com/questions/26690932/opencv-rectangle-with-dotted-or-dashed-lines 45 | def draw_dotted_line(image, pt1, pt2, color, thickness=1, gap=20): 46 | 47 | dist =((pt1[0]-pt2[0])**2+(pt1[1]-pt2[1])**2)**.5 48 | pts= [] 49 | for i in np.arange(0,dist,gap): 50 | r=i/dist 51 | x=int((pt1[0]*(1-r)+pt2[0]*r)+.5) 52 | y=int((pt1[1]*(1-r)+pt2[1]*r)+.5) 53 | p = (x,y) 54 | pts.append(p) 55 | 56 | if gap==1: 57 | for p in pts: 58 | cv2.circle(image,p,thickness,color,-1,cv2.LINE_AA) 59 | else: 60 | def pairwise(iterable): 61 | "s -> (s0, s1), (s2, s3), (s4, s5), ..." 62 | a = iter(iterable) 63 | return zip(a, a) 64 | 65 | for p, q in pairwise(pts): 66 | cv2.line(image,p, q, color,thickness,cv2.LINE_AA) 67 | 68 | 69 | def draw_dotted_poly(image, pts, color, thickness=1, gap=20): 70 | s=pts[0] 71 | e=pts[0] 72 | pts.append(pts.pop(0)) 73 | for p in pts: 74 | s=e 75 | e=p 76 | draw_dotted_line(image,s,e,color,thickness,gap) 77 | 78 | 79 | def draw_dotted_rect(image, pt1, pt2, color, thickness=1, gap=3): 80 | pts = [pt1,(pt2[0],pt1[1]),pt2,(pt1[0],pt2[1])] 81 | draw_dotted_poly(image, pts, color, thickness, gap) 82 | 83 | def draw_screen_rect(image, pt1, pt2, color, alpha=0.5): 84 | x1, y1 = pt1 85 | x2, y2 = pt2 86 | image[y1:y2,x1:x2,:] = (1-alpha)*image[y1:y2,x1:x2,:] + (alpha)*np.array(color, np.uint8) 87 | 88 | 89 | 90 | # def draw_mask(image, mask, color=(255,255,255), α=1, β=0.25, λ=0., threshold=32 ): 91 | # # image * α + mask * β + λ 92 | # 93 | # if threshold is None: 94 | # mask = mask/255 95 | # else: 96 | # mask = clean_mask(mask,threshold,1) 97 | # 98 | # mask = np.dstack((color[0]*mask,color[1]*mask,color[2]*mask)).astype(np.uint8) 99 | # image[...] = cv2.addWeighted(image, α, mask, β, λ) 100 | # 101 | 102 | 103 | # def draw_contour(image, mask, color=(0,255,0), thickness=1, threshold=127): 104 | # ret, thresh = cv2.threshold(mask,threshold,255,0) 105 | # ret = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) 106 | # hierarchy = ret[0] 107 | # contours = ret[1] 108 | # #image[...]=image 109 | # cv2.drawContours(image, contours, -1, color, thickness, cv2.LINE_AA) 110 | # ## drawContours(image, contours, contourIdx, color, thickness=None, lineType=None, hierarchy=None, maxLevel=None, offset=None): # real signature unknown; restored from __doc__ 111 | # 112 | # 113 | 114 | 115 | def to_color(s, color=None): 116 | 117 | if type(color) in [str] or color is None: 118 | #https://matplotlib.org/xkcd/examples/color/colormaps_reference.html 119 | 120 | if color is None: color='cool' 121 | color = matplotlib.get_cmap(color)(s) 122 | b = int(255*color[2]) 123 | g = int(255*color[1]) 124 | r = int(255*color[0]) 125 | 126 | elif type(color) in [list,tuple]: 127 | b = int(s*color[0]) 128 | g = int(s*color[1]) 129 | r = int(s*color[2]) 130 | 131 | return b,g,r 132 | 133 | 134 | 135 | # main ################################################################# 136 | if __name__ == '__main__': 137 | print( '%s: calling main function ... ' % os.path.basename(__file__)) 138 | 139 | 140 | image = np.zeros((50,50,3), np.uint8) 141 | cv2.rectangle(image, (0,0),(49,49), (0,0,255),1) #inclusive 142 | 143 | image[8,8]=[255,255,255] 144 | 145 | image_show('image',image,10) 146 | cv2.waitKey(0) 147 | 148 | 149 | print('\nsucess!') -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/build_predictions.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | from functools import partial 4 | from timeit import default_timer as timer 5 | from time import time 6 | import warnings 7 | warnings.filterwarnings("ignore") 8 | 9 | from mpnn_model.common import * 10 | from mpnn_model.common_constants import * 11 | from mpnn_model.train_loss import lmae_criterion 12 | from mpnn_model.callback import * 13 | 14 | from time import time 15 | ############################################################################################################# 16 | # # 17 | # Get prediction function # 18 | # # 19 | ############################################################################################################# 20 | def do_test(net, test_loader, test_len, num_output, predict_type, grm, normalize=False, gaussrank=False): 21 | """ 22 | do_test -> return list of (indices, predictions) 23 | 24 | Input arguments: 25 | net (nn.module) : the graph neural network model 26 | test_loader (Dataloader): Test data loader 27 | test_len (int): length of test dataset 28 | """ 29 | 30 | test_num = 0 31 | test_predict = [] 32 | test_coupling_type = [] 33 | test_coupling_value = [] 34 | test_id = [] 35 | test_contributions = [] 36 | molecule_representation = [] 37 | num_batches = 0 38 | test_loss = 0 39 | start = timer() 40 | for b, (((node, edge, edge_index, node_index, coupling_index, type_, atomic), targets), infor) in enumerate(test_loader): 41 | net.eval() 42 | with torch.no_grad(): 43 | 44 | coupling_value = targets[0] 45 | predict = net(node, edge, edge_index, node_index, coupling_index, type_, atomic) 46 | 47 | if predict_type: 48 | predict = torch.gather(predict[0], 1, targets[3].unsqueeze(1)).view(-1) 49 | predict = [predict, [], []] 50 | 51 | if normalize: 52 | coupling_mean = torch.gather(COUPLING_TYPE_MEAN, 0, targets[3]) 53 | coupling_std = torch.gather(COUPLING_TYPE_STD, 0, targets[3]) 54 | predict = (predict * coupling_std) + coupling_mean 55 | coupling_value = (coupling_value * coupling_std) + coupling_mean 56 | predict = [predict, [], []] 57 | 58 | loss = lmae_criterion(predict, coupling_value, coupling_value, [], []) 59 | 60 | 61 | batch_size = test_loader.batch_size 62 | test_id.extend(list(infor.data.cpu().numpy())) 63 | 64 | test_predict.append(predict[0].data.cpu().numpy()) 65 | molecule_representation.append(net.pool.data.cpu().numpy()) 66 | 67 | test_coupling_type.append(coupling_index[:,-2].data.cpu().numpy()) 68 | test_coupling_value.append(coupling_value.data.cpu().numpy()) 69 | 70 | test_loss += loss.item()*batch_size 71 | test_num = len(test_id) 72 | num_batches += batch_size 73 | 74 | print('\r %8d/%8d %0.2f %s'%( test_num, test_len, test_num/test_len, 75 | time_to_str(timer()-start,'min')),end='',flush=True) 76 | 77 | pass 78 | 79 | test_loss = test_loss/num_batches 80 | print('\n') 81 | 82 | print('predict') 83 | predict = np.concatenate(test_predict) 84 | 85 | if num_output==5: 86 | contributions = np.concatenate(test_contributions) 87 | else: 88 | contributions = [] 89 | 90 | test_coupling_value = np.concatenate(test_coupling_value) 91 | test_coupling_type = np.concatenate(test_coupling_type).astype(np.int32) 92 | molecule_representation = np.concatenate(molecule_representation) 93 | 94 | # convert gaussrank test predictions to their actual values 95 | if gaussrank: 96 | print('compute the reverse frame') 97 | reverse_frame = get_reverse_frame(test_id, predict, test_coupling_type, test_coupling_value, grm) 98 | predict = reverse_frame['scalar_coupling_constant'].values 99 | 100 | else: 101 | print('build preds frame') 102 | reverse_frame = pd.DataFrame(predict) 103 | reverse_frame['type'] = test_coupling_type 104 | reverse_frame.columns = ['scalar_coupling_constant', 'type_ind'] 105 | reverse_frame['id'] = test_id 106 | reverse_frame['true_scalar_coupling_constant'] = test_coupling_value 107 | 108 | 109 | mae, log_mae = compute_kaggle_metric(reverse_frame.scalar_coupling_constant, reverse_frame.true_scalar_coupling_constant, reverse_frame.type_ind) 110 | 111 | print('Compute lmae per type') 112 | num_target = NUM_COUPLING_TYPE 113 | for t in range(NUM_COUPLING_TYPE): 114 | if mae[t] is None: 115 | mae[t] = 0 116 | log_mae[t] = 0 117 | num_target -= 1 118 | 119 | mae_mean, log_mae_mean = sum(mae)/NUM_COUPLING_TYPE, sum(log_mae)/NUM_COUPLING_TYPE 120 | 121 | loss_ = log_mae + [ test_loss, mae_mean, log_mae_mean ] 122 | 123 | return loss_, reverse_frame, contributions, molecule_representation -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/callback.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | from fastai.callbacks import SaveModelCallback 5 | from fastai.callbacks import Callback 6 | from fastai.torch_core import add_metrics 7 | 8 | import torch 9 | from torch import nn 10 | import torch.nn.functional as F 11 | 12 | import pdb 13 | 14 | from mpnn_model.common_constants import NUM_COUPLING_TYPE, COUPLING_TYPE_MEAN, COUPLING_TYPE_STD, REVERSE_COUPLING_TYPE 15 | 16 | 17 | __all__ = ['get_reverse_frame', 'lmae', 'compute_kaggle_metric', 'LMAE'] 18 | 19 | # reverse the gaussrank predictions to the actual distribution 20 | def get_reverse_frame(test_id, predictions, coupling_type, target, grm,): 21 | preds = pd.DataFrame(predictions) 22 | preds['type_ind'] = coupling_type 23 | preds.columns = ['prediction', 'type_ind'] 24 | preds['type'] = preds['type_ind'].map(REVERSE_COUPLING_TYPE) 25 | preds['id'] = test_id 26 | preds['true_scalar_coupling_constant'] = target 27 | preds['scalar_coupling_constant'] = grm.convert_df(preds, from_coupling=False) 28 | return preds 29 | 30 | 31 | # Compute lmae of scalar coupling with respect to the type ` 32 | def lmae(truth,pred,types): 33 | # Log of the Mean Absolute Error 34 | # will make it per type later 35 | df = pd.DataFrame({'truth':truth,'pred':pred,'types':types}) 36 | df['err'] = np.abs(df['truth']-df['pred']) 37 | x = df.groupby('types')['err'].mean().values 38 | x = np.log(1e-8+x) 39 | return np.mean(x) 40 | 41 | 42 | # lmae w.r.t 8 coupling types : kaggle metric 43 | def compute_kaggle_metric(predict, coupling_value, coupling_type): 44 | """ 45 | predict lmae loss w.r.t the coupling type 46 | 47 | Arguments: 48 | - predict: type(Array) array of scalar coupling predictions returned by the model 49 | - coupling_value: type(Array ) True coupling values 50 | - coupling_type: type(Array) True coupling type 51 | Returns: 52 | - mae, log_mae : the mean and log mean absolute error between predictions and true labels 53 | """ 54 | 55 | mae = [None]*NUM_COUPLING_TYPE 56 | log_mae = [None]*NUM_COUPLING_TYPE 57 | diff = np.fabs(predict-coupling_value) 58 | for t in range(NUM_COUPLING_TYPE): 59 | index = np.where(coupling_type==t)[0] 60 | if len(index)>0: 61 | m = diff[index].mean() 62 | log_m = np.log(m+1e-8) 63 | 64 | mae[t] = m 65 | log_mae[t] = log_m 66 | else: 67 | pass 68 | return mae, log_mae 69 | 70 | 71 | # Callback to calculate LMAE at the end of each epoch 72 | class LMAE(Callback): 73 | ''' 74 | Comput LMAE for the prediction of the coupling value 75 | ''' 76 | _order = -20 #Needs to run before the recorder 77 | 78 | def __init__(self, learn,grm, predict_type=False, normalize_coupling=False, coupling_rank=True, **kwargs): 79 | self.learn = learn 80 | self.predict_type = predict_type 81 | self.normalize_coupling = normalize_coupling 82 | self.grm = grm 83 | self.coupling_rank = coupling_rank 84 | def on_train_begin(self, **kwargs): self.learn.recorder.add_metric_names(['LMAE']) 85 | def on_epoch_begin(self, **kwargs): self.output, self.target, self.types = [], [], [] 86 | 87 | def on_batch_end(self, last_target, last_output, train, **kwargs): 88 | if not train: 89 | self.target.append(last_target[0]) 90 | self.types.append(last_target[3]) 91 | if self.predict_type: 92 | coupling = torch.gather(last_output[0], 1, last_target[3].unsqueeze(1)).view(-1) 93 | self.output.append(coupling) 94 | else: 95 | self.output.append(last_output[0]) 96 | 97 | def on_epoch_end(self, last_metrics, **kwargs): 98 | if len(self.output) > 0: 99 | output = torch.cat(self.output) 100 | target = torch.cat(self.target) 101 | types = torch.cat(self.types) 102 | 103 | if self.normalize_coupling : 104 | # Denormalize w.r.t to type 105 | means = torch.gather(COUPLING_TYPE_MEAN, 0, types) 106 | stds = torch.gather(COUPLING_TYPE_STD, 0, types) 107 | output = (output * stds) + means 108 | target = (target * stds) + means 109 | metric = lmae(output.data.cpu().numpy(), target.data.cpu().numpy(), types.data.cpu().numpy()) 110 | 111 | elif self.coupling_rank: 112 | # Reverse using grm mapping frames 113 | preds = pd.DataFrame(output.data.cpu().numpy()) 114 | preds['type'] = types.data.cpu().numpy() 115 | preds.columns = ['prediction', 'type'] 116 | preds['type'] = preds['type'].map(REVERSE_COUPLING_TYPE) 117 | preds['true_scalar_coupling_constant'] = target.data.cpu().numpy() 118 | preds['scalar_coupling_constant'] = self.grm.convert_df(preds, from_coupling=False) 119 | 120 | # compute metric for reversed scalar coupling predictions 121 | metric = lmae(preds['scalar_coupling_constant'], preds['true_scalar_coupling_constant'], preds['type']) 122 | 123 | else: 124 | preds = output.data.cpu().numpy().reshape(-1,) 125 | type_ = types.data.cpu().numpy() 126 | targets = target.data.cpu().numpy() 127 | metric = lmae(targets, preds, type_) 128 | 129 | return add_metrics(last_metrics, [metric]) 130 | -------------------------------------------------------------------------------- /champs-scalar-coupling/save_pretrained_single_models.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 5, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "GPU_id = 1\n", 11 | "os.environ['CUDA_VISIBLE_DEVICES'] = str(GPU_id)\n", 12 | "\n", 13 | "from torch_scatter import *\n", 14 | "from torch_geometric.utils import scatter_\n", 15 | "import torch\n", 16 | "import torch.nn as nn\n", 17 | "import numbers\n", 18 | "\n", 19 | "import torch\n", 20 | "from torch import _utils\n", 21 | "from fastai.torch_core import to_device\n", 22 | "import torch.nn.functional as F \n", 23 | "\n", 24 | "from fastai.basic_data import DataBunch\n", 25 | "from fastai.basic_data import *\n", 26 | "from fastai.tabular import *\n", 27 | "from fastai import *\n", 28 | "\n", 29 | "import copy\n", 30 | "import warnings\n", 31 | "warnings.filterwarnings(\"ignore\") \n", 32 | "\n", 33 | "from mpnn_model.common import *\n", 34 | "from mpnn_model.helpers import load_cfg\n", 35 | "from mpnn_model.model import *\n", 36 | "from mpnn_model.common_constants import * \n", 37 | "\n", 38 | "DATA_DIR = '/rapids/notebooks/srabhi/champs-2019/input'\n" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 6, 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "model_dict = { '1JHC': 'lmae', '2JHC': 'lmae', '3JHC': 'lmae', '3JHH': 'lmae',\n", 48 | " '1JHN': 'mlmae' , '2JHN':'mlmae' , '3JHN':'mlmae', '2JHH':'mlmae'}" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 7, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "# to modify with the resulting model of self_training \n", 58 | "best_all_type_model_lmae = './models/mpnn_gauss_rank_LMAE_2CE_RNN_NODE_TYPE_ATOMIC_SEQ_BOOTSTRAP_V2_'\n", 59 | "best_all_type_model_mlmae = './models/mpnn_gauss_rank_MLMAE_2CE_RNN_NODE_TYPE_ATOMIC_SEQ_BOOTSTRAP_V2_'" 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": {}, 65 | "source": [ 66 | "### Without GAUSSRANK : " 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 10, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "def save_model(fold, coupling_type, loss_criterion='lmae'): \n", 76 | " cfg = load_cfg('./experiments/MPNN_RNN_MAE_WO_GAUSSRANK_SINGLE_TYPE.yaml')\n", 77 | " if loss_criterion == 'lmae': \n", 78 | " pretrained_dict = torch.load(best_all_type_model_lmae+'_fold_%s_final_save.pth'%fold).state_dict()\n", 79 | " else:\n", 80 | " pretrained_dict = torch.load(best_all_type_model_lmae+'_fold_%s_final_save.pth'%fold).state_dict()\n", 81 | " ind = COUPLING_TYPE.index(coupling_type)\n", 82 | " net= Net(cfg, y_range=[COUPLING_MIN_[ind], COUPLING_MAX_[ind]])\n", 83 | " # 1. filter out unnecessary keys\n", 84 | " pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict and not k.startswith('predict')}\n", 85 | " # 2. overwrite entries in the existing state dict\n", 86 | " model_dict.update(pretrained_dict) \n", 87 | " # 3. load the new state dict\n", 88 | " net.load_state_dict(pretrained_dict, strict=False)\n", 89 | " # save model \n", 90 | " torch.save(net, 'pre_trained_models/' + 'coupling_%s_%s_fold_%s_wo_gaussrank.pth'%(coupling_type, loss_criterion, fold))" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 11, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "for fold in range(4):\n", 100 | " for type_, criterion in model_dict.items():\n", 101 | " save_model(fold, type_, criterion)" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "metadata": {}, 107 | "source": [ 108 | "### With Gaussrank " 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": 12, 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "def save_model(fold, coupling_type, loss_criterion='lmae'): \n", 118 | " cfg = load_cfg('/rapids/notebooks/srabhi/champs-2019/CherKeng_solution/fastai_code/experiments/MPNN_RNN_MAE_GAUSSRANK_SINGLE_TYPE.yaml')\n", 119 | " if loss_criterion == 'lmae': \n", 120 | " pretrained_dict = torch.load('models/mpnn_gauss_rank_LMAE_2CE_RNN_NODE_TYPE_ATOMIC_SEQ_BOOTSTRAP_V2__fold_%s_final_save.pth'%fold).state_dict()\n", 121 | " else:\n", 122 | " pretrained_dict = torch.load('models/mpnn_gauss_rank_MLMAE_2CE_RNN_NODE_TYPE_ATOMIC_SEQ_BOOTSTRAP_V2__fold_%s_final_save.pth'%fold).state_dict()\n", 123 | " ind = COUPLING_TYPE.index(coupling_type)\n", 124 | " net= Net(cfg, y_range=[-2.326753765513524, 2.3267537655135464])\n", 125 | " # 1. filter out unnecessary keys\n", 126 | " pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict and not k.startswith('predict')}\n", 127 | " # 2. overwrite entries in the existing state dict\n", 128 | " model_dict.update(pretrained_dict) \n", 129 | " # 3. load the new state dict\n", 130 | " net.load_state_dict(pretrained_dict, strict=False)\n", 131 | " # save model \n", 132 | " torch.save(net, 'pre_trained_models/' + 'coupling_%s_%s_fold_%s_gaussrank.pth'%(coupling_type, loss_criterion, fold))" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 13, 138 | "metadata": {}, 139 | "outputs": [], 140 | "source": [ 141 | "for fold in range(4):\n", 142 | " for type_, criterion in model_dict.items():\n", 143 | " save_model(fold, type_, criterion)" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": null, 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [] 152 | } 153 | ], 154 | "metadata": { 155 | "kernelspec": { 156 | "display_name": "Python 3", 157 | "language": "python", 158 | "name": "python3" 159 | }, 160 | "language_info": { 161 | "codemirror_mode": { 162 | "name": "ipython", 163 | "version": 3 164 | }, 165 | "file_extension": ".py", 166 | "mimetype": "text/x-python", 167 | "name": "python", 168 | "nbconvert_exporter": "python", 169 | "pygments_lexer": "ipython3", 170 | "version": "3.7.3" 171 | } 172 | }, 173 | "nbformat": 4, 174 | "nbformat_minor": 4 175 | } 176 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/common_model.py: -------------------------------------------------------------------------------- 1 | from mpnn_model.common import * 2 | 3 | from torch_scatter import * 4 | from torch_geometric.utils import scatter_ 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | import numbers 9 | 10 | # Fast ai 11 | from fastai.tabular import * 12 | from fastai.callbacks import SaveModelCallback 13 | 14 | 15 | __all__ = ['LinearBn', 'MlpBn', 'CustomTabularModel', 'get_node_encoder', 'get_edge_encoder',] 16 | 17 | 18 | ############################################################################################################# 19 | # # 20 | # Linear batch-norm layers # 21 | # # 22 | ############################################################################################################# 23 | 24 | class LinearBn(nn.Module): 25 | ''' 26 | Batch norm dense layer 27 | Arguments: 28 | - in_channel: int, Input dimension 29 | - out_channel: int, Output dimension 30 | - act: str, Activation function to apply to the output of batch normalizaiton. 31 | ''' 32 | def __init__(self, in_channel, out_channel, act=None): 33 | super(LinearBn, self).__init__() 34 | self.linear = nn.Linear(in_channel, out_channel, bias=False) 35 | self.bn = nn.BatchNorm1d(out_channel, eps=1e-05, momentum=0.1) 36 | if act is not None : 37 | self.act = F.__dict__[act] 38 | else: 39 | self.act = act 40 | 41 | def forward(self, x): 42 | x = self.linear(x) 43 | if self.bn is not None: 44 | x = self.bn(x) 45 | if self.act is not None: 46 | x = self.act(x) 47 | return x 48 | 49 | 50 | class MlpBn(nn.Module): 51 | ''' Fully connected feed forward neural network: stacked batch norm layers with dropout 52 | Args: 53 | input_dim (int32): the dimension of input 54 | dimensions (int32): the dimension of hiddenlayers. 55 | act (string): Activation function to apply to the output of each layer. 56 | dropout (float): the dropout probabily to apply to each layer. 57 | ''' 58 | def __init__(self, 59 | input_dim, 60 | dimensions, 61 | activation='Relu', 62 | dropout=0.): 63 | 64 | super(MlpBn, self).__init__() 65 | self.input_dim = input_dim 66 | self.dimensions = dimensions 67 | self.activation = activation 68 | self.dropout = dropout 69 | 70 | # Modules 71 | self.linears = nn.ModuleList([LinearBn(input_dim, dimensions[0], act=activation)]) 72 | for din, dout in zip(dimensions[:-1], dimensions[1:]): 73 | self.linears.append(LinearBn(din, dout, act=self.activation)) 74 | 75 | def forward(self, x): 76 | for i,lin in enumerate(self.linears): 77 | x = lin(x) 78 | if self.dropout > 0: 79 | x = F.dropout(x, self.dropout, training=self.training) 80 | return x 81 | 82 | ############################################################################################################# 83 | # # 84 | # Tabular model # 85 | # # 86 | ############################################################################################################# 87 | 88 | class CustomTabularModel(nn.Module): 89 | "Basic model for tabular data." 90 | def __init__(self, emb_szs:ListSizes, n_cont:int, out_sz:int, layers:Collection[int], ps:Collection[float]=None, 91 | emb_drop:float=0., y_range:OptRange=None, use_bn:bool=True, bn_final:bool=False): 92 | super().__init__() 93 | ps = ifnone(ps, [0]*len(layers)) 94 | ps = listify(ps, layers) 95 | #self.bsn = BatchSwapNoise(0.15) 96 | self.embeds = nn.ModuleList([embedding(ni, nf) for ni,nf in emb_szs]) 97 | self.emb_drop = nn.Dropout(emb_drop) 98 | self.bn_cont = nn.BatchNorm1d(n_cont) 99 | n_emb = sum(e.embedding_dim for e in self.embeds) 100 | self.n_emb,self.n_cont,self.y_range = n_emb,n_cont,y_range 101 | sizes = self.get_sizes(layers, out_sz) 102 | actns = [nn.ReLU(inplace=True) for _ in range(len(sizes)-2)] + [None] 103 | layers = [] 104 | for i,(n_in,n_out,dp,act) in enumerate(zip(sizes[:-1],sizes[1:],[0.]+ps,actns)): 105 | layers += bn_drop_lin(n_in, n_out, bn=use_bn and i!=0, p=dp, actn=act) 106 | if bn_final: layers.append(nn.BatchNorm1d(sizes[-1])) 107 | layers = layers[:-2] 108 | self.layers = nn.Sequential(*layers) 109 | 110 | def get_sizes(self, layers, out_sz): 111 | return [self.n_emb + self.n_cont] + layers + [out_sz] 112 | 113 | def forward(self, x_cat:Tensor, x_cont:Tensor) -> Tensor: 114 | #self.bsn(x_cat) 115 | 116 | if self.n_emb != 0: 117 | x = [e(x_cat[:,i]) for i,e in enumerate(self.embeds)] 118 | x = torch.cat(x, 1) 119 | x = self.emb_drop(x) 120 | if self.n_cont != 0: 121 | x_cont = self.bn_cont(x_cont) 122 | x = torch.cat([x, x_cont], 1) if self.n_emb != 0 else x_cont 123 | x = self.layers(x) 124 | return x 125 | 126 | 127 | ############################################################################################################# 128 | # # 129 | # Node and edge encoders # 130 | # # 131 | ############################################################################################################# 132 | 133 | def get_node_encoder(encoding, emb_sz, n_cont, node_dim, layers, activation, dropout=0.): 134 | ''' 135 | - Get the MLP network to process nodes features and build node representation 136 | ''' 137 | if encoding == 'one_hot': 138 | return MlpBn(node_dim, dimensions=layers, activation=activation, dropout=dropout) 139 | elif encoding== 'label': 140 | # embed symbol, acceptor, donor, aromatic, hybridization 141 | # emb_sz = [(6,4), (3,3), (3,3), (3,3), (5,4)] 142 | return CustomTabularModel(emb_szs = emb_sz, out_sz=2, n_cont=n_cont, layers=layers, ps=[dropout], emb_drop=0.) 143 | 144 | 145 | def get_edge_encoder(encoding, emb_sz, n_cont, node_dim, edge_dim, layers, activation, dropout=0.): 146 | ''' 147 | Get the MLP network to process edges features and build matrix representation 148 | Arguments: 149 | - encoding: str, the encoding of categorical variables : "label" vs "one_hot" 150 | - emb_sz: list of tuples, the embedding size of each categorical variable 151 | - n_cont: int, the number of continious variables 152 | - node_dim: int, the dimension of node's representation 153 | - edge_dim: int, the input dimension of edge's features 154 | - layers: list of int, the dimensions of hidden layers 155 | - activation: str, the activation to apply for layers. 156 | - dropout: [float], dropout of each hidden layer. 157 | ''' 158 | if encoding == 'one_hot': 159 | return MlpBn(edge_dim, dimensions=layers+[node_dim*node_dim], activation=activation, dropout=dropout) 160 | 161 | elif encoding== 'label': 162 | # emb_sz = [(5,8)] 163 | return CustomTabularModel(emb_szs = emb_sz, n_cont=n_cont , out_sz=2, layers=layers+[node_dim*node_dim], ps=[dropout], emb_drop=0.) 164 | -------------------------------------------------------------------------------- /pytorch/optimizers/adam.py: -------------------------------------------------------------------------------- 1 | import math 2 | import os 3 | from distutils.util import strtobool 4 | import torch 5 | from torch.optim.optimizer import Optimizer 6 | from torch.hub import _check_module_exists 7 | 8 | NUMBA_CUDA_EXIST = False 9 | NUMBA_CUDA_THREAD_PER_BLOCK = 512 10 | if not strtobool(os.environ.get('NO_NUMBA', 'n')) and _check_module_exists("numba.cuda"): 11 | import numba.cuda 12 | NUMBA_CUDA_EXIST = numba.cuda.is_available() 13 | 14 | @numba.cuda.jit() 15 | def numba_cuda_kernel(param, grad, exp_avg, exp_avg_sq, beta1, 16 | beta2, step_size, bias_correction2, eps, 17 | weight_decay): 18 | i = numba.cuda.grid(1) 19 | if i >= param.size: 20 | return 21 | 22 | if weight_decay != 0: 23 | grad[i] += weight_decay * param[i] 24 | 25 | exp_avg[i] = exp_avg[i] * beta1 + (1 - beta1) * grad[i] 26 | exp_avg_sq[i] = exp_avg_sq[i] * beta2 + (1 - beta2) * grad[i] * grad[i] 27 | 28 | denom = math.sqrt(exp_avg_sq[i]) / bias_correction2 + eps 29 | param[i] = param[i] + (-step_size) * (exp_avg[i] / denom) 30 | 31 | 32 | class Adam(Optimizer): 33 | r"""Implements Adam algorithm. 34 | 35 | It has been proposed in `Adam: A Method for Stochastic Optimization`_. 36 | 37 | Arguments: 38 | params (iterable): iterable of parameters to optimize or dicts defining 39 | parameter groups 40 | lr (float, optional): learning rate (default: 1e-3) 41 | betas (Tuple[float, float], optional): coefficients used for computing 42 | running averages of gradient and its square (default: (0.9, 0.999)) 43 | eps (float, optional): term added to the denominator to improve 44 | numerical stability (default: 1e-8) 45 | weight_decay (float, optional): weight decay (L2 penalty) (default: 0) 46 | amsgrad (boolean, optional): whether to use the AMSGrad variant of this 47 | algorithm from the paper `On the Convergence of Adam and Beyond`_ 48 | (default: False) 49 | 50 | .. _Adam\: A Method for Stochastic Optimization: 51 | https://arxiv.org/abs/1412.6980 52 | .. _On the Convergence of Adam and Beyond: 53 | https://openreview.net/forum?id=ryQu7f-RZ 54 | """ 55 | 56 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, 57 | weight_decay=0, amsgrad=False): 58 | if not 0.0 <= lr: 59 | raise ValueError("Invalid learning rate: {}".format(lr)) 60 | if not 0.0 <= eps: 61 | raise ValueError("Invalid epsilon value: {}".format(eps)) 62 | if not 0.0 <= betas[0] < 1.0: 63 | raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) 64 | if not 0.0 <= betas[1] < 1.0: 65 | raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) 66 | defaults = dict(lr=lr, betas=betas, eps=eps, 67 | weight_decay=weight_decay, amsgrad=amsgrad) 68 | super(Adam, self).__init__(params, defaults) 69 | 70 | def __setstate__(self, state): 71 | super(Adam, self).__setstate__(state) 72 | for group in self.param_groups: 73 | group.setdefault('amsgrad', False) 74 | 75 | def step(self, closure=None): 76 | """Performs a single optimization step. 77 | 78 | Arguments: 79 | closure (callable, optional): A closure that reevaluates the model 80 | and returns the loss. 81 | """ 82 | 83 | # In order to reduce Numba overhead, we save the device arrays 84 | # between calls to `step()` in `_nbstate`. 85 | self._nbstate = getattr(self, '_nbstate', {}) 86 | 87 | loss = None 88 | if closure is not None: 89 | loss = closure() 90 | 91 | for group in self.param_groups: 92 | for param in group['params']: 93 | if param.grad is None: 94 | continue 95 | 96 | # Perform optimization step 97 | grad = param.grad.data 98 | p = param.data 99 | if grad.is_sparse: 100 | raise RuntimeError('Adam does not support sparse gradients,' 101 | 'please consider SparseAdam instead') 102 | amsgrad = group['amsgrad'] 103 | 104 | state = self.state[param] 105 | 106 | # State initialization 107 | if len(state) == 0: 108 | state['step'] = 0 109 | # Exponential moving average of gradient values 110 | state['exp_avg'] = torch.zeros_like(p) 111 | # Exponential moving average of squared gradient values 112 | state['exp_avg_sq'] = torch.zeros_like(p) 113 | if amsgrad: 114 | # Maintains max of all exp. moving avg. of sq. grad. values 115 | state['max_exp_avg_sq'] = torch.zeros_like(p) 116 | elif NUMBA_CUDA_EXIST and numba.cuda.is_cuda_array(p.data): 117 | self._nbstate[param] = { 118 | 'param': numba.cuda.as_cuda_array(p.data.flatten()), 119 | 'grad': numba.cuda.as_cuda_array(grad.flatten()), 120 | 'exp_avg': numba.cuda.as_cuda_array(state['exp_avg'].data.flatten()), 121 | 'exp_avg_sq': numba.cuda.as_cuda_array(state['exp_avg_sq']. 122 | data.flatten()), 123 | 'blockspergrid': math.ceil(p.data.numel() / NUMBA_CUDA_THREAD_PER_BLOCK) 124 | } 125 | 126 | weight_decay = group['weight_decay'] 127 | eps = group['eps'] 128 | beta1, beta2 = group['betas'] 129 | 130 | state['step'] += 1 131 | bias_correction1 = 1 - beta1 ** state['step'] 132 | bias_correction2 = math.sqrt(1 - beta2 ** state['step']) 133 | step_size = group['lr'] / bias_correction1 134 | 135 | if param in self._nbstate: 136 | s = self._nbstate[param] 137 | numba_cuda_kernel[s['blockspergrid'], 138 | NUMBA_CUDA_THREAD_PER_BLOCK](s['param'], 139 | s['grad'], 140 | s['exp_avg'], 141 | s['exp_avg_sq'], 142 | beta1, beta2, 143 | step_size, 144 | bias_correction2, 145 | eps, weight_decay) 146 | else: 147 | if weight_decay != 0: 148 | grad.add_(weight_decay, p.data) 149 | exp_avg = state['exp_avg'].data 150 | exp_avg_sq = state['exp_avg_sq'].data 151 | # Decay the first and second moment running average coefficient 152 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 153 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 154 | if amsgrad: 155 | max_exp_avg_sq = state['max_exp_avg_sq'] 156 | # Maintains the maximum of all 2nd moment running avg. till now 157 | torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq) 158 | # Use the max. for normalizing running avg. of gradient 159 | denom = (max_exp_avg_sq.sqrt() / bias_correction2).add_(eps) 160 | else: 161 | denom = (exp_avg_sq.sqrt() / bias_correction2).add_(eps) 162 | p.data.addcdiv_(-step_size, exp_avg, denom) 163 | 164 | return loss 165 | -------------------------------------------------------------------------------- /pytorch/optimizers/adamw.py: -------------------------------------------------------------------------------- 1 | import math 2 | import os 3 | from distutils.util import strtobool 4 | import torch 5 | from torch.optim.optimizer import Optimizer 6 | from torch.hub import _check_module_exists 7 | 8 | NUMBA_CUDA_EXIST = False 9 | NUMBA_CUDA_THREAD_PER_BLOCK = 512 10 | if not strtobool(os.environ.get('NO_NUMBA', 'n')) and _check_module_exists("numba.cuda"): 11 | import numba.cuda 12 | NUMBA_CUDA_EXIST = numba.cuda.is_available() 13 | 14 | @numba.cuda.jit() 15 | def numba_cuda_kernel(param, grad, exp_avg, exp_avg_sq, beta1, 16 | beta2, step_size, bias_correction2, eps, 17 | weight_decay): 18 | i = numba.cuda.grid(1) 19 | if i >= param.size: 20 | return 21 | 22 | exp_avg[i] = exp_avg[i] * beta1 + (1 - beta1) * grad[i] 23 | exp_avg_sq[i] = exp_avg_sq[i] * beta2 + (1 - beta2) * grad[i] * grad[i] 24 | 25 | denom = math.sqrt(exp_avg_sq[i]) / bias_correction2 + eps 26 | param[i] *= weight_decay 27 | param[i] = param[i] + (-step_size) * (exp_avg[i] / denom) 28 | 29 | 30 | class AdamW(Optimizer): 31 | r"""Implements AdamW algorithm. 32 | 33 | The original Adam algorithm was proposed in `Adam: A Method for Stochastic Optimization`_. 34 | The AdamW variant was proposed in `Decoupled Weight Decay Regularization`_. 35 | 36 | Arguments: 37 | params (iterable): iterable of parameters to optimize or dicts defining 38 | parameter groups 39 | lr (float, optional): learning rate (default: 1e-3) 40 | betas (Tuple[float, float], optional): coefficients used for computing 41 | running averages of gradient and its square (default: (0.9, 0.999)) 42 | eps (float, optional): term added to the denominator to improve 43 | numerical stability (default: 1e-8) 44 | weight_decay (float, optional): weight decay coefficient (default: 1e-2) 45 | amsgrad (boolean, optional): whether to use the AMSGrad variant of this 46 | algorithm from the paper `On the Convergence of Adam and Beyond`_ 47 | (default: False) 48 | 49 | .. _Adam\: A Method for Stochastic Optimization: 50 | https://arxiv.org/abs/1412.6980 51 | .. _Decoupled Weight Decay Regularization: 52 | https://arxiv.org/abs/1711.05101 53 | .. _On the Convergence of Adam and Beyond: 54 | https://openreview.net/forum?id=ryQu7f-RZ 55 | """ 56 | 57 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, 58 | weight_decay=1e-2, amsgrad=False): 59 | if not 0.0 <= lr: 60 | raise ValueError("Invalid learning rate: {}".format(lr)) 61 | if not 0.0 <= eps: 62 | raise ValueError("Invalid epsilon value: {}".format(eps)) 63 | if not 0.0 <= betas[0] < 1.0: 64 | raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) 65 | if not 0.0 <= betas[1] < 1.0: 66 | raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) 67 | defaults = dict(lr=lr, betas=betas, eps=eps, 68 | weight_decay=weight_decay, amsgrad=amsgrad) 69 | super(AdamW, self).__init__(params, defaults) 70 | 71 | def __setstate__(self, state): 72 | super(AdamW, self).__setstate__(state) 73 | for group in self.param_groups: 74 | group.setdefault('amsgrad', False) 75 | 76 | def step(self, closure=None): 77 | """Performs a single optimization step. 78 | 79 | Arguments: 80 | closure (callable, optional): A closure that reevaluates the model 81 | and returns the loss. 82 | """ 83 | 84 | # In order to reduce Numba overhead, we save the device arrays 85 | # between calls to `step()` in `_nbstate`. 86 | self._nbstate = getattr(self, '_nbstate', {}) 87 | 88 | loss = None 89 | if closure is not None: 90 | loss = closure() 91 | 92 | for group in self.param_groups: 93 | for param in group['params']: 94 | if param.grad is None: 95 | continue 96 | 97 | # Perform optimization step 98 | grad = param.grad.data 99 | p = param.data 100 | if grad.is_sparse: 101 | raise RuntimeError('Adam does not support sparse gradients,' 102 | 'please consider SparseAdam instead') 103 | amsgrad = group['amsgrad'] 104 | 105 | state = self.state[param] 106 | 107 | # State initialization 108 | if len(state) == 0: 109 | state['step'] = 0 110 | # Exponential moving average of gradient values 111 | state['exp_avg'] = torch.zeros_like(p) 112 | # Exponential moving average of squared gradient values 113 | state['exp_avg_sq'] = torch.zeros_like(p) 114 | if amsgrad: 115 | # Maintains max of all exp. moving avg. of sq. grad. values 116 | state['max_exp_avg_sq'] = torch.zeros_like(p) 117 | elif NUMBA_CUDA_EXIST and numba.cuda.is_cuda_array(p.data): 118 | self._nbstate[param] = { 119 | 'param': numba.cuda.as_cuda_array(p.data.flatten()), 120 | 'grad': numba.cuda.as_cuda_array(grad.flatten()), 121 | 'exp_avg': numba.cuda.as_cuda_array(state['exp_avg'].data.flatten()), 122 | 'exp_avg_sq': numba.cuda.as_cuda_array(state['exp_avg_sq'] 123 | .data.flatten()), 124 | 'blockspergrid': math.ceil(p.data.numel() / NUMBA_CUDA_THREAD_PER_BLOCK) 125 | } 126 | 127 | weight_decay = 1 - group['lr'] * group['weight_decay'] 128 | eps = group['eps'] 129 | beta1, beta2 = group['betas'] 130 | 131 | state['step'] += 1 132 | bias_correction1 = 1 - beta1 ** state['step'] 133 | bias_correction2 = math.sqrt(1 - beta2 ** state['step']) 134 | step_size = group['lr'] / bias_correction1 135 | 136 | if param in self._nbstate: 137 | s = self._nbstate[param] 138 | numba_cuda_kernel[s['blockspergrid'], 139 | NUMBA_CUDA_THREAD_PER_BLOCK](s['param'], 140 | s['grad'], 141 | s['exp_avg'], 142 | s['exp_avg_sq'], 143 | beta1, beta2, 144 | step_size, 145 | bias_correction2, 146 | eps, weight_decay) 147 | else: 148 | exp_avg = state['exp_avg'].data 149 | exp_avg_sq = state['exp_avg_sq'].data 150 | # Decay the first and second moment running average coefficient 151 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 152 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 153 | if amsgrad: 154 | max_exp_avg_sq = state['max_exp_avg_sq'] 155 | # Maintains the maximum of all 2nd moment running avg. till now 156 | torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq) 157 | # Use the max. for normalizing running avg. of gradient 158 | denom = (max_exp_avg_sq.sqrt() / bias_correction2).add_(eps) 159 | else: 160 | denom = (exp_avg_sq.sqrt() / bias_correction2).add_(eps) 161 | 162 | # Perform stepweight decay 163 | p.data.mul_(weight_decay) 164 | 165 | p.data.addcdiv_(-step_size, exp_avg, denom) 166 | 167 | return loss 168 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/train_loss.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | import torch 5 | from torch import nn 6 | import torch.nn.functional as F 7 | 8 | ############################################################################################################# 9 | # # 10 | # Loss functions # 11 | # # 12 | ############################################################################################################# 13 | # lmae for single model: 1-type prediction 14 | def lmae_criterion(predict, coupling_value, coupling_rank, coupling_contribution, coupling_type,): 15 | ''' 16 | lmae between regression predictions and true scalar coupling constant 17 | ''' 18 | coupling_preds, contribution_preds, type_preds = predict 19 | predict = coupling_preds.view(-1) 20 | truth = coupling_value.view(-1) 21 | assert(predict.shape==truth.shape) 22 | loss = torch.abs(predict-truth) 23 | loss = loss.mean() 24 | loss = torch.log(loss+1e-8) 25 | return loss 26 | 27 | def lmae(coupling_preds, coupling_value): 28 | predict = coupling_preds.view(-1) 29 | truth = coupling_value.view(-1) 30 | assert(predict.shape==truth.shape) 31 | loss = torch.abs(predict-truth) 32 | loss = loss.mean() 33 | loss = torch.log(loss+1e-8) 34 | return loss 35 | 36 | # lmae for multi-type model 37 | def train_criterion(predict, 38 | coupling_value, coupling_rank, coupling_contribution, coupling_type , 39 | criterion='lmae', 40 | num_output= 1, 41 | gaussrank=True, 42 | pred_type= False): 43 | ''' 44 | The loss to be used for training the model w.r.t to flags: pred_type, num_output 45 | 46 | TODO : Include per-type loss training 47 | ''' 48 | 49 | coupling_preds, contribution_preds, type_preds = predict 50 | 51 | if not gaussrank: 52 | coupling_rank = coupling_value 53 | 54 | # fix the regression loss to use : mse or lmae 55 | if criterion == 'mse': 56 | l = nn.MSELoss() 57 | 58 | elif criterion == 'lmae': 59 | l = lmae_criterion 60 | 61 | elif criterion == 'mlmae2ce': 62 | 63 | cross_entropy_loss = torch.nn.CrossEntropyLoss()(type_preds, coupling_type) 64 | abs_diff = torch.abs(coupling_preds - coupling_rank.view(-1,1).expand(coupling_preds.size())) 65 | 66 | if criterion == 'mse': 67 | abs_diff = abs_diff**2 68 | 69 | proba_types = F.softmax(type_preds) 70 | weighted_diff = torch.mul(abs_diff, proba_types).sum(dim=1) 71 | 72 | unique_labels, labels_count = coupling_type.unique(dim=0, return_counts=True) 73 | res = torch.zeros(unique_labels.max()+1, dtype=torch.float, device='cuda') 74 | res = res.scatter_add_(0, coupling_type, weighted_diff) 75 | res = res[unique_labels] 76 | res = res.div(labels_count.float()) 77 | res = res.log().mean() 78 | 79 | return res + 2 * cross_entropy_loss 80 | 81 | elif criterion == 'mlmaeo2ce': 82 | cross_entropy_loss = torch.nn.CrossEntropyLoss()(type_preds, coupling_type) 83 | abs_diff = torch.abs(coupling_preds - coupling_rank.view(-1,1).expand(coupling_preds.size())) 84 | 85 | proba_types = F.softmax(type_preds) 86 | weighted_diff = torch.mul(abs_diff, proba_types).sum(dim=1) 87 | 88 | unique_labels, labels_count = coupling_type.unique(dim=0, return_counts=True) 89 | res = torch.zeros(unique_labels.max()+1, dtype=torch.float, device='cuda') 90 | res = res.scatter_add_(0, coupling_type, weighted_diff) 91 | res = res[unique_labels] 92 | res = res.div(labels_count.float()) 93 | res = res.log().mean() 94 | 95 | if cross_entropy_loss >= 0.05: 96 | return res + 2 * cross_entropy_loss 97 | else: 98 | return res 99 | 100 | elif criterion == 'mlmaeo2ceh': 101 | cross_entropy_loss = torch.nn.CrossEntropyLoss()(type_preds, coupling_type) 102 | abs_diff = torch.abs(coupling_preds - coupling_rank.view(-1,1).expand(coupling_preds.size())) 103 | 104 | proba_types = F.softmax(type_preds) 105 | 106 | weighted_diff = torch.sum((torch.index_select(abs_diff,1,torch.argmax(type_preds, dim=1))*torch.eye(len(coupling_type), device='cuda')), dim=1) 107 | 108 | unique_labels, labels_count = coupling_type.unique(dim=0, return_counts=True) 109 | res = torch.zeros(unique_labels.max()+1, dtype=torch.float, device='cuda') 110 | res = res.scatter_add_(0, coupling_type, weighted_diff) 111 | res = res[unique_labels] 112 | res = res.div(labels_count.float()) 113 | res = res.log().mean() 114 | 115 | if cross_entropy_loss >= 0.05: 116 | return res + 2 * cross_entropy_loss 117 | else: 118 | return res 119 | 120 | elif criterion == 'mlmaeo2ceha' or criterion == 'wmlmaeo2ceha': 121 | cross_entropy_loss = torch.nn.CrossEntropyLoss()(type_preds, coupling_type) 122 | abs_diff = torch.abs(coupling_preds - coupling_rank.view(-1,1).expand(coupling_preds.size())) 123 | 124 | proba_types = F.softmax(type_preds) 125 | 126 | weighted_diff = torch.sum((torch.index_select(abs_diff,1,coupling_type)*torch.eye(len(coupling_type), device='cuda')), dim=1) 127 | 128 | unique_labels, labels_count = coupling_type.unique(dim=0, return_counts=True) 129 | res = torch.zeros(unique_labels.max()+1, dtype=torch.float, device='cuda') 130 | res = res.scatter_add_(0, coupling_type, weighted_diff) 131 | if criterion == 'wmlmaeo2ceha': 132 | res = res * torch.tensor([10.,.1,.1,.1,.1,.1,.1,.1], dtype=torch.float, device='cuda') 133 | res = res[unique_labels] 134 | res = res.div(labels_count.float()) 135 | if criterion == 'wmlmaeo2ceha': 136 | res = res*res 137 | res = res.mean() 138 | else: 139 | res = res.log().mean() 140 | 141 | if cross_entropy_loss >= 0.05: 142 | return res + 2 * cross_entropy_loss 143 | else: 144 | return res 145 | 146 | elif criterion == 'lmaeo2ceha': 147 | cross_entropy_loss = torch.nn.CrossEntropyLoss()(type_preds, coupling_type) 148 | abs_diff = torch.abs(coupling_preds - coupling_rank.view(-1,1).expand(coupling_preds.size())) 149 | 150 | proba_types = F.softmax(type_preds) 151 | 152 | weighted_diff = torch.sum((torch.index_select(abs_diff,1,coupling_type)*torch.eye(len(coupling_type), device='cuda')), dim=1) 153 | 154 | res = torch.log(weighted_diff.mean()) 155 | 156 | if cross_entropy_loss >= 0.05: 157 | return res + 2 * cross_entropy_loss 158 | else: 159 | return res 160 | 161 | elif criterion == 'lmae_embed_type': 162 | return lmae(coupling_preds, coupling_rank) 163 | 164 | else: 165 | raise Exception(f"""{criterion} is not handled""") 166 | 167 | if pred_type: 168 | cross_entropy_loss = torch.nn.CrossEntropyLoss()(type_preds, coupling_type) 169 | abs_diff = torch.abs(coupling_preds - coupling_rank.view(-1,1).expand(coupling_preds.size())) 170 | 171 | if criterion == 'mse': 172 | abs_diff = abs_diff**2 173 | 174 | proba_types = F.softmax(type_preds) 175 | weighted_diff = torch.mul(abs_diff, proba_types).sum(dim=1) 176 | weighted_loss = torch.log(weighted_diff.mean()) 177 | 178 | weighted_loss = weighted_loss + 2 * cross_entropy_loss 179 | return weighted_loss 180 | 181 | elif num_output == 5: 182 | loss_coupling = l(coupling_preds, coupling_rank) 183 | loss_fc = l(contribution_preds[:, 0], coupling_contribution[:, 0]) 184 | loss_sd = l(contribution_preds[:, 1], coupling_contribution[:, 1]) 185 | loss_pso = l(contribution_preds[:, 2], coupling_contribution[:, 2]) 186 | loss_dso = l(contribution_preds[:, 3], coupling_contribution[:, 3]) 187 | return loss_coupling + (0.1 * (loss_fc + loss_sd + loss_pso + loss_dso) / 4) 188 | 189 | elif num_output ==1 : 190 | return l(coupling_preds, coupling_rank) 191 | 192 | else: 193 | raise Exception(f"""{num_output} is not handled""") -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/radam.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | from torch.optim.optimizer import Optimizer, required 4 | 5 | class RAdam(Optimizer): 6 | 7 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): 8 | defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) 9 | self.buffer = [[None, None, None] for ind in range(10)] 10 | super(RAdam, self).__init__(params, defaults) 11 | 12 | def __setstate__(self, state): 13 | super(RAdam, self).__setstate__(state) 14 | 15 | def step(self, closure=None): 16 | 17 | loss = None 18 | if closure is not None: 19 | loss = closure() 20 | 21 | for group in self.param_groups: 22 | 23 | for p in group['params']: 24 | if p.grad is None: 25 | continue 26 | grad = p.grad.data.float() 27 | if grad.is_sparse: 28 | raise RuntimeError('RAdam does not support sparse gradients') 29 | 30 | p_data_fp32 = p.data.float() 31 | 32 | state = self.state[p] 33 | 34 | if len(state) == 0: 35 | state['step'] = 0 36 | state['exp_avg'] = torch.zeros_like(p_data_fp32) 37 | state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) 38 | else: 39 | state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) 40 | state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) 41 | 42 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 43 | beta1, beta2 = group['betas'] 44 | 45 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 46 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 47 | 48 | state['step'] += 1 49 | buffered = self.buffer[int(state['step'] % 10)] 50 | if state['step'] == buffered[0]: 51 | N_sma, step_size = buffered[1], buffered[2] 52 | else: 53 | buffered[0] = state['step'] 54 | beta2_t = beta2 ** state['step'] 55 | N_sma_max = 2 / (1 - beta2) - 1 56 | N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t) 57 | buffered[1] = N_sma 58 | 59 | # more conservative since it's an approximated value 60 | if N_sma >= 5: 61 | step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step']) 62 | else: 63 | step_size = group['lr'] / (1 - beta1 ** state['step']) 64 | buffered[2] = step_size 65 | 66 | if group['weight_decay'] != 0: 67 | p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32) 68 | 69 | # more conservative since it's an approximated value 70 | if N_sma >= 5: 71 | denom = exp_avg_sq.sqrt().add_(group['eps']) 72 | p_data_fp32.addcdiv_(-step_size, exp_avg, denom) 73 | else: 74 | p_data_fp32.add_(-step_size, exp_avg) 75 | 76 | p.data.copy_(p_data_fp32) 77 | 78 | return loss 79 | 80 | class PlainRAdam(Optimizer): 81 | 82 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0): 83 | defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) 84 | 85 | super(PlainRAdam, self).__init__(params, defaults) 86 | 87 | def __setstate__(self, state): 88 | super(PlainRAdam, self).__setstate__(state) 89 | 90 | def step(self, closure=None): 91 | 92 | loss = None 93 | if closure is not None: 94 | loss = closure() 95 | 96 | for group in self.param_groups: 97 | 98 | for p in group['params']: 99 | if p.grad is None: 100 | continue 101 | grad = p.grad.data.float() 102 | if grad.is_sparse: 103 | raise RuntimeError('RAdam does not support sparse gradients') 104 | 105 | p_data_fp32 = p.data.float() 106 | 107 | state = self.state[p] 108 | 109 | if len(state) == 0: 110 | state['step'] = 0 111 | state['exp_avg'] = torch.zeros_like(p_data_fp32) 112 | state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) 113 | else: 114 | state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) 115 | state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) 116 | 117 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 118 | beta1, beta2 = group['betas'] 119 | 120 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 121 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 122 | 123 | state['step'] += 1 124 | beta2_t = beta2 ** state['step'] 125 | N_sma_max = 2 / (1 - beta2) - 1 126 | N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t) 127 | 128 | if group['weight_decay'] != 0: 129 | p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32) 130 | 131 | # more conservative since it's an approximated value 132 | if N_sma >= 5: 133 | step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step']) 134 | denom = exp_avg_sq.sqrt().add_(group['eps']) 135 | p_data_fp32.addcdiv_(-step_size, exp_avg, denom) 136 | else: 137 | step_size = group['lr'] / (1 - beta1 ** state['step']) 138 | p_data_fp32.add_(-step_size, exp_avg) 139 | 140 | p.data.copy_(p_data_fp32) 141 | 142 | return loss 143 | 144 | 145 | class AdamW(Optimizer): 146 | 147 | def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, warmup = 0): 148 | defaults = dict(lr=lr, betas=betas, eps=eps, 149 | weight_decay=weight_decay, warmup = warmup) 150 | super(AdamW, self).__init__(params, defaults) 151 | 152 | def __setstate__(self, state): 153 | super(AdamW, self).__setstate__(state) 154 | 155 | def step(self, closure=None): 156 | loss = None 157 | if closure is not None: 158 | loss = closure() 159 | 160 | for group in self.param_groups: 161 | 162 | for p in group['params']: 163 | if p.grad is None: 164 | continue 165 | grad = p.grad.data.float() 166 | if grad.is_sparse: 167 | raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead') 168 | 169 | p_data_fp32 = p.data.float() 170 | 171 | state = self.state[p] 172 | 173 | if len(state) == 0: 174 | state['step'] = 0 175 | state['exp_avg'] = torch.zeros_like(p_data_fp32) 176 | state['exp_avg_sq'] = torch.zeros_like(p_data_fp32) 177 | else: 178 | state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32) 179 | state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32) 180 | 181 | exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] 182 | beta1, beta2 = group['betas'] 183 | 184 | state['step'] += 1 185 | 186 | exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) 187 | exp_avg.mul_(beta1).add_(1 - beta1, grad) 188 | 189 | denom = exp_avg_sq.sqrt().add_(group['eps']) 190 | bias_correction1 = 1 - beta1 ** state['step'] 191 | bias_correction2 = 1 - beta2 ** state['step'] 192 | 193 | if group['warmup'] > state['step']: 194 | scheduled_lr = 1e-8 + state['step'] * group['lr'] / group['warmup'] 195 | else: 196 | scheduled_lr = group['lr'] 197 | 198 | step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1 199 | 200 | if group['weight_decay'] != 0: 201 | p_data_fp32.add_(-group['weight_decay'] * scheduled_lr, p_data_fp32) 202 | 203 | p_data_fp32.addcdiv_(-step_size, exp_avg, denom) 204 | 205 | p.data.copy_(p_data_fp32) 206 | 207 | return loss -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/message_passing.py: -------------------------------------------------------------------------------- 1 | from mpnn_model.common_model import * 2 | from mpnn_model.common import * 3 | 4 | from torch_scatter import * 5 | from torch_geometric.utils import scatter_ 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | import numbers 10 | 11 | 12 | __all__ = ['message_pass' , 'MessagePassing', 'GRUUpdate', 'Set2Set'] 13 | 14 | ############################################################################################################# 15 | # # 16 | # MPNN- PHASE1 : Message Passing # 17 | # # 18 | ############################################################################################################# 19 | def message_pass(node_states, edge_index, a_in): 20 | """Computes a_t from h_{t-1}, see bottom of page 3 in the paper. 21 | a_t = sum_w A(e_vw) . h^t 22 | Args: 23 | node_states: [batch_size*num_nodes, node_dim] tensor (h_{t-1}) 24 | a_in (torch.float32): [batch_size*num_nodes, node_dim, node_dim]: Encoded edge matrix 25 | edge_index [batch_size*num_edges, 2]: the indices of edges 26 | Returns: 27 | messages (torch.float32): [batch_size*num_nodes, node_dim] For each pair 28 | of nodes in the graph a message is sent along both the incoming edge. 29 | """ 30 | num_node, node_dim = node_states.shape 31 | edge_index = edge_index.t().contiguous() 32 | x_i = torch.index_select(node_states, 0, edge_index[0]) 33 | message = torch.matmul( x_i.view(-1,1,node_dim), a_in).view(-1, node_dim) 34 | message = scatter_('mean', message, edge_index[1], dim_size=num_node) 35 | return message 36 | 37 | class MessagePassing(nn.Module): 38 | ''' 39 | A feed forward neural network is applied to each edge in the adjacency matrix, 40 | which is assumed to be vector valued. It maps the edge vector to a 41 | node_dim x node_dim matrix, denoted NN(e). The message from node v -> w is 42 | then NN(e) h_v. This is a generalization of the message function in the 43 | GG-NN paper, which embeds the discrete edge label as a matrix. 44 | ''' 45 | def __init__(self, ConfigParams): 46 | ''' 47 | 48 | ''' 49 | super(MessagePassing, self).__init__() 50 | self.encoding = ConfigParams['model']['mpnn']['node_encoder']['encoding'] 51 | self.edge_encoder = get_edge_encoder(**ConfigParams['model']['mpnn']['edge_encoder']) 52 | self.node_dim = ConfigParams['model']['mpnn']['edge_encoder']['node_dim'] 53 | self.device = ConfigParams['train']['device'] 54 | 55 | if self.device == 'cuda': 56 | self.bias = nn.Parameter(torch.Tensor(self.node_dim)).cuda() 57 | else: 58 | self.bias = nn.Parameter(torch.Tensor(self.node_dim)) 59 | 60 | 61 | self.bias.data.uniform_(-1.0 / math.sqrt(self.node_dim), 1.0 / math.sqrt(self.node_dim)) 62 | 63 | self._a_in = [] 64 | 65 | def _pre_encode_edges(self, edge): 66 | ''' 67 | Args: 68 | edge: [batch_size*num_edges, edge_dim] edge features 69 | Return: 70 | A neural representation of the edge festures where each vector is represented as 71 | matrix of shape node_dim x node_dim 72 | ''' 73 | if self.encoding == 'label': 74 | edge_cat = edge[:, 0].long().view(-1,1) 75 | edge_cont = edge[:, 1:].float() 76 | edge = self.edge_encoder(edge_cat, edge_cont).view(-1,self.node_dim,self.node_dim) 77 | 78 | elif self.encoding == 'one_hot': 79 | edge = self.edge_encoder(edge).view(-1, self.node_dim, self.node_dim) 80 | 81 | self._a_in = edge 82 | 83 | def forward(self, node_states, edge_index, edge, reuse_graph_tensors=True): 84 | ''' 85 | Args: 86 | node_states: [batch_size*num_nodes, node_dim] tensor (h_{t-1}) 87 | edge_in: [batch_size*num_nodes, edge_dim] (torch.int32) 88 | reuse_graph_tensors: Boolean to indicate whether or not the self._a_in 89 | should be reused or not. Should be set to False on first call, and True 90 | on subsequent calls. 91 | Returns: 92 | message_t: [batch_size * num_nodes, node_dim] which is the node representations 93 | after a single propgation step 94 | ''' 95 | if not reuse_graph_tensors: 96 | self._pre_encode_edges(edge) 97 | new_state = message_pass(node_states, edge_index, self._a_in) 98 | return F.relu(new_state + self.bias) 99 | 100 | ############################################################################################################# 101 | # # 102 | # MPNN- PHASE2 : Updage nodes states # 103 | # # 104 | ############################################################################################################# 105 | 106 | class GRUUpdate(nn.Module): 107 | def __init__(self, ConfigParams): 108 | super(GRUUpdate, self).__init__() 109 | 110 | self.node_dim = ConfigParams['model']['mpnn']['edge_encoder']['node_dim'] 111 | self.gru = nn.GRU(self.node_dim, self.node_dim, batch_first=False, bidirectional=False) 112 | 113 | def forward(self, messages, node_states): 114 | """Build the fprop graph. 115 | Args: 116 | node_states: [batch_size*num_nodes, node_dim] tensor (h_{t-1}) 117 | messages: [batch_size*num_nodes, node_dim] (a_t from the GGNN paper) 118 | Returns: 119 | updated_states: [batch_size*num_nodes, node_dim] 120 | """ 121 | num_node, node_dim = node_states.shape 122 | 123 | update, _ = self.gru(messages.view(1,-1,self.node_dim), 124 | node_states.view(1,num_node,-1)) 125 | 126 | return update.view(-1,node_dim) 127 | 128 | ############################################################################################################# 129 | # # 130 | # MPNN- PHASE3 : Readout function # 131 | # # 132 | ############################################################################################################# 133 | 134 | class Set2Set(torch.nn.Module): 135 | 136 | def softmax(self, x, index, num=None): 137 | x = x - scatter_max(x, index, dim=0, dim_size=num)[0][index] 138 | x = x.exp() 139 | x = x / (scatter_add(x, index, dim=0, dim_size=num)[index] + 1e-16) 140 | return x 141 | 142 | def __init__(self, in_channel, processing_step=1, num_layer = 1, batch_size=32): 143 | super(Set2Set, self).__init__() 144 | 145 | out_channel = 2 * in_channel 146 | self.processing_step = processing_step 147 | self.batch_size = batch_size 148 | self.in_channel = in_channel 149 | self.out_channel = out_channel 150 | self.num_layer = num_layer 151 | self.lstm = torch.nn.LSTM(out_channel, in_channel, num_layer) 152 | self.lstm.reset_parameters() 153 | 154 | 155 | def forward(self, x, batch_index): 156 | h = (x.new_zeros((self.num_layer, self.batch_size, self.in_channel)), 157 | x.new_zeros((self.num_layer, self.batch_size, self.in_channel))) 158 | # zeros of shape: bs x 2*node_dim : init q_star 159 | q_star = x.new_zeros(self.batch_size, self.out_channel) 160 | 161 | # n readout steps 162 | for i in range(self.processing_step): 163 | # read from memory 164 | q, h = self.lstm(q_star.unsqueeze(0), h) 165 | q = q.view(self.batch_size, -1) 166 | #energies : dot product between input_set and q 167 | e = (x * q[batch_index]).sum(dim=-1, keepdim=True) #shape = num_node x 1 168 | # Compute attention 169 | a = self.softmax(e, batch_index, num=self.batch_size) #shape = num_node x 1 170 | #compute readout 171 | r = scatter_add(a * x, batch_index, dim=0, dim_size=self.batch_size) #apply attention #shape = batch_size x ... 172 | #update q_star 173 | q_star = torch.cat([q, r], dim=-1) 174 | # print(q_star.shape) 175 | return q_star 176 | 177 | -------------------------------------------------------------------------------- /champs-scalar-coupling/build_data/lib/net/rate.py: -------------------------------------------------------------------------------- 1 | # learning rate schduler 2 | from lib.include import * 3 | 4 | # http://elgoacademy.org/anatomy-matplotlib-part-1/ 5 | def plot_rates(fig, lrs, title=''): 6 | 7 | N = len(lrs) 8 | epoches = np.arange(0,N) 9 | 10 | 11 | #get limits 12 | max_lr = np.max(lrs) 13 | xmin=0 14 | xmax=N 15 | dx=2 16 | 17 | ymin=0 18 | ymax=max_lr*1.2 19 | dy=(ymax-ymin)/10 20 | dy=10**math.ceil(math.log10(dy)) 21 | 22 | ax = fig.add_subplot(111) 23 | #ax = fig.gca() 24 | ax.set_axisbelow(True) 25 | ax.minorticks_on() 26 | ax.set_xticks(np.arange(xmin,xmax+0.0001, dx)) 27 | ax.set_yticks(np.arange(ymin,ymax+0.0001, dy)) 28 | ax.set_xlim(xmin,xmax+0.0001) 29 | ax.set_ylim(ymin,ymax+0.0001) 30 | ax.grid(b=True, which='minor', color='black', alpha=0.1, linestyle='dashed') 31 | ax.grid(b=True, which='major', color='black', alpha=0.4, linestyle='dashed') 32 | 33 | ax.set_xlabel('iter') 34 | ax.set_ylabel('learning rate') 35 | ax.set_title(title) 36 | ax.plot(epoches, lrs) 37 | 38 | 39 | 40 | ## simple stepping rates 41 | class StepScheduler(): 42 | def __init__(self, pairs): 43 | super(StepScheduler, self).__init__() 44 | 45 | N=len(pairs) 46 | rates=[] 47 | steps=[] 48 | for n in range(N): 49 | steps.append(pairs[n][0]) 50 | rates.append(pairs[n][1]) 51 | 52 | self.rates = rates 53 | self.steps = steps 54 | 55 | def __call__(self, epoch): 56 | 57 | N = len(self.steps) 58 | lr = -1 59 | for n in range(N): 60 | if epoch >= self.steps[n]: 61 | lr = self.rates[n] 62 | return lr 63 | 64 | def __str__(self): 65 | string = 'Step Learning Rates\n' \ 66 | + 'rates=' + str(['%7.4f' % i for i in self.rates]) + '\n' \ 67 | + 'steps=' + str(['%7.0f' % i for i in self.steps]) + '' 68 | return string 69 | 70 | 71 | ## https://github.com/pytorch/tutorials/blob/master/beginner_source/transfer_learning_tutorial.py 72 | class DecayScheduler(): 73 | def __init__(self, base_lr, decay, step): 74 | super(DecayScheduler, self).__init__() 75 | self.step = step 76 | self.decay = decay 77 | self.base_lr = base_lr 78 | 79 | def get_rate(self, epoch): 80 | lr = self.base_lr * (self.decay**(epoch // self.step)) 81 | return lr 82 | 83 | 84 | 85 | def __str__(self): 86 | string = '(Exp) Decay Learning Rates\n' \ 87 | + 'base_lr=%0.3f, decay=%0.3f, step=%0.3f'%(self.base_lr, self.decay, self.step) 88 | return string 89 | 90 | 91 | 92 | 93 | # 'Cyclical Learning Rates for Training Neural Networks'- Leslie N. Smith, arxiv 2017 94 | # https://arxiv.org/abs/1506.01186 95 | # https://github.com/bckenstler/CLR 96 | 97 | class CyclicScheduler1(): 98 | 99 | def __init__(self, min_lr=0.001, max_lr=0.01, period=10 ): 100 | super(CyclicScheduler, self).__init__() 101 | 102 | self.min_lr = min_lr 103 | self.max_lr = max_lr 104 | self.period = period 105 | 106 | def __call__(self, time): 107 | 108 | #sawtooth 109 | #r = (1-(time%self.period)/self.period) 110 | 111 | #cosine 112 | time= time%self.period 113 | r = (np.cos(time/self.period *PI)+1)/2 114 | 115 | lr = self.min_lr + r*(self.max_lr-self.min_lr) 116 | return lr 117 | 118 | def __str__(self): 119 | string = 'CyclicScheduler\n' \ 120 | + 'min_lr=%0.3f, max_lr=%0.3f, period=%8.1f'%(self.min_lr, self.max_lr, self.period) 121 | return string 122 | 123 | 124 | class CyclicScheduler2(): 125 | 126 | def __init__(self, min_lr=0.001, max_lr=0.01, period=10, max_decay=0.99, warm_start=0 ): 127 | super(CyclicScheduler, self).__init__() 128 | 129 | self.min_lr = min_lr 130 | self.max_lr = max_lr 131 | self.period = period 132 | self.max_decay = max_decay 133 | self.warm_start = warm_start 134 | self.cycle = -1 135 | 136 | def __call__(self, time): 137 | if time= step: 228 | # lr = rate 229 | # 230 | # return lr 231 | # 232 | # 233 | # 234 | # def __str__(self): 235 | # string = 'CyclicScheduler\n' \ 236 | # + 'lrs =' + str(['%7.4f' % i for i in self.lrs]) + '\n' \ 237 | # + 'steps=' + str(['%7.0f' % i for i in self.steps]) + '\n' \ 238 | # + 'period=%8.1f'%(self.period) 239 | # return string 240 | 241 | 242 | class NullScheduler(): 243 | def __init__(self, lr=0.01 ): 244 | super(NullScheduler, self).__init__() 245 | self.lr = lr 246 | self.cycle = 0 247 | 248 | def __call__(self, time): 249 | return self.lr 250 | 251 | def __str__(self): 252 | string = 'NullScheduler\n' \ 253 | + 'lr=%0.5f '%(self.lr) 254 | return string 255 | 256 | 257 | # net ------------------------------------ 258 | # https://github.com/pytorch/examples/blob/master/imagenet/main.py ############### 259 | def adjust_learning_rate(optimizer, lr): 260 | for param_group in optimizer.param_groups: 261 | param_group['lr'] = lr 262 | 263 | def get_learning_rate(optimizer): 264 | lr=[] 265 | for param_group in optimizer.param_groups: 266 | lr +=[ param_group['lr'] ] 267 | 268 | assert(len(lr)==1) #we support only one param_group 269 | lr = lr[0] 270 | 271 | return lr 272 | 273 | 274 | 275 | 276 | 277 | # main ################################################################# 278 | if __name__ == '__main__': 279 | print( '%s: calling main function ... ' % os.path.basename(__file__)) 280 | 281 | num_iters=125 282 | 283 | scheduler = StepScheduler([ (0,0.1), (10,0.01), (25,0.005), (35,0.001), (40,0.0001), (43,-1)]) 284 | #scheduler = DecayScheduler(base_lr=0.1, decay=0.32, step=10) 285 | #scheduler = CyclicScheduler(min_lr=0.0001, max_lr=0.01, period=30., warm_start=5) ##exp_range ##triangular2 286 | #scheduler = CyclicScheduler([ (0,0.1), (25,0.01), (45,0.005)], period=50., warm_start=5) ##exp_range ##triangular2 287 | 288 | 289 | lrs = np.zeros((num_iters),np.float32) 290 | for iter in range(num_iters): 291 | 292 | lr = scheduler(iter) 293 | lrs[iter] = lr 294 | if lr<0: 295 | num_iters = iter 296 | break 297 | #print ('iter=%02d, lr=%f %d'%(iter,lr, scheduler.cycle)) 298 | 299 | 300 | #plot 301 | fig = plt.figure() 302 | plot_rates(fig, lrs, title=str(scheduler)) 303 | plt.show() 304 | 305 | 306 | # https://github.com/Jiaming-Liu/pytorch-lr-scheduler/blob/master/lr_scheduler.py 307 | # PVANET plateau lr policy 308 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/lib/net/rate.py: -------------------------------------------------------------------------------- 1 | # learning rate schduler 2 | from mpnn_model.lib.include import * 3 | 4 | # http://elgoacademy.org/anatomy-matplotlib-part-1/ 5 | def plot_rates(fig, lrs, title=''): 6 | 7 | N = len(lrs) 8 | epoches = np.arange(0,N) 9 | 10 | 11 | #get limits 12 | max_lr = np.max(lrs) 13 | xmin=0 14 | xmax=N 15 | dx=2 16 | 17 | ymin=0 18 | ymax=max_lr*1.2 19 | dy=(ymax-ymin)/10 20 | dy=10**math.ceil(math.log10(dy)) 21 | 22 | ax = fig.add_subplot(111) 23 | #ax = fig.gca() 24 | ax.set_axisbelow(True) 25 | ax.minorticks_on() 26 | ax.set_xticks(np.arange(xmin,xmax+0.0001, dx)) 27 | ax.set_yticks(np.arange(ymin,ymax+0.0001, dy)) 28 | ax.set_xlim(xmin,xmax+0.0001) 29 | ax.set_ylim(ymin,ymax+0.0001) 30 | ax.grid(b=True, which='minor', color='black', alpha=0.1, linestyle='dashed') 31 | ax.grid(b=True, which='major', color='black', alpha=0.4, linestyle='dashed') 32 | 33 | ax.set_xlabel('iter') 34 | ax.set_ylabel('learning rate') 35 | ax.set_title(title) 36 | ax.plot(epoches, lrs) 37 | 38 | 39 | 40 | ## simple stepping rates 41 | class StepScheduler(): 42 | def __init__(self, pairs): 43 | super(StepScheduler, self).__init__() 44 | 45 | N=len(pairs) 46 | rates=[] 47 | steps=[] 48 | for n in range(N): 49 | steps.append(pairs[n][0]) 50 | rates.append(pairs[n][1]) 51 | 52 | self.rates = rates 53 | self.steps = steps 54 | 55 | def __call__(self, epoch): 56 | 57 | N = len(self.steps) 58 | lr = -1 59 | for n in range(N): 60 | if epoch >= self.steps[n]: 61 | lr = self.rates[n] 62 | return lr 63 | 64 | def __str__(self): 65 | string = 'Step Learning Rates\n' \ 66 | + 'rates=' + str(['%7.4f' % i for i in self.rates]) + '\n' \ 67 | + 'steps=' + str(['%7.0f' % i for i in self.steps]) + '' 68 | return string 69 | 70 | 71 | ## https://github.com/pytorch/tutorials/blob/master/beginner_source/transfer_learning_tutorial.py 72 | class DecayScheduler(): 73 | def __init__(self, base_lr, decay, step): 74 | super(DecayScheduler, self).__init__() 75 | self.step = step 76 | self.decay = decay 77 | self.base_lr = base_lr 78 | 79 | def get_rate(self, epoch): 80 | lr = self.base_lr * (self.decay**(epoch // self.step)) 81 | return lr 82 | 83 | 84 | 85 | def __str__(self): 86 | string = '(Exp) Decay Learning Rates\n' \ 87 | + 'base_lr=%0.3f, decay=%0.3f, step=%0.3f'%(self.base_lr, self.decay, self.step) 88 | return string 89 | 90 | 91 | 92 | 93 | # 'Cyclical Learning Rates for Training Neural Networks'- Leslie N. Smith, arxiv 2017 94 | # https://arxiv.org/abs/1506.01186 95 | # https://github.com/bckenstler/CLR 96 | 97 | class CyclicScheduler1(): 98 | 99 | def __init__(self, min_lr=0.001, max_lr=0.01, period=10 ): 100 | super(CyclicScheduler, self).__init__() 101 | 102 | self.min_lr = min_lr 103 | self.max_lr = max_lr 104 | self.period = period 105 | 106 | def __call__(self, time): 107 | 108 | #sawtooth 109 | #r = (1-(time%self.period)/self.period) 110 | 111 | #cosine 112 | time= time%self.period 113 | r = (np.cos(time/self.period *PI)+1)/2 114 | 115 | lr = self.min_lr + r*(self.max_lr-self.min_lr) 116 | return lr 117 | 118 | def __str__(self): 119 | string = 'CyclicScheduler\n' \ 120 | + 'min_lr=%0.3f, max_lr=%0.3f, period=%8.1f'%(self.min_lr, self.max_lr, self.period) 121 | return string 122 | 123 | 124 | class CyclicScheduler2(): 125 | 126 | def __init__(self, min_lr=0.001, max_lr=0.01, period=10, max_decay=0.99, warm_start=0 ): 127 | super(CyclicScheduler, self).__init__() 128 | 129 | self.min_lr = min_lr 130 | self.max_lr = max_lr 131 | self.period = period 132 | self.max_decay = max_decay 133 | self.warm_start = warm_start 134 | self.cycle = -1 135 | 136 | def __call__(self, time): 137 | if time= step: 228 | # lr = rate 229 | # 230 | # return lr 231 | # 232 | # 233 | # 234 | # def __str__(self): 235 | # string = 'CyclicScheduler\n' \ 236 | # + 'lrs =' + str(['%7.4f' % i for i in self.lrs]) + '\n' \ 237 | # + 'steps=' + str(['%7.0f' % i for i in self.steps]) + '\n' \ 238 | # + 'period=%8.1f'%(self.period) 239 | # return string 240 | 241 | 242 | class NullScheduler(): 243 | def __init__(self, lr=0.01 ): 244 | super(NullScheduler, self).__init__() 245 | self.lr = lr 246 | self.cycle = 0 247 | 248 | def __call__(self, time): 249 | return self.lr 250 | 251 | def __str__(self): 252 | string = 'NullScheduler\n' \ 253 | + 'lr=%0.5f '%(self.lr) 254 | return string 255 | 256 | 257 | # net ------------------------------------ 258 | # https://github.com/pytorch/examples/blob/master/imagenet/main.py ############### 259 | def adjust_learning_rate(optimizer, lr): 260 | for param_group in optimizer.param_groups: 261 | param_group['lr'] = lr 262 | 263 | def get_learning_rate(optimizer): 264 | lr=[] 265 | for param_group in optimizer.param_groups: 266 | lr +=[ param_group['lr'] ] 267 | 268 | assert(len(lr)==1) #we support only one param_group 269 | lr = lr[0] 270 | 271 | return lr 272 | 273 | 274 | 275 | 276 | 277 | # main ################################################################# 278 | if __name__ == '__main__': 279 | print( '%s: calling main function ... ' % os.path.basename(__file__)) 280 | 281 | num_iters=125 282 | 283 | scheduler = StepScheduler([ (0,0.1), (10,0.01), (25,0.005), (35,0.001), (40,0.0001), (43,-1)]) 284 | #scheduler = DecayScheduler(base_lr=0.1, decay=0.32, step=10) 285 | #scheduler = CyclicScheduler(min_lr=0.0001, max_lr=0.01, period=30., warm_start=5) ##exp_range ##triangular2 286 | #scheduler = CyclicScheduler([ (0,0.1), (25,0.01), (45,0.005)], period=50., warm_start=5) ##exp_range ##triangular2 287 | 288 | 289 | lrs = np.zeros((num_iters),np.float32) 290 | for iter in range(num_iters): 291 | 292 | lr = scheduler(iter) 293 | lrs[iter] = lr 294 | if lr<0: 295 | num_iters = iter 296 | break 297 | #print ('iter=%02d, lr=%f %d'%(iter,lr, scheduler.cycle)) 298 | 299 | 300 | #plot 301 | fig = plt.figure() 302 | plot_rates(fig, lrs, title=str(scheduler)) 303 | plt.show() 304 | 305 | 306 | # https://github.com/Jiaming-Liu/pytorch-lr-scheduler/blob/master/lr_scheduler.py 307 | # PVANET plateau lr policy 308 | -------------------------------------------------------------------------------- /champs-scalar-coupling/mpnn_model/data_collate.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | from mpnn_model.common import * 5 | from mpnn_model.common_constants import * 6 | from mpnn_model.data import * 7 | 8 | import copy 9 | 10 | DATA_DIR = '/rapids/notebooks/srabhi/champs-2019/input' 11 | 12 | __all__ = ['tensor_collate_rnn', 'tensor_collate_baseline'] 13 | 14 | def tensor_collate_rnn(batch, batch_size, COUPLING_MAX, mode='train'): 15 | """ 16 | Function to apply dynamic padding of each batch in order to prepare inputs of the graph neural network 17 | 18 | node, edge_feats, edge_index, node_index, coupling_index), targets 19 | Returns: 20 | X: 21 | node [batch_size*molecules_num_nodes, node_dim] : nodes states, variable size per batch 22 | edge_feats [batch_size*molecules_num_edges, edge_dim] : edges states, variable size per batch 23 | edge_index [batch_size*molecules_num_edges, 2] : Index edges of the same molecule 24 | node_index [batch_size*molecules_num_nodes, 1] : Index nodes of the same molecule 25 | coupling_index 26 | Y: 27 | targets [N_coupling, 4]: tuple of four targets (scalar_coupling, coupling_gaussrank, coupling_contribs, coupling_type) 28 | Test info: 29 | infor: the ids of the coupling needed for build submission files # N_coupling, 30 | """ 31 | 32 | batch_node, batch_edge, batch_coupling, batch_num_node, batch_num_edge, batch_num_coupling = batch 33 | batch_node = batch_node.reshape(-1, NODE_MAX, 7).float() 34 | batch_edge = batch_edge.reshape(-1, EDGE_MAX, 5) 35 | batch_coupling = batch_coupling.reshape(-1, COUPLING_MAX, 21) 36 | batch_size = batch_node.shape[0] 37 | 38 | #### Create nodes / edges / coupling masks : optimized V1 : TO be optimized 39 | mask = torch.cat([F.pad(torch.ones(i, device='cuda'), (0,NODE_MAX-i)).unsqueeze(0) for i in batch_num_node], dim=0) 40 | mask_coupling = torch.cat([F.pad(torch.ones(i, device='cuda'), (0,COUPLING_MAX-i)).unsqueeze(0) for i in batch_num_coupling], dim=0) 41 | mask_edge = torch.cat([F.pad(torch.ones(i, device='cuda'), (0,EDGE_MAX-i)).unsqueeze(0) for i in batch_num_edge], dim=0) 42 | 43 | #### Build the output X: 44 | # Get effective nodes / edges / coupling values : without padding 45 | node = batch_node[mask.bool()].view(-1, 7) 46 | edge = batch_edge[mask_edge.bool()].view(-1, 5) 47 | coupling = batch_coupling[mask_coupling.bool()].view(-1, 21) 48 | 49 | # node indices track nodes of the same molecule 50 | node_index = mask.nonzero()[:, 0] 51 | 52 | # Get edges feats and indices 53 | edge_feats = edge[:, 2:] 54 | edge_index = edge[:, :2].long() 55 | 56 | # Get coupling path index 57 | coupling_index = coupling[:, 10:14].long() 58 | num_coupling = coupling_index.shape[0] 59 | 60 | #get sequence of coupling type 61 | pad_vector = torch.zeros(num_coupling, device='cuda').long()-1 62 | coupling_type_sequence = torch.cat([pad_vector.view(-1,1), coupling[:, 14:17].long()], 1) 63 | 64 | 65 | # batch_coupling_index : track coupling values of the same molecule 66 | batch_coupling_index = mask_coupling.nonzero()[:, 0] 67 | 68 | # offset edge and coupling indices w.r.t to N of nodes in each molecule 69 | offset = torch.cat([torch.zeros(1, device='cuda').long(), batch_num_node[:-1]]).cumsum(0) 70 | #edge 71 | expanded_offset = torch.cat([torch.zeros(num_edges, device='cuda')+offset[mol_index] for mol_index,num_edges in 72 | enumerate(batch_num_edge)], 0) 73 | edge_index = torch.add(edge_index, expanded_offset.unsqueeze(1).long()) 74 | #coupling 75 | expanded_offset=torch.cat([torch.zeros(n_coupling, device='cuda')+offset[mol_index] for mol_index,n_coupling in 76 | enumerate(batch_num_coupling)], 0) 77 | coupling_index = torch.add(coupling_index, expanded_offset.unsqueeze(1).long()) 78 | # type_id 79 | coupling_type = coupling[:, 2].long() 80 | 81 | # new coupling index: atom_0, atom_1, atom_2, atom_3, coupling_type, batch_index 82 | coupling_index = torch.cat([coupling_index, coupling_type.view(-1,1) , batch_coupling_index.view(-1, 1)], -1) 83 | 84 | # get sequence of atomic number 85 | coupling_atomic = coupling[:, 17:].long() 86 | 87 | #### Get Targets 88 | # 4 coupling contirbutions 89 | coupling_contribution = coupling[:, 5:9] 90 | #coupling value 91 | coupling_value = coupling[:, 3] 92 | #gauss_rank 93 | gaussrank = coupling[:, 4] 94 | targets = [coupling_value.float(), gaussrank.float(), coupling_contribution.float(), coupling_type.long()] 95 | 96 | #### ids for inference time 97 | infor = coupling[ : , 9] 98 | 99 | # mode flag to return additional information for test data 100 | if mode == 'test': 101 | return (node, edge_feats, edge_index, node_index, coupling_index, coupling_type_sequence, coupling_atomic), targets, infor 102 | 103 | return (node, edge_feats, edge_index, node_index, coupling_index, coupling_type_sequence, coupling_atomic), targets 104 | 105 | def tensor_collate_baseline(batch, batch_size, COUPLING_MAX, mode='train'): 106 | """ 107 | Function to apply dynamic padding of each batch in order to prepare inputs of the graph neural network 108 | 109 | node, edge_feats, edge_index, node_index, coupling_index), targets 110 | Returns: 111 | X: 112 | node [batch_size*molecules_num_nodes, node_dim] : nodes states, variable size per batch 113 | edge_feats [batch_size*molecules_num_edges, edge_dim] : edges states, variable size per batch 114 | edge_index [batch_size*molecules_num_edges, 2] : Index edges of the same molecule 115 | node_index [batch_size*molecules_num_nodes, 1] : Index nodes of the same molecule 116 | coupling_index 117 | Y: 118 | targets [N_coupling, 4]: tuple of four targets (scalar_coupling, coupling_gaussrank, coupling_contribs, coupling_type) 119 | Test info: 120 | infor: the ids of the coupling needed for build submission files # N_coupling, 121 | """ 122 | 123 | batch_node, batch_edge, batch_coupling, batch_num_node, batch_num_edge, batch_num_coupling = batch 124 | batch_node = batch_node.reshape(-1, NODE_MAX, 7).float() 125 | batch_edge = batch_edge.reshape(-1, EDGE_MAX, 5) 126 | batch_coupling = batch_coupling.reshape(-1, COUPLING_MAX, 10) 127 | batch_size = batch_node.shape[0] 128 | 129 | #### Create nodes / edges / coupling masks : optimized V1 : TO be optimized 130 | mask = torch.cat([F.pad(torch.ones(i, device='cuda'), (0,NODE_MAX-i)).unsqueeze(0) for i in batch_num_node], dim=0) 131 | mask_coupling = torch.cat([F.pad(torch.ones(i, device='cuda'), (0,COUPLING_MAX-i)).unsqueeze(0) for i in batch_num_coupling], dim=0) 132 | mask_edge = torch.cat([F.pad(torch.ones(i, device='cuda'), (0,EDGE_MAX-i)).unsqueeze(0) for i in batch_num_edge], dim=0) 133 | 134 | #### Build the output X: 135 | # Get effective nodes / edges / coupling values : without padding 136 | node = batch_node[mask.bool()].view(-1, 7) 137 | edge = batch_edge[mask_edge.bool()].view(-1, 5) 138 | coupling = batch_coupling[mask_coupling.bool()].view(-1, 10) 139 | 140 | # node indices track nodes of the same molecule 141 | node_index = mask.nonzero()[:, 0] 142 | 143 | # Get edges feats and indices 144 | edge_feats = edge[:, 2:] 145 | edge_index = edge[:, :2].long() 146 | 147 | # Get coupling index 148 | coupling_index = coupling[:, :2].long() 149 | # batch_coupling_index : track coupling values of the same molecule 150 | batch_coupling_index = mask_coupling.nonzero()[:, 0] 151 | 152 | # offset edge and coupling indices w.r.t to N of nodes in each molecule 153 | offset = torch.cat([torch.zeros(1, device='cuda').long(), batch_num_node[:-1]]).cumsum(0) 154 | #edge 155 | expanded_offset = torch.cat([torch.zeros(num_edges, device='cuda')+offset[mol_index] for mol_index,num_edges in 156 | enumerate(batch_num_edge)], 0) 157 | edge_index = torch.add(edge_index, expanded_offset.unsqueeze(1).long()) 158 | #coupling 159 | expanded_offset=torch.cat([torch.zeros(n_coupling, device='cuda')+offset[mol_index] for mol_index,n_coupling in 160 | enumerate(batch_num_coupling)], 0) 161 | coupling_index = torch.add(coupling_index, expanded_offset.unsqueeze(1).long()) 162 | # type_id 163 | coupling_type = coupling[:, 2].long() 164 | 165 | # new coupling index: atom_0, atom_1, coupling_type, batch_index 166 | coupling_index = torch.cat([coupling_index, coupling_type.view(-1,1) , batch_coupling_index.view(-1, 1)], -1) 167 | 168 | 169 | #### Get Targets 170 | # 4 coupling contirbutions 171 | coupling_contribution = coupling[:, 5:9] 172 | #coupling value 173 | coupling_value = coupling[:, 3] 174 | #gauss_rank 175 | gaussrank = coupling[:, 4] 176 | targets = [coupling_value.float(), gaussrank.float(), coupling_contribution.float(), coupling_type.long()] 177 | 178 | #### ids for inference time 179 | infor = coupling[ : , 9] 180 | 181 | 182 | # We don't use sequence data of the shortest path 183 | coupling_type_sequence, coupling_atomic = [], [] 184 | # mode flag to return additional information for test data 185 | if mode == 'test': 186 | return (node, edge_feats, edge_index, node_index, coupling_index, coupling_type_sequence, coupling_atomic), targets, infor 187 | 188 | return (node, edge_feats, edge_index, node_index, coupling_index, coupling_type_sequence, coupling_atomic), targets 189 | 190 | -------------------------------------------------------------------------------- /champs-scalar-coupling/build_data/create_parquet.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # 4 | # 5 | # This module aims to create a parquet file from .pkl graph files 6 | # 7 | # It also gives the possibility to compute the gaussrank and create 8 | # train/validation .parquet files for each fold with. each line represents 9 | # a molecules and its information in the following order : 10 | # 'molecule_name', 11 | # 'num_node', 'num_edge','num_coupling', 12 | # 'node_dim', 'edge_dim','coupling_dim', 13 | # 'node_0', ...., 'node_NODE_MAX*7' 14 | # 'edge_0', ...., 'edge_EDGE_MAX*5' 15 | # 'coupling_0', ...., 'coupling_COUPLING_MAX*9' 16 | # 'gaussrank_0', ....., 'gaussrank_COUPLING_MAX' 17 | # 18 | # 19 | # 20 | ##################################################################################### 21 | 22 | 23 | import glob 24 | from parallel_process import parallel_process 25 | from data import * 26 | from common import * 27 | from GaussRank import * 28 | 29 | from functools import partial 30 | import pandas as pd 31 | from time import time 32 | 33 | NODE_MAX = 32 34 | EDGE_MAX = 816 35 | COUPLING_MAX = 136 36 | 37 | 38 | def get_one_vector_from_graph(molecule_file): 39 | ''' 40 | - molecule file: path to %molecule_name.pickle 41 | Returns: 42 | Convert the pickled graph to a padded vector with all the molecule information 43 | ''' 44 | molecule_name = molecule_file.split('/')[-1].strip('.pickle') 45 | graph = read_pickle_from_file(molecule_file) 46 | molecule_name = graph.molecule_name 47 | node_feats = np.concatenate(graph.node,-1) 48 | edge_feats = np.concatenate(graph.edge,-1) 49 | edge_feats = np.concatenate([graph.edge_index, edge_feats], -1) 50 | 51 | coupling = np.concatenate([graph.coupling.index, graph.coupling.type.reshape(-1, 1), 52 | graph.coupling.value.reshape(-1,1), graph.coupling.contribution, 53 | graph.coupling.id.reshape(-1,1)], -1) 54 | 55 | num_node, node_dim = node_feats.shape 56 | num_edge, edge_dim = edge_feats.shape 57 | num_coupling, coupling_dim = coupling.shape 58 | 59 | infor = [molecule_name, num_node, num_edge, num_coupling, node_dim, edge_dim, coupling_dim] 60 | return infor, node_feats.reshape(num_node*node_dim), edge_feats.reshape(num_edge*edge_dim), coupling.reshape(num_coupling*coupling_dim) 61 | 62 | def build_general_frame(graph_dir, parquet_dir='/rapids/notebooks/srabhi/champs-2019/input/parquet/'): 63 | ''' 64 | Args: 65 | - graph_dir to use for getting molecule information: 66 | - graph1: one_hot encoding for categorical values + acutal value of scalar coupling constant 67 | - graph2: label encoding for cats + actual scalar coupling 68 | - graph3: one_hot encoding for cats + normalized scalar coupling 69 | - graph4: label encoding for cats + normalized scalar coupling 70 | - parquet_dir: 71 | - output directory where to store the general parquet frame 72 | ''' 73 | 74 | files = glob.glob(graph_dir+'/*.pickle') 75 | tabular_data = parallel_process(files, get_one_vector_from_graph) 76 | 77 | nodes = [] 78 | infos = [] 79 | edges = [] 80 | coupling = [] 81 | for i in tabular_data: 82 | infos.append(i[0]) 83 | nodes.append(i[1]) 84 | edges.append(i[2]) 85 | coupling.append(i[3]) 86 | info_frame, node_frame, edge_frame, coupling_frame = (pd.DataFrame(infos,columns=['molecule_name', 'num_node', 'num_edge', 87 | 'num_coupling', 'node_dim', 'edge_dim', 'coupling_dim']), 88 | pd.DataFrame(nodes), pd.DataFrame(edges), pd.DataFrame(coupling)) 89 | 90 | ### Get a multiple 8 for gpu ops 91 | # pad 29 nodes to node_max 32 : 92 | pad_cols = 21 93 | d = dict.fromkeys([str(i) for i in range(node_frame.shape[1], node_frame.shape[1]+pad_cols)], 0.0) 94 | node_frame = node_frame.assign(**d).fillna(0.0) 95 | 96 | # pad edge_max 812 to 816 97 | pad_cols = 20 98 | d = dict.fromkeys([str(i) for i in range(edge_frame.shape[1], edge_frame.shape[1]+pad_cols)], 0.0) 99 | edge_frame = edge_frame.assign(**d).fillna(0.0) 100 | 101 | # pad coupling_max to 136 102 | pad_cols = 9 103 | d = dict.fromkeys([str(i) for i in range(coupling_frame.shape[1], coupling_frame.shape[1]+pad_cols)], 0.0) 104 | coupling_frame = coupling_frame.assign(**d).fillna(0.0) 105 | 106 | # concat the whole frame 107 | general_frame = pd.concat([info_frame, node_frame, edge_frame, coupling_frame], axis=1) 108 | general_frame = general_frame.fillna(0.0) 109 | 110 | print('Dataframe created for %s molecules' %general_frame.shape[0]) 111 | cols = ['molecule_name', 'num_node', 'num_edge', 'num_coupling', 'node_dim', 'edge_dim', 'coupling_dim'] + \ 112 | ['node_%s'%i for i in range(NODE_MAX*7)] + ['edge_%s'%i for i in range(EDGE_MAX*5)] + ['coupling_%s'%i for i in range(COUPLING_MAX*9)] 113 | general_frame.columns = cols 114 | general_frame.to_parquet(os.path.join(parquet_dir, 'general_frame.parquet')) 115 | 116 | 117 | def build_test_data(data, DATA_DIR = '/rapids/notebooks/srabhi/champs-2019/input'): 118 | #data = gd.read_parquet(DATA_DIR +'/parquet/general_frame.parquet') 119 | csv = 'test' 120 | df = pd.read_csv(DATA_DIR + '/csv/%s.csv'%csv) 121 | id_test = gd.DataFrame() 122 | mol_test = df.molecule_name.unique() 123 | id_test['molecule_name'] = mol_test 124 | test_data = id_test.merge(data, on='molecule_name', how='left') 125 | tmp = pd.DataFrame(np.zeros((45772, 136), dtype=float)) 126 | tmp.columns = ['gaussrank_%s'%i for i in range(136)] 127 | tmp = gd.from_pandas(tmp) 128 | tmp['molecule_name'] = test_data.molecule_name 129 | test = tmp.merge(test_data, on='molecule_name', how='left') 130 | test.to_parquet(DATA_DIR +'/parquet/test_frame.parquet') 131 | 132 | 133 | def build_cv_ranks_parquet(data, fold, DATA_DIR = '/rapids/notebooks/srabhi/champs-2019/input'): 134 | print(fold) 135 | ### Get data 136 | split_train = 'train_split_by_mol_hash.%s.npy'%fold 137 | split_valid = 'valid_split_by_mol_hash.%s.npy'%fold 138 | id_train_ = np.load(DATA_DIR + '/split/%s'%split_train,allow_pickle=True) 139 | id_valid_ = np.load(DATA_DIR + '/split/%s'%split_valid,allow_pickle=True) 140 | df = pd.read_csv(DATA_DIR + '/csv/train.csv') 141 | #data = gd.read_parquet(DATA_DIR+'/parquet/general_frame.parquet') 142 | 143 | train = df[df.molecule_name.isin(id_train_)] 144 | validation = df[df.molecule_name.isin(id_valid_)] 145 | 146 | # Get GaussRank of coupling values 147 | t0 = time() 148 | grm = GaussRankMap() 149 | transformed_training = grm.fit_training(train, reset=True) 150 | transformed_validation = grm.convert_df(validation, from_coupling=True) 151 | validation['transformed_coupling'] = transformed_validation 152 | train['transformed_coupling'] = transformed_training 153 | print('Getting gaussrank transformation for train/validation data took %s seconds' %(time()-t0)) 154 | print(grm.coupling_order) 155 | # Get the rank coupling values at the molecule level and pad coupling rank values to 136 : 156 | validation_gaussrank = validation.groupby('molecule_name').apply(lambda x : x['transformed_coupling'].values) 157 | train_gaussrank = train.groupby('molecule_name').apply(lambda x : x['transformed_coupling'].values) 158 | 159 | val_ranks = pd.DataFrame(validation_gaussrank.tolist()).fillna(0.0) 160 | num_cols = val_ranks.shape[1] 161 | pad_cols = 136 - num_cols 162 | d = dict.fromkeys([str(i) for i in range(num_cols, num_cols+pad_cols)], 0.0) 163 | val_ranks = val_ranks.assign(**d) 164 | val_ranks = val_ranks.astype(float) 165 | val_ranks.columns = ['gaussrank_%s'%i for i in range(136)] 166 | val_ranks['molecule_name'] = validation_gaussrank.index 167 | 168 | train_ranks = pd.DataFrame(train_gaussrank.tolist()).fillna(0.0) 169 | num_cols = train_ranks.shape[1] 170 | pad_cols = 136 - num_cols 171 | d = dict.fromkeys([str(i) for i in range(num_cols, num_cols+pad_cols)], 0.0) 172 | train_ranks = train_ranks.assign(**d) 173 | train_ranks = train_ranks.astype(float) 174 | train_ranks.columns = ['gaussrank_%s'%i for i in range(136)] 175 | train_ranks['molecule_name'] = train_gaussrank.index 176 | 177 | # Merge with node /edge/coupling frame 178 | id_valid = gd.DataFrame() 179 | id_valid['molecule_name'] = id_valid_ 180 | valid_data = id_valid.merge(data, on='molecule_name', how='left') 181 | 182 | id_valid = gd.DataFrame() 183 | id_valid['molecule_name'] = id_valid_ 184 | valid_data = id_valid.merge(data, on='molecule_name', how='left').to_pandas() 185 | validation_frame = pd.merge(valid_data, val_ranks, on='molecule_name', how='left') 186 | 187 | 188 | # Merge with node /edge/coupling frame 189 | id_train= gd.DataFrame() 190 | id_train['molecule_name'] = id_train_ 191 | train_data = id_valid.merge(data, on='molecule_name', how='left') 192 | 193 | id_train = gd.DataFrame() 194 | id_train['molecule_name'] = id_train_ 195 | train_data = id_train.merge(data, on='molecule_name', how='left').to_pandas() 196 | training_frame = pd.merge(train_data, train_ranks, on='molecule_name', how='left') 197 | 198 | # Save parquet files for fold 199 | parquet_dir = DATA_DIR + '/parquet/fold_%s' %fold 200 | if not os.path.exists(parquet_dir): 201 | os.makedirs(parquet_dir) 202 | training_frame.to_parquet(parquet_dir+'/train.parquet') 203 | validation_frame.to_parquet(parquet_dir+'/validation.parquet') 204 | 205 | # save mapping 206 | for i, (type_, frame) in enumerate(zip(grm.coupling_order, grm.training_maps)): 207 | frame.to_csv(parquet_dir+'/mapping_type_%s_order_%s.csv'%(type_, i), index=False) 208 | pass 209 | -------------------------------------------------------------------------------- /RecSys2020Tutorial/00_0_Initial.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# The MIT License (MIT)\n", 10 | "\n", 11 | "# Copyright (c) 2020, NVIDIA CORPORATION.\n", 12 | "\n", 13 | "# Permission is hereby granted, free of charge, to any person obtaining a copy of\n", 14 | "# this software and associated documentation files (the \"Software\"), to deal in\n", 15 | "# the Software without restriction, including without limitation the rights to\n", 16 | "# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\n", 17 | "# the Software, and to permit persons to whom the Software is furnished to do so,\n", 18 | "# subject to the following conditions:\n", 19 | "\n", 20 | "# The above copyright notice and this permission notice shall be included in all\n", 21 | "# copies or substantial portions of the Software.\n", 22 | "\n", 23 | "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", 24 | "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\n", 25 | "# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\n", 26 | "# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\n", 27 | "# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n", 28 | "# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "# Tutorial: Feature Engineering for Recommender Systems\n", 36 | "\n", 37 | "# 0. Initial Data Loading" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "metadata": {}, 43 | "source": [ 44 | "In our tutorial, we use the [eCommerce behavior data from multi category store](https://www.kaggle.com/mkechinov/ecommerce-behavior-data-from-multi-category-store) from [REES46 Marketing Platform](https://rees46.com/) as our dataset. We define our own goal and filter the dataset accordingly. This jupyter notebook provides the code to preprocess the dataset and generate the train, validation and test sets for the remainder of the tutorial.

\n", 45 | "\n", 46 | "\n", 47 | "For our tutorial, we decided that our goal is to predict if a user purchased an item:\n", 48 | "\n", 49 | "
  • Positive: User purchased an item\n", 50 | "
  • Negative: User added an item to the cart, but did not purchase it (in the same session) \n", 51 | "

    \n", 52 | "We split the dataset into train, validation and test set by the timestamp:\n", 53 | "
  • Training: October 2019 - February 2020\n", 54 | "
  • Validation: March 2020\n", 55 | "
  • Test: April 2020\n", 56 | "

    \n", 57 | "We remove AddToCart Events from a session, if in the same session the same item was purchased." 58 | ] 59 | }, 60 | { 61 | "cell_type": "markdown", 62 | "metadata": {}, 63 | "source": [ 64 | "First, download all csv files included in the Google Drive folder." 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 1, 70 | "metadata": {}, 71 | "outputs": [], 72 | "source": [ 73 | "import pandas as pd\n", 74 | "import glob" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 2, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "list_files = glob.glob('./data/*.csv')" 84 | ] 85 | }, 86 | { 87 | "cell_type": "markdown", 88 | "metadata": {}, 89 | "source": [ 90 | "Next, we process a single .csv file and extract/filter the rows" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 3, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "def process_files(file):\n", 100 | " df_tmp = pd.read_csv(file)\n", 101 | " df_tmp['session_purchase'] = df_tmp['user_session'] + '_' + df_tmp['product_id'].astype(str)\n", 102 | " df_purchase = df_tmp[df_tmp['event_type']=='purchase']\n", 103 | " df_cart = df_tmp[df_tmp['event_type']=='cart']\n", 104 | " df_purchase = df_purchase[df_purchase['session_purchase'].isin(df_cart['session_purchase'])]\n", 105 | " df_cart = df_cart[~(df_cart['session_purchase'].isin(df_purchase['session_purchase']))]\n", 106 | " df_cart['target'] = 0\n", 107 | " df_purchase['target'] = 1\n", 108 | " df = pd.concat([df_cart, df_purchase])\n", 109 | " df = df.drop('category_id', axis=1)\n", 110 | " df = df.drop('session_purchase', axis=1)\n", 111 | " df[['cat_0', 'cat_1', 'cat_2', 'cat_3']] = df['category_code'].str.split(\"\\.\", n = 3, expand = True).fillna('NA')\n", 112 | " df['brand'] = df['brand'].fillna('NA')\n", 113 | " df = df.drop('category_code', axis=1)\n", 114 | " df['timestamp'] = pd.to_datetime(df['event_time'].str.replace(' UTC', ''))\n", 115 | " df['ts_hour'] = df['timestamp'].dt.hour\n", 116 | " df['ts_minute'] = df['timestamp'].dt.minute\n", 117 | " df['ts_weekday'] = df['timestamp'].dt.weekday\n", 118 | " df['ts_day'] = df['timestamp'].dt.day\n", 119 | " df['ts_month'] = df['timestamp'].dt.month\n", 120 | " df['ts_year'] = df['timestamp'].dt.year\n", 121 | " df.to_csv('./' + file.replace('../data/', ''), index=False)" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 10, 127 | "metadata": {}, 128 | "outputs": [ 129 | { 130 | "data": { 131 | "text/plain": [ 132 | "['./2019-Dec.csv',\n", 133 | " './2020-Jan.csv',\n", 134 | " './2020-Feb.csv',\n", 135 | " './2019-Oct.csv',\n", 136 | " './2020-Mar.csv',\n", 137 | " './2020-Apr.csv',\n", 138 | " './2019-Nov.csv']" 139 | ] 140 | }, 141 | "execution_count": 10, 142 | "metadata": {}, 143 | "output_type": "execute_result" 144 | } 145 | ], 146 | "source": [ 147 | "list_files" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 4, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "for file in list_files:\n", 157 | " print(file)\n", 158 | " process_files(file)" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "Finally, we load all preprocessed csv files, combine them and create our train, validation and test sets." 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": 13, 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "lp = []\n", 175 | "list_files = glob.glob('./*.csv')" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": 14, 181 | "metadata": {}, 182 | "outputs": [ 183 | { 184 | "name": "stderr", 185 | "output_type": "stream", 186 | "text": [ 187 | "/conda/envs/rapids/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3146: DtypeWarning: Columns (11) have mixed types.Specify dtype option on import or set low_memory=False.\n", 188 | " interactivity=interactivity, compiler=compiler, result=result)\n" 189 | ] 190 | } 191 | ], 192 | "source": [ 193 | "for file in list_files:\n", 194 | " lp.append(pd.read_csv(file))" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 16, 200 | "metadata": {}, 201 | "outputs": [], 202 | "source": [ 203 | "df = pd.concat(lp)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": 17, 209 | "metadata": {}, 210 | "outputs": [ 211 | { 212 | "data": { 213 | "text/plain": [ 214 | "(16695562, 19)" 215 | ] 216 | }, 217 | "execution_count": 17, 218 | "metadata": {}, 219 | "output_type": "execute_result" 220 | } 221 | ], 222 | "source": [ 223 | "df.shape" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": 21, 229 | "metadata": {}, 230 | "outputs": [], 231 | "source": [ 232 | "df_test = df[df['ts_month']==4]\n", 233 | "df_valid = df[df['ts_month']==3]\n", 234 | "df_train = df[(df['ts_month']!=3)&(df['ts_month']!=4)]" 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": 22, 240 | "metadata": {}, 241 | "outputs": [ 242 | { 243 | "data": { 244 | "text/plain": [ 245 | "((11461357, 19), (2461719, 19), (2772486, 19))" 246 | ] 247 | }, 248 | "execution_count": 22, 249 | "metadata": {}, 250 | "output_type": "execute_result" 251 | } 252 | ], 253 | "source": [ 254 | "df_train.shape, df_valid.shape, df_test.shape" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": 25, 260 | "metadata": {}, 261 | "outputs": [], 262 | "source": [ 263 | "df_train.to_parquet('./data/train.parquet', index=False)" 264 | ] 265 | }, 266 | { 267 | "cell_type": "code", 268 | "execution_count": 28, 269 | "metadata": {}, 270 | "outputs": [], 271 | "source": [ 272 | "df_valid.to_parquet('./data/valid.parquet', index=False)" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": 27, 278 | "metadata": {}, 279 | "outputs": [], 280 | "source": [ 281 | "df_test.to_parquet('./data/test.parquet', index=False)" 282 | ] 283 | } 284 | ], 285 | "metadata": { 286 | "kernelspec": { 287 | "display_name": "Python 3", 288 | "language": "python", 289 | "name": "python3" 290 | }, 291 | "language_info": { 292 | "codemirror_mode": { 293 | "name": "ipython", 294 | "version": 3 295 | }, 296 | "file_extension": ".py", 297 | "mimetype": "text/x-python", 298 | "name": "python", 299 | "nbconvert_exporter": "python", 300 | "pygments_lexer": "ipython3", 301 | "version": "3.7.8" 302 | } 303 | }, 304 | "nbformat": 4, 305 | "nbformat_minor": 4 306 | } 307 | --------------------------------------------------------------------------------