├── .gitignore ├── GraphZooPoster.pdf ├── LICENSE ├── README.md ├── _config.yml ├── graphzoo ├── __init__.py ├── config.py ├── dataloader │ ├── __init__.py │ ├── dataloader.py │ └── download.py ├── layers │ ├── __init__.py │ ├── att_layers.py │ ├── hyp_att_layers.py │ ├── hyp_layers.py │ └── layers.py ├── manifolds │ ├── __init__.py │ ├── base.py │ ├── euclidean.py │ ├── hyperboloid.py │ └── poincare.py ├── models │ ├── __init__.py │ ├── base_models.py │ ├── decoders.py │ └── encoders.py ├── optimizers │ ├── __init__.py │ ├── radam.py │ └── rsgd.py ├── trainers │ ├── __init__.py │ └── train.py └── utils │ ├── __init__.py │ ├── eval_utils.py │ ├── math_utils.py │ └── train_utils.py ├── pyproject.toml ├── setup.py └── tutorials ├── linkprediction └── hgcn.ipynb └── nodeclassification └── hgcn.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.log 3 | .ipynb_checkpoints/ 4 | *.DS_Store 5 | graphzoo/log/* -------------------------------------------------------------------------------- /GraphZooPoster.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AnoushkaVyas/GraphZoo/3934f145c25a79388feb3adf02e9f1ea56f6e424/GraphZooPoster.pdf -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Anoushka Vyas 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GraphZoo 2 | > PyTorch version of [GraphZoo](https://github.com/AnoushkaVyas/GraphZoo). 3 | 4 | > Facilitating learning, using, and designing graph processing pipelines/models systematically. 5 | 6 | We present a novel framework GraphZoo, that makes learning, using, and designing graph processing pipelines/models systematic by abstraction over the redundant components. The framework contains a powerful library that supports several hyperbolic manifolds and an easy-to-use modular framework to perform graph processing tasks which aids researchers in different components, namely, (i) reproduce evaluation pipelines of state-of-the-art approaches, (ii) design new hyperbolic or Euclidean graph networks and compare them against the state-of-the art approaches on standard benchmarks, (iii) add custom datasets for evaluation, (iv) add new tasks and evaluation criteria. For more details check out our [paper](https://www2022.thewebconf.org/PaperFiles/54.pdf) and [poster](https://github.com/AnoushkaVyas/GraphZoo/blob/main/GraphZooPoster.pdf). 7 | 8 | ## Installation 9 | 10 | ### Using Github source: 11 | ``` 12 | git clone https://github.com/AnoushkaVyas/GraphZoo.git 13 | cd GraphZoo 14 | python setup.py install 15 | ``` 16 | 17 | ### Using Pypi: 18 | 19 | ``` 20 | pip install graphzoo 21 | ``` 22 | 23 | ## Getting Started in 60 Seconds 24 | 25 | To train a Hyperbolic Graph Convolutional Networks model for node classification task on Cora dataset, make use of GraphZoo APIs customized loss functions and evaluation metrics for this task. 26 | 27 | Download data: 28 | 29 | ```python 30 | import graphzoo as gz 31 | import torch 32 | from graphzoo.config import parser 33 | 34 | params = parser.parse_args(args=[]) 35 | params.download_folder ='./data/' 36 | gz.dataloader.download_and_extract(params) 37 | ``` 38 | Prepare input data: 39 | 40 | ```python 41 | params.dataset='cora' 42 | params.task='nc' 43 | params.datapath='data/cora' 44 | data = gz.dataloader.DataLoader(params) 45 | ``` 46 | 47 | Initialize the model and fine-tune the hyperparameters: 48 | 49 | ```python 50 | params.model='HGCN' 51 | params.manifold='PoincareBall' 52 | params.dim=128 53 | model= gz.models.NCModel(params) 54 | ``` 55 | 56 | `Trainer` is used to control the training flow: 57 | 58 | ```python 59 | optimizer = gz.optimizers.RiemannianAdam(params=model.parameters(), lr=params.lr, weight_decay=params.weight_decay) 60 | trainer=gz.trainers.Trainer(params,model,optimizer,data) 61 | trainer.run() 62 | trainer.evaluate() 63 | ``` 64 | ## Getting Started Using Command Line 65 | To train a Hyperbolic Graph Convolutional Networks model for node classification task on Cora dataset using command line: 66 | 67 | ```python 68 | cd GraphZoo 69 | python graphzoo/trainers/train.py --task nc --dataset cora --datapath ./data/cora --download_folder ./data/ --model HGCN --lr 0.01 --dim 16 --num-layers 2 --act relu --bias 1 --dropout 0.5 --weight-decay 0.001 --manifold PoincareBall --log-freq 5 --cuda 0 --c None 70 | ``` 71 | ## Customizing Input Arguments 72 | 73 | Various flags can be modified in the `graphzoo.config` module by the user. 74 | 75 | ### Download Data 76 | 77 | ```python 78 | """ 79 | GraphZoo Download and Extract 80 | 81 | Input Parameters 82 | ---------- 83 | 'dataset': (None, 'which dataset to use, can be any of [cora, pubmed, airport, disease_nc, disease_lp] (type: str)') 84 | 'download_folder': (None, 'path to the folder for raw data (type: str)') 85 | 86 | API Input Parameters 87 | ---------- 88 | args: list of above defined input parameters from `graphzoo.config` 89 | 90 | """ 91 | 92 | ``` 93 | 94 | ### Preparing Data 95 | 96 | ```python 97 | """ 98 | GraphZoo Dataloader 99 | 100 | Input Parameters 101 | ---------- 102 | 'dataset': ('cora', 'which dataset to use, can be any of [cora, pubmed, airport, disease_nc, disease_lp, ppi, citeseer, webkb] (type: str)'), 103 | 'datapath': (None, 'path to raw data (type: str)'), 104 | 'val-prop': (0.05, 'proportion of validation edges for link prediction (type:float)'), 105 | 'test-prop': (0.1, 'proportion of test edges for link prediction (type: float)'), 106 | 'use-feats': (1, 'whether to use node features (1) or not (0 in case of Shallow methods) (type: int)'), 107 | 'normalize-feats': (1, 'whether to normalize input node features (1) or not (0) (type: int)'), 108 | 'normalize-adj': (1, 'whether to row-normalize the adjacency matrix (1) or not(0) (type: int)'), 109 | 'split-seed': (1234, 'seed for data splits (train/test/val) (type: int)') 110 | 111 | API Input Parameters 112 | ---------- 113 | args: list of above defined input parameters from `graphzoo.config` 114 | 115 | """ 116 | 117 | ``` 118 | 119 | ### Models 120 | 121 | ```python 122 | """ 123 | Base model for graph embedding tasks 124 | 125 | Input Parameters 126 | ---------- 127 | 'task': ('nc', 'which tasks to train on, can be any of [lp, nc] (type: str)'), 128 | 'model': ('HGCN', 'which encoder to use, can be any of [Shallow, MLP, HNN, GCN, GAT, HGCN,HGAT] (type: str)'), 129 | 'dim': (128, 'embedding dimension (type: int)'), 130 | 'manifold': ('PoincareBall', 'which manifold to use, can be any of [Euclidean, Hyperboloid, PoincareBall] (type: str)'), 131 | 'c': (1.0, 'hyperbolic radius, set to None for trainable curvature (type: float)'), 132 | 'r': (2.0, 'fermi-dirac decoder parameter for lp (type: float)'), 133 | 't': (1.0, 'fermi-dirac decoder parameter for lp (type: float)'), 134 | 'pretrained-embeddings': (None, 'path to pretrained embeddings (.npy file) for Shallow node classification (type: str)'), 135 | 'num-layers': (2, 'number of hidden layers in encoder (type: int)'), 136 | 'bias': (1, 'whether to use bias (1) or not (0) (type: int)'), 137 | 'act': ('relu', 'which activation function to use or None for no activation (type: str)'), 138 | 'n-heads': (4, 'number of attention heads for graph attention networks, must be a divisor dim (type: int)'), 139 | 'alpha': (0.2, 'alpha for leakyrelu in graph attention networks (type: float)'), 140 | 'use-att': (0, 'whether to use hyperbolic attention (1) or not (0) (type: int)'), 141 | 'local-agg': (0, 'whether to local tangent space aggregation (1) or not (0) (type: int)') 142 | 143 | API Input Parameters 144 | ---------- 145 | args: list of above defined input parameters from `graphzoo.config` 146 | """ 147 | ``` 148 | 149 | ### Trainer 150 | 151 | ```python 152 | """ 153 | GraphZoo Trainer 154 | 155 | Input Parameters 156 | ---------- 157 | 'lr': (0.05, 'initial learning rate (type: float)'), 158 | 'dropout': (0.0, 'dropout probability (type: float)'), 159 | 'cuda': (-1, 'which cuda device to use or -1 for cpu training (type: int)'), 160 | 'repeat': (10, 'number of times to repeat the experiment (type: int)'), 161 | 'optimizer': ('Adam', 'which optimizer to use, can be any of [Adam, RiemannianAdam, RiemannianSGD] (type: str)'), 162 | 'epochs': (5000, 'maximum number of epochs to train for (type:int)'), 163 | 'weight-decay': (0.0, 'l2 regularization strength (type: float)'), 164 | 'momentum': (0.999, 'momentum in optimizer (type: float)'), 165 | 'patience': (100, 'patience for early stopping (type: int)'), 166 | 'seed': (1234, 'seed for training (type: int)'), 167 | 'log-freq': (5, 'how often to compute print train/val metrics in epochs (type: int)'), 168 | 'eval-freq': (1, 'how often to compute val metrics in epochs (type: int)'), 169 | 'save': (0, '1 to save model and logs and 0 otherwise (type: int)'), 170 | 'save-dir': (None, 'path to save training logs and model weights (type: str)'), 171 | 'lr-reduce-freq': (None, 'reduce lr every lr-reduce-freq or None to keep lr constant (type: int)'), 172 | 'gamma': (0.5, 'gamma for lr scheduler (type: float)'), 173 | 'grad-clip': (None, 'max norm for gradient clipping, or None for no gradient clipping (type: float)'), 174 | 'min-epochs': (100, 'do not early stop before min-epochs (type: int)'), 175 | 'betas': ((0.9, 0.999), 'coefficients used for computing running averages of gradient and its square (type: Tuple[float, float])'), 176 | 'eps': (1e-8, 'term added to the denominator to improve numerical stability (type: float)'), 177 | 'amsgrad': (False, 'whether to use the AMSGrad variant of this algorithm from the paper `On the Convergence of Adam and Beyond` (type: bool)'), 178 | 'stabilize': (None, 'stabilize parameters if they are off-manifold due to numerical reasons every ``stabilize`` steps (type: int)'), 179 | 'dampening': (0,'dampening for momentum (type: float)'), 180 | 'nesterov': (False,'enables Nesterov momentum (type: bool)') 181 | 182 | API Input Parameters 183 | ---------- 184 | args: list of above defined input parameters from `graphzoo.config` 185 | optimizer: a :class:`optim.Optimizer` instance 186 | model: a :class:`BaseModel` instance 187 | 188 | """ 189 | ``` 190 | 191 | ## Customizing the Framework 192 | 193 | ### Adding Custom Dataset 194 | 195 | 1. Add the dataset files in the `data` folder of the source code. 196 | 2. To run this code on new datasets, please add corresponding data processing and loading in `load_data_nc` and `load_data_lp` functions in `dataloader/dataloader.py` in the source code. 197 | 198 | Output format for node classification dataloader is: 199 | 200 | ``` 201 | data = {'adj_train': adj, 'features': features, 'labels': labels, 'idx_train': idx_train, 'idx_val': idx_val, 'idx_test': idx_test} 202 | ``` 203 | Output format for link prediction dataloader is: 204 | 205 | ``` 206 | data = {'adj_train': adj_train, 'features': features, ‘train_edges’: train_edges, ‘train_edges_false’: train_edges_false, ‘val_edges’: val_edges, ‘val_edges_false’: val_edges_false, ‘test_edges’: test_edges, ‘test_edges_false’: test_edges_false, 'adj_train_norm':adj_train_norm} 207 | ``` 208 | 209 | ### Adding Custom Layers 210 | 211 | 1. Attention layers can be added in `layers/att_layers.py` in the source code by adding a class in the file. 212 | 2. Hyperbolic layers can be added in `layers/hyp_layers.py` in the source code by adding a class in the file. 213 | 3. Other layers like a single GCN layer can be added in `layers/layers.py` in the source code by adding a class in the file. 214 | 215 | ### Adding Custom Models 216 | 217 | 1. After adding custom layers, custom models can be added in `models/encoders.py` in the source code by adding a class in the file. 218 | 2. After adding custom layers, custom decoders to calculate the final output can be added in `models/decoders.py` in the source code by adding a class in the file. 219 | 220 | ## Datasets 221 | 222 | The included datasets are the following and they need to be downloaded from the [link](https://github.com/AnoushkaVyas/GraphZoo/releases/tag/Datasets): 223 | 1. Cora 224 | 2. Pubmed 225 | 3. Disease 226 | 4. Airport 227 | 5. PPI 228 | 6. Webkb 229 | 7. Citeseer 230 | 231 | ## Models 232 | 233 | ### Shallow Methods 234 | 1. Shallow Euclidean 235 | 2. Shallow Hyperbolic 236 | 237 | ### Neural Network Methods 238 | 1. Multi-Layer Perceptron (MLP) 239 | 2. Hyperbolic Neural Networks (HNN) 240 | 241 | ### Graph Neural Network Methods 242 | 1. Graph Convolutional Neural Networks (GCN) 243 | 2. Graph Attention Networks (GAT) 244 | 3. Hyperbolic Graph Convolutions (HGCN) 245 | 4. Hyperbolic Graph Attention Networks (HGAT) 246 | 247 | 248 | ## Package References 249 | 250 | [Tutorials](https://github.com/AnoushkaVyas/GraphZoo/tree/main/tutorials) 251 | 252 | [Documentation](https://github.com/AnoushkaVyas/GraphZoo#customizing-input-arguments) 253 | 254 | ## Code References 255 | 256 | Some of the code was forked from the following repositories. 257 | - [hgcn](https://github.com/HazyResearch/hgcn) 258 | - [hgat](https://github.com/oom-debugger/hyperbolic-layers/tree/8ead8b713fee28f830dd8b33a1468082e0eeae50/py_hnn) 259 | - [geoopt](https://github.com/geoopt/geoopt) 260 | - [pygcn](https://github.com/tkipf/pygcn/tree/master/pygcn) 261 | - [gae](https://github.com/tkipf/gae/tree/master/gae) 262 | - [hyperbolic-image-embeddings](https://github.com/leymir/hyperbolic-image-embeddings) 263 | - [pyGAT](https://github.com/Diego999/pyGAT) 264 | - [poincare-embeddings](https://github.com/facebookresearch/poincare-embeddings) 265 | 266 | ## Citation 267 | If you use GraphZooZoo in your research, please use the following BibTex entry. 268 | 269 | ``` 270 | @inproceedings{10.1145/3487553.3524241, 271 | author = {Vyas, Anoushka and Choudhary, Nurendra and Khatir, Mehrdad and Reddy, Chandan K.}, 272 | title = {GraphZoo: A Development Toolkit for Graph Neural Networks with Hyperbolic Geometries}, 273 | year = {2022}, 274 | isbn = {9781450391306}, 275 | publisher = {Association for Computing Machinery}, 276 | address = {New York, NY, USA}, 277 | url = {https://doi.org/10.1145/3487553.3524241}, 278 | doi = {10.1145/3487553.3524241}, 279 | booktitle = {Companion Proceedings of the Web Conference 2022}, 280 | keywords = {graph learning, graph neural network, hyperbolic models, software}, 281 | location = {Lyon, France}, 282 | series = {WWW '22} 283 | } 284 | ``` 285 | 286 | 287 | ## License 288 | [MIT](https://opensource.org/licenses/MIT) 289 | 290 | Copyright (c) 2022 Anoushka Vyas 291 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-modernist -------------------------------------------------------------------------------- /graphzoo/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | from . import dataloader 4 | from . import layers 5 | from . import manifolds 6 | from . import trainers 7 | from . import optimizers 8 | from . import utils 9 | from . import models 10 | -------------------------------------------------------------------------------- /graphzoo/config.py: -------------------------------------------------------------------------------- 1 | """Input parameters for the library""" 2 | import argparse 3 | from graphzoo.utils.train_utils import add_flags_from_config 4 | 5 | config_args = { 6 | 'training_config': { 7 | 'lr': (0.05, 'initial learning rate (type: float)'), 8 | 'dropout': (0.0, 'dropout probability (type: float)'), 9 | 'cuda': (-1, 'which cuda device to use or -1 for cpu training (type: int)'), 10 | 'repeat': (10, 'number of times to repeat the experiment (type: int)'), 11 | 'optimizer': ('Adam', 'which optimizer to use, can be any of [Adam, RiemannianAdam, RiemannianSGD] (type: str)'), 12 | 'epochs': (5000, 'maximum number of epochs to train for (type:int)'), 13 | 'weight-decay': (0.0, 'l2 regularization strength (type: float)'), 14 | 'momentum': (0.999, 'momentum in optimizer (type: float)'), 15 | 'patience': (100, 'patience for early stopping (type: int)'), 16 | 'seed': (1234, 'seed for training (type: int)'), 17 | 'log-freq': (5, 'how often to compute print train/val metrics in epochs (type: int)'), 18 | 'eval-freq': (1, 'how often to compute val metrics in epochs (type: int)'), 19 | 'save': (0, '1 to save model and logs and 0 otherwise (type: int)'), 20 | 'save-dir': (None, 'path to save training logs and model weights (type: str)'), 21 | 'lr-reduce-freq': (None, 'reduce lr every lr-reduce-freq or None to keep lr constant (type: int)'), 22 | 'gamma': (0.5, 'gamma for lr scheduler (type: float)'), 23 | 'grad-clip': (None, 'max norm for gradient clipping, or None for no gradient clipping (type: float)'), 24 | 'min-epochs': (100, 'do not early stop before min-epochs (type: int)'), 25 | 'betas': ((0.9, 0.999), 'coefficients used for computing running averages of gradient and its square (type: Tuple[float, float])'), 26 | 'eps': (1e-8, 'term added to the denominator to improve numerical stability (type: float)'), 27 | 'amsgrad': (False, 'whether to use the AMSGrad variant of this algorithm from the paper `On the Convergence of Adam and Beyond` (type: bool)'), 28 | 'stabilize': (None, 'stabilize parameters if they are off-manifold due to numerical reasons every ``stabilize`` steps (type: int)'), 29 | 'dampening': (0,'dampening for momentum (type: float)'), 30 | 'nesterov': (False,'enables Nesterov momentum (type: bool)') 31 | }, 32 | 'model_config': { 33 | 'task': ('nc', 'which tasks to train on, can be any of [lp, nc] (type: str)'), 34 | 'model': ('HGCN', 'which encoder to use, can be any of [Shallow, MLP, HNN, GCN, GAT, HGCN,HGAT] (type: str)'), 35 | 'dim': (128, 'embedding dimension (type: int)'), 36 | 'manifold': ('PoincareBall', 'which manifold to use, can be any of [Euclidean, Hyperboloid, PoincareBall] (type: str)'), 37 | 'c': (1.0, 'hyperbolic radius, set to None for trainable curvature (type: float)'), 38 | 'r': (2.0, 'fermi-dirac decoder parameter for lp (type: float)'), 39 | 't': (1.0, 'fermi-dirac decoder parameter for lp (type: float)'), 40 | 'pretrained-embeddings': (None, 'path to pretrained embeddings (.npy file) for Shallow node classification (type: str)'), 41 | 'num-layers': (2, 'number of hidden layers in encoder (type: int)'), 42 | 'bias': (1, 'whether to use bias (1) or not (0) (type: int)'), 43 | 'act': ('relu', 'which activation function to use or None for no activation (type: str)'), 44 | 'n-heads': (4, 'number of attention heads for graph attention networks, must be a divisor dim (type: int)'), 45 | 'alpha': (0.2, 'alpha for leakyrelu in graph attention networks (type: float)'), 46 | 'use-att': (0, 'whether to use hyperbolic attention (1) or not (0) (type: int)'), 47 | 'local-agg': (0, 'whether to local tangent space aggregation (1) or not (0) (type: int)') 48 | }, 49 | 'data_config': { 50 | 'dataset': ('cora', 'which dataset to use, can be any of [cora, pubmed, airport, disease_nc, disease_lp, ppi, citeseer, webkb] (type: str)'), 51 | 'datapath': (None, 'path to raw data (type: str)'), 52 | 'download_folder': (None, 'path to the folder for raw data (type: str)'), 53 | 'val-prop': (0.05, 'proportion of validation edges for link prediction (type:float)'), 54 | 'test-prop': (0.1, 'proportion of test edges for link prediction (type: float)'), 55 | 'use-feats': (1, 'whether to use node features (1) or not (0 in case of Shallow methods) (type: int)'), 56 | 'normalize-feats': (1, 'whether to normalize input node features (1) or not (0) (type: int)'), 57 | 'normalize-adj': (1, 'whether to row-normalize the adjacency matrix (1) or not(0) (type: int)'), 58 | 'split-seed': (1234, 'seed for data splits (train/test/val) (type: int)') 59 | } 60 | } 61 | 62 | parser = argparse.ArgumentParser() 63 | for _, config_dict in config_args.items(): 64 | parser = add_flags_from_config(parser, config_dict) 65 | -------------------------------------------------------------------------------- /graphzoo/dataloader/__init__.py: -------------------------------------------------------------------------------- 1 | from .dataloader import * 2 | from .download import * -------------------------------------------------------------------------------- /graphzoo/dataloader/dataloader.py: -------------------------------------------------------------------------------- 1 | """ 2 | Data utils functions for pre-processing and data loading 3 | """ 4 | import os 5 | import pickle as pkl 6 | import sys 7 | import networkx as nx 8 | import numpy as np 9 | import scipy.sparse as sp 10 | import torch 11 | 12 | def DataLoader(args): 13 | 14 | """ 15 | GraphZoo Dataloader 16 | 17 | Input Parameters 18 | ---------- 19 | 'dataset': ('cora', 'which dataset to use, can be any of [cora, pubmed, airport, disease_nc, disease_lp, ppi, citeseer, webkb] (type: str)'), 20 | 'datapath': (None, 'path to raw data (type: str)'), 21 | 'val-prop': (0.05, 'proportion of validation edges for link prediction (type:float)'), 22 | 'test-prop': (0.1, 'proportion of test edges for link prediction (type: float)'), 23 | 'use-feats': (1, 'whether to use node features (1) or not (0 in case of Shallow methods) (type: int)'), 24 | 'normalize-feats': (1, 'whether to normalize input node features (1) or not (0) (type: int)'), 25 | 'normalize-adj': (1, 'whether to row-normalize the adjacency matrix (1) or not(0) (type: int)'), 26 | 'split-seed': (1234, 'seed for data splits (train/test/val) (type: int)') 27 | 28 | API Input Parameters 29 | ---------- 30 | args: list of above defined input parameters from `graphzoo.config` 31 | 32 | """ 33 | if args.task == 'nc': 34 | data = load_data_nc(args.dataset, args.use_feats, args.datapath, args.split_seed) 35 | args.n_classes = int(data['labels'].max() + 1) 36 | else: 37 | data = load_data_lp(args.dataset, args.use_feats, args.datapath) 38 | adj = data['adj_train'] 39 | adj_train, train_edges, train_edges_false, val_edges, val_edges_false, test_edges, test_edges_false = mask_edges( 40 | adj, args.val_prop, args.test_prop, args.split_seed 41 | ) 42 | data['adj_train'] = adj_train 43 | data['train_edges'], data['train_edges_false'] = train_edges, train_edges_false 44 | data['val_edges'], data['val_edges_false'] = val_edges, val_edges_false 45 | data['test_edges'], data['test_edges_false'] = test_edges, test_edges_false 46 | 47 | data['adj_train_norm'], data['features'] = process( 48 | data['adj_train'], data['features'], args.normalize_adj,args.normalize_feats 49 | ) 50 | if args.dataset == 'airport': 51 | data['features'] = augment(data['adj_train'], data['features']) 52 | 53 | args.n_nodes, args.feat_dim = data['features'].shape 54 | return data 55 | 56 | # ############### LINK PREDICTION DATA LOADERS #################################### 57 | 58 | 59 | def load_data_lp(dataset, use_feats, data_path): 60 | if dataset in ['cora', 'citeseer', 'pubmed']: 61 | adj, features = load_citation_data(dataset, use_feats, data_path)[:2] 62 | elif dataset == 'ppi': 63 | adj, features = load_ppi_data(data_path)[:2] 64 | elif dataset == 'webkb': 65 | adj, features = load_webkb_data(dataset,data_path)[:2] 66 | elif dataset == 'disease_lp': 67 | adj, features = load_synthetic_data(dataset, use_feats, data_path)[:2] 68 | elif dataset == 'airport': 69 | adj, features = load_data_airport(dataset, data_path, return_label=False) 70 | else: 71 | raise FileNotFoundError('Dataset {} is not supported.'.format(dataset)) 72 | data = {'adj_train': adj, 'features': features} 73 | return data 74 | 75 | 76 | # ############### NODE CLASSIFICATION DATA LOADERS #################################### 77 | 78 | 79 | def load_data_nc(dataset, use_feats, data_path, split_seed): 80 | if dataset in ['cora', 'citeseer', 'pubmed']: 81 | adj, features, labels, idx_train, idx_val, idx_test =load_citation_data( 82 | dataset, use_feats, data_path) 83 | elif dataset == 'ppi': 84 | adj, features, labels = load_ppi_data(data_path) 85 | val_prop, test_prop = 0.15, 0.15 86 | idx_val, idx_test, idx_train = split_data(labels, val_prop, test_prop, seed=split_seed) 87 | elif dataset == 'webkb': 88 | adj, features, labels = load_webkb_data(dataset,data_path) 89 | val_prop, test_prop = 0.15, 0.15 90 | idx_val, idx_test, idx_train = split_data(labels, val_prop, test_prop, seed=split_seed) 91 | else: 92 | if dataset == 'disease_nc': 93 | adj, features, labels = load_synthetic_data(dataset, use_feats, data_path) 94 | val_prop, test_prop = 0.10, 0.60 95 | elif dataset == 'airport': 96 | adj, features, labels = load_data_airport(dataset, data_path, return_label=True) 97 | val_prop, test_prop = 0.15, 0.15 98 | else: 99 | raise FileNotFoundError('Dataset {} is not supported.'.format(dataset)) 100 | idx_val, idx_test, idx_train = split_data(labels, val_prop, test_prop, seed=split_seed) 101 | 102 | labels = torch.LongTensor(labels) 103 | data = {'adj_train': adj, 'features': features, 'labels': labels, 'idx_train': idx_train, 'idx_val': idx_val, 'idx_test': idx_test} 104 | return data 105 | 106 | # ############### FEATURES PROCESSING #################################### 107 | 108 | 109 | def process(adj, features, normalize_adj, normalize_feats): 110 | if sp.isspmatrix(features): 111 | features = np.array(features.todense()) 112 | if normalize_feats: 113 | features = normalize(features) 114 | features = torch.Tensor(features) 115 | if normalize_adj: 116 | adj = normalize(adj + sp.eye(adj.shape[0])) 117 | adj = sparse_mx_to_torch_sparse_tensor(adj) 118 | return adj, features 119 | 120 | 121 | def normalize(mx): 122 | """Row-normalize sparse matrix.""" 123 | rowsum = np.array(mx.sum(1)) 124 | r_inv = np.power(rowsum, -1).flatten() 125 | r_inv[np.isinf(r_inv)] = 0. 126 | r_mat_inv = sp.diags(r_inv) 127 | mx = r_mat_inv.dot(mx) 128 | return mx 129 | 130 | 131 | def sparse_mx_to_torch_sparse_tensor(sparse_mx): 132 | """Convert a scipy sparse matrix to a torch sparse tensor.""" 133 | sparse_mx = sparse_mx.tocoo() 134 | indices = torch.from_numpy( 135 | np.vstack((sparse_mx.row, sparse_mx.col)).astype(np.int64) 136 | ) 137 | values = torch.Tensor(sparse_mx.data) 138 | shape = torch.Size(sparse_mx.shape) 139 | return torch.sparse.FloatTensor(indices, values, shape) 140 | 141 | 142 | def augment(adj, features, normalize_feats=True): 143 | deg = np.squeeze(np.sum(adj, axis=0).astype(int)) 144 | deg[deg > 5] = 5 145 | deg_onehot = torch.tensor(np.eye(6)[deg], dtype=torch.float).squeeze() 146 | const_f = torch.ones(features.size(0), 1) 147 | features = torch.cat((features, deg_onehot, const_f), dim=1) 148 | return features 149 | 150 | def encode_onehot(labels): 151 | # The classes must be sorted before encoding to enable static class encoding. 152 | # In other words, make sure the first class always maps to index 0. 153 | classes = sorted(list(set(labels))) 154 | classes_dict = {c: np.identity(len(classes))[i, :] for i, c in enumerate(classes)} 155 | labels_onehot = list(map(classes_dict.get, labels))# np.array(, dtype=np.int32) 156 | labels_index = list(list(i).index(1) for i in labels_onehot) 157 | return np.array(labels_index, dtype=np.int32)# labels_onehot 158 | 159 | 160 | def normalize_adj(mx): 161 | """Row-normalize sparse matrix""" 162 | rowsum = np.array(mx.sum(1)) 163 | r_inv_sqrt = np.power(rowsum, -0.5).flatten() 164 | r_inv_sqrt[np.isinf(r_inv_sqrt)] = 0. 165 | r_mat_inv_sqrt = sp.diags(r_inv_sqrt) 166 | return mx.dot(r_mat_inv_sqrt).transpose().dot(r_mat_inv_sqrt) 167 | 168 | 169 | def normalize_features(mx): 170 | """Row-normalize sparse matrix""" 171 | rowsum = np.array(mx.sum(1)) 172 | r_inv = np.power(rowsum, -1).flatten() 173 | r_inv[np.isinf(r_inv)] = 0. 174 | r_mat_inv = sp.diags(r_inv) 175 | mx = r_mat_inv.dot(mx) 176 | return mx 177 | 178 | # ############### DATA SPLITS ##################################################### 179 | 180 | def mask_edges(adj, val_prop, test_prop, seed): 181 | np.random.seed(seed) # get tp edges 182 | x, y = sp.triu(adj).nonzero() 183 | pos_edges = np.array(list(zip(x, y))) 184 | np.random.shuffle(pos_edges) 185 | # get tn edges 186 | x, y = sp.triu(sp.csr_matrix(1. - adj.toarray())).nonzero() 187 | neg_edges = np.array(list(zip(x, y))) 188 | np.random.shuffle(neg_edges) 189 | 190 | m_pos = len(pos_edges) 191 | n_val = int(m_pos * val_prop) 192 | n_test = int(m_pos * test_prop) 193 | val_edges, test_edges, train_edges = pos_edges[:n_val], pos_edges[n_val:n_test + n_val], pos_edges[n_test + n_val:] 194 | val_edges_false, test_edges_false = neg_edges[:n_val], neg_edges[n_val:n_test + n_val] 195 | train_edges_false = np.concatenate([neg_edges, val_edges, test_edges], axis=0) 196 | adj_train = sp.csr_matrix((np.ones(train_edges.shape[0]), (train_edges[:, 0], train_edges[:, 1])), shape=adj.shape) 197 | adj_train = adj_train + adj_train.T 198 | return adj_train, torch.LongTensor(train_edges), torch.LongTensor(train_edges_false), torch.LongTensor(val_edges), \ 199 | torch.LongTensor(val_edges_false), torch.LongTensor(test_edges), torch.LongTensor( 200 | test_edges_false) 201 | 202 | 203 | def split_data(labels, val_prop, test_prop, seed): 204 | np.random.seed(seed) 205 | nb_nodes = labels.shape[0] 206 | all_idx = np.arange(nb_nodes) 207 | pos_idx = labels.nonzero()[0] 208 | neg_idx = (1. - labels).nonzero()[0] 209 | np.random.shuffle(pos_idx) 210 | np.random.shuffle(neg_idx) 211 | pos_idx = pos_idx.tolist() 212 | neg_idx = neg_idx.tolist() 213 | nb_pos_neg = min(len(pos_idx), len(neg_idx)) 214 | nb_val = round(val_prop * nb_pos_neg) 215 | nb_test = round(test_prop * nb_pos_neg) 216 | idx_val_pos, idx_test_pos, idx_train_pos = pos_idx[:nb_val], pos_idx[nb_val:nb_val + nb_test], pos_idx[ 217 | nb_val + nb_test:] 218 | idx_val_neg, idx_test_neg, idx_train_neg = neg_idx[:nb_val], neg_idx[nb_val:nb_val + nb_test], neg_idx[ 219 | nb_val + nb_test:] 220 | return idx_val_pos + idx_val_neg, idx_test_pos + idx_test_neg, idx_train_pos + idx_train_neg 221 | 222 | 223 | # ############### DATASETS #################################### 224 | 225 | def load_citation_data(dataset_str, use_feats, data_path): 226 | names = ['x', 'y', 'tx', 'ty', 'allx', 'ally', 'graph'] 227 | objects = [] 228 | for i in range(len(names)): 229 | with open(os.path.join(data_path, "ind.{}.{}".format(dataset_str, names[i])), 'rb') as f: 230 | if sys.version_info > (3, 0): 231 | objects.append(pkl.load(f, encoding='latin1')) 232 | else: 233 | objects.append(pkl.load(f)) 234 | 235 | x, y, tx, ty, allx, ally, graph = tuple(objects) 236 | test_idx_reorder = parse_index_file(os.path.join(data_path, "ind.{}.test.index".format(dataset_str))) 237 | test_idx_range = np.sort(test_idx_reorder) 238 | 239 | if dataset_str == 'citeseer': 240 | # Fix citeseer dataset (there are some isolated nodes in the graph) 241 | # Find isolated nodes, add them as zero-vecs into the right position 242 | test_idx_range_full = range(min(test_idx_reorder), max(test_idx_reorder)+1) 243 | tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1])) 244 | tx_extended[test_idx_range-min(test_idx_range), :] = tx 245 | tx = tx_extended 246 | ty_extended = np.zeros((len(test_idx_range_full), y.shape[1])) 247 | ty_extended[test_idx_range-min(test_idx_range), :] = ty 248 | ty = ty_extended 249 | 250 | features = sp.vstack((allx, tx)).tolil() 251 | features[test_idx_reorder, :] = features[test_idx_range, :] 252 | 253 | labels = np.vstack((ally, ty)) 254 | labels[test_idx_reorder, :] = labels[test_idx_range, :] 255 | labels = np.argmax(labels, 1) 256 | 257 | idx_test = test_idx_range.tolist() 258 | idx_train = list(range(len(y))) 259 | idx_val = range(len(y), len(y) + 500) 260 | 261 | adj = nx.adjacency_matrix(nx.from_dict_of_lists(graph)) 262 | if not use_feats: 263 | features = sp.eye(adj.shape[0]) 264 | return adj, features, labels, idx_train, idx_val, idx_test 265 | 266 | def parse_index_file(filename): 267 | index = [] 268 | for line in open(filename): 269 | index.append(int(line.strip())) 270 | return index 271 | 272 | def bin_feat(feat, bins): 273 | digitized = np.digitize(feat, bins) 274 | return digitized - digitized.min() 275 | 276 | def load_synthetic_data(dataset_str, use_feats, data_path): 277 | object_to_idx = {} 278 | idx_counter = 0 279 | edges = [] 280 | with open(os.path.join(data_path, "{}.edges.csv".format(dataset_str)), 'r') as f: 281 | all_edges = f.readlines() 282 | for line in all_edges: 283 | n1, n2 = line.rstrip().split(',') 284 | if n1 in object_to_idx: 285 | i = object_to_idx[n1] 286 | else: 287 | i = idx_counter 288 | object_to_idx[n1] = i 289 | idx_counter += 1 290 | if n2 in object_to_idx: 291 | j = object_to_idx[n2] 292 | else: 293 | j = idx_counter 294 | object_to_idx[n2] = j 295 | idx_counter += 1 296 | edges.append((i, j)) 297 | adj = np.zeros((len(object_to_idx), len(object_to_idx))) 298 | for i, j in edges: 299 | adj[i, j] = 1. # comment this line for directed adjacency matrix 300 | adj[j, i] = 1. 301 | if use_feats: 302 | features = sp.load_npz(os.path.join(data_path, "{}.feats.npz".format(dataset_str))) 303 | else: 304 | features = sp.eye(adj.shape[0]) 305 | labels = np.load(os.path.join(data_path, "{}.labels.npy".format(dataset_str))) 306 | return sp.csr_matrix(adj), features, labels 307 | 308 | 309 | def load_data_airport(dataset_str, data_path, return_label=False): 310 | graph = pkl.load(open(os.path.join(data_path, dataset_str + '.p'), 'rb')) 311 | adj = nx.adjacency_matrix(graph) 312 | features = np.array([graph.nodes[u]['feat'] for u in graph.nodes()]) 313 | if return_label: 314 | label_idx = 4 315 | labels = features[:, label_idx] 316 | features = features[:, :label_idx] 317 | labels = bin_feat(labels, bins=[7.0/7, 8.0/7, 9.0/7]) 318 | return sp.csr_matrix(adj), features, labels 319 | else: 320 | return sp.csr_matrix(adj), features 321 | 322 | def load_ppi_data(data_path): 323 | features_ppi=np.load(data_path+"/features.npy") 324 | edges=np.loadtxt(data_path+"/edges.txt") 325 | labels = np.loadtxt(data_path+"/node2label.txt",delimiter=" ") 326 | labels = labels[:,1] 327 | adj = np.zeros((len(features_ppi), len(features_ppi))) 328 | for item in edges: 329 | adj[int(list(item)[0]), int(list(item)[1])] = 1. 330 | adj[int(list(item)[1]), int(list(item)[0])] = 1. 331 | graph = nx.from_numpy_matrix(adj) 332 | global G 333 | G = graph 334 | return sp.csr_matrix(adj), features_ppi, labels 335 | 336 | def load_webkb_data(dataset,data_path): 337 | idx_features_labels = np.genfromtxt("{}/{}.content".format(data_path, dataset), dtype=np.dtype(str)) 338 | features = sp.csr_matrix(idx_features_labels[:, 1:-1], dtype=np.float32) 339 | labels = encode_onehot(idx_features_labels[:, -1]) 340 | idx = np.array(idx_features_labels[:, 0], dtype=np.int32) 341 | idx_map = {j: i for i, j in enumerate(idx)} 342 | edges_unordered = np.genfromtxt("{}/{}.cites".format(data_path, dataset), dtype=np.int32) 343 | edges = np.array(list(map(idx_map.get, edges_unordered.flatten())), dtype=np.int32).reshape(edges_unordered.shape) 344 | adj = sp.coo_matrix((np.ones(edges.shape[0]), (edges[:, 0], edges[:, 1])), shape=(labels.shape[0], labels.shape[0]), dtype=np.float32) 345 | adj = adj + adj.T.multiply(adj.T > adj) - adj.multiply(adj.T > adj) 346 | features = normalize_features(features) 347 | adj = normalize_adj(adj + sp.eye(adj.shape[0])) 348 | graph = nx.from_numpy_matrix(np.array(adj.todense())) 349 | global G 350 | G = graph 351 | return adj, features, labels 352 | -------------------------------------------------------------------------------- /graphzoo/dataloader/download.py: -------------------------------------------------------------------------------- 1 | """ 2 | Data utils functions for downloading and extracting datasets 3 | """ 4 | import os 5 | from pathlib import Path 6 | from zipfile import ZipFile 7 | import requests 8 | 9 | url = { 'cora': "https://github.com/AnoushkaVyas/GraphZoo/releases/download/Datasets/cora.zip", 10 | 'airport': "https://github.com/AnoushkaVyas/GraphZoo/releases/download/Datasets/airport.zip", 11 | 'citeseer': "https://github.com/AnoushkaVyas/GraphZoo/releases/download/Datasets/citeseer.zip", 12 | 'disease_lp': "https://github.com/AnoushkaVyas/GraphZoo/releases/download/Datasets/disease_lp.zip", 13 | 'disease_nc': "https://github.com/AnoushkaVyas/GraphZoo/releases/download/Datasets/disease_nc.zip", 14 | 'ppi': "https://github.com/AnoushkaVyas/GraphZoo/releases/download/Datasets/ppi.zip", 15 | 'pubmed': "https://github.com/AnoushkaVyas/GraphZoo/releases/download/Datasets/pubmed.zip", 16 | 'webkb': "https://github.com/AnoushkaVyas/GraphZoo/releases/download/Datasets/webkb.zip" 17 | } 18 | 19 | def get_url(args): 20 | """ 21 | Gets the URL of the dataset from the dictionary 22 | """ 23 | if args.dataset not in list(url.keys()): 24 | raise ValueError('unknown dataset') 25 | return url[args.dataset] 26 | 27 | def download_and_extract(args): 28 | 29 | """ 30 | GraphZoo Download and Extract 31 | 32 | Input Parameters 33 | ---------- 34 | 'dataset': (None, 'which dataset to use, can be any of [cora, pubmed, airport, disease_nc, disease_lp] (type: str)') 35 | 'download_folder': (None, 'path to the folder for raw data (type: str)') 36 | 37 | API Input Parameters 38 | ---------- 39 | args: list of above defined input parameters from `graphzoo.config` 40 | 41 | """ 42 | 43 | if not args.download_folder: 44 | path = os.path.join(os.getcwd()+'/data/') 45 | 46 | else: 47 | path = args.download_folder 48 | 49 | filename = args.dataset+'.zip' 50 | 51 | fn = os.path.join(path, filename) 52 | 53 | if not (os.path.exists(path) and os.path.isdir(path)): 54 | os.makedirs(path, exist_ok=True) 55 | 56 | if not os.path.isfile(fn): 57 | print('%s does not exist..downloading..' % fn) 58 | url = get_url(args) 59 | f_remote = requests.get(url, stream=True) 60 | sz = f_remote.headers.get('content-length') 61 | assert f_remote.status_code == 200, 'fail to open {}'.format(url) 62 | with open(fn, 'wb') as writer: 63 | for chunk in f_remote.iter_content(chunk_size=1024*1024): 64 | writer.write(chunk) 65 | print('Download finished. Unzipping the file... (%s)' % fn) 66 | if not os.path.isfile(fn): 67 | raise ValueError('Download unsuccessful!') 68 | 69 | else: 70 | print('zip file already exists! Unzipping...') 71 | 72 | 73 | file = ZipFile(fn) 74 | file.extractall(path=str(path)) 75 | file.close() 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /graphzoo/layers/__init__.py: -------------------------------------------------------------------------------- 1 | from .layers import * 2 | from .att_layers import * 3 | from .hyp_layers import * 4 | from .hyp_att_layers import * 5 | -------------------------------------------------------------------------------- /graphzoo/layers/att_layers.py: -------------------------------------------------------------------------------- 1 | """Attention layers (some modules are copied from https://github.com/Diego999/pyGAT)""" 2 | import numpy as np 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | _MIN_NORM = 1e-8 7 | 8 | class DenseAtt(nn.Module): 9 | def __init__(self, in_features, dropout): 10 | super(DenseAtt, self).__init__() 11 | self.dropout = dropout 12 | self.linear = nn.Linear(2 * in_features, 1, bias=True) 13 | self.in_features = in_features 14 | 15 | def forward (self, x, adj): 16 | n = x.size(0) 17 | # n x 1 x d 18 | x_left = torch.unsqueeze(x, 1) 19 | x_left = x_left.expand(-1, n, -1) 20 | # 1 x n x d 21 | x_right = torch.unsqueeze(x, 0) 22 | x_right = x_right.expand(n, -1, -1) 23 | 24 | x_cat = torch.cat((x_left, x_right), dim=2) 25 | att_adj = self.linear(x_cat).squeeze() 26 | att_adj = F.sigmoid(att_adj) 27 | att_adj = torch.mul(adj.to_dense(), att_adj) 28 | return att_adj 29 | 30 | 31 | class SpecialSpmmFunction(torch.autograd.Function): 32 | """Special function for only sparse region backpropataion layer""" 33 | 34 | @staticmethod 35 | def forward(ctx, indices, values, shape, b): 36 | assert indices.requires_grad == False 37 | a = torch.sparse_coo_tensor(indices, values, shape) 38 | ctx.save_for_backward(a, b) 39 | ctx.N = shape[0] 40 | return torch.matmul(a, b) 41 | 42 | @staticmethod 43 | def backward(ctx, grad_output): 44 | a, b = ctx.saved_tensors 45 | grad_values = grad_b = None 46 | if ctx.needs_input_grad[1]: 47 | grad_a_dense = grad_output.matmul(b.t()) 48 | edge_idx = a._indices()[0, :] * ctx.N + a._indices()[1, :] 49 | grad_values = grad_a_dense.view(-1)[edge_idx] 50 | if ctx.needs_input_grad[3]: 51 | grad_b = a.t().matmul(grad_output) 52 | return None, grad_values, None, grad_b 53 | 54 | 55 | class SpecialSpmm(nn.Module): 56 | def forward(self, indices, values, shape, b): 57 | return SpecialSpmmFunction.apply(indices, values, shape, b) 58 | 59 | 60 | class SpGraphAttentionLayer(nn.Module): 61 | """ 62 | Sparse version GAT layer, similar to https://arxiv.org/abs/1710.10903 63 | """ 64 | 65 | def __init__(self, in_features, out_features, dropout, alpha, activation): 66 | super(SpGraphAttentionLayer, self).__init__() 67 | self.in_features = in_features 68 | self.out_features = out_features 69 | self.alpha = alpha 70 | 71 | self.W = nn.Parameter(torch.zeros(size=(in_features, out_features))) 72 | nn.init.xavier_normal_(self.W.data, gain=1.414) 73 | 74 | self.a = nn.Parameter(torch.zeros(size=(1, 2 * out_features))) 75 | nn.init.xavier_normal_(self.a.data, gain=1.414) 76 | 77 | self.dropout = nn.Dropout(dropout) 78 | self.leakyrelu = nn.LeakyReLU(self.alpha) 79 | self.special_spmm = SpecialSpmm() 80 | self.act = activation 81 | 82 | def forward(self, input, adj): 83 | N = input.size()[0] 84 | edge = adj._indices() 85 | 86 | h = torch.mm(input, self.W) 87 | # h: N x out 88 | assert not torch.isnan(h).any() 89 | 90 | # Self-attention on the nodes - Shared attention mechanism 91 | edge_h = torch.cat((h[edge[0, :], :], h[edge[1, :], :]), dim=1).t() 92 | # edge: 2*D x E 93 | 94 | edge_e = torch.exp(-self.leakyrelu(self.a.mm(edge_h).squeeze())) 95 | assert not torch.isnan(edge_e).any() 96 | # edge_e: E 97 | 98 | ones = torch.ones(size=(N, 1)) 99 | if h.is_cuda: 100 | ones = ones.cuda() 101 | e_rowsum = self.special_spmm(edge, edge_e, torch.Size([N, N]), ones) 102 | # e_rowsum: N x 1 103 | 104 | edge_e = self.dropout(edge_e) 105 | # edge_e: E 106 | 107 | h_prime = self.special_spmm(edge, edge_e, torch.Size([N, N]), h) 108 | assert not torch.isnan(h_prime).any() 109 | # h_prime: N x out 110 | 111 | h_prime = h_prime.div(e_rowsum.clamp_min(_MIN_NORM)) 112 | # h_prime: N x out 113 | assert not torch.isnan(h_prime).any() 114 | return self.act(h_prime) 115 | 116 | def __repr__(self): 117 | return self.__class__.__name__ + ' (' + str(self.in_features) + ' -> ' + str(self.out_features) + ')' 118 | 119 | 120 | class GraphAttentionLayer(nn.Module): 121 | def __init__(self, input_dim, output_dim, dropout, activation, alpha, nheads, concat): 122 | """Sparse version of GAT""" 123 | super(GraphAttentionLayer, self).__init__() 124 | self.dropout = dropout 125 | self.output_dim = output_dim 126 | self.attentions = [SpGraphAttentionLayer(input_dim, 127 | output_dim, 128 | dropout=dropout, 129 | alpha=alpha, 130 | activation=activation) for _ in range(nheads)] 131 | self.concat = concat 132 | for i, attention in enumerate(self.attentions): 133 | self.add_module('attention_{}'.format(i), attention) 134 | 135 | def forward(self, input): 136 | x, adj = input 137 | x = F.dropout(x, self.dropout, training=self.training) 138 | if self.concat: 139 | h = torch.cat([att(x, adj) for att in self.attentions], dim=1) 140 | else: 141 | h_cat = torch.cat([att(x, adj).view((-1, self.output_dim, 1)) for att in self.attentions], dim=2) 142 | h = torch.mean(h_cat, dim=2) 143 | h = F.dropout(h, self.dropout, training=self.training) 144 | return (h, adj) 145 | -------------------------------------------------------------------------------- /graphzoo/layers/hyp_att_layers.py: -------------------------------------------------------------------------------- 1 | """Hyperbolic Graph Attention layers""" 2 | import torch 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | from torch.nn.modules.module import Module 6 | from graphzoo.layers.hyp_layers import HypLinear, HypAct 7 | from graphzoo.manifolds.poincare import PoincareBall 8 | from graphzoo.manifolds.hyperboloid import Hyperboloid 9 | from graphzoo.manifolds.euclidean import Euclidean 10 | from graphzoo.layers.att_layers import SpecialSpmm 11 | 12 | class AdjustableModule(nn.Module): 13 | 14 | def __init__(self, curvature): 15 | super(AdjustableModule, self).__init__() 16 | self.curvature = curvature 17 | 18 | def update_curvature(self, curvature): 19 | self.curvature = curvature 20 | 21 | 22 | class SpGraphAttentionLayer(AdjustableModule): 23 | """ 24 | Sparse version GAT layer, similar to https://arxiv.org/abs/1710.10903 25 | """ 26 | 27 | def __init__(self, manifold, in_features, out_features, dropout, alpha, activation, curvature = 1, use_bias=False): 28 | super(SpGraphAttentionLayer, self).__init__(curvature) 29 | self.in_features = in_features 30 | self.out_features = out_features 31 | self.alpha = alpha 32 | self.use_bias = use_bias 33 | self.manifold = manifold 34 | self.linear = HypLinear(manifold, in_features, out_features, self.curvature, dropout, use_bias=use_bias) 35 | 36 | self.a = nn.Parameter(torch.zeros(size=(1, 2 * out_features))) 37 | nn.init.xavier_normal_(self.a.data, gain=1.414) 38 | 39 | self.dropout = nn.Dropout(dropout) 40 | self.leakyrelu = nn.LeakyReLU(self.alpha) 41 | self.special_spmm = SpecialSpmm() 42 | self.act = activation 43 | 44 | def update_curvature(self, c): 45 | super(SpGraphAttentionLayer, self).update_curvature(c) 46 | self.linear.update_curvature(c) 47 | 48 | def forward(self, input, adj): 49 | N = input.size()[0] 50 | edge = adj._indices() 51 | h = self.linear(input) 52 | 53 | # h: N x out 54 | assert not torch.isnan(h).any() 55 | 56 | # Self-attention on the nodes - Shared attention mechanism 57 | edge_h = -self.manifold.sqdist(h[edge[0, :], :], h[edge[1, :], :], c=self.curvature).unsqueeze(0) 58 | 59 | ########################Euclidean Block (START)######################## 60 | # convert h to Euclidean space. 61 | h = self.manifold.proj_tan0(self.manifold.logmap0(h, c=self.curvature), c=self.curvature) 62 | 63 | edge_e = torch.exp(-self.leakyrelu(edge_h.squeeze())) 64 | 65 | assert not torch.isnan(edge_e).any() 66 | # edge_e: E 67 | 68 | ones = torch.ones(size=(N, 1)) 69 | if h.is_cuda: 70 | ones = ones.cuda() 71 | e_rowsum = self.special_spmm(indices=edge, values=edge_e, shape=torch.Size([N, N]), b=ones) 72 | # e_rowsum: N x 1 73 | 74 | edge_e = self.dropout(edge_e) 75 | # edge_e: E 76 | 77 | h_prime = self.special_spmm(indices=edge, values=edge_e, shape=torch.Size([N, N]), b=h) 78 | assert not torch.isnan(h_prime).any() 79 | # h_prime: N x out 80 | 81 | h_prime = h_prime.div(e_rowsum) 82 | # h_prime: N x out 83 | assert not torch.isnan(h_prime).any() 84 | out = self.act(h_prime) 85 | ########################Euclidean Block (END)########################## 86 | # convert h back to Hyperbolic space (from Euclidean space). 87 | out = self.manifold.proj(self.manifold.expmap0(self.manifold.proj_tan0(out, self.curvature), c=self.curvature), c=self.curvature) 88 | return out 89 | 90 | def __repr__(self): 91 | return self.__class__.__name__ + ' (' + str(self.in_features) + ' -> ' + str(self.out_features) + ')' 92 | 93 | 94 | class GraphAttentionLayer(AdjustableModule): 95 | def __init__(self, manifold, input_dim, output_dim, dropout, activation, alpha, nheads, concat, curvature, use_bias): 96 | """Sparse version of GAT.""" 97 | super(GraphAttentionLayer, self).__init__(curvature) 98 | self.dropout = dropout 99 | self.output_dim = output_dim 100 | self.nheads = nheads 101 | self.manifold = manifold 102 | if self.nheads > 1 and isinstance(manifold, Hyperboloid): 103 | tmp_dim = (self.output_dim-1) * nheads + 1 104 | self.linear_out = HypLinear(manifold, tmp_dim, self.nheads*self.output_dim, self.curvature, dropout=0, use_bias=False) 105 | 106 | self.attentions = [SpGraphAttentionLayer( 107 | manifold, 108 | input_dim, 109 | output_dim, 110 | dropout=dropout, 111 | alpha=alpha, 112 | activation=activation, 113 | curvature=curvature, 114 | use_bias=use_bias) for _ in range(nheads)] 115 | self.concat = concat 116 | for i, attention in enumerate(self.attentions): 117 | self.add_module('attention_{}'.format(i), attention) 118 | 119 | def update_curvature(self, c): 120 | super(GraphAttentionLayer, self).update_curvature(c) 121 | for layer in self.attentions: 122 | layer.update_curvature(c) 123 | 124 | def forward(self, input): 125 | x, adj = input 126 | if torch.any(torch.isnan(x)): 127 | raise ValueError('input tensor has NaaN values') 128 | x = F.dropout(x, self.dropout, training=self.training) 129 | if self.concat: 130 | if isinstance(self.manifold, Euclidean): 131 | h = torch.cat([att(x, adj) for att in self.attentions], dim=1) 132 | elif self.nheads > 1: 133 | h = torch.stack([att(x, adj) for att in self.attentions], dim=-2) 134 | h = self.manifold.concat(h, c=self.curvature).squeeze() 135 | if isinstance(self.manifold, Hyperboloid): 136 | h = self.linear_out(h) 137 | else: # No concat 138 | h = self.attentions[0](x, adj) 139 | else: 140 | raise ValueError('aggregation is not supported ') 141 | h = F.dropout(h, self.dropout, training=self.training) 142 | return (h, adj) 143 | 144 | class SharedSelfAttention(Module): 145 | """ 146 | Hyperbolic attention layer with self-attention matrix. 147 | """ 148 | def __init__(self, manifold, input_dim, output_dim, curvature, activation=None, alpha=0.2, dropout=0.1, use_bias=True): 149 | super(SharedSelfAttention, self).__init__() 150 | self.curvature = curvature 151 | self.input_dim = input_dim 152 | self.output_dim = output_dim 153 | self.manifold = manifold 154 | 155 | # As the first step, we create an attention matrix using a linear layer 156 | # followed by a leakyReLU. 157 | # inspired from "Graph Attention Networks" by P. Veickovic ICLR 2018. 158 | 159 | # Note: the paper uses shared attention matrix, which means it is the same W 160 | # for all inputs of all nodes. W_dim(in_dim=d_model, out_dim=d_k) 161 | # However, if we want to have node-specific attention then 162 | # W_dim(graph_nodes * d_model, graph_nodes * graph_nodes * self.d_k) 163 | 164 | self.att_input_linear = HypLinear( 165 | manifold=self.manifold, 166 | in_features=self.input_dim, 167 | out_features=self.output_dim, 168 | c=self.curvature, 169 | dropout=dropout, 170 | use_bias=use_bias) 171 | nn.init.xavier_uniform(self.att_input_linear.weight) 172 | 173 | self.hyp_act = None 174 | if activation: 175 | self.hyp_act = HypAct( 176 | manifold=self.manifold, 177 | c_in=self.curvature, 178 | c_out=self.curvature, 179 | act=activation) 180 | 181 | 182 | def __repr__(self): 183 | return self.__class__.__name__ + ' (' + str(self.input_dim) + ' -> ' + str(self.output_dim) + ')' 184 | 185 | class SharedSelfAttentionV0(SharedSelfAttention): 186 | """ 187 | Hyperbolic attention layer with self-attention matrix. 188 | 189 | Uses mobius midpoint for calculating attention coefficient. 190 | """ 191 | def __init__(self, manifold, input_dim, output_dim, curvature, activation=None, alpha=0.2, dropout=0.1, use_bias=True): 192 | super(SharedSelfAttentionV0, self).__init__(manifold, input_dim, 193 | output_dim, curvature, activation, alpha, dropout, use_bias) 194 | 195 | def forward(self, hyp_features, edges): 196 | if torch.any(torch.isnan(hyp_features)): 197 | raise ValueError('input to SharedSelfAttentionV0 has NaaN values') 198 | att_per_node = self.att_input_linear(hyp_features) 199 | # Alternatively you cann pass hyp_features from a seperate linear layer 200 | # to create reduced_hyp_features 201 | reduced_hyp_features = att_per_node 202 | 203 | # create adjaceny matrix from edge info 204 | mask = edges.to_dense().transpose(0,1) 205 | if torch.nonzero(mask).numel() == 0: 206 | raise ValueError('adjacency matrix must have at least 1 edge.') 207 | hyp_att_embeddings = [] 208 | 209 | for src_node, incoming_edges in enumerate(mask): 210 | # calculate the activation for each node 211 | masked_v = [] 212 | masked_a = [] 213 | for tgt_node, val in enumerate(incoming_edges): 214 | if val > 0.01: 215 | # we define attention coefficient with the following formula. 216 | coef = -1 * val * Hyperboloid().sqdist(att_per_node[tgt_node], att_per_node[src_node], c=self.curvature) 217 | if torch.isnan(coef): 218 | raise ValueError('we cannot have attentions coeficinet as NaaN') 219 | masked_a.append(coef) 220 | masked_v.append(reduced_hyp_features[tgt_node]) 221 | if not masked_a and not masked_v: 222 | raise ValueError( 223 | 'A graph node must have at least one incoming edge.') 224 | 225 | masked_a = torch.FloatTensor(torch.stack(masked_a).squeeze(-1)) 226 | masked_v = torch.stack(masked_v) 227 | # Note since for attention matrix we use linear layer which includes 228 | # droupout rate as well. we omit the separate drop out layer. 229 | # project the hyperbolic vector to poincare model. 230 | poincare_v = PoincareBall().from_hyperboloid(x=masked_v, c=self.curvature) 231 | # calculate attention embeddings for each node. 232 | att_embed = PoincareBall().mobius_midpoint(a=masked_a, v=poincare_v) 233 | hyp_att_em = Hyperboloid().from_poincare(att_embed, c=self.curvature) 234 | hyp_att_embeddings.append(hyp_att_em) 235 | 236 | hyp_att_embeddings = torch.stack(hyp_att_embeddings) 237 | 238 | if self.hyp_act: 239 | hyp_att_embeddings = self.hyp_act(hyp_att_embeddings) 240 | return hyp_att_embeddings 241 | 242 | 243 | class MultiHeadGraphAttentionLayer(Module): 244 | 245 | def __init__(self, manifold, input_dim, output_dim, dropout, curvature=1, activation=None, alpha=0.2, nheads=1, concat=None, self_attention_version='v0'): 246 | """Sparse version of GAT.""" 247 | super(MultiHeadGraphAttentionLayer, self).__init__() 248 | if self_attention_version == 'v0': 249 | self_attention_layer_class = SharedSelfAttentionV0 250 | else: 251 | raise ValueError('Unknown self-attention version!') 252 | self.manifold = manifold 253 | self.dropout = dropout 254 | self.output_dim = output_dim 255 | self.curvature = curvature 256 | self.manifold = Hyperboloid 257 | self.attentions = [self_attention_layer_class( 258 | manifold=self.manifold, 259 | input_dim=input_dim, 260 | output_dim=self.output_dim, 261 | curvature=self.curvature, 262 | alpha=alpha, 263 | activation=activation, 264 | dropout=self.dropout, 265 | use_bias=False) for _ in range(nheads)] 266 | self.linear_out = None 267 | if nheads > 1: 268 | self.linear_out = HypLinear( 269 | manifold=self.manifold, 270 | in_features=nheads * (self.output_dim - 1) + 1, 271 | out_features=nheads * self.output_dim, 272 | c=self.curvature, 273 | dropout=0.0, 274 | use_bias=False) 275 | 276 | for i, attention in enumerate(self.attentions): 277 | self.add_module('attention_{}'.format(i), attention) 278 | 279 | def forward(self, input): 280 | x, adj = input 281 | if torch.any(torch.isnan(x)): 282 | raise ValueError('input tensor has NaaN values') 283 | x = F.dropout(x, self.dropout, training=self.training) 284 | p_h = PoincareBall().from_hyperboloid(torch.stack([att(x, adj) for att in self.attentions], dim=1)) 285 | p_h = PoincareBall().concat(p_h) 286 | h = Hyperboloid().from_poincare(p_h) 287 | 288 | if self.linear_out: 289 | h = self.linear_out(h) 290 | 291 | h = F.dropout(h, self.dropout, training=self.training) 292 | return (h, adj) 293 | 294 | class HypGraphSelfAttentionLayer(Module): 295 | """ 296 | Hyperbolic attention layer with node-specific attention matrix. 297 | """ 298 | def __init__(self, graph_size, vector_dim, curvature, dropout=0.1, use_bias=False): 299 | super(HypGraphSelfAttentionLayer, self).__init__() 300 | self.curvature = curvature 301 | self.vector_dim = vector_dim 302 | self.graph_dim = graph_size 303 | # As the first step, we create an attention matrix using a linear layer 304 | # followed by a leakyReLU. 305 | # inspired from "Graph Attention Networks" by P. Veickovic ICLR 2018. 306 | self.input_linear = HypLinear( 307 | manifold=PoincareBall, 308 | in_features=self.graph_dim * self.vector_dim, 309 | out_features=self.graph_dim * self.graph_dim * self.vector_dim, 310 | c=self.curvature, 311 | dropout=dropout, 312 | use_bias=use_bias) 313 | 314 | self.att_out_linear = HypLinear( 315 | manifold=PoincareBall, 316 | in_features=self.graph_dim * self.vector_dim, 317 | out_features=self.graph_dim * self.vector_dim, 318 | c=self.curvature, 319 | dropout=False, 320 | use_bias=use_bias) 321 | 322 | @classmethod 323 | def graph_attention(self, a, v, mask): 324 | """calculare the graph attention for a single node. 325 | 326 | Note: it is based on the eq.8, eq.9 in "Hyperbolic graph attention network" paper. 327 | 328 | args: 329 | a: attention coefficient vector of dim(M,M,N). 330 | v: M*N dimensional matrix, where M is the number of 331 | vectors of dim N. 332 | mask: a vector of dim (M, M) that indicates the connection map of 333 | the nodes in the graph. 334 | returns: 335 | a vector of dim(M,N) corresponding to the attention embeddings for 336 | the given node. 337 | """ 338 | 339 | masked_v = [] 340 | masked_a = [] 341 | h = [] 342 | for i, _ in enumerate(v): 343 | a_i = a[i].view() 344 | mask_i = mask[i].view() 345 | # For each node we extract the nodes in the connection map, then 346 | # we calculate the mid-point for that node. This needs to be 347 | # repeated for all nodes in the graph. 348 | for idx, mask_bit in enumerate(mask_i): 349 | if mask_bit == 1: 350 | masked_v.append(v[idx]) 351 | masked_a.append(a_i[idx]) 352 | h.append(PoincareBall()._mobius_midpoint(v=torch.stack(masked_v)), a=torch.stack(masked_a)) 353 | return torch.stack(h) 354 | 355 | def forward(self, input_vectors, mask): 356 | # project the hyperbolic vector to poincare model. 357 | poincare_in = PoincareBall().proj(x=input_vectors, c=self.curvature) 358 | att_coeff = self.input_linear(poincare_in) 359 | att_vectors = self.graph_attention(a=att_coeff, v=poincare_in, mask=self.mask) 360 | return PoincareBall().to_hyperboloid(x=self.att_out_linear(att_vectors), c=self.curvature) -------------------------------------------------------------------------------- /graphzoo/layers/hyp_layers.py: -------------------------------------------------------------------------------- 1 | """Hyperbolic layers.""" 2 | import math 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import torch.nn.init as init 7 | from torch.nn.modules.module import Module 8 | from graphzoo.layers.att_layers import DenseAtt 9 | 10 | def get_dim_act_curv(args): 11 | """ 12 | Helper function to get dimension and activation at every layer 13 | :param args: 14 | :return: 15 | """ 16 | if not args.act: 17 | act = lambda x: x 18 | else: 19 | act = getattr(F, args.act) 20 | acts = [act] * (args.num_layers - 1) 21 | dims = [args.feat_dim] + ([args.dim] * (args.num_layers - 1)) 22 | if args.task in ['lp', 'rec']: 23 | dims += [args.dim] 24 | acts += [act] 25 | n_curvatures = args.num_layers 26 | else: 27 | n_curvatures = args.num_layers - 1 28 | if args.c is None: 29 | # create list of trainable curvature parameters 30 | curvatures = [nn.Parameter(torch.Tensor([1.])) for _ in range(n_curvatures)] 31 | else: 32 | # fixed curvature 33 | curvatures = [torch.tensor([args.c]) for _ in range(n_curvatures)] 34 | if not args.cuda == -1: 35 | curvatures = [curv.to(args.device) for curv in curvatures] 36 | return dims, acts, curvatures 37 | 38 | 39 | class HNNLayer(nn.Module): 40 | """ 41 | Hyperbolic neural networks layer 42 | """ 43 | 44 | def __init__(self, manifold, in_features, out_features, c, dropout, act, use_bias): 45 | super(HNNLayer, self).__init__() 46 | self.linear = HypLinear(manifold, in_features, out_features, c, dropout, use_bias) 47 | self.hyp_act = HypAct(manifold, c, c, act) 48 | 49 | def forward(self, x): 50 | h = self.linear.forward(x) 51 | h = self.hyp_act.forward(h) 52 | return h 53 | 54 | 55 | class HyperbolicGraphConvolution(nn.Module): 56 | """ 57 | Hyperbolic graph convolution layer 58 | """ 59 | 60 | def __init__(self, manifold, in_features, out_features, c_in, c_out, dropout, act, use_bias, use_att, local_agg): 61 | super(HyperbolicGraphConvolution, self).__init__() 62 | self.linear = HypLinear(manifold, in_features, out_features, c_in, dropout, use_bias) 63 | self.agg = HypAgg(manifold, c_in, out_features, dropout, use_att, local_agg) 64 | self.hyp_act = HypAct(manifold, c_in, c_out, act) 65 | 66 | def forward(self, input): 67 | x, adj = input 68 | h = self.linear.forward(x) 69 | h = self.agg.forward(h, adj) 70 | h = self.hyp_act.forward(h) 71 | output = h, adj 72 | return output 73 | 74 | 75 | class HypLinear(nn.Module): 76 | """ 77 | Hyperbolic linear layer 78 | """ 79 | 80 | def __init__(self, manifold, in_features, out_features, c, dropout, use_bias): 81 | super(HypLinear, self).__init__() 82 | self.manifold = manifold 83 | self.in_features = in_features 84 | self.out_features = out_features 85 | self.c = c 86 | self.dropout = dropout 87 | self.use_bias = use_bias 88 | self.bias = nn.Parameter(torch.Tensor(out_features)) 89 | self.weight = nn.Parameter(torch.Tensor(out_features, in_features)) 90 | self.reset_parameters() 91 | 92 | def reset_parameters(self): 93 | init.xavier_uniform_(self.weight, gain=math.sqrt(2)) 94 | init.constant_(self.bias, 0) 95 | 96 | def forward(self, x): 97 | drop_weight = F.dropout(self.weight, self.dropout, training=self.training) 98 | mv = self.manifold.mobius_matvec(drop_weight, x, self.c) 99 | res = self.manifold.proj(mv, self.c) 100 | if self.use_bias: 101 | bias = self.manifold.proj_tan0(self.bias.view(1, -1), self.c) 102 | hyp_bias = self.manifold.expmap0(bias, self.c) 103 | hyp_bias = self.manifold.proj(hyp_bias, self.c) 104 | res = self.manifold.mobius_add(res, hyp_bias, c=self.c) 105 | res = self.manifold.proj(res, self.c) 106 | return res 107 | 108 | def extra_repr(self): 109 | return 'in_features={}, out_features={}, c={}'.format( 110 | self.in_features, self.out_features, self.c 111 | ) 112 | 113 | 114 | class HypAgg(Module): 115 | """ 116 | Hyperbolic aggregation layer 117 | """ 118 | 119 | def __init__(self, manifold, c, in_features, dropout, use_att, local_agg): 120 | super(HypAgg, self).__init__() 121 | self.manifold = manifold 122 | self.c = c 123 | 124 | self.in_features = in_features 125 | self.dropout = dropout 126 | self.local_agg = local_agg 127 | self.use_att = use_att 128 | if self.use_att: 129 | self.att = DenseAtt(in_features, dropout) 130 | 131 | def forward(self, x, adj): 132 | x_tangent = self.manifold.logmap0(x, c=self.c) 133 | if self.use_att: 134 | if self.local_agg: 135 | x_local_tangent = [] 136 | for i in range(x.size(0)): 137 | x_local_tangent.append(self.manifold.logmap(x[i], x, c=self.c)) 138 | x_local_tangent = torch.stack(x_local_tangent, dim=0) 139 | adj_att = self.att(x_tangent, adj) 140 | att_rep = adj_att.unsqueeze(-1) * x_local_tangent 141 | support_t = torch.sum(adj_att.unsqueeze(-1) * x_local_tangent, dim=1) 142 | output = self.manifold.proj(self.manifold.expmap(x, support_t, c=self.c), c=self.c) 143 | return output 144 | else: 145 | adj_att = self.att(x_tangent, adj) 146 | support_t = torch.matmul(adj_att, x_tangent) 147 | else: 148 | support_t = torch.spmm(adj, x_tangent) 149 | output = self.manifold.proj(self.manifold.expmap0(support_t, c=self.c), c=self.c) 150 | return output 151 | 152 | def extra_repr(self): 153 | return 'c={}'.format(self.c) 154 | 155 | 156 | class HypAct(Module): 157 | """ 158 | Hyperbolic activation layer 159 | """ 160 | 161 | def __init__(self, manifold, c_in, c_out, act): 162 | super(HypAct, self).__init__() 163 | self.manifold = manifold 164 | self.c_in = c_in 165 | self.c_out = c_out 166 | self.act = act 167 | 168 | def forward(self, x): 169 | xt = self.act(self.manifold.logmap0(x, c=self.c_in)) 170 | xt = self.manifold.proj_tan0(xt, c=self.c_out) 171 | return self.manifold.proj(self.manifold.expmap0(xt, c=self.c_out), c=self.c_out) 172 | 173 | def extra_repr(self): 174 | return 'c_in={}, c_out={}'.format( 175 | self.c_in, self.c_out 176 | ) 177 | -------------------------------------------------------------------------------- /graphzoo/layers/layers.py: -------------------------------------------------------------------------------- 1 | """Euclidean layers""" 2 | import math 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | from torch.nn.modules.module import Module 7 | from torch.nn.parameter import Parameter 8 | 9 | 10 | def get_dim_act(args): 11 | """ 12 | Helper function to get dimension and activation at every layer. 13 | :param args: 14 | :return: 15 | """ 16 | if not args.act: 17 | act = lambda x: x 18 | else: 19 | act = getattr(F, args.act) 20 | acts = [act] * (args.num_layers - 1) 21 | dims = [args.feat_dim] + ([args.dim] * (args.num_layers - 1)) 22 | if args.task in ['lp', 'rec']: 23 | dims += [args.dim] 24 | acts += [act] 25 | return dims, acts 26 | 27 | 28 | class GraphConvolution(Module): 29 | """ 30 | Simple GCN layer 31 | """ 32 | 33 | def __init__(self, in_features, out_features, dropout, act, use_bias): 34 | super(GraphConvolution, self).__init__() 35 | self.dropout = dropout 36 | self.linear = nn.Linear(in_features, out_features, use_bias) 37 | self.act = act 38 | self.in_features = in_features 39 | self.out_features = out_features 40 | 41 | def forward(self, input): 42 | x, adj = input 43 | hidden = self.linear.forward(x) 44 | hidden = F.dropout(hidden, self.dropout, training=self.training) 45 | if adj.is_sparse: 46 | support = torch.spmm(adj, hidden) 47 | else: 48 | support = torch.mm(adj, hidden) 49 | output = self.act(support), adj 50 | return output 51 | 52 | def extra_repr(self): 53 | return 'input_dim={}, output_dim={}'.format( 54 | self.in_features, self.out_features 55 | ) 56 | 57 | 58 | class Linear(Module): 59 | """ 60 | Simple Linear layer with dropout 61 | """ 62 | 63 | def __init__(self, in_features, out_features, dropout, act, use_bias): 64 | super(Linear, self).__init__() 65 | self.dropout = dropout 66 | self.linear = nn.Linear(in_features, out_features, use_bias) 67 | self.act = act 68 | 69 | def forward(self, x): 70 | hidden = self.linear.forward(x) 71 | hidden = F.dropout(hidden, self.dropout, training=self.training) 72 | out = self.act(hidden) 73 | return out 74 | 75 | 76 | class FermiDiracDecoder(Module): 77 | """Fermi Dirac to compute edge probabilities based on distances""" 78 | 79 | def __init__(self, r, t): 80 | super(FermiDiracDecoder, self).__init__() 81 | self.r = r 82 | self.t = t 83 | 84 | def forward(self, dist): 85 | probs = 1. / (torch.exp((dist - self.r) / self.t) + 1.0) 86 | return probs 87 | 88 | -------------------------------------------------------------------------------- /graphzoo/manifolds/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import * 2 | from .euclidean import Euclidean 3 | from .hyperboloid import Hyperboloid 4 | from .poincare import PoincareBall 5 | -------------------------------------------------------------------------------- /graphzoo/manifolds/base.py: -------------------------------------------------------------------------------- 1 | """Base manifold""" 2 | from torch.nn import Parameter 3 | from typing import Tuple 4 | import torch 5 | 6 | class Manifold(object): 7 | """ 8 | Abstract class to define operations on a manifold 9 | """ 10 | 11 | def __init__(self): 12 | super().__init__() 13 | self.eps = 10e-8 14 | 15 | def sqdist(self, p1, p2, c): 16 | """Squared distance between pairs of points""" 17 | raise NotImplementedError 18 | 19 | def egrad2rgrad(self, p, dp, c): 20 | """Converts Euclidean Gradient to Riemannian Gradients""" 21 | raise NotImplementedError 22 | 23 | def proj(self, p, c): 24 | """Projects point p on the manifold""" 25 | raise NotImplementedError 26 | 27 | def proj_tan(self, u, p, c): 28 | """Projects u on the tangent space of p""" 29 | raise NotImplementedError 30 | 31 | def proj_tan0(self, u, c): 32 | """Projects u on the tangent space of the origin""" 33 | raise NotImplementedError 34 | 35 | def expmap(self, u, p, c): 36 | """Exponential map of u at point p""" 37 | raise NotImplementedError 38 | 39 | def logmap(self, p1, p2, c): 40 | """Logarithmic map of point p1 at point p2""" 41 | raise NotImplementedError 42 | 43 | def expmap0(self, u, c): 44 | """Exponential map of u at the origin""" 45 | raise NotImplementedError 46 | 47 | def logmap0(self, p, c): 48 | """Logarithmic map of point p at the origin""" 49 | raise NotImplementedError 50 | 51 | def mobius_add(self, x, y, c, dim=-1): 52 | """Adds points x and y""" 53 | raise NotImplementedError 54 | 55 | def mobius_matvec(self, m, x, c): 56 | """Performs hyperboic martrix-vector multiplication""" 57 | raise NotImplementedError 58 | 59 | def init_weights(self, w, c, irange=1e-5): 60 | """Initializes random weigths on the manifold""" 61 | raise NotImplementedError 62 | 63 | def inner(self, p, c, u, v=None, keepdim=False): 64 | """Inner product for tangent vectors at point x""" 65 | raise NotImplementedError 66 | 67 | def ptransp(self, x, y, u, c): 68 | """Parallel transport of u from x to y""" 69 | raise NotImplementedError 70 | 71 | def ptransp0(self, x, u, c): 72 | """Parallel transport of u from the origin to y""" 73 | raise NotImplementedError 74 | 75 | def retr(self, x: torch.Tensor, u: torch.Tensor) -> torch.Tensor: 76 | """ 77 | Perform a retraction from point :math:`x` with given direction :math:`u` 78 | 79 | Parameters 80 | ---------- 81 | x : torch.Tensor 82 | point on the manifold 83 | u : torch.Tensor 84 | tangent vector at point :math:`x` 85 | 86 | Returns 87 | ------- 88 | torch.Tensor 89 | transported point 90 | """ 91 | raise NotImplementedError 92 | 93 | def transp(self, x: torch.Tensor, y: torch.Tensor, v: torch.Tensor) -> torch.Tensor: 94 | r""" 95 | Perform vector transport :math:`\mathfrak{T}_{x\to y}(v)`. 96 | Parameters 97 | ---------- 98 | x : torch.Tensor 99 | start point on the manifold 100 | y : torch.Tensor 101 | target point on the manifold 102 | v : torch.Tensor 103 | tangent vector at point :math:`x` 104 | Returns 105 | ------- 106 | torch.Tensor 107 | transported tensor 108 | """ 109 | raise NotImplementedError 110 | 111 | def retr_transp( 112 | self, x: torch.Tensor, u: torch.Tensor, v: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: 113 | """ 114 | Perform a retraction + vector transport at once. 115 | Parameters 116 | ---------- 117 | x : torch.Tensor 118 | point on the manifold 119 | u : torch.Tensor 120 | tangent vector at point :math:`x` 121 | v : torch.Tensor 122 | tangent vector at point :math:`x` to be transported 123 | 124 | Returns 125 | ------- 126 | Tuple[torch.Tensor, torch.Tensor] 127 | transported point and vectors 128 | Notes 129 | ----- 130 | Sometimes this is a far more optimal way to preform retraction + vector transport 131 | """ 132 | y = self.retr(x, u) 133 | v_transp = self.transp(x, y, v) 134 | return y, v_transp 135 | 136 | class ManifoldParameter(Parameter): 137 | """ 138 | Subclass of torch.nn.Parameter for Riemannian optimization. 139 | """ 140 | def __new__(cls, data, requires_grad, manifold, c): 141 | return Parameter.__new__(cls, data, requires_grad) 142 | 143 | def __init__(self, data, requires_grad, manifold, c): 144 | self.c = c 145 | self.manifold = manifold 146 | 147 | def __repr__(self): 148 | return '{} Parameter containing:\n'.format(self.manifold.name) + super(Parameter, self).__repr__() 149 | -------------------------------------------------------------------------------- /graphzoo/manifolds/euclidean.py: -------------------------------------------------------------------------------- 1 | """Euclidean manifold.""" 2 | from graphzoo.manifolds.base import Manifold 3 | import torch 4 | from graphzoo.utils.train_utils import broadcast_shapes 5 | 6 | class Euclidean(Manifold): 7 | """ 8 | Euclidean Manifold class 9 | """ 10 | 11 | def __init__(self): 12 | super(Euclidean, self).__init__() 13 | self.name = 'Euclidean' 14 | 15 | def normalize(self, p): 16 | dim = p.size(-1) 17 | p.view(-1, dim).renorm_(2, 0, 1.) 18 | return p 19 | 20 | def sqdist(self, p1, p2, c): 21 | return (p1 - p2).pow(2).sum(dim=-1) 22 | 23 | def egrad2rgrad(self, p, dp, c): 24 | return dp 25 | 26 | def proj(self, p, c): 27 | return p 28 | 29 | def proj_tan(self, u, p, c): 30 | return u 31 | 32 | def proj_tan0(self, u, c): 33 | return u 34 | 35 | def expmap(self, u, p, c): 36 | return p + u 37 | 38 | def logmap(self, p1, p2, c): 39 | return p2 - p1 40 | 41 | def expmap0(self, u, c): 42 | return u 43 | 44 | def logmap0(self, p, c): 45 | return p 46 | 47 | def mobius_add(self, x, y, c, dim=-1): 48 | return x + y 49 | 50 | def mobius_matvec(self, m, x, c): 51 | mx = x @ m.transpose(-1, -2) 52 | return mx 53 | 54 | def init_weights(self, w, c, irange=1e-5): 55 | w.data.uniform_(-irange, irange) 56 | return w 57 | 58 | def inner(self, p, c, u, v=None, keepdim=False): 59 | if v is None: 60 | v = u 61 | return (u * v).sum(dim=-1, keepdim=keepdim) 62 | 63 | def ptransp(self, x, y, v, c): 64 | return v 65 | 66 | def ptransp0(self, x, v, c): 67 | return x + v 68 | 69 | def retr(self, x: torch.Tensor, u: torch.Tensor) -> torch.Tensor: 70 | return x + u 71 | 72 | def transp(self, x: torch.Tensor, y: torch.Tensor, v: torch.Tensor) -> torch.Tensor: 73 | target_shape = broadcast_shapes(x.shape, y.shape, v.shape) 74 | return v.expand(target_shape) -------------------------------------------------------------------------------- /graphzoo/manifolds/hyperboloid.py: -------------------------------------------------------------------------------- 1 | """Hyperboloid manifold""" 2 | import torch 3 | from graphzoo.manifolds.base import Manifold 4 | from graphzoo.utils.math_utils import arcosh, cosh, sinh 5 | from graphzoo.utils.train_utils import broadcast_shapes 6 | from graphzoo.manifolds.poincare import PoincareBall 7 | 8 | class Hyperboloid(Manifold): 9 | """ 10 | Hyperboloid manifold class 11 | 12 | We use the following convention: -x0^2 + x1^2 + ... + xd^2 = -K 13 | 14 | c = 1 / K is the hyperbolic curvature 15 | """ 16 | 17 | def __init__(self): 18 | super(Hyperboloid, self).__init__() 19 | self.name = 'Hyperboloid' 20 | self.eps = {torch.float32: 1e-7, torch.float64: 1e-15} 21 | self.min_norm = 1e-15 22 | self.max_norm = 1e6 23 | 24 | def minkowski_dot(self, x, y, keepdim=True): 25 | res = torch.sum(x * y, dim=-1) - 2 * x[..., 0] * y[..., 0] 26 | if keepdim: 27 | res = res.view(res.shape + (1,)) 28 | return res 29 | 30 | def minkowski_norm(self, u, keepdim=True): 31 | dot = self.minkowski_dot(u, u, keepdim=keepdim) 32 | return torch.sqrt(torch.clamp(dot, min=self.eps[u.dtype])) 33 | 34 | def sqdist(self, x, y, c): 35 | K = 1. / c 36 | prod = self.minkowski_dot(x, y) 37 | theta = torch.clamp(-prod / K, min=1.0 + self.eps[x.dtype]) 38 | sqdist = K * arcosh(theta) ** 2 39 | # clamp distance to avoid nans in Fermi-Dirac decoder 40 | return torch.clamp(sqdist, max=50.0) 41 | 42 | def proj(self, x, c): 43 | K = 1. / c 44 | d = x.size(-1) - 1 45 | y = x.narrow(-1, 1, d) 46 | y_sqnorm = torch.norm(y, p=2, dim=1, keepdim=True) ** 2 47 | mask = torch.ones_like(x) 48 | mask[:, 0] = 0 49 | vals = torch.zeros_like(x) 50 | vals[:, 0:1] = torch.sqrt(torch.clamp(K + y_sqnorm, min=self.eps[x.dtype])) 51 | return vals + mask * x 52 | 53 | def proj_tan(self, u, x, c): 54 | K = 1. / c 55 | d = x.size(1) - 1 56 | ux = torch.sum(x.narrow(-1, 1, d) * u.narrow(-1, 1, d), dim=1, keepdim=True) 57 | mask = torch.ones_like(u) 58 | mask[:, 0] = 0 59 | vals = torch.zeros_like(u) 60 | vals[:, 0:1] = ux / torch.clamp(x[:, 0:1], min=self.eps[x.dtype]) 61 | return vals + mask * u 62 | 63 | def proj_tan0(self, u, c): 64 | narrowed = u.narrow(-1, 0, 1) 65 | vals = torch.zeros_like(u) 66 | vals[:, 0:1] = narrowed 67 | return u - vals 68 | 69 | def expmap(self, u, x, c): 70 | K = 1. / c 71 | sqrtK = K ** 0.5 72 | normu = self.minkowski_norm(u) 73 | normu = torch.clamp(normu, max=self.max_norm) 74 | theta = normu / sqrtK 75 | theta = torch.clamp(theta, min=self.min_norm) 76 | result = cosh(theta) * x + sinh(theta) * u / theta 77 | return self.proj(result, c) 78 | 79 | def logmap(self, x, y, c): 80 | K = 1. / c 81 | xy = torch.clamp(self.minkowski_dot(x, y) + K, max=-self.eps[x.dtype]) - K 82 | u = y + xy * x * c 83 | normu = self.minkowski_norm(u) 84 | normu = torch.clamp(normu, min=self.min_norm) 85 | dist = self.sqdist(x, y, c) ** 0.5 86 | result = dist * u / normu 87 | return self.proj_tan(result, x, c) 88 | 89 | def expmap0(self, u, c): 90 | K = 1. / c 91 | sqrtK = K ** 0.5 92 | d = u.size(-1) - 1 93 | x = u.narrow(-1, 1, d).view(-1, d) 94 | x_norm = torch.norm(x, p=2, dim=1, keepdim=True) 95 | x_norm = torch.clamp(x_norm, min=self.min_norm) 96 | theta = x_norm / sqrtK 97 | res = torch.ones_like(u) 98 | res[:, 0:1] = sqrtK * cosh(theta) 99 | res[:, 1:] = sqrtK * sinh(theta) * x / x_norm 100 | return self.proj(res, c) 101 | 102 | def logmap0(self, x, c): 103 | K = 1. / c 104 | sqrtK = K ** 0.5 105 | d = x.size(-1) - 1 106 | y = x.narrow(-1, 1, d).view(-1, d) 107 | y_norm = torch.norm(y, p=2, dim=1, keepdim=True) 108 | y_norm = torch.clamp(y_norm, min=self.min_norm) 109 | res = torch.zeros_like(x) 110 | theta = torch.clamp(x[:, 0:1] / sqrtK, min=1.0 + self.eps[x.dtype]) 111 | res[:, 1:] = sqrtK * arcosh(theta) * y / y_norm 112 | return res 113 | 114 | def mobius_add(self, x, y, c): 115 | u = self.logmap0(y, c) 116 | v = self.ptransp0(x, u, c) 117 | return self.expmap(v, x, c) 118 | 119 | def mobius_matvec(self, m, x, c): 120 | u = self.logmap0(x, c) 121 | mu = u @ m.transpose(-1, -2) 122 | return self.expmap0(mu, c) 123 | 124 | def ptransp(self, x, y, u, c): 125 | logxy = self.logmap(x, y, c) 126 | logyx = self.logmap(y, x, c) 127 | sqdist = torch.clamp(self.sqdist(x, y, c), min=self.min_norm) 128 | alpha = self.minkowski_dot(logxy, u) / sqdist 129 | res = u - alpha * (logxy + logyx) 130 | return self.proj_tan(res, y, c) 131 | 132 | def ptransp0(self, x, u, c): 133 | K = 1. / c 134 | sqrtK = K ** 0.5 135 | x0 = x.narrow(-1, 0, 1) 136 | d = x.size(-1) - 1 137 | y = x.narrow(-1, 1, d) 138 | y_norm = torch.clamp(torch.norm(y, p=2, dim=1, keepdim=True), min=self.min_norm) 139 | y_normalized = y / y_norm 140 | v = torch.ones_like(x) 141 | v[:, 0:1] = - y_norm 142 | v[:, 1:] = (sqrtK - x0) * y_normalized 143 | alpha = torch.sum(y_normalized * u[:, 1:], dim=1, keepdim=True) / sqrtK 144 | res = u - alpha * v 145 | return self.proj_tan(res, x, c) 146 | 147 | def to_poincare(self, x, c): 148 | K = 1. / c 149 | sqrtK = K ** 0.5 150 | d = x.size(-1) - 1 151 | return sqrtK * x.narrow(-1, 1, d) / (x[:, 0:1] + sqrtK) 152 | 153 | def retr(self, x: torch.Tensor, u: torch.Tensor) -> torch.Tensor: 154 | return x + u 155 | 156 | def transp(self, x: torch.Tensor, y: torch.Tensor, v: torch.Tensor) -> torch.Tensor: 157 | target_shape = broadcast_shapes(x.shape, y.shape, v.shape) 158 | return v.expand(target_shape) 159 | 160 | def concat(self, v, c): 161 | """ 162 | Note that the output dimension is (input_dim-1) * n + 1 163 | """ 164 | p = PoincareBall().from_hyperboloid(v, c) 165 | p = PoincareBall().concat(p) 166 | return Hyperboloid().from_poincare(p, c) 167 | 168 | def from_poincare(self, x, c=1, ideal=False): 169 | """Convert from Poincare ball model to hyperboloid model. 170 | 171 | Note: converting a point from poincare ball to hyperbolic is 172 | reversible, i.e. p == to_poincare(from_poincare(p)). 173 | 174 | Args: 175 | x: torch.tensor of shape (..., dim) 176 | ideal: boolean. Should be True if the input vectors are ideal points, False otherwise 177 | Returns: 178 | torch.tensor of shape (..., dim+1) 179 | To do: 180 | Add some capping to make things numerically stable. This is only needed in the case ideal == False 181 | """ 182 | if ideal: 183 | t = torch.ones(x.shape[:-1], device=x.device).unsqueeze(-1) 184 | return torch.cat((t, x), dim=-1) 185 | else: 186 | K = 1./ c 187 | sqrtK = K ** 0.5 188 | eucl_squared_norm = (x * x).sum(dim=-1, keepdim=True) 189 | return sqrtK * torch.cat((K + eucl_squared_norm, 2 * sqrtK * x), dim=-1) / (K - eucl_squared_norm).clamp_min(self.min_norm) 190 | -------------------------------------------------------------------------------- /graphzoo/manifolds/poincare.py: -------------------------------------------------------------------------------- 1 | """Poincare ball manifold""" 2 | import torch 3 | import scipy.special as sc 4 | from graphzoo.manifolds.base import Manifold 5 | from graphzoo.utils.math_utils import artanh, tanh 6 | from graphzoo.utils.train_utils import broadcast_shapes 7 | 8 | class PoincareBall(Manifold): 9 | """ 10 | PoicareBall Manifold class 11 | 12 | We use the following convention: x0^2 + x1^2 + ... + xd^2 < 1 / c 13 | 14 | Note that 1/sqrt(c) is the Poincare ball radius 15 | 16 | """ 17 | 18 | def __init__(self, ): 19 | super(PoincareBall, self).__init__() 20 | self.name = 'PoincareBall' 21 | self.min_norm = 1e-5 22 | self.eps = {torch.float32: 4e-3, torch.float64: 1e-5} 23 | 24 | def sqdist(self, p1, p2, c): 25 | sqrt_c = c ** 0.5 26 | dist_c = artanh( 27 | sqrt_c * self.mobius_add(-p1, p2, c, dim=-1).norm(dim=-1, p=2, keepdim=False) 28 | ) 29 | dist = dist_c * 2 / sqrt_c 30 | return dist ** 2 31 | 32 | def _lambda_x(self, x, c): 33 | x_sqnorm = torch.sum(x.data.pow(2), dim=-1, keepdim=True) 34 | return 2 / (1. - c * x_sqnorm).clamp_min(self.min_norm) 35 | 36 | def egrad2rgrad(self, p, dp, c): 37 | lambda_p = self._lambda_x(p, c) 38 | dp /= lambda_p.pow(2) 39 | return dp 40 | 41 | def proj(self, x, c): 42 | norm = torch.clamp_min(x.norm(dim=-1, keepdim=True, p=2), self.min_norm) 43 | maxnorm = (1 - self.eps[x.dtype]) / (c ** 0.5) 44 | cond = norm > maxnorm 45 | projected = x / norm * maxnorm 46 | return torch.where(cond, projected, x) 47 | 48 | def proj_tan(self, u, p, c): 49 | return u 50 | 51 | def proj_tan0(self, u, c): 52 | return u 53 | 54 | def expmap(self, u, p, c): 55 | sqrt_c = c ** 0.5 56 | u_norm = u.norm(dim=-1, p=2, keepdim=True).clamp_min(self.min_norm) 57 | second_term = ( 58 | tanh(sqrt_c / 2 * self._lambda_x(p, c) * u_norm) 59 | * u 60 | / (sqrt_c * u_norm) 61 | ) 62 | gamma_1 = self.mobius_add(p, second_term, c) 63 | return gamma_1 64 | 65 | def logmap(self, p1, p2, c): 66 | sub = self.mobius_add(-p1, p2, c) 67 | sub_norm = sub.norm(dim=-1, p=2, keepdim=True).clamp_min(self.min_norm) 68 | lam = self._lambda_x(p1, c) 69 | sqrt_c = c ** 0.5 70 | return 2 / sqrt_c / lam * artanh(sqrt_c * sub_norm) * sub / sub_norm 71 | 72 | def expmap0(self, u, c): 73 | sqrt_c = c ** 0.5 74 | u_norm = torch.clamp_min(u.norm(dim=-1, p=2, keepdim=True), self.min_norm) 75 | gamma_1 = tanh(sqrt_c * u_norm) * u / (sqrt_c * u_norm) 76 | return gamma_1 77 | 78 | def logmap0(self, p, c): 79 | sqrt_c = c ** 0.5 80 | p_norm = p.norm(dim=-1, p=2, keepdim=True).clamp_min(self.min_norm) 81 | scale = 1. / sqrt_c * artanh(sqrt_c * p_norm) / p_norm 82 | return scale * p 83 | 84 | def mobius_add(self, x, y, c, dim=-1): 85 | x2 = x.pow(2).sum(dim=dim, keepdim=True) 86 | y2 = y.pow(2).sum(dim=dim, keepdim=True) 87 | xy = (x * y).sum(dim=dim, keepdim=True) 88 | num = (1 + 2 * c * xy + c * y2) * x + (1 - c * x2) * y 89 | denom = 1 + 2 * c * xy + c ** 2 * x2 * y2 90 | return num / denom.clamp_min(self.min_norm) 91 | 92 | def mobius_matvec(self, m, x, c): 93 | sqrt_c = c ** 0.5 94 | x_norm = x.norm(dim=-1, keepdim=True, p=2).clamp_min(self.min_norm) 95 | mx = x @ m.transpose(-1, -2) 96 | mx_norm = mx.norm(dim=-1, keepdim=True, p=2).clamp_min(self.min_norm) 97 | res_c = tanh(mx_norm / x_norm * artanh(sqrt_c * x_norm)) * mx / (mx_norm * sqrt_c) 98 | cond = (mx == 0).prod(-1, keepdim=True, dtype=torch.uint8) 99 | res_0 = torch.zeros(1, dtype=res_c.dtype, device=res_c.device) 100 | res = torch.where(cond, res_0, res_c) 101 | return res 102 | 103 | def init_weights(self, w, c, irange=1e-5): 104 | w.data.uniform_(-irange, irange) 105 | return w 106 | 107 | def _gyration(self, u, v, w, c, dim: int = -1): 108 | u2 = u.pow(2).sum(dim=dim, keepdim=True) 109 | v2 = v.pow(2).sum(dim=dim, keepdim=True) 110 | uv = (u * v).sum(dim=dim, keepdim=True) 111 | uw = (u * w).sum(dim=dim, keepdim=True) 112 | vw = (v * w).sum(dim=dim, keepdim=True) 113 | c2 = c ** 2 114 | a = -c2 * uw * v2 + c * vw + 2 * c2 * uv * vw 115 | b = -c2 * vw * u2 - c * uw 116 | d = 1 + 2 * c * uv + c2 * u2 * v2 117 | return w + 2 * (a * u + b * v) / d.clamp_min(self.min_norm) 118 | 119 | def inner(self, x, c, u, v=None, keepdim=False): 120 | if v is None: 121 | v = u 122 | lambda_x = self._lambda_x(x, c) 123 | return lambda_x ** 2 * (u * v).sum(dim=-1, keepdim=keepdim) 124 | 125 | def ptransp(self, x, y, u, c): 126 | lambda_x = self._lambda_x(x, c) 127 | lambda_y = self._lambda_x(y, c) 128 | return self._gyration(y, -x, u, c) * lambda_x / lambda_y 129 | 130 | def ptransp_(self, x, y, u, c): 131 | lambda_x = self._lambda_x(x, c) 132 | lambda_y = self._lambda_x(y, c) 133 | return self._gyration(y, -x, u, c) * lambda_x / lambda_y 134 | 135 | def ptransp0(self, x, u, c): 136 | lambda_x = self._lambda_x(x, c) 137 | return 2 * u / lambda_x.clamp_min(self.min_norm) 138 | 139 | def to_hyperboloid(self, x, c): 140 | K = 1./ c 141 | sqrtK = K ** 0.5 142 | sqnorm = torch.norm(x, p=2, dim=1, keepdim=True) ** 2 143 | return sqrtK * torch.cat([K + sqnorm, 2 * sqrtK * x], dim=1) / (K - sqnorm) 144 | 145 | def retr(self, x: torch.Tensor, u: torch.Tensor) -> torch.Tensor: 146 | return x + u 147 | 148 | def transp(self, x: torch.Tensor, y: torch.Tensor, v: torch.Tensor) -> torch.Tensor: 149 | target_shape = broadcast_shapes(x.shape, y.shape, v.shape) 150 | return v.expand(target_shape) 151 | 152 | def from_hyperboloid(self, x, c=1, ideal=False): 153 | """Convert from hyperboloid model to Poincare ball model. 154 | 155 | Note: converting a point from hyperbolic to poincare ball, for a random 156 | vector is a not reversivle i.e. p != from_poincare(to_poincare(p)). 157 | p must be in hyperbolic plane to satisfy the reversiblity. 158 | 159 | Args: 160 | x: torch.tensor of shape (..., Minkowski_dim), where Minkowski_dim >= 3 161 | ideal: boolean. Should be True if the input vectors are ideal points, False otherwise 162 | Returns: 163 | torch.tensor of shape (..., Minkowski_dim - 1) 164 | """ 165 | if ideal: 166 | return x[..., 1:] / (x[..., 0].unsqueeze(-1)).clamp_min(self.min_norm) 167 | else: 168 | sqrtK = (1. / c) ** 0.5 169 | return sqrtK * x[..., 1:] / (sqrtK + x[..., 0].unsqueeze(-1)).clamp_min(self.min_norm) 170 | 171 | def concat(self, v, c = None): 172 | """Concatnates a matrix of dim (M, N) across the last dimension. 173 | 174 | Note: of the inputs are given as a batch, it assumes that dim (B, M, N) 175 | where B is the batch size. 176 | 177 | The concat operation is based on "Hyperbolic Neural Network++" paper. 178 | Args: 179 | v: a tensor of dim(M,N) where M is the number of vectors to be 180 | concatnated, and N is the dimension of the vectors. 181 | 182 | Returns: 183 | A tensor of dim(M*N) (or dim(B, M*N in case of batch inputs)) in 184 | the poincare ball of the same radius. 185 | """ 186 | del c 187 | concat_dim = 1 if len(v.shape) == 3 else 0 188 | a = sc.beta(v.shape[concat_dim]*v.shape[-1]/2, 0.5) / sc.beta(v.shape[-1]/2, 0.5) 189 | # Note that the following multiplication should not be a mobius mul. 190 | # It is there to normalize the raduis of the new PoincareBall to <= 1. 191 | return torch.cat(tensors=torch.unbind(v*a, dim=concat_dim), dim=concat_dim) -------------------------------------------------------------------------------- /graphzoo/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .base_models import * 2 | from .decoders import * 3 | from .encoders import * 4 | 5 | -------------------------------------------------------------------------------- /graphzoo/models/base_models.py: -------------------------------------------------------------------------------- 1 | """ 2 | Base model class 3 | """ 4 | import numpy as np 5 | from sklearn.metrics import roc_auc_score, average_precision_score 6 | import torch 7 | import torch.nn as nn 8 | import torch.nn.functional as F 9 | from graphzoo.layers.layers import FermiDiracDecoder 10 | from graphzoo import manifolds 11 | import graphzoo.models.encoders as encoders 12 | from graphzoo.models.decoders import model2decoder 13 | from graphzoo.utils.eval_utils import acc_f1 14 | 15 | 16 | class BaseModel(nn.Module): 17 | """ 18 | Base model for graph embedding tasks 19 | 20 | Input Parameters 21 | ---------- 22 | 'task': ('nc', 'which tasks to train on, can be any of [lp, nc] (type: str)'), 23 | 'model': ('HGCN', 'which encoder to use, can be any of [Shallow, MLP, HNN, GCN, GAT, HGCN,HGAT] (type: str)'), 24 | 'dim': (128, 'embedding dimension (type: int)'), 25 | 'manifold': ('PoincareBall', 'which manifold to use, can be any of [Euclidean, Hyperboloid, PoincareBall] (type: str)'), 26 | 'c': (1.0, 'hyperbolic radius, set to None for trainable curvature (type: float)'), 27 | 'r': (2.0, 'fermi-dirac decoder parameter for lp (type: float)'), 28 | 't': (1.0, 'fermi-dirac decoder parameter for lp (type: float)'), 29 | 'pretrained-embeddings': (None, 'path to pretrained embeddings (.npy file) for Shallow node classification (type: str)'), 30 | 'num-layers': (2, 'number of hidden layers in encoder (type: int)'), 31 | 'bias': (1, 'whether to use bias (1) or not (0) (type: int)'), 32 | 'act': ('relu', 'which activation function to use or None for no activation (type: str)'), 33 | 'n-heads': (4, 'number of attention heads for graph attention networks, must be a divisor dim (type: int)'), 34 | 'alpha': (0.2, 'alpha for leakyrelu in graph attention networks (type: float)'), 35 | 'use-att': (0, 'whether to use hyperbolic attention (1) or not (0) (type: int)'), 36 | 'local-agg': (0, 'whether to local tangent space aggregation (1) or not (0) (type: int)') 37 | 38 | API Input Parameters 39 | ---------- 40 | args: list of above defined input parameters from `graphzoo.config` 41 | """ 42 | 43 | def __init__(self, args): 44 | super(BaseModel, self).__init__() 45 | self.manifold_name = args.manifold 46 | if args.c is not None: 47 | self.c = torch.tensor([args.c]) 48 | if not args.cuda == -1: 49 | self.c = self.c.to('cuda:' + str(args.cuda)) 50 | else: 51 | self.c = nn.Parameter(torch.Tensor([1.])) 52 | self.manifold = getattr(manifolds, self.manifold_name)() 53 | if self.manifold.name == 'Hyperboloid': 54 | args.feat_dim = args.feat_dim + 1 55 | 56 | self.nnodes = args.n_nodes 57 | self.encoder = getattr(encoders, args.model)(self.c, args) 58 | 59 | def encode(self, x, adj): 60 | if self.manifold.name == 'Hyperboloid': 61 | o = torch.zeros_like(x) 62 | x = torch.cat([o[:, 0:1], x], dim=1) 63 | h = self.encoder.encode(x, adj) 64 | return h 65 | 66 | def compute_metrics(self, embeddings, data, split): 67 | raise NotImplementedError 68 | 69 | def init_metric_dict(self): 70 | raise NotImplementedError 71 | 72 | def has_improved(self, m1, m2): 73 | raise NotImplementedError 74 | 75 | 76 | class NCModel(BaseModel): 77 | """ 78 | Base model for node classification task 79 | """ 80 | 81 | def __init__(self, args): 82 | super(NCModel, self).__init__(args) 83 | self.decoder = model2decoder[args.model](self.c, args) 84 | 85 | if args.n_classes > 2: 86 | self.f1_average = 'micro' 87 | else: 88 | self.f1_average = 'binary' 89 | 90 | self.weights = torch.Tensor([1.] * args.n_classes) 91 | 92 | if not args.cuda == -1: 93 | self.weights = self.weights.to('cuda:' + str(args.cuda)) 94 | 95 | def decode(self, h, adj, idx): 96 | output = self.decoder.decode(h, adj) 97 | return F.log_softmax(output[idx], dim=1) 98 | 99 | def compute_metrics(self, embeddings, data, split): 100 | idx = data[f'idx_{split}'] 101 | output = self.decode(embeddings, data['adj_train_norm'], idx) 102 | loss = F.nll_loss(output, data['labels'][idx], self.weights) 103 | acc, f1 = acc_f1(output, data['labels'][idx], average=self.f1_average) 104 | metrics = {'loss': loss, 'acc': acc, 'f1': f1} 105 | return metrics 106 | 107 | def init_metric_dict(self): 108 | return {'acc': -1, 'f1': -1} 109 | 110 | def has_improved(self, m1, m2): 111 | return m1["f1"] < m2["f1"] 112 | 113 | 114 | class LPModel(BaseModel): 115 | """ 116 | Base model for link prediction task 117 | """ 118 | 119 | def __init__(self, args): 120 | super(LPModel, self).__init__(args) 121 | self.dc = FermiDiracDecoder(r=args.r, t=args.t) 122 | 123 | def decode(self, h, idx): 124 | if self.manifold_name == 'Euclidean': 125 | h = self.manifold.normalize(h) 126 | emb_in = h[idx[:, 0], :] 127 | emb_out = h[idx[:, 1], :] 128 | sqdist = self.manifold.sqdist(emb_in, emb_out, self.c) 129 | probs = self.dc.forward(sqdist) 130 | return probs 131 | 132 | def compute_metrics(self, embeddings, data, split): 133 | if split == 'train': 134 | nb_false_edges = len(data['train_edges_false']) 135 | nb_edges = len(data['train_edges']) 136 | edges_false = data[f'{split}_edges_false'][np.random.randint(0, nb_false_edges, nb_edges)] 137 | else: 138 | edges_false = data[f'{split}_edges_false'] 139 | pos_scores = self.decode(embeddings, data[f'{split}_edges']) 140 | neg_scores = self.decode(embeddings, edges_false) 141 | loss = F.binary_cross_entropy(pos_scores, torch.ones_like(pos_scores)) 142 | loss += F.binary_cross_entropy(neg_scores, torch.zeros_like(neg_scores)) 143 | if pos_scores.is_cuda: 144 | pos_scores = pos_scores.cpu() 145 | neg_scores = neg_scores.cpu() 146 | labels = [1] * pos_scores.shape[0] + [0] * neg_scores.shape[0] 147 | preds = list(pos_scores.data.numpy()) + list(neg_scores.data.numpy()) 148 | roc = roc_auc_score(labels, preds) 149 | ap = average_precision_score(labels, preds) 150 | metrics = {'loss': loss, 'roc': roc, 'ap': ap} 151 | return metrics 152 | 153 | def init_metric_dict(self): 154 | return {'roc': -1, 'ap': -1} 155 | 156 | def has_improved(self, m1, m2): 157 | return 0.5 * (m1['roc'] + m1['ap']) < 0.5 * (m2['roc'] + m2['ap']) 158 | 159 | -------------------------------------------------------------------------------- /graphzoo/models/decoders.py: -------------------------------------------------------------------------------- 1 | """Graph decoders""" 2 | from graphzoo import manifolds 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | from graphzoo.layers.att_layers import GraphAttentionLayer 6 | from graphzoo.layers.layers import GraphConvolution, Linear 7 | from graphzoo.layers.hyp_att_layers import GraphAttentionLayer as HGraphAttentionLayer 8 | 9 | class Decoder(nn.Module): 10 | """ 11 | Decoder abstract class for node classification tasks 12 | """ 13 | 14 | def __init__(self, c): 15 | super(Decoder, self).__init__() 16 | self.c = c 17 | 18 | def decode(self, x, adj): 19 | if self.decode_adj: 20 | input = (x, adj) 21 | probs, _ = self.cls.forward(input) 22 | else: 23 | probs = self.cls.forward(x) 24 | return probs 25 | 26 | 27 | class GCNDecoder(Decoder): 28 | """ 29 | Graph Convolution Decoder 30 | """ 31 | 32 | def __init__(self, c, args): 33 | super(GCNDecoder, self).__init__(c) 34 | act = lambda x: x 35 | self.cls = GraphConvolution(args.dim, args.n_classes, args.dropout, act, args.bias) 36 | self.decode_adj = True 37 | 38 | 39 | class GATDecoder(Decoder): 40 | """ 41 | Graph Attention Decoder 42 | """ 43 | 44 | def __init__(self, c, args): 45 | super(GATDecoder, self).__init__(c) 46 | self.cls = GraphAttentionLayer(args.dim, args.n_classes, args.dropout, F.elu, args.alpha, 1, True) 47 | self.decode_adj = True 48 | 49 | 50 | class LinearDecoder(Decoder): 51 | """ 52 | MLP Decoder for Hyperbolic/Euclidean node classification models 53 | """ 54 | 55 | def __init__(self, c, args): 56 | super(LinearDecoder, self).__init__(c) 57 | self.manifold = getattr(manifolds, args.manifold)() 58 | self.input_dim = args.dim 59 | self.output_dim = args.n_classes 60 | self.bias = args.bias 61 | self.cls = Linear(self.input_dim, self.output_dim, args.dropout, lambda x: x, self.bias) 62 | self.decode_adj = False 63 | 64 | def decode(self, x, adj): 65 | h = self.manifold.proj_tan0(self.manifold.logmap0(x, c=self.c), c=self.c) 66 | return super(LinearDecoder, self).decode(h, adj) 67 | 68 | def extra_repr(self): 69 | return 'in_features={}, out_features={}, bias={}, c={}'.format( 70 | self.input_dim, self.output_dim, self.bias, self.c 71 | ) 72 | 73 | class HGATDecoder(Decoder): 74 | """ 75 | Graph Attention Decoder. 76 | """ 77 | 78 | def __init__(self, c, args): 79 | super(HGATDecoder, self).__init__(c) 80 | self.manifold = getattr(manifolds, args.manifold)() 81 | self.cls = HGraphAttentionLayer( 82 | manifold=self.manifold, 83 | input_dim=args.dim, 84 | output_dim=args.n_classes, 85 | dropout=args.dropout, 86 | activation=F.elu, 87 | alpha=args.alpha, 88 | nheads=1, 89 | concat=True, 90 | curvature=self.c, 91 | use_bias= args.bias) 92 | self.decode_adj = True 93 | 94 | def decode(self, x, adj): 95 | x = super(HGATDecoder, self).decode(x, adj) 96 | return self.manifold.proj_tan0(self.manifold.logmap0(x, c=self.c), c=self.c) 97 | 98 | model2decoder = { 99 | 'GCN': GCNDecoder, 100 | 'GAT': GATDecoder, 101 | 'HNN': LinearDecoder, 102 | 'HGCN': LinearDecoder, 103 | 'HGAT': HGATDecoder, 104 | 'MLP': LinearDecoder, 105 | 'Shallow': LinearDecoder 106 | } 107 | 108 | -------------------------------------------------------------------------------- /graphzoo/models/encoders.py: -------------------------------------------------------------------------------- 1 | """Graph encoders""" 2 | import numpy as np 3 | import torch 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | from graphzoo import manifolds 7 | from graphzoo.layers.att_layers import GraphAttentionLayer 8 | import graphzoo.layers.hyp_layers as hyp_layers 9 | from graphzoo.layers.layers import GraphConvolution, Linear, get_dim_act 10 | import graphzoo.utils.math_utils as pmath 11 | from graphzoo.layers.hyp_att_layers import GraphAttentionLayer as HGATlayer 12 | 13 | class Encoder(nn.Module): 14 | """ 15 | Encoder abstract class 16 | """ 17 | 18 | def __init__(self, c): 19 | super(Encoder, self).__init__() 20 | self.c = c 21 | 22 | def encode(self, x, adj): 23 | if self.encode_graph: 24 | input = (x, adj) 25 | output, _ = self.layers.forward(input) 26 | else: 27 | output = self.layers.forward(x) 28 | return output 29 | 30 | class MLP(Encoder): 31 | """ 32 | Multi-layer perceptron 33 | """ 34 | 35 | def __init__(self, c, args): 36 | super(MLP, self).__init__(c) 37 | assert args.num_layers > 0 38 | dims, acts = get_dim_act(args) 39 | layers = [] 40 | for i in range(len(dims) - 1): 41 | in_dim, out_dim = dims[i], dims[i + 1] 42 | act = acts[i] 43 | layers.append(Linear(in_dim, out_dim, args.dropout, act, args.bias)) 44 | self.layers = nn.Sequential(*layers) 45 | self.encode_graph = False 46 | 47 | 48 | class HNN(Encoder): 49 | """ 50 | Hyperbolic Neural Networks 51 | """ 52 | 53 | def __init__(self, c, args): 54 | super(HNN, self).__init__(c) 55 | self.manifold = getattr(manifolds, args.manifold)() 56 | assert args.num_layers > 1 57 | dims, acts, _ = hyp_layers.get_dim_act_curv(args) 58 | hnn_layers = [] 59 | for i in range(len(dims) - 1): 60 | in_dim, out_dim = dims[i], dims[i + 1] 61 | act = acts[i] 62 | hnn_layers.append( 63 | hyp_layers.HNNLayer( 64 | self.manifold, in_dim, out_dim, self.c, args.dropout, act, args.bias) 65 | ) 66 | self.layers = nn.Sequential(*hnn_layers) 67 | self.encode_graph = False 68 | 69 | def encode(self, x, adj): 70 | x_hyp = self.manifold.proj(self.manifold.expmap0(self.manifold.proj_tan0(x, self.c), c=self.c), c=self.c) 71 | return super(HNN, self).encode(x_hyp, adj) 72 | 73 | class GCN(Encoder): 74 | """ 75 | Graph Convolution Networks 76 | """ 77 | 78 | def __init__(self, c, args): 79 | super(GCN, self).__init__(c) 80 | assert args.num_layers > 0 81 | dims, acts = get_dim_act(args) 82 | gc_layers = [] 83 | for i in range(len(dims) - 1): 84 | in_dim, out_dim = dims[i], dims[i + 1] 85 | act = acts[i] 86 | gc_layers.append(GraphConvolution(in_dim, out_dim, args.dropout, act, args.bias)) 87 | self.layers = nn.Sequential(*gc_layers) 88 | self.encode_graph = True 89 | 90 | 91 | class HGCN(Encoder): 92 | """ 93 | Hyperbolic-GCN 94 | """ 95 | 96 | def __init__(self, c, args): 97 | super(HGCN, self).__init__(c) 98 | self.manifold = getattr(manifolds, args.manifold)() 99 | assert args.num_layers > 1 100 | dims, acts, self.curvatures = hyp_layers.get_dim_act_curv(args) 101 | self.curvatures.append(self.c) 102 | hgc_layers = [] 103 | for i in range(len(dims) - 1): 104 | c_in, c_out = self.curvatures[i], self.curvatures[i + 1] 105 | in_dim, out_dim = dims[i], dims[i + 1] 106 | act = acts[i] 107 | hgc_layers.append( 108 | hyp_layers.HyperbolicGraphConvolution( 109 | self.manifold, in_dim, out_dim, c_in, c_out, args.dropout, act, args.bias, args.use_att, args.local_agg 110 | ) 111 | ) 112 | self.layers = nn.Sequential(*hgc_layers) 113 | self.encode_graph = True 114 | 115 | def encode(self, x, adj): 116 | x_tan = self.manifold.proj_tan0(x, self.curvatures[0]) 117 | x_hyp = self.manifold.expmap0(x_tan, c=self.curvatures[0]) 118 | x_hyp = self.manifold.proj(x_hyp, c=self.curvatures[0]) 119 | return super(HGCN, self).encode(x_hyp, adj) 120 | 121 | 122 | class GAT(Encoder): 123 | """ 124 | Graph Attention Networks 125 | """ 126 | 127 | def __init__(self, c, args): 128 | super(GAT, self).__init__(c) 129 | assert args.num_layers > 0 130 | dims, acts = get_dim_act(args) 131 | gat_layers = [] 132 | for i in range(len(dims) - 1): 133 | in_dim, out_dim = dims[i], dims[i + 1] 134 | act = acts[i] 135 | assert dims[i + 1] % args.n_heads == 0 136 | out_dim = dims[i + 1] // args.n_heads 137 | concat = True 138 | gat_layers.append( 139 | GraphAttentionLayer(in_dim, out_dim, args.dropout, act, args.alpha, args.n_heads, concat)) 140 | self.layers = nn.Sequential(*gat_layers) 141 | self.encode_graph = True 142 | 143 | 144 | class Shallow(Encoder): 145 | """ 146 | Shallow Embedding method 147 | Learns embeddings or loads pretrained embeddings and uses an MLP for classification 148 | """ 149 | 150 | def __init__(self, c, args): 151 | super(Shallow, self).__init__(c) 152 | self.manifold = getattr(manifolds, args.manifold)() 153 | self.use_feats = args.use_feats 154 | weights = torch.Tensor(args.n_nodes, args.dim) 155 | if not args.pretrained_embeddings: 156 | weights = self.manifold.init_weights(weights, self.c) 157 | trainable = True 158 | else: 159 | weights = torch.Tensor(np.load(args.pretrained_embeddings)) 160 | assert weights.shape[0] == args.n_nodes, "The embeddings you passed seem to be for another dataset." 161 | trainable = False 162 | self.lt = manifolds.ManifoldParameter(weights, trainable, self.manifold, self.c) 163 | self.all_nodes = torch.LongTensor(list(range(args.n_nodes))) 164 | layers = [] 165 | if args.pretrained_embeddings is not None and args.num_layers > 0: 166 | # MLP layers after pre-trained embeddings 167 | dims, acts = get_dim_act(args) 168 | if self.use_feats: 169 | dims[0] = args.feat_dim + weights.shape[1] 170 | else: 171 | dims[0] = weights.shape[1] 172 | for i in range(len(dims) - 1): 173 | in_dim, out_dim = dims[i], dims[i + 1] 174 | act = acts[i] 175 | layers.append(Linear(in_dim, out_dim, args.dropout, act, args.bias)) 176 | self.layers = nn.Sequential(*layers) 177 | self.encode_graph = False 178 | 179 | def encode(self, x, adj): 180 | h = self.lt[self.all_nodes, :] 181 | if self.use_feats: 182 | h = torch.cat((h, x), 1) 183 | return super(Shallow, self).encode(h, adj) 184 | 185 | class HGAT(Encoder): 186 | """ 187 | Hyperbolic Graph Attention Networks. 188 | """ 189 | 190 | def __init__(self, c, args): 191 | super(HGAT, self).__init__(c) 192 | assert args.num_layers > 0 193 | self.manifold = getattr(manifolds, args.manifold)() 194 | dims, acts, self.curvatures = hyp_layers.get_dim_act_curv(args) 195 | gat_layers = [] 196 | for i in range(len(dims) - 1): 197 | in_dim, out_dim = dims[i], dims[i + 1] 198 | act = acts[i] 199 | assert dims[i + 1] % args.n_heads == 0 200 | out_dim = dims[i + 1] // args.n_heads 201 | concat = True 202 | gat_layers.append( 203 | HGATlayer( 204 | manifold=self.manifold, 205 | input_dim=in_dim, 206 | output_dim=out_dim, 207 | dropout=args.dropout, 208 | activation=act, 209 | alpha=args.alpha, 210 | nheads=args.n_heads, 211 | concat=concat, 212 | curvature=self.curvatures[i], 213 | use_bias=args.bias)) 214 | 215 | self.layers = nn.Sequential(*gat_layers) 216 | self.encode_graph = True 217 | 218 | def update_curvature(self, c): 219 | self.c = torch.Tensor([c]) 220 | for idx, _ in enumerate(self.curvatures): 221 | self.curvatures[idx] = self.c 222 | for layer in self.layers: 223 | layer.update_curvature(self.c) 224 | 225 | def encode(self, x, adj): 226 | x_tan = self.manifold.proj_tan0(x, self.curvatures[0]) 227 | x_hyp = self.manifold.expmap0(x_tan, c=self.curvatures[0]) 228 | x_hyp = self.manifold.proj(x_hyp, c=self.curvatures[0]) 229 | x = self.manifold.proj(self.manifold.expmap0(self.manifold.proj_tan0(x, self.curvatures[0]), c=self.curvatures[0]), c=self.curvatures[0]) 230 | return super(HGAT, self).encode(x_hyp, adj) 231 | -------------------------------------------------------------------------------- /graphzoo/optimizers/__init__.py: -------------------------------------------------------------------------------- 1 | from .radam import * 2 | from .rsgd import * -------------------------------------------------------------------------------- /graphzoo/optimizers/radam.py: -------------------------------------------------------------------------------- 1 | """Riemannian Adam optimizer geoopt implementation (https://github.com/geoopt/).""" 2 | import torch.optim 3 | from graphzoo.manifolds import Euclidean, ManifoldParameter 4 | 5 | # in order not to create it at each iteration 6 | _default_manifold = Euclidean() 7 | 8 | 9 | class OptimMixin(object): 10 | def __init__(self, *args, stabilize=None, **kwargs): 11 | self._stabilize = stabilize 12 | super().__init__(*args, **kwargs) 13 | 14 | def stabilize_group(self, group): 15 | pass 16 | 17 | def stabilize(self): 18 | """Stabilize parameters if they are off-manifold due to numerical reasons""" 19 | 20 | for group in self.param_groups: 21 | self.stabilize_group(group) 22 | 23 | 24 | def copy_or_set_(dest, source): 25 | """ 26 | A workaround to respect strides of :code:`dest` when copying :code:`source` 27 | (https://github.com/geoopt/geoopt/issues/70) 28 | Parameters 29 | ---------- 30 | dest : torch.Tensor 31 | Destination tensor where to store new data 32 | source : torch.Tensor 33 | Source data to put in the new tensor 34 | Returns 35 | ------- 36 | dest 37 | torch.Tensor, modified inplace 38 | """ 39 | if dest.stride() != source.stride(): 40 | return dest.copy_(source) 41 | else: 42 | return dest.set_(source) 43 | 44 | 45 | class RiemannianAdam(OptimMixin, torch.optim.Adam): 46 | """ 47 | Riemannian Adam with the same API as :class:`torch.optim.Adam` 48 | 49 | Parameters 50 | ---------- 51 | params : iterable 52 | iterable of parameters to optimize or dicts defining 53 | parameter groups 54 | lr : float (optional) 55 | learning rate (default: 1e-3) 56 | betas : Tuple[float, float] (optional) 57 | coefficients used for computing 58 | running averages of gradient and its square (default: (0.9, 0.999)) 59 | eps : float (optional) 60 | term added to the denominator to improve 61 | numerical stability (default: 1e-8) 62 | weight_decay : float (optional) 63 | weight decay (L2 penalty) (default: 0) 64 | amsgrad : bool (optional) 65 | whether to use the AMSGrad variant of this 66 | algorithm from the paper `On the Convergence of Adam and Beyond`_ 67 | (default: False) 68 | 69 | Other Parameters 70 | ---------------- 71 | stabilize : int 72 | stabilize parameters if they are off-manifold due to numerical 73 | reasons every ``stabilize`` steps (default: ``None`` -- no stabilize) 74 | On the Convergence of Adam and Beyond: 75 | https://openreview.net/forum?id=ryQu7f-RZ 76 | """ 77 | 78 | def step(self, closure=None): 79 | """Performs a single optimization step. 80 | Arguments 81 | --------- 82 | closure : callable (optional) 83 | A closure that reevaluates the model 84 | and returns the loss. 85 | """ 86 | loss = None 87 | if closure is not None: 88 | loss = closure() 89 | with torch.no_grad(): 90 | for group in self.param_groups: 91 | if "step" not in group: 92 | group["step"] = 0 93 | betas = group["betas"] 94 | weight_decay = group["weight_decay"] 95 | eps = group["eps"] 96 | learning_rate = group["lr"] 97 | amsgrad = group["amsgrad"] 98 | for point in group["params"]: 99 | grad = point.grad 100 | if grad is None: 101 | continue 102 | if isinstance(point, (ManifoldParameter)): 103 | manifold = point.manifold 104 | c = point.c 105 | else: 106 | manifold = _default_manifold 107 | c = None 108 | if grad.is_sparse: 109 | raise RuntimeError( 110 | "Riemannian Adam does not support sparse gradients yet (PR is welcome)" 111 | ) 112 | 113 | state = self.state[point] 114 | 115 | # State initialization 116 | if len(state) == 0: 117 | state["step"] = 0 118 | # Exponential moving average of gradient values 119 | state["exp_avg"] = torch.zeros_like(point) 120 | # Exponential moving average of squared gradient values 121 | state["exp_avg_sq"] = torch.zeros_like(point) 122 | if amsgrad: 123 | # Maintains max of all exp. moving avg. of sq. grad. values 124 | state["max_exp_avg_sq"] = torch.zeros_like(point) 125 | # make local variables for easy access 126 | exp_avg = state["exp_avg"] 127 | exp_avg_sq = state["exp_avg_sq"] 128 | # actual step 129 | grad.add_(weight_decay, point) 130 | grad = manifold.egrad2rgrad(point, grad, c) 131 | exp_avg.mul_(betas[0]).add_(1 - betas[0], grad) 132 | exp_avg_sq.mul_(betas[1]).add_( 133 | 1 - betas[1], manifold.inner(point, c, grad, keepdim=True) 134 | ) 135 | if amsgrad: 136 | max_exp_avg_sq = state["max_exp_avg_sq"] 137 | # Maintains the maximum of all 2nd moment running avg. till now 138 | torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq) 139 | # Use the max. for normalizing running avg. of gradient 140 | denom = max_exp_avg_sq.sqrt().add_(eps) 141 | else: 142 | denom = exp_avg_sq.sqrt().add_(eps) 143 | group["step"] += 1 144 | bias_correction1 = 1 - betas[0] ** group["step"] 145 | bias_correction2 = 1 - betas[1] ** group["step"] 146 | step_size = ( 147 | learning_rate * bias_correction2 ** 0.5 / bias_correction1 148 | ) 149 | 150 | # copy the state, we need it for retraction 151 | # get the direction for ascend 152 | direction = exp_avg / denom 153 | # transport the exponential averaging to the new point 154 | new_point = manifold.proj(manifold.expmap(-step_size * direction, point, c), c) 155 | exp_avg_new = manifold.ptransp(point, new_point, exp_avg, c) 156 | # use copy only for user facing point 157 | copy_or_set_(point, new_point) 158 | exp_avg.set_(exp_avg_new) 159 | 160 | group["step"] += 1 161 | if self._stabilize is not None and group["step"] % self._stabilize == 0: 162 | self.stabilize_group(group) 163 | return loss 164 | 165 | @torch.no_grad() 166 | def stabilize_group(self, group): 167 | for p in group["params"]: 168 | if not isinstance(p, ManifoldParameter): 169 | continue 170 | state = self.state[p] 171 | if not state: # due to None grads 172 | continue 173 | manifold = p.manifold 174 | c = p.c 175 | exp_avg = state["exp_avg"] 176 | copy_or_set_(p, manifold.proj(p, c)) 177 | exp_avg.set_(manifold.proj_tan(exp_avg, u, c)) -------------------------------------------------------------------------------- /graphzoo/optimizers/rsgd.py: -------------------------------------------------------------------------------- 1 | """Riemannian Stochastic Gradient Descent optimizer geoopt implementation (https://github.com/geoopt/).""" 2 | import torch 3 | from graphzoo.manifolds import Euclidean, ManifoldParameter 4 | 5 | _default_manifold = Euclidean() 6 | 7 | 8 | class OptimMixin(object): 9 | _default_manifold = Euclidean() 10 | 11 | def __init__(self, *args, stabilize=None, **kwargs): 12 | self._stabilize = stabilize 13 | super().__init__(*args, **kwargs) 14 | 15 | def add_param_group(self, param_group: dict): 16 | param_group.setdefault("stabilize", self._stabilize) 17 | return super().add_param_group(param_group) 18 | 19 | def stabilize_group(self, group): 20 | pass 21 | 22 | def stabilize(self): 23 | """Stabilize parameters if they are off-manifold due to numerical reasons""" 24 | for group in self.param_groups: 25 | self.stabilize_group(group) 26 | 27 | 28 | class RiemannianSGD(OptimMixin, torch.optim.Optimizer): 29 | """ 30 | Riemannian Stochastic Gradient Descent with the same API as :class:`torch.optim.SGD` 31 | 32 | Parameters 33 | ---------- 34 | params : iterable 35 | iterable of parameters to optimize or dicts defining 36 | parameter groups 37 | lr : float 38 | learning rate 39 | momentum : float (optional) 40 | momentum factor (default: 0) 41 | weight_decay : float (optional) 42 | weight decay (L2 penalty) (default: 0) 43 | dampening : float (optional) 44 | dampening for momentum (default: 0) 45 | nesterov : bool (optional) 46 | enables Nesterov momentum (default: False) 47 | 48 | Other Parameters 49 | ---------------- 50 | stabilize : int 51 | stabilize parameters if they are off-manifold due to numerical 52 | reasons every ``stabilize`` steps (default: ``None`` -- no stabilize) 53 | On the Convergence of Adam and Beyond: 54 | https://openreview.net/forum?id=ryQu7f-RZ 55 | """ 56 | 57 | def __init__( 58 | self, 59 | params, 60 | lr, 61 | momentum=0, 62 | dampening=0, 63 | weight_decay=0, 64 | nesterov=False, 65 | stabilize=None, 66 | ): 67 | if lr < 0.0: 68 | raise ValueError("Invalid learning rate: {}".format(lr)) 69 | if momentum < 0.0: 70 | raise ValueError("Invalid momentum value: {}".format(momentum)) 71 | if weight_decay < 0.0: 72 | raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) 73 | 74 | defaults = dict( 75 | lr=lr, 76 | momentum=momentum, 77 | dampening=dampening, 78 | weight_decay=weight_decay, 79 | nesterov=nesterov, 80 | ) 81 | if nesterov and (momentum <= 0 or dampening != 0): 82 | raise ValueError("Nesterov momentum requires a momentum and zero dampening") 83 | super().__init__(params, defaults, stabilize=stabilize) 84 | 85 | def step(self, closure=None): 86 | loss = None 87 | if closure is not None: 88 | loss = closure() 89 | with torch.no_grad(): 90 | for group in self.param_groups: 91 | if "step" not in group: 92 | group["step"] = 0 93 | weight_decay = group["weight_decay"] 94 | momentum = group["momentum"] 95 | dampening = group["dampening"] 96 | nesterov = group["nesterov"] 97 | learning_rate = group["lr"] 98 | group["step"] += 1 99 | 100 | for point in group["params"]: 101 | grad = point.grad 102 | if grad is None: 103 | continue 104 | if grad.is_sparse: 105 | raise RuntimeError( 106 | "RiemannianSGD does not support sparse gradients, use SparseRiemannianSGD instead" 107 | ) 108 | state = self.state[point] 109 | 110 | # State initialization 111 | if len(state) == 0: 112 | if momentum > 0: 113 | state["momentum_buffer"] = grad.clone() 114 | 115 | if isinstance(point, ManifoldParameter): 116 | manifold = point.manifold 117 | c = point.c 118 | else: 119 | manifold = _default_manifold 120 | c = None 121 | 122 | grad.add_(point, alpha=weight_decay) 123 | grad = manifold.egrad2rgrad(point, grad, c) 124 | 125 | if momentum > 0: 126 | momentum_buffer = state["momentum_buffer"] 127 | momentum_buffer.mul_(momentum).add_(grad, alpha=1 - dampening) 128 | if nesterov: 129 | grad = grad.add_(momentum_buffer, alpha=momentum) 130 | else: 131 | grad = momentum_buffer 132 | # we have all the things projected 133 | new_point, new_momentum_buffer = manifold.retr_transp( 134 | point, -learning_rate * grad, momentum_buffer 135 | ) 136 | momentum_buffer.copy_(new_momentum_buffer) 137 | # use copy only for user facing point 138 | point.copy_(new_point) 139 | else: 140 | new_point = manifold.retr(point, -learning_rate * grad) 141 | point.copy_(new_point) 142 | 143 | if ( 144 | group["stabilize"] is not None 145 | and group["step"] % group["stabilize"] == 0 146 | ): 147 | self.stabilize_group(group) 148 | return loss 149 | 150 | @torch.no_grad() 151 | def stabilize_group(self, group): 152 | for p in group["params"]: 153 | if not isinstance(p, ManifoldParameter): 154 | continue 155 | manifold = p.manifold 156 | momentum = group["momentum"] 157 | p.copy_(manifold.projx(p)) 158 | if momentum > 0: 159 | param_state = self.state[p] 160 | if not param_state: # due to None grads 161 | continue 162 | if "momentum_buffer" in param_state: 163 | buf = param_state["momentum_buffer"] 164 | buf.copy_(manifold.proju(p, buf)) -------------------------------------------------------------------------------- /graphzoo/trainers/__init__.py: -------------------------------------------------------------------------------- 1 | from .train import Trainer -------------------------------------------------------------------------------- /graphzoo/trainers/train.py: -------------------------------------------------------------------------------- 1 | """GraphZoo trainer""" 2 | from __future__ import division 3 | from __future__ import print_function 4 | import datetime 5 | import json 6 | import logging 7 | from operator import ne 8 | import os 9 | import pickle 10 | import time 11 | import numpy as np 12 | import torch 13 | import torch.optim as optim 14 | from graphzoo.optimizers.radam import RiemannianAdam 15 | from graphzoo.optimizers.rsgd import RiemannianSGD 16 | from graphzoo.config import parser 17 | from graphzoo.models.base_models import NCModel, LPModel 18 | from graphzoo.utils.train_utils import get_dir_name, format_metrics 19 | from graphzoo.dataloader.dataloader import DataLoader 20 | from graphzoo.dataloader.download import download_and_extract 21 | 22 | class Trainer: 23 | """ 24 | GraphZoo Trainer 25 | 26 | Input Parameters 27 | ---------- 28 | 'lr': (0.05, 'initial learning rate (type: float)'), 29 | 'dropout': (0.0, 'dropout probability (type: float)'), 30 | 'cuda': (-1, 'which cuda device to use or -1 for cpu training (type: int)'), 31 | 'repeat': (10, 'number of times to repeat the experiment (type: int)'), 32 | 'optimizer': ('Adam', 'which optimizer to use, can be any of [Adam, RiemannianAdam, RiemannianSGD] (type: str)'), 33 | 'epochs': (5000, 'maximum number of epochs to train for (type:int)'), 34 | 'weight-decay': (0.0, 'l2 regularization strength (type: float)'), 35 | 'momentum': (0.999, 'momentum in optimizer (type: float)'), 36 | 'patience': (100, 'patience for early stopping (type: int)'), 37 | 'seed': (1234, 'seed for training (type: int)'), 38 | 'log-freq': (5, 'how often to compute print train/val metrics in epochs (type: int)'), 39 | 'eval-freq': (1, 'how often to compute val metrics in epochs (type: int)'), 40 | 'save': (0, '1 to save model and logs and 0 otherwise (type: int)'), 41 | 'save-dir': (None, 'path to save training logs and model weights (type: str)'), 42 | 'lr-reduce-freq': (None, 'reduce lr every lr-reduce-freq or None to keep lr constant (type: int)'), 43 | 'gamma': (0.5, 'gamma for lr scheduler (type: float)'), 44 | 'grad-clip': (None, 'max norm for gradient clipping, or None for no gradient clipping (type: float)'), 45 | 'min-epochs': (100, 'do not early stop before min-epochs (type: int)'), 46 | 'betas': ((0.9, 0.999), 'coefficients used for computing running averages of gradient and its square (type: Tuple[float, float])'), 47 | 'eps': (1e-8, 'term added to the denominator to improve numerical stability (type: float)'), 48 | 'amsgrad': (False, 'whether to use the AMSGrad variant of this algorithm from the paper `On the Convergence of Adam and Beyond` (type: bool)'), 49 | 'stabilize': (None, 'stabilize parameters if they are off-manifold due to numerical reasons every ``stabilize`` steps (type: int)'), 50 | 'dampening': (0,'dampening for momentum (type: float)'), 51 | 'nesterov': (False,'enables Nesterov momentum (type: bool)') 52 | 53 | API Input Parameters 54 | ---------- 55 | args: list of above defined input parameters from `graphzoo.config` 56 | optimizer: a :class:`optim.Optimizer` instance 57 | model: a :class:`BaseModel` instance 58 | 59 | """ 60 | def __init__(self,args,model, optimizer,data): 61 | 62 | self.args=args 63 | self.model=model 64 | self.optimizer =optimizer 65 | self.data=data 66 | self.best_test_metrics = None 67 | self.best_emb = None 68 | self.best_val_metrics = self.model.init_metric_dict() 69 | 70 | np.random.seed(self.args.seed) 71 | torch.manual_seed(self.args.seed) 72 | 73 | if int(self.args.cuda) >= 0: 74 | torch.cuda.manual_seed(self.args.seed) 75 | 76 | if args.cuda is None: 77 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 78 | 79 | else: 80 | device = 'cuda:' + str(args.cuda) if int(args.cuda) >= 0 else 'cpu' 81 | 82 | logging.getLogger().setLevel(logging.INFO) 83 | if self.args.save: 84 | if not self.args.save_dir: 85 | dt = datetime.datetime.now() 86 | date = f"{dt.year}_{dt.month}_{dt.day}" 87 | models_dir = os.path.join(os.getcwd(), self.args.dataset, self.args.task,self.args.model, date) 88 | self.save_dir = get_dir_name(models_dir) 89 | else: 90 | self.save_dir = self.args.save_dir 91 | logging.basicConfig(level=logging.INFO, 92 | handlers=[ 93 | logging.FileHandler(os.path.join(self.save_dir, 'log.txt')), 94 | logging.StreamHandler() 95 | ]) 96 | 97 | logging.info(f'Using: {device}') 98 | logging.info("Using seed {}.".format(self.args.seed)) 99 | 100 | if not self.args.lr_reduce_freq: 101 | self.lr_reduce_freq = self.args.epochs 102 | 103 | logging.info(str(self.model)) 104 | 105 | self.lr_scheduler = torch.optim.lr_scheduler.StepLR( 106 | self.optimizer, 107 | step_size=int(self.lr_reduce_freq), 108 | gamma=float(self.args.gamma) 109 | ) 110 | tot_params = sum([np.prod(p.size()) for p in self.model.parameters()]) 111 | logging.info(f"Total number of parameters: {tot_params}") 112 | if self.args.cuda is not None and int(self.args.cuda) >= 0 : 113 | os.environ['CUDA_VISIBLE_DEVICES'] = str(self.args.cuda) 114 | self.model = self.model.to(device) 115 | for x, val in self.data.items(): 116 | if torch.is_tensor(self.data[x]): 117 | self.data[x] = self.data[x].to(device) 118 | 119 | def run(self): 120 | """ 121 | Train model. 122 | 123 | The processes: 124 | Run each epoch -> Run scheduler -> Should stop early? 125 | """ 126 | t_total = time.time() 127 | counter = 0 128 | for epoch in range(self.args.epochs): 129 | t = time.time() 130 | self.model.train() 131 | self.optimizer.zero_grad() 132 | embeddings = self.model.encode(self.data['features'], self.data['adj_train_norm']) 133 | train_metrics = self.model.compute_metrics(embeddings, self.data, 'train') 134 | train_metrics['loss'].backward() 135 | if self.args.grad_clip is not None: 136 | max_norm = float(self.args.grad_clip) 137 | all_params = list(self.model.parameters()) 138 | for param in all_params: 139 | torch.nn.utils.clip_grad_norm_(param, max_norm) 140 | self.optimizer.step() 141 | self.lr_scheduler.step() 142 | if (epoch + 1) % self.args.log_freq == 0: 143 | logging.info(" ".join(['Epoch: {:04d}'.format(epoch + 1), 144 | 'lr: {}'.format(self.lr_scheduler.get_lr()[0]), 145 | format_metrics(train_metrics, 'train'), 146 | 'time: {:.4f}s'.format(time.time() - t) 147 | ])) 148 | if (epoch + 1) % self.args.eval_freq == 0: 149 | self.model.eval() 150 | embeddings = self.model.encode(self.data['features'], self.data['adj_train_norm']) 151 | val_metrics = self.model.compute_metrics(embeddings, self.data, 'val') 152 | if (epoch + 1) % self.args.log_freq == 0: 153 | logging.info(" ".join(['Epoch: {:04d}'.format(epoch + 1), format_metrics(val_metrics, 'val')])) 154 | if self.model.has_improved(self.best_val_metrics, val_metrics): 155 | self.best_test_metrics = self.model.compute_metrics(embeddings, self.data, 'test') 156 | self.best_emb = embeddings.cpu() 157 | if self.args.save: 158 | np.save(os.path.join(self.save_dir, 'embeddings.npy'), self.best_emb.detach().numpy()) 159 | self.best_val_metrics = val_metrics 160 | counter = 0 161 | else: 162 | counter += 1 163 | if counter == self.args.patience and epoch > self.args.min_epochs: 164 | logging.info("Early stopping") 165 | break 166 | 167 | logging.info("Optimization Finished!") 168 | logging.info("Total time elapsed: {:.4f}s".format(time.time() - t_total)) 169 | 170 | def evaluate(self): 171 | """ 172 | Evaluate the model. 173 | """ 174 | if not self.best_test_metrics: 175 | self.model.eval() 176 | self.best_emb = self.model.encode(self.data['features'], self.data['adj_train_norm']) 177 | self.best_test_metrics = self.model.compute_metrics(self.best_emb, self.data, 'test') 178 | logging.info(" ".join(["Val set results:", format_metrics(self.best_val_metrics, 'val')])) 179 | logging.info(" ".join(["Test set results:", format_metrics(self.best_test_metrics, 'test')])) 180 | if self.args.save: 181 | np.save(os.path.join(self.save_dir, 'embeddings.npy'), self.best_emb.cpu().detach().numpy()) 182 | if hasattr(self.model.encoder, 'att_adj'): 183 | filename = os.path.join(self.save_dir, self.args.dataset + '_att_adj.p') 184 | pickle.dump(self.model.encoder.att_adj.cpu().to_dense(), open(filename, 'wb')) 185 | print('Dumped attention adj: ' + filename) 186 | 187 | json.dump(vars(self.args), open(os.path.join(self.save_dir, 'config.json'), 'w')) 188 | torch.save(self.model.state_dict(), os.path.join(self.save_dir, 'model.pth')) 189 | logging.info(f"Saved model in {self.save_dir}") 190 | return self.best_test_metrics 191 | 192 | 193 | if __name__ == '__main__': 194 | 195 | """ 196 | Main function to run command line evaluations 197 | 198 | Note 199 | ---------- 200 | Metrics averaged over repetitions are F1 score for node classification (accuracy for cora and pubmed), 201 | ROC for link prediction. Metrics to be averaged can be changed easily in the code. 202 | """ 203 | args = parser.parse_args() 204 | result_list=[] 205 | 206 | args = parser.parse_args() 207 | 208 | download_and_extract(args) 209 | 210 | data=DataLoader(args) 211 | 212 | for i in range(args.repeat): 213 | 214 | if args.task=='nc': 215 | model=NCModel(args) 216 | else: 217 | model=LPModel(args) 218 | 219 | if args.optimizer=='Adam': 220 | optimizer = optim.Adam(params=model.parameters(), lr=args.lr, weight_decay=args.weight_decay, 221 | betas=args.betas, eps=args.eps, amsgrad=args.amsgrad) 222 | if args.optimizer =='RiemannianAdam': 223 | optimizer=RiemannianAdam(params=model.parameters(), lr=args.lr, weight_decay=args.weight_decay, 224 | betas=args.betas, eps=args.eps ,amsgrad=args.amsgrad, 225 | stabilize=args.stabilize) 226 | if args.optimizer =='RiemannianSGD': 227 | optimizer=RiemannianSGD(params=model.parameters(), lr=args.lr, weight_decay=args.weight_decay, 228 | momentum=args.momentum, dampening=args.dampening, nesterov=args.nesterov, 229 | stabilize=args.stabilize) 230 | 231 | trainer=Trainer(args,model, optimizer,data) 232 | trainer.run() 233 | result=trainer.evaluate() 234 | 235 | if args.task=='nc' and args.dataset in ['cora','pubmed']: 236 | result_list.append(100*result['acc']) 237 | 238 | elif args.task=='nc' and args.dataset not in ['cora','pubmed']: 239 | result_list.append(100*result['f1']) 240 | 241 | else: 242 | result_list.append(100*result['roc']) 243 | 244 | result_list=torch.FloatTensor(result_list) 245 | print("Score",torch.mean(result_list),"Error",torch.std(result_list)) 246 | -------------------------------------------------------------------------------- /graphzoo/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .eval_utils import acc_f1 2 | from .math_utils import * 3 | from .math_utils import * 4 | from .train_utils import * -------------------------------------------------------------------------------- /graphzoo/utils/eval_utils.py: -------------------------------------------------------------------------------- 1 | """Evaluation metric for node classification""" 2 | from sklearn.metrics import average_precision_score, accuracy_score, f1_score 3 | 4 | def acc_f1(output, labels, average='binary'): 5 | preds = output.max(1)[1].type_as(labels) 6 | if preds.is_cuda: 7 | preds = preds.cpu() 8 | labels = labels.cpu() 9 | accuracy = accuracy_score(preds, labels) 10 | f1 = f1_score(preds, labels, average=average) 11 | return accuracy, f1 12 | 13 | -------------------------------------------------------------------------------- /graphzoo/utils/math_utils.py: -------------------------------------------------------------------------------- 1 | """Math utils functions""" 2 | import torch 3 | 4 | def cosh(x, clamp=15): 5 | return x.clamp(-clamp, clamp).cosh() 6 | 7 | 8 | def sinh(x, clamp=15): 9 | return x.clamp(-clamp, clamp).sinh() 10 | 11 | 12 | def tanh(x, clamp=15): 13 | return x.clamp(-clamp, clamp).tanh() 14 | 15 | 16 | def arcosh(x): 17 | return Arcosh.apply(x) 18 | 19 | 20 | def arsinh(x): 21 | return Arsinh.apply(x) 22 | 23 | 24 | def artanh(x): 25 | return Artanh.apply(x) 26 | 27 | 28 | class Artanh(torch.autograd.Function): 29 | @staticmethod 30 | def forward(ctx, x): 31 | x = x.clamp(-1 + 1e-15, 1 - 1e-15) 32 | ctx.save_for_backward(x) 33 | z = x.double() 34 | return (torch.log_(1 + z).sub_(torch.log_(1 - z))).mul_(0.5).to(x.dtype) 35 | 36 | @staticmethod 37 | def backward(ctx, grad_output): 38 | input, = ctx.saved_tensors 39 | return grad_output / (1 - input ** 2) 40 | 41 | 42 | class Arsinh(torch.autograd.Function): 43 | @staticmethod 44 | def forward(ctx, x): 45 | ctx.save_for_backward(x) 46 | z = x.double() 47 | return (z + torch.sqrt_(1 + z.pow(2))).clamp_min_(1e-15).log_().to(x.dtype) 48 | 49 | @staticmethod 50 | def backward(ctx, grad_output): 51 | input, = ctx.saved_tensors 52 | return grad_output / (1 + input ** 2) ** 0.5 53 | 54 | 55 | class Arcosh(torch.autograd.Function): 56 | @staticmethod 57 | def forward(ctx, x): 58 | x = x.clamp(min=1.0 + 1e-15) 59 | ctx.save_for_backward(x) 60 | z = x.double() 61 | return (z + torch.sqrt_(z.pow(2) - 1)).clamp_min_(1e-15).log_().to(x.dtype) 62 | 63 | @staticmethod 64 | def backward(ctx, grad_output): 65 | input, = ctx.saved_tensors 66 | return grad_output / (input ** 2 - 1) ** 0.5 67 | 68 | -------------------------------------------------------------------------------- /graphzoo/utils/train_utils.py: -------------------------------------------------------------------------------- 1 | """Utility functions required for training""" 2 | import os 3 | import numpy as np 4 | import torch 5 | import torch.nn.functional as F 6 | import torch.nn.modules.loss 7 | from typing import Tuple 8 | import itertools 9 | 10 | def format_metrics(metrics, split): 11 | """Format metric in metric dict for logging.""" 12 | return " ".join( 13 | ["{}_{}: {:.4f}".format(split, metric_name, metric_val) for metric_name, metric_val in metrics.items()]) 14 | 15 | 16 | def get_dir_name(models_dir): 17 | """ 18 | Gets a directory to save the model. 19 | 20 | If the directory already exists, then append a new integer to the end of 21 | it. This method is useful so that we don't overwrite existing models 22 | when launching new jobs. 23 | 24 | Args: 25 | models_dir: The directory where all the models are. 26 | 27 | Returns: 28 | The name of a new directory to save the training logs and model weights. 29 | """ 30 | if not os.path.exists(models_dir): 31 | save_dir = os.path.join(models_dir, '0') 32 | os.makedirs(save_dir) 33 | else: 34 | existing_dirs = np.array( 35 | [ 36 | d 37 | for d in os.listdir(models_dir) 38 | if os.path.isdir(os.path.join(models_dir, d)) 39 | ] 40 | ).astype(np.int) 41 | if len(existing_dirs) > 0: 42 | dir_id = str(existing_dirs.max() + 1) 43 | else: 44 | dir_id = "1" 45 | save_dir = os.path.join(models_dir, dir_id) 46 | os.makedirs(save_dir) 47 | return save_dir 48 | 49 | 50 | def add_flags_from_config(parser, config_dict): 51 | """ 52 | Adds a flag (and default value) to an ArgumentParser for each parameter in a config 53 | """ 54 | 55 | def OrNone(default): 56 | def func(x): 57 | # Convert "none" to proper None object 58 | if x.lower() == "none": 59 | return None 60 | # If default is None (and x is not None), return x without conversion as str 61 | elif default is None: 62 | return str(x) 63 | # Otherwise, default has non-None type; convert x to that type 64 | else: 65 | return type(default)(x) 66 | 67 | return func 68 | 69 | for param in config_dict: 70 | default, description = config_dict[param] 71 | try: 72 | if isinstance(default, dict): 73 | parser = add_flags_from_config(parser, default) 74 | elif isinstance(default, list): 75 | if len(default) > 0: 76 | # pass a list as argument 77 | parser.add_argument( 78 | f"--{param}", 79 | action="append", 80 | type=type(default[0]), 81 | default=default, 82 | help=description 83 | ) 84 | else: 85 | pass 86 | parser.add_argument(f"--{param}", action="append", default=default, help=description) 87 | else: 88 | pass 89 | parser.add_argument(f"--{param}", type=OrNone(default), default=default, help=description) 90 | except argparse.ArgumentError: 91 | print( 92 | f"Could not add flag for param {param} because it was already present." 93 | ) 94 | return parser 95 | 96 | 97 | def broadcast_shapes(*shapes: Tuple[int]) -> Tuple[int]: 98 | """Apply numpy broadcasting rules to shapes.""" 99 | result = [] 100 | for dims in itertools.zip_longest(*map(reversed, shapes), fillvalue=1): 101 | dim: int = 1 102 | for d in dims: 103 | if dim != 1 and d != 1 and d != dim: 104 | raise ValueError("Shapes can't be broadcasted") 105 | elif d > dim: 106 | dim = d 107 | result.append(dim) 108 | return tuple(reversed(result)) -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "graphzoo" 7 | version = "0.0.1" 8 | authors = [ 9 | { name="Anoushka Vyas", email="anoushkavyas@gmail.com" }, 10 | ] 11 | description = "A hyperbolic toolkit" 12 | readme = "README.md" 13 | license = { file="LICENSE" } 14 | requires-python = ">=3.5" 15 | classifiers = [ 16 | "Programming Language :: Python :: 3", 17 | "License :: OSI Approved :: MIT License", 18 | "Operating System :: OS Independent", 19 | ] 20 | 21 | [project.urls] 22 | "Homepage" = "https://github.com/AnoushkaVyas/GraphZoo" 23 | "Bug Tracker" = "https://github.com/AnoushkaVyas/GraphZoo" 24 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from setuptools import find_packages 3 | 4 | 5 | with open("README.md", 'r') as fh: 6 | long_description = fh.read() 7 | 8 | setup(name = "graphzoo", 9 | version = "0.0.1", 10 | author='Anoushka', 11 | maintainer='Anoushka', 12 | description = "A PyTorch library for hyperbolic neural networks.", 13 | long_description=long_description, 14 | long_description_content_type = "text/markdown", 15 | packages = find_packages(), 16 | url='https://github.com/AnoushkaVyas/GraphZoo.git', 17 | include_package_data=True, 18 | install_requires = [ 19 | 'numpy>=1.20.1', 20 | 'scikit-learn>=0.24.1', 21 | 'torch>=1.10.1', 22 | 'torchvision>=0.8.2' 23 | 'networkx>=2.5' 24 | ], 25 | classifiers=[ 26 | "Intended Audience :: Developers", 27 | "Intended Audience :: Education", 28 | "Intended Audience :: Science/Research", 29 | "Programming Language :: Python :: 3.5", 30 | "Programming Language :: Python :: 3.6", 31 | "Programming Language :: Python :: 3.7", 32 | "Programming Language :: Python :: 3.8", 33 | "Topic :: Scientific/Engineering :: Mathematics", 34 | "Topic :: Software Development :: Libraries :: Python Modules", 35 | "Topic :: Software Development :: Libraries", 36 | ], 37 | license="MIT", 38 | ) -------------------------------------------------------------------------------- /tutorials/nodeclassification/hgcn.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "22adf6c2", 6 | "metadata": {}, 7 | "source": [ 8 | "# Importing Libraries" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": 1, 14 | "id": "98864f51", 15 | "metadata": { 16 | "scrolled": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "import graphzoo as gz\n", 21 | "import torch\n", 22 | "from graphzoo.config import parser" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "id": "af6d6752", 28 | "metadata": {}, 29 | "source": [ 30 | "# Defining Parameters\n", 31 | "\n", 32 | "The config file in the source is used to define the parameters. Users can define it in the following way using the library." 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 2, 38 | "id": "cc63b449", 39 | "metadata": { 40 | "scrolled": true 41 | }, 42 | "outputs": [], 43 | "source": [ 44 | "#Defining Parameters\n", 45 | "params = parser.parse_args(args=[])\n", 46 | "\n", 47 | "#Data Parameters\n", 48 | "params.dataset='cora'\n", 49 | "params.datapath='data/cora'\n", 50 | "\n", 51 | "#Model Parameters\n", 52 | "params.task='nc'\n", 53 | "params.model='HGCN'\n", 54 | "params.manifold='PoincareBall'\n", 55 | "params.dim=128\n", 56 | "\n", 57 | "#Training Parameters\n", 58 | "params.lr=0.01\n", 59 | "params.weight_decay=0.001\n", 60 | "params.dropout=0.5" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "id": "cba8b5ac", 66 | "metadata": {}, 67 | "source": [ 68 | "# Loading Data\n", 69 | "Using the data parameters defined by the user, raw data is converted to desired graph data which is feature matrix, adjacency matrix and labels. After that the data is split into train, validation and test set depending on parameters given by the user." 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 3, 75 | "id": "65df9dbf", 76 | "metadata": { 77 | "scrolled": true 78 | }, 79 | "outputs": [], 80 | "source": [ 81 | "data = gz.dataloader.DataLoader(params)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "id": "bd67844b", 87 | "metadata": {}, 88 | "source": [ 89 | "# Building Model" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": 4, 95 | "id": "d1728e5b", 96 | "metadata": { 97 | "scrolled": true 98 | }, 99 | "outputs": [], 100 | "source": [ 101 | "model= gz.models.NCModel(params)" 102 | ] 103 | }, 104 | { 105 | "cell_type": "markdown", 106 | "id": "285dab6b", 107 | "metadata": {}, 108 | "source": [ 109 | "# Defining Optimizer" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 5, 115 | "id": "c9fffcaa", 116 | "metadata": { 117 | "scrolled": true 118 | }, 119 | "outputs": [], 120 | "source": [ 121 | "#Defining Optimizer\n", 122 | "optimizer = gz.optimizers.RiemannianAdam(params=model.parameters(), \n", 123 | " lr=params.lr, weight_decay=params.weight_decay)\n" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "id": "e1b31cb6", 129 | "metadata": {}, 130 | "source": [ 131 | "# Training and Testing Model" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": 6, 137 | "id": "a689c7ef", 138 | "metadata": { 139 | "scrolled": false 140 | }, 141 | "outputs": [ 142 | { 143 | "name": "stderr", 144 | "output_type": "stream", 145 | "text": [ 146 | "INFO:root:Using: cpu\n", 147 | "INFO:root:Using seed 1234.\n", 148 | "INFO:root:NCModel(\n", 149 | " (encoder): HGCN(\n", 150 | " (layers): Sequential(\n", 151 | " (0): HyperbolicGraphConvolution(\n", 152 | " (linear): HypLinear(in_features=1433, out_features=128, c=tensor([1.]))\n", 153 | " (agg): HypAgg(c=tensor([1.]))\n", 154 | " (hyp_act): HypAct(c_in=tensor([1.]), c_out=tensor([1.]))\n", 155 | " )\n", 156 | " )\n", 157 | " )\n", 158 | " (decoder): LinearDecoder(\n", 159 | " in_features=128, out_features=7, bias=1, c=tensor([1.])\n", 160 | " (cls): Linear(\n", 161 | " (linear): Linear(in_features=128, out_features=7, bias=True)\n", 162 | " )\n", 163 | " )\n", 164 | ")\n", 165 | "INFO:root:Total number of parameters: 184455\n", 166 | "/opt/anaconda3/lib/python3.8/site-packages/graphzoo-0.0.1-py3.8.egg/graphzoo/manifolds/poincare.py:99: UserWarning: where received a uint8 condition tensor. This behavior is deprecated and will be removed in a future version of PyTorch. Use a boolean condition instead. (Triggered internally at ../aten/src/ATen/native/TensorCompare.cpp:328.)\n", 167 | "/opt/anaconda3/lib/python3.8/site-packages/graphzoo-0.0.1-py3.8.egg/graphzoo/optimizers/radam.py:129: UserWarning: This overload of add_ is deprecated:\n", 168 | "\tadd_(Number alpha, Tensor other)\n", 169 | "Consider using one of the following signatures instead:\n", 170 | "\tadd_(Tensor other, *, Number alpha) (Triggered internally at ../torch/csrc/utils/python_arg_parser.cpp:1050.)\n", 171 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 172 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 173 | "INFO:root:Epoch: 0005 lr: 0.01 train_loss: 1.9406 train_acc: 0.1786 train_f1: 0.1786 time: 0.0769s\n", 174 | "INFO:root:Epoch: 0005 val_loss: 1.9325 val_acc: 0.2800 val_f1: 0.2800\n", 175 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 176 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 177 | "INFO:root:Epoch: 0010 lr: 0.01 train_loss: 1.9418 train_acc: 0.1429 train_f1: 0.1429 time: 0.0785s\n", 178 | "INFO:root:Epoch: 0010 val_loss: 1.9321 val_acc: 0.3160 val_f1: 0.3160\n", 179 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 180 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 181 | "INFO:root:Epoch: 0015 lr: 0.01 train_loss: 1.9404 train_acc: 0.1571 train_f1: 0.1571 time: 0.0774s\n", 182 | "INFO:root:Epoch: 0015 val_loss: 1.9316 val_acc: 0.3180 val_f1: 0.3180\n", 183 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 184 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 185 | "INFO:root:Epoch: 0020 lr: 0.01 train_loss: 1.9327 train_acc: 0.2500 train_f1: 0.2500 time: 0.0761s\n", 186 | "INFO:root:Epoch: 0020 val_loss: 1.9308 val_acc: 0.3900 val_f1: 0.3900\n", 187 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 188 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 189 | "INFO:root:Epoch: 0025 lr: 0.01 train_loss: 1.9354 train_acc: 0.1571 train_f1: 0.1571 time: 0.0765s\n", 190 | "INFO:root:Epoch: 0025 val_loss: 1.9303 val_acc: 0.3620 val_f1: 0.3620\n", 191 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 192 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 193 | "INFO:root:Epoch: 0030 lr: 0.01 train_loss: 1.9211 train_acc: 0.3000 train_f1: 0.3000 time: 0.0759s\n", 194 | "INFO:root:Epoch: 0030 val_loss: 1.9379 val_acc: 0.2940 val_f1: 0.2940\n", 195 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 196 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 197 | "INFO:root:Epoch: 0035 lr: 0.01 train_loss: 1.9066 train_acc: 0.3429 train_f1: 0.3429 time: 0.0781s\n", 198 | "INFO:root:Epoch: 0035 val_loss: 1.9347 val_acc: 0.3160 val_f1: 0.3160\n", 199 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 200 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 201 | "INFO:root:Epoch: 0040 lr: 0.01 train_loss: 1.9013 train_acc: 0.2786 train_f1: 0.2786 time: 0.0771s\n", 202 | "INFO:root:Epoch: 0040 val_loss: 1.9235 val_acc: 0.3640 val_f1: 0.3640\n", 203 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 204 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 205 | "INFO:root:Epoch: 0045 lr: 0.01 train_loss: 1.8790 train_acc: 0.3286 train_f1: 0.3286 time: 0.0828s\n", 206 | "INFO:root:Epoch: 0045 val_loss: 1.9060 val_acc: 0.4140 val_f1: 0.4140\n", 207 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 208 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 209 | "INFO:root:Epoch: 0050 lr: 0.01 train_loss: 1.8338 train_acc: 0.4714 train_f1: 0.4714 time: 0.0800s\n", 210 | "INFO:root:Epoch: 0050 val_loss: 1.8745 val_acc: 0.5940 val_f1: 0.5940\n", 211 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 212 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 213 | "INFO:root:Epoch: 0055 lr: 0.01 train_loss: 1.7947 train_acc: 0.4643 train_f1: 0.4643 time: 0.0838s\n", 214 | "INFO:root:Epoch: 0055 val_loss: 1.8320 val_acc: 0.6900 val_f1: 0.6900\n", 215 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 216 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 217 | "INFO:root:Epoch: 0060 lr: 0.01 train_loss: 1.6752 train_acc: 0.5429 train_f1: 0.5429 time: 0.0792s\n", 218 | "INFO:root:Epoch: 0060 val_loss: 1.7879 val_acc: 0.6300 val_f1: 0.6300\n", 219 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 220 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 221 | "INFO:root:Epoch: 0065 lr: 0.01 train_loss: 1.6270 train_acc: 0.4786 train_f1: 0.4786 time: 0.0797s\n", 222 | "INFO:root:Epoch: 0065 val_loss: 1.7243 val_acc: 0.6340 val_f1: 0.6340\n", 223 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 224 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 225 | "INFO:root:Epoch: 0070 lr: 0.01 train_loss: 1.5454 train_acc: 0.4286 train_f1: 0.4286 time: 0.0760s\n", 226 | "INFO:root:Epoch: 0070 val_loss: 1.6261 val_acc: 0.7120 val_f1: 0.7120\n", 227 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 228 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 229 | "INFO:root:Epoch: 0075 lr: 0.01 train_loss: 1.4295 train_acc: 0.4571 train_f1: 0.4571 time: 0.0762s\n", 230 | "INFO:root:Epoch: 0075 val_loss: 1.5127 val_acc: 0.7360 val_f1: 0.7360\n", 231 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 232 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 233 | "INFO:root:Epoch: 0080 lr: 0.01 train_loss: 1.2277 train_acc: 0.5714 train_f1: 0.5714 time: 0.0761s\n", 234 | "INFO:root:Epoch: 0080 val_loss: 1.4126 val_acc: 0.7480 val_f1: 0.7480\n", 235 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 236 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 237 | "INFO:root:Epoch: 0085 lr: 0.01 train_loss: 1.1056 train_acc: 0.6214 train_f1: 0.6214 time: 0.0785s\n" 238 | ] 239 | }, 240 | { 241 | "name": "stderr", 242 | "output_type": "stream", 243 | "text": [ 244 | "INFO:root:Epoch: 0085 val_loss: 1.3170 val_acc: 0.7380 val_f1: 0.7380\n", 245 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 246 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 247 | "INFO:root:Epoch: 0090 lr: 0.01 train_loss: 1.0884 train_acc: 0.6000 train_f1: 0.6000 time: 0.0794s\n", 248 | "INFO:root:Epoch: 0090 val_loss: 1.1993 val_acc: 0.7620 val_f1: 0.7620\n", 249 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 250 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 251 | "INFO:root:Epoch: 0095 lr: 0.01 train_loss: 1.0434 train_acc: 0.6071 train_f1: 0.6071 time: 0.0771s\n", 252 | "INFO:root:Epoch: 0095 val_loss: 1.1194 val_acc: 0.7680 val_f1: 0.7680\n", 253 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 254 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 255 | "INFO:root:Epoch: 0100 lr: 0.01 train_loss: 1.0486 train_acc: 0.5500 train_f1: 0.5500 time: 0.0758s\n", 256 | "INFO:root:Epoch: 0100 val_loss: 1.0589 val_acc: 0.7680 val_f1: 0.7680\n", 257 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 258 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 259 | "INFO:root:Epoch: 0105 lr: 0.01 train_loss: 0.9647 train_acc: 0.5571 train_f1: 0.5571 time: 0.0761s\n", 260 | "INFO:root:Epoch: 0105 val_loss: 1.0121 val_acc: 0.7600 val_f1: 0.7600\n", 261 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 262 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 263 | "INFO:root:Epoch: 0110 lr: 0.01 train_loss: 0.8277 train_acc: 0.6786 train_f1: 0.6786 time: 0.0759s\n", 264 | "INFO:root:Epoch: 0110 val_loss: 0.9623 val_acc: 0.7660 val_f1: 0.7660\n", 265 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 266 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 267 | "INFO:root:Epoch: 0115 lr: 0.01 train_loss: 0.9118 train_acc: 0.6000 train_f1: 0.6000 time: 0.0760s\n", 268 | "INFO:root:Epoch: 0115 val_loss: 0.9117 val_acc: 0.7700 val_f1: 0.7700\n", 269 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 270 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 271 | "INFO:root:Epoch: 0120 lr: 0.01 train_loss: 0.8403 train_acc: 0.6286 train_f1: 0.6286 time: 0.0762s\n", 272 | "INFO:root:Epoch: 0120 val_loss: 0.8965 val_acc: 0.7780 val_f1: 0.7780\n", 273 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 274 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 275 | "INFO:root:Epoch: 0125 lr: 0.01 train_loss: 0.7800 train_acc: 0.6571 train_f1: 0.6571 time: 0.0762s\n", 276 | "INFO:root:Epoch: 0125 val_loss: 0.8740 val_acc: 0.7880 val_f1: 0.7880\n", 277 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 278 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 279 | "INFO:root:Epoch: 0130 lr: 0.01 train_loss: 0.8298 train_acc: 0.6000 train_f1: 0.6000 time: 0.0760s\n", 280 | "INFO:root:Epoch: 0130 val_loss: 0.8445 val_acc: 0.7840 val_f1: 0.7840\n", 281 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 282 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 283 | "INFO:root:Epoch: 0135 lr: 0.01 train_loss: 0.7723 train_acc: 0.6429 train_f1: 0.6429 time: 0.0763s\n", 284 | "INFO:root:Epoch: 0135 val_loss: 0.8294 val_acc: 0.7740 val_f1: 0.7740\n", 285 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 286 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 287 | "INFO:root:Epoch: 0140 lr: 0.01 train_loss: 0.7845 train_acc: 0.6286 train_f1: 0.6286 time: 0.0754s\n", 288 | "INFO:root:Epoch: 0140 val_loss: 0.8211 val_acc: 0.7660 val_f1: 0.7660\n", 289 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 290 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 291 | "INFO:root:Epoch: 0145 lr: 0.01 train_loss: 0.7662 train_acc: 0.6000 train_f1: 0.6000 time: 0.0757s\n", 292 | "INFO:root:Epoch: 0145 val_loss: 0.8192 val_acc: 0.7660 val_f1: 0.7660\n", 293 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 294 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 295 | "INFO:root:Epoch: 0150 lr: 0.01 train_loss: 0.7771 train_acc: 0.6143 train_f1: 0.6143 time: 0.0775s\n", 296 | "INFO:root:Epoch: 0150 val_loss: 0.8056 val_acc: 0.7680 val_f1: 0.7680\n", 297 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 298 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 299 | "INFO:root:Epoch: 0155 lr: 0.01 train_loss: 0.7410 train_acc: 0.6357 train_f1: 0.6357 time: 0.0846s\n", 300 | "INFO:root:Epoch: 0155 val_loss: 0.8006 val_acc: 0.7820 val_f1: 0.7820\n", 301 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 302 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 303 | "INFO:root:Epoch: 0160 lr: 0.01 train_loss: 0.7039 train_acc: 0.6786 train_f1: 0.6786 time: 0.0771s\n", 304 | "INFO:root:Epoch: 0160 val_loss: 0.7945 val_acc: 0.7800 val_f1: 0.7800\n", 305 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 306 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 307 | "INFO:root:Epoch: 0165 lr: 0.01 train_loss: 0.7472 train_acc: 0.6286 train_f1: 0.6286 time: 0.0841s\n", 308 | "INFO:root:Epoch: 0165 val_loss: 0.7763 val_acc: 0.7620 val_f1: 0.7620\n", 309 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 310 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 311 | "INFO:root:Epoch: 0170 lr: 0.01 train_loss: 0.7063 train_acc: 0.7071 train_f1: 0.7071 time: 0.0756s\n", 312 | "INFO:root:Epoch: 0170 val_loss: 0.7738 val_acc: 0.7700 val_f1: 0.7700\n", 313 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 314 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 315 | "INFO:root:Epoch: 0175 lr: 0.01 train_loss: 0.7393 train_acc: 0.6286 train_f1: 0.6286 time: 0.0757s\n", 316 | "INFO:root:Epoch: 0175 val_loss: 0.7709 val_acc: 0.7720 val_f1: 0.7720\n", 317 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 318 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 319 | "INFO:root:Epoch: 0180 lr: 0.01 train_loss: 0.7000 train_acc: 0.6286 train_f1: 0.6286 time: 0.0757s\n", 320 | "INFO:root:Epoch: 0180 val_loss: 0.7760 val_acc: 0.7720 val_f1: 0.7720\n", 321 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 322 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 323 | "INFO:root:Epoch: 0185 lr: 0.01 train_loss: 0.7839 train_acc: 0.5929 train_f1: 0.5929 time: 0.0764s\n" 324 | ] 325 | }, 326 | { 327 | "name": "stderr", 328 | "output_type": "stream", 329 | "text": [ 330 | "INFO:root:Epoch: 0185 val_loss: 0.7854 val_acc: 0.7620 val_f1: 0.7620\n", 331 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 332 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 333 | "INFO:root:Epoch: 0190 lr: 0.01 train_loss: 0.7346 train_acc: 0.5786 train_f1: 0.5786 time: 0.0768s\n", 334 | "INFO:root:Epoch: 0190 val_loss: 0.7983 val_acc: 0.7620 val_f1: 0.7620\n", 335 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 336 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 337 | "INFO:root:Epoch: 0195 lr: 0.01 train_loss: 0.7911 train_acc: 0.6429 train_f1: 0.6429 time: 0.0767s\n", 338 | "INFO:root:Epoch: 0195 val_loss: 0.7940 val_acc: 0.7760 val_f1: 0.7760\n", 339 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 340 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 341 | "INFO:root:Epoch: 0200 lr: 0.01 train_loss: 0.7981 train_acc: 0.6214 train_f1: 0.6214 time: 0.0758s\n", 342 | "INFO:root:Epoch: 0200 val_loss: 0.7856 val_acc: 0.7800 val_f1: 0.7800\n", 343 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 344 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 345 | "INFO:root:Epoch: 0205 lr: 0.01 train_loss: 0.6945 train_acc: 0.6286 train_f1: 0.6286 time: 0.0763s\n", 346 | "INFO:root:Epoch: 0205 val_loss: 0.7858 val_acc: 0.7720 val_f1: 0.7720\n", 347 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 348 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 349 | "INFO:root:Epoch: 0210 lr: 0.01 train_loss: 0.6424 train_acc: 0.6643 train_f1: 0.6643 time: 0.0850s\n", 350 | "INFO:root:Epoch: 0210 val_loss: 0.7897 val_acc: 0.7680 val_f1: 0.7680\n", 351 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 352 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 353 | "INFO:root:Epoch: 0215 lr: 0.01 train_loss: 0.7426 train_acc: 0.6286 train_f1: 0.6286 time: 0.0759s\n", 354 | "INFO:root:Epoch: 0215 val_loss: 0.7976 val_acc: 0.7580 val_f1: 0.7580\n", 355 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 356 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 357 | "INFO:root:Epoch: 0220 lr: 0.01 train_loss: 0.7281 train_acc: 0.6357 train_f1: 0.6357 time: 0.0757s\n", 358 | "INFO:root:Epoch: 0220 val_loss: 0.7933 val_acc: 0.7600 val_f1: 0.7600\n", 359 | "/opt/anaconda3/lib/python3.8/site-packages/torch/optim/lr_scheduler.py:369: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n", 360 | " warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n", 361 | "INFO:root:Epoch: 0225 lr: 0.01 train_loss: 0.6514 train_acc: 0.6714 train_f1: 0.6714 time: 0.0768s\n", 362 | "INFO:root:Epoch: 0225 val_loss: 0.7912 val_acc: 0.7680 val_f1: 0.7680\n", 363 | "INFO:root:Early stopping\n", 364 | "INFO:root:Optimization Finished!\n", 365 | "INFO:root:Total time elapsed: 28.0648s\n", 366 | "INFO:root:Val set results: val_loss: 0.8740 val_acc: 0.7880 val_f1: 0.7880\n", 367 | "INFO:root:Test set results: test_loss: 0.8494 test_acc: 0.7890 test_f1: 0.7890\n" 368 | ] 369 | }, 370 | { 371 | "data": { 372 | "text/plain": [ 373 | "{'loss': tensor(0.8494, grad_fn=),\n", 374 | " 'acc': 0.789,\n", 375 | " 'f1': 0.7890000000000001}" 376 | ] 377 | }, 378 | "execution_count": 6, 379 | "metadata": {}, 380 | "output_type": "execute_result" 381 | } 382 | ], 383 | "source": [ 384 | "#Training and Testing\n", 385 | "trainer=gz.trainers.Trainer(params,model,optimizer,data)\n", 386 | "trainer.run()\n", 387 | "trainer.evaluate()" 388 | ] 389 | } 390 | ], 391 | "metadata": { 392 | "kernelspec": { 393 | "display_name": "Python 3", 394 | "language": "python", 395 | "name": "python3" 396 | }, 397 | "language_info": { 398 | "codemirror_mode": { 399 | "name": "ipython", 400 | "version": 3 401 | }, 402 | "file_extension": ".py", 403 | "mimetype": "text/x-python", 404 | "name": "python", 405 | "nbconvert_exporter": "python", 406 | "pygments_lexer": "ipython3", 407 | "version": "3.8.8" 408 | } 409 | }, 410 | "nbformat": 4, 411 | "nbformat_minor": 5 412 | } 413 | --------------------------------------------------------------------------------