├── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── clients ├── __init__.py ├── client_constants.py ├── es_client.py ├── s3_client.py └── ses_client.py ├── common.py ├── competitions ├── __init__.py ├── carvana.py ├── dogscats.py ├── planet.py └── team │ ├── __init__.py │ └── brendan.py ├── config.py ├── constants.py ├── datasets ├── __init__.py ├── data_aug.py ├── data_folds.py ├── data_loaders.py ├── data_utils.py ├── datasets.py └── metadata.py ├── docs ├── email.png ├── kibana1.png ├── kibana2.png ├── visdom.png └── visdom2.png ├── ensembles ├── __init__.py ├── ens_utils.py └── ensemble.py ├── experiments ├── __init__.py ├── exp_builder.py ├── exp_config.py ├── exp_history.py ├── exp_utils.py └── experiment.py ├── explore-carvana.ipynb ├── explore-dogscats.ipynb ├── explore-planet.ipynb ├── init_project.py ├── metrics ├── __init__.py ├── evaluate.py ├── loss_functions.py ├── metric.py ├── metric_builder.py └── metric_utils.py ├── models ├── __init__.py ├── builder.py ├── layers.py ├── resnet.py ├── simplenet.py ├── unet.py └── utils.py ├── notifications ├── __init__.py ├── email_constants.py └── emailer.py ├── predictions ├── __init__.py ├── pred_builder.py ├── pred_constants.py ├── pred_utils.py └── prediction.py ├── requirements.txt ├── submissions ├── __init__.py └── utils.py ├── tests └── unit_tests │ └── training │ └── test_learning_rates.py ├── torchsample ├── __init__.py ├── callbacks.py ├── constraints.py ├── datasets.py ├── functions │ ├── __init__.py │ └── affine.py ├── initializers.py ├── metrics.py ├── modules │ ├── __init__.py │ ├── _utils.py │ └── module_trainer.py ├── regularizers.py ├── samplers.py ├── transforms │ ├── __init__.py │ ├── affine_transforms.py │ ├── distortion_transforms.py │ ├── image_transforms.py │ └── tensor_transforms.py ├── utils.py └── version.py ├── training ├── __init__.py ├── learning_rates.py ├── pseudolabels.py ├── trainers.py └── utils.py ├── utils ├── __init__.py ├── files.py ├── general.py ├── imgs.py ├── logger.py ├── multitasking.py └── widgets.py └── visualizers ├── __init__.py ├── kibana.py ├── vis_utils.py └── viz.py /.gitignore: -------------------------------------------------------------------------------- 1 | *gz 2 | *.swp 3 | .ipynb_checkpoints/ 4 | __pycache__ 5 | *~ 6 | *pyc 7 | .cache 8 | .vscode 9 | 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Brendan Fortuner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Summary 2 | Pytorch Kaggle starter is a framework for managing experiments in Kaggle competitions. It reduces time to first submission by providing a suite of helper functions for model training, data loading, adjusting learning rates, making predictions, ensembling models, and formatting submissions. 3 | 4 | Inside are example Jupyter notebooks walking through how to get strong scores on popular competitions: 5 | 6 | * [Dogs vs Cats Redux](https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition) - Top 8% 7 | * [Planet Amazon Rainforest](https://www.kaggle.com/c/planet-understanding-the-amazon-from-space) - Top 15% 8 | * [Carvana Image Segmentation](https://www.kaggle.com/c/carvana-image-masking-challenge) - WIP 9 | 10 | These notebooks outline basic, single-model submissions. Scores can be improved significantly by ensembling models and using test-time augmentation. 11 | 12 | ## Features 13 | 14 | 1. **Experiments** - Launch experiments from python dictionaries inside jupyter notebooks or python scripts. Attach Visualizers (Visdom, Kibana), Metrics (Accuracy, F2, Loss), or external datastores (S3, Elasticsearch) 15 | 2. **Monitoring** - Track experiments from your phone or web-browser in real-time with Visdom, a lightweight visualization framework from Facebook 16 | 3. **Notifications** - Receive email notifications when experiments complete or fail 17 | 4. **Sharing** - Upload experiments, predictions and ensembles to S3 for other users to download 18 | 5. **Analysis** - Compare experiments across users with Kibana. Design custom dashboards for specific competitions 19 | 6. **Helpers** - Reduce time to submission with helper code for common tasks--custom datasets, metrics, storing predictions, ensembling models, making submissions, and more. 20 | 7. **Torchsample** - Includes the latest release of ncullen93's [torchsample](https://github.com/ncullen93/torchsample) project for additional trainer helpers and data augmentations. 21 | 22 | ## Requirements 23 | 24 | 1. [Anaconda](https://www.continuum.io/downloads) with Python3 25 | 2. [Pytorch](http://pytorch.org/) 26 | 3. Other requirements: ```pip install -r requirements.txt``` 27 | 4. conda install -c menpo opencv 28 | 5. Server with GPU and Cuda installed 29 | 30 | ## Datasets 31 | To get started you'll need to move all training and test images to the `project_root/datasets/inputs` directory (then either trn_jpg tst_jpg subdirectories). Running the first cell of each notebook creates the directory structure outlined in the `config.py` file. 32 | 33 | There is no need to create separate directories for classes or validation sets. This is handled by the data_fold.py module and the FileDataset, which expects a list of filepaths and targets. After trying out a lot of approaches, I found this to be the easiest and most extensible. You'll sometimes need to generate a `metadata.csv` file separately if Kaggle didn't provide one. This sort of competition-specific code can live in the `competitions/` directory. 34 | 35 | ## Visdom 36 | Visualize experiment progress on your phone with Facebook's new [Visdom](https://github.com/facebookresearch/visdom) framework. 37 | 38 |  39 | 40 | ## Kibana 41 | Spin up an [Elasticsearch](https://www.elastic.co/) cluster locally or on AWS to start visualizing or tracking experiments. Create custom dashboards with [Kibana's](https://www.elastic.co/products/kibana) easy-to-use drag and drop chart creation tools. 42 | 43 |  44 | 45 | Filter and sort experiments, zoom to a specific time period, or aggregate metrics across experiments and see updates in real time. 46 | 47 |  48 | 49 | ## Emails 50 | Receive emails when experiments compete or fail using AWS SES service. 51 | 52 |  53 | 54 | ## Kaggle CLI 55 | Quickly download and submit with the kaggle cli tool. 56 | 57 | ``` 58 | kg download -c dogs-vs-cats-redux-kernels-edition -v -u USERNAME -p PASSWORD 59 | kg submit -m 'my sub' -c dogs-vs-cats-redux-kernels-edition -v -u USERNAME -p PASSWORD my_exp_tst.csv 60 | ``` 61 | 62 | ## Best practices 63 | 64 | * Use systemd for always running Visdom and Jupyter servers 65 | 66 | 67 | ## Unit Tests 68 | 69 | Run tests with: 70 | ``` 71 | python -m pytest tests/ 72 | ``` 73 | 74 | Other run commands: 75 | ``` 76 | python -m pytest tests/ (all tests) 77 | python -m pytest -k filenamekeyword (tests matching keyword) 78 | python -m pytest tests/utils/test_sample.py (single test file) 79 | python -m pytest tests/utils/test_sample.py::test_answer_correct (single test method) 80 | python -m pytest --resultlog=testlog.log tests/ (log output to file) 81 | python -m pytest -s tests/ (print output to console) 82 | ``` 83 | 84 | ## TODO 85 | 86 | * Add TTA (test time augmentation) example 87 | * Add Pseudolabeling example 88 | * Add Knowledge Distillation example 89 | * Add Multi-input/Multi-target examples 90 | * Add stacking helper functions 91 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------------------------------- /clients/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/clients/__init__.py -------------------------------------------------------------------------------- /clients/client_constants.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | 3 | # AWS Config 4 | AWS_REGION = cfg.AWS_REGION 5 | AWS_ACCESS_KEY = cfg.AWS_ACCESS_KEY 6 | AWS_SECRET_KEY = cfg.AWS_SECRET_KEY 7 | TIMEZONE = cfg.TIMEZONE 8 | 9 | # S3 Config 10 | S3_BUCKET = 'kaggle{:s}'.format(cfg.PROJECT_NAME) 11 | EXPERIMENT_CONFIG_PREFIX = 'experiment_configs/' 12 | EXPERIMENT_HISTORY_PREFIX = 'experiment_histories/' 13 | EXPERIMENT_PREFIX = 'experiments/' 14 | PREDICTION_PREFIX = 'predictions/' 15 | ENSEMBLE_PREFIX = 'ensembles/' 16 | 17 | # Elasticsearch Config 18 | ES_EXPERIMENT_HISTORY_INDEX = 'kaggle-{:s}-history'.format(cfg.PROJECT_NAME) 19 | ES_EXPERIMENT_CONFIG_INDEX = 'kaggle-{:s}-config'.format(cfg.PROJECT_NAME) 20 | ES_PREDICTIONS_INDEX = 'kaggle-{:s}-predictions'.format(cfg.PROJECT_NAME) 21 | ES_EXPERIMENT_HISTORY_DOC_TYPE = 'history' 22 | ES_EXPERIMENT_CONFIG_DOC_TYPE = 'config' 23 | ES_PREDICTIONS_DOC_TYPE = 'prediction' 24 | ES_ENDPOINT = cfg.ES_ENDPOINT 25 | ES_PORT = cfg.ES_PORT 26 | 27 | # SES Config 28 | AWS_SES_REGION = cfg.AWS_SES_REGION 29 | ADMIN_EMAIL = cfg.ADMIN_EMAIL 30 | USER_EMAIL = cfg.USER_EMAIL 31 | EMAIL_CHARSET = 'UTF-8' 32 | -------------------------------------------------------------------------------- /clients/es_client.py: -------------------------------------------------------------------------------- 1 | from elasticsearch import Elasticsearch 2 | from datetime import datetime 3 | import pytz 4 | 5 | from .client_constants import * 6 | 7 | 8 | def upload_experiment_history(config, history): 9 | index_docs(history.to_doc(config), ES_EXPERIMENT_HISTORY_INDEX, 10 | ES_EXPERIMENT_HISTORY_DOC_TYPE) 11 | 12 | 13 | def upload_experiment_config(config): 14 | index_doc(config.to_doc(), ES_EXPERIMENT_CONFIG_INDEX, 15 | ES_EXPERIMENT_CONFIG_DOC_TYPE) 16 | 17 | 18 | def upload_prediction(pred): 19 | index_doc(pred.to_doc(), ES_PREDICTIONS_INDEX, 20 | ES_PREDICTIONS_DOC_TYPE) 21 | 22 | 23 | def delete_experiment(config): 24 | delete_experiment_by_id(config.get_id()) 25 | 26 | 27 | def delete_experiment_by_id(exp_id): 28 | r1 = delete_by_field(ES_EXPERIMENT_HISTORY_INDEX, 29 | ES_EXPERIMENT_HISTORY_DOC_TYPE, 30 | ES_EXP_KEY_FIELD, exp_id) 31 | r2 = delete_by_field(ES_EXPERIMENT_CONFIG_INDEX, 32 | ES_EXPERIMENT_CONFIG_DOC_TYPE, 33 | ES_EXP_KEY_FIELD, exp_id) 34 | return r1,r2 35 | 36 | 37 | def delete_experiment_by_field(field, value): 38 | r1 = delete_by_field(ES_EXPERIMENT_HISTORY_INDEX, 39 | ES_EXPERIMENT_HISTORY_DOC_TYPE, 40 | field, value) 41 | r2 = delete_by_field(ES_EXPERIMENT_CONFIG_INDEX, 42 | ES_EXPERIMENT_CONFIG_DOC_TYPE, 43 | field, value) 44 | return r1,r2 45 | 46 | 47 | # API 48 | # http://elasticsearch-py.readthedocs.io/en/master/api.html 49 | 50 | def get_client(): 51 | return Elasticsearch([ 52 | {'host': ES_ENDPOINT, 'port': ES_PORT}, 53 | ]) 54 | 55 | def create_index(name, shards=2, replicas=1): 56 | es = get_client() 57 | ok = es.indices.create(name, body={ 58 | "settings" : { 59 | "index" : { 60 | "number_of_shards" : shards, 61 | "number_of_replicas" : replicas 62 | } 63 | } 64 | })['acknowledged'] 65 | assert ok is True 66 | return ok 67 | 68 | def delete_index(name): 69 | ok = get_client().indices.delete(name)['acknowledged'] 70 | assert ok is True 71 | return ok 72 | 73 | def delete_docs_by_ids(index_name, doc_ids): 74 | pass 75 | 76 | def delete_by_field(index_name, doc_type, field, value): 77 | query = { 78 | "query": { 79 | "match" : { 80 | field : value 81 | } 82 | } 83 | } 84 | es = get_client() 85 | r = es.delete_by_query(index=index_name, doc_type=doc_type, body=query) 86 | return r 87 | 88 | def search_by_field(index_name, doc_type, field, value): 89 | query = { 90 | "term" : { 91 | field : value 92 | } 93 | } 94 | print(query) 95 | es = get_client() 96 | resp = es.search(index=index_name, doc_type=doc_type, body=query) 97 | return resp 98 | 99 | def get_doc(index_name, doc_key): 100 | return get_client().get(index_name, id=doc_key) 101 | 102 | def search(index_name, query, metadata_only=False, n_docs=10): 103 | es = get_client() 104 | filters = [] 105 | if metadata_only: 106 | filters = ['hits.hits._id', 'hits.hits._type'] 107 | return es.search(index=index_name, filter_path=filters, 108 | body=query, size=n_docs) 109 | 110 | def index_doc(doc, index_name, doc_type): 111 | assert 'key' in doc 112 | es = get_client() 113 | doc['uploaded'] = datetime.now(pytz.timezone(TIMEZONE)) 114 | es.index(index=index_name, doc_type=doc_type, body=doc, id=doc['key']) 115 | 116 | def index_docs(docs, index_name, doc_type): 117 | # There exists a bulk API, but this is fine for now 118 | for doc in docs: 119 | index_doc(doc, index_name, doc_type) 120 | 121 | def get_mappings(index_name): 122 | # Shows the keys and data types in an index 123 | return get_client().indices.get_mapping(index_name) 124 | 125 | def doc_exists(index_name, doc_type, doc_id): 126 | return get_client().exists(index_name, doc_type, doc_id) 127 | 128 | def health(): 129 | return get_client().cluster.health(wait_for_status='yellow', 130 | request_timeout=1) 131 | 132 | def ping(): 133 | return get_client().ping() 134 | 135 | -------------------------------------------------------------------------------- /clients/s3_client.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import constants as c 3 | from .client_constants import * 4 | 5 | 6 | # List Files 7 | 8 | def list_experiment_configs(): 9 | return list_fnames(EXPERIMENT_CONFIG_PREFIX, 10 | c.EXPERIMENT_CONFIG_FILE_EXT) 11 | 12 | def list_experiments(): 13 | return list_fnames(EXPERIMENT_PREFIX, c.EXP_FILE_EXT) 14 | 15 | def list_predictions(): 16 | return list_fnames(PREDICTION_PREFIX, c.PRED_FILE_EXT) 17 | 18 | def list_fnames(prefix, postfix): 19 | keys = get_keys(prefix=prefix) 20 | names = [] 21 | for k in keys: 22 | names.append(k.replace(prefix,'').replace(postfix,'')) 23 | return names 24 | 25 | 26 | # Download 27 | 28 | def download_experiment(dest_fpath, exp_name, bucket=S3_BUCKET): 29 | key = EXPERIMENT_PREFIX+exp_name + c.EXP_FILE_EXT 30 | download_file(dest_fpath, key, bucket=bucket) 31 | 32 | def download_experiment_config(dest_fpath, exp_name, bucket=S3_BUCKET): 33 | key = EXPERIMENT_CONFIG_PREFIX+exp_name+c.EXPERIMENT_CONFIG_FILE_EXT 34 | download_file(dest_fpath, key, bucket=bucket) 35 | 36 | def download_prediction(dest_fpath, pred_name, bucket=S3_BUCKET): 37 | key = PREDICTION_PREFIX+pred_name+c.PRED_FILE_EXT 38 | download_file(dest_fpath, key, bucket=bucket) 39 | 40 | 41 | # Read Object directly from S3 42 | 43 | def fetch_experiment_history(exp_name, bucket=S3_BUCKET): 44 | key = EXPERIMENT_HISTORY_PREFIX+exp_name+c.EXPERIMENT_HISTORY_FILE_EXT 45 | return get_object_str(key, bucket) 46 | 47 | def fetch_experiment_config(exp_name, bucket=S3_BUCKET): 48 | key = EXPERIMENT_CONFIG_PREFIX+exp_name+c.EXPERIMENT_CONFIG_FILE_EXT 49 | return get_object_str(key, bucket) 50 | 51 | 52 | # Upload 53 | 54 | def upload_experiment(src_fpath, exp_name, bucket=S3_BUCKET): 55 | key = EXPERIMENT_PREFIX+exp_name+c.EXP_FILE_EXT 56 | upload_file(src_fpath, key, bucket=bucket) 57 | 58 | def upload_experiment_config(src_fpath, exp_name, bucket=S3_BUCKET): 59 | key = EXPERIMENT_CONFIG_PREFIX+exp_name+c.EXPERIMENT_CONFIG_FILE_EXT 60 | upload_file(src_fpath, key, bucket=bucket) 61 | 62 | def upload_experiment_history(src_fpath, exp_name, bucket=S3_BUCKET): 63 | key = EXPERIMENT_HISTORY_PREFIX+exp_name+c.EXPERIMENT_HISTORY_FILE_EXT 64 | upload_file(src_fpath, key, bucket=bucket) 65 | 66 | def upload_prediction(src_fpath, pred_name, bucket=S3_BUCKET): 67 | key = PREDICTION_PREFIX+pred_name+c.PRED_FILE_EXT 68 | upload_file(src_fpath, key, bucket=bucket) 69 | 70 | 71 | # Cleanup 72 | 73 | def delete_experiment(exp_name): 74 | exp_config_key = (EXPERIMENT_CONFIG_PREFIX + exp_name 75 | + c.EXPERIMENT_CONFIG_FILE_EXT) 76 | exp_history_key = (EXPERIMENT_HISTORY_PREFIX + exp_name 77 | + c.EXPERIMENT_HISTORY_FILE_EXT) 78 | delete_object(S3_BUCKET, key=exp_config_key) 79 | delete_object(S3_BUCKET, key=exp_history_key) 80 | 81 | 82 | # Base Helpers 83 | 84 | def get_client(): 85 | return boto3.client('s3', aws_access_key_id=AWS_ACCESS_KEY, 86 | aws_secret_access_key=AWS_SECRET_KEY) 87 | 88 | def get_resource(): 89 | return boto3.resource('s3', aws_access_key_id=AWS_ACCESS_KEY, 90 | aws_secret_access_key=AWS_SECRET_KEY) 91 | 92 | def get_buckets(): 93 | return get_client().list_buckets() 94 | 95 | def get_object_str(key, bucket=S3_BUCKET): 96 | s3 = get_resource() 97 | obj = s3.Object(bucket, key) 98 | return obj.get()['Body'].read().decode('utf-8') 99 | 100 | def get_keys(prefix, bucket=S3_BUCKET): 101 | objs = get_objects(prefix, bucket) 102 | keys = [] 103 | if 'Contents' not in objs: 104 | return keys 105 | for obj in objs['Contents']: 106 | keys.append(obj['Key']) 107 | return keys 108 | 109 | def download_file(dest_fpath, s3_fpath, bucket=S3_BUCKET): 110 | get_client().download_file(Filename=dest_fpath, 111 | Bucket=bucket, 112 | Key=s3_fpath) 113 | 114 | def upload_file(src_fpath, s3_fpath, bucket=S3_BUCKET): 115 | get_client().upload_file(Filename=src_fpath, 116 | Bucket=bucket, 117 | Key=s3_fpath) 118 | 119 | def get_download_url(s3_path, bucket=S3_BUCKET, expiry=86400): 120 | return get_client().generate_presigned_url( 121 | ClientMethod='get_object', 122 | Params={'Bucket': bucket, 123 | 'Key': s3_path}, 124 | ExpiresIn=expiry 125 | ) 126 | 127 | #key = 'experiment_configs/JeremyCNN-SGD-lr1-wd0001-bs32-id6E878.json' 128 | def delete_object(bucket, key): 129 | return get_client().delete_object( 130 | Bucket=bucket, 131 | Key=key 132 | ) 133 | -------------------------------------------------------------------------------- /clients/ses_client.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from .client_constants import * 3 | 4 | 5 | def get_client(): 6 | return boto3.client('ses', aws_access_key_id=AWS_ACCESS_KEY, 7 | aws_secret_access_key=AWS_SECRET_KEY, 8 | region_name=AWS_SES_REGION) 9 | 10 | 11 | def send_email(subject, body, to_email, from_email=ADMIN_EMAIL): 12 | response = get_client().send_email( 13 | Source=from_email, 14 | Destination={ 15 | 'ToAddresses': [ 16 | to_email, 17 | ], 18 | 'CcAddresses': [], 19 | 'BccAddresses': [] 20 | }, 21 | Message={ 22 | 'Subject': { 23 | 'Data': subject, 24 | 'Charset': EMAIL_CHARSET 25 | }, 26 | 'Body': { 27 | 'Text': { 28 | 'Data': body, 29 | 'Charset': EMAIL_CHARSET 30 | }, 31 | 'Html': { 32 | 'Data': body, 33 | 'Charset': EMAIL_CHARSET 34 | } 35 | } 36 | } 37 | ) 38 | return response 39 | 40 | -------------------------------------------------------------------------------- /common.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import pandas as pd 4 | import time 5 | import importlib 6 | import torch 7 | import random 8 | import pickle 9 | import math 10 | import matplotlib.pyplot as plt 11 | import socket 12 | import datetime 13 | from PIL import Image 14 | from collections import Counter 15 | from glob import glob 16 | from IPython.display import FileLink 17 | 18 | import torch 19 | import torch.nn.functional as F 20 | from torchvision import transforms 21 | from torch.backends import cudnn 22 | from torch.autograd import Variable 23 | from torch import optim 24 | from torch import nn 25 | import torchvision 26 | import torchsample 27 | 28 | import config as cfg 29 | import constants as c 30 | 31 | import competitions 32 | 33 | import ensembles 34 | from ensembles import ens_utils 35 | 36 | import datasets 37 | from datasets import data_aug 38 | from datasets import data_folds 39 | from datasets import data_loaders 40 | from datasets import metadata 41 | 42 | from experiments.experiment import Experiment 43 | from experiments import exp_utils, exp_builder 44 | 45 | import models.builder 46 | import models.resnet 47 | import models.unet 48 | import models.utils 49 | 50 | from metrics import evaluate 51 | from metrics import metric_utils 52 | from metrics import metric 53 | from metrics import loss_functions 54 | 55 | import predictions 56 | from predictions import pred_utils 57 | 58 | import submissions 59 | 60 | import training 61 | from training import learning_rates 62 | from training import trainers 63 | 64 | import visualizers 65 | from visualizers.viz import Viz 66 | from visualizers.kibana import Kibana 67 | from visualizers import vis_utils 68 | 69 | import utils 70 | -------------------------------------------------------------------------------- /competitions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/competitions/__init__.py -------------------------------------------------------------------------------- /competitions/carvana.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import shutil 4 | import gzip 5 | 6 | import utils.files 7 | import config as cfg 8 | import constants as c 9 | from datasets import datasets 10 | from datasets import data_loaders 11 | import predictions 12 | import training 13 | import submissions 14 | 15 | 16 | 17 | def get_submission_lines(pred_arr, fnames): 18 | lines = [] 19 | for i in range(len(pred_arr)): 20 | rle = submissions.run_length_encode(pred_arr[i]) 21 | lines.append(fnames[i]+','+rle) 22 | return lines 23 | 24 | 25 | def make_submission(pred, block_size=10000, header=None, compress=True): 26 | meta = pred.attrs['meta'] 27 | print("Preds", pred.shape, meta['name'], meta['dset']) 28 | input_fnames = meta['input_fnames'] 29 | sub_fpath = os.path.join(cfg.PATHS['submissions'], meta['name']+c.SUBMISSION_FILE_EXT) 30 | lines = [] if header is None else [header] 31 | 32 | i = 0 33 | while i < len(pred): 34 | start = time.time() 35 | pred_block = pred[i:i+block_size].squeeze().astype('uint8') 36 | newlines = get_submission_lines(pred_block, input_fnames[i:i+block_size]) 37 | lines.extend(newlines) 38 | i += block_size 39 | print(training.get_time_msg(start)) 40 | 41 | sub_fpath = utils.files.write_lines(sub_fpath, lines, compress) 42 | return sub_fpath 43 | 44 | 45 | def get_block_predict_dataloaders(dataset, block_size, batch_size): 46 | loaders = [] 47 | i = 0 48 | while i < len(dataset): 49 | inp_fpaths = dataset.input_fpaths[i:i+block_size] 50 | tar_fpaths = (None if dataset.target_fpaths is None 51 | else dataset.target_fpaths[i:i+block_size]) 52 | block_dset = datasets.ImageTargetDataset(inp_fpaths, tar_fpaths, 53 | 'pil', 'pil', input_transform=dataset.input_transform, 54 | target_transform=dataset.target_transform, 55 | joint_transform=dataset.joint_transform) 56 | block_loader = data_loaders.get_data_loader(block_dset, batch_size, 57 | shuffle=False, n_workers=2, pin_memory=False) 58 | loaders.append(block_loader) 59 | i += block_size 60 | return loaders 61 | 62 | 63 | def predict_binary_mask_blocks(name, dset, model, dataset, block_size, 64 | batch_size, threshold, W=None, H=None): 65 | pred_fpath = os.path.join(cfg.PATHS['predictions'], name + '_' 66 | + dset + c.PRED_FILE_EXT) 67 | if os.path.exists(pred_fpath): 68 | print('Pred file exists. Overwriting') 69 | time.sleep(2) 70 | shutil.rmtree(pred_fpath) 71 | 72 | loaders = get_block_predict_dataloaders(dataset, block_size, batch_size) 73 | input_fnames = utils.files.get_fnames_from_fpaths(dataset.input_fpaths) 74 | target_fnames = (None if dataset.target_fpaths is None else 75 | utils.files.get_fnames_from_fpaths(dataset.target_fpaths)) 76 | meta = { 77 | 'name': name, 78 | 'dset': dset, 79 | 'input_fnames': input_fnames, 80 | 'target_fnames': target_fnames 81 | } 82 | 83 | i = 0 84 | for loader in loaders: 85 | print('Predicting block_{:d}, inputs: {:d}'.format(i, len(loader.dataset))) 86 | start = time.time() 87 | pred_block = predictions.get_mask_predictions( 88 | model, loader, threshold, W, H).astype('uint8') 89 | if i == 0: 90 | preds = predictions.save_pred(pred_fpath, pred_block, meta) 91 | else: 92 | preds = predictions.append_to_pred(preds, pred_block) 93 | i += 1 94 | print(training.get_time_msg(start)) 95 | return pred_fpath 96 | 97 | 98 | def upsample_preds(preds, block_size, W, H): 99 | meta = preds[0].attrs['meta'].copy() 100 | n_inputs = preds[0].shape[0] 101 | up_fpath = os.path.join(cfg.PATHS['predictions'], 102 | meta['name']+'_up'+c.PRED_FILE_EXT) 103 | print("inputs", n_inputs, "preds",len(preds), up_fpath) 104 | 105 | if os.path.exists(up_fpath): 106 | print('Ens file exists. Overwriting') 107 | time.sleep(2) 108 | shutil.rmtree(up_fpath) 109 | 110 | i = 0 111 | start = time.time() 112 | while i < n_inputs: 113 | up_block = predictions.resize_batch( 114 | preds[i:i+block_size], W, H).astype('uint8') 115 | if i == 0: 116 | up_pred = predictions.save_pred(up_fpath, up_block, meta) 117 | else: 118 | up_pred = predictions.append_to_pred(up_pred, up_block) 119 | i += block_size 120 | 121 | print(utils.logger.get_time_msg(start)) 122 | return up_pred 123 | 124 | 125 | 126 | def get_and_write_probabilities_to_bcolz(): 127 | """If I need extra speed""" 128 | pass -------------------------------------------------------------------------------- /competitions/dogscats.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import pandas as pd 4 | 5 | import config as cfg 6 | import constants as c 7 | 8 | import datasets.metadata as meta 9 | import utils 10 | 11 | 12 | LABEL_NAMES = ['cat', 'dog'] 13 | LABEL_TO_IDX = meta.get_labels_to_idxs(LABEL_NAMES) 14 | IDX_TO_LABEL = meta.get_idxs_to_labels(LABEL_NAMES) 15 | SUB_HEADER = 'id,label' 16 | 17 | 18 | def make_metadata_file(): 19 | ''' 20 | First move the cats/dogs data in train.zip 21 | to `catsdogs/datasets/inputs/trn_jpg` 22 | ''' 23 | train_path = cfg.PATHS['datasets']['inputs']['trn_jpg'] 24 | _, fnames = utils.files.get_paths_to_files( 25 | train_path, strip_ext=True) 26 | lines = [] 27 | for name in fnames: 28 | label = name.split('.')[0] 29 | lines.append('{:s},{:s}\n'.format(name, label)) 30 | with open(cfg.METADATA_PATH, 'w') as f: 31 | f.writelines(lines) -------------------------------------------------------------------------------- /competitions/planet.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import pandas as pd 4 | 5 | import config as cfg 6 | import constants as c 7 | 8 | import datasets.metadata as meta 9 | import utils 10 | 11 | 12 | LABEL_NAMES = [ 13 | 'clear','partly_cloudy','haze','cloudy','primary','agriculture','road','water', 14 | 'cultivation','habitation','bare_ground','selective_logging','artisinal_mine','blooming', 15 | 'slash_burn','blow_down','conventional_mine'] 16 | LABEL_TO_IDX = meta.get_labels_to_idxs(LABEL_NAMES) 17 | IDX_TO_LABEL = meta.get_idxs_to_labels(LABEL_NAMES) 18 | SUB_HEADER = 'image_name,tags' 19 | -------------------------------------------------------------------------------- /competitions/team/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/competitions/team/__init__.py -------------------------------------------------------------------------------- /competitions/team/brendan.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/competitions/team/brendan.py -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import socket 3 | import init_project 4 | import constants as c 5 | 6 | # Main config 7 | HOSTNAME = socket.gethostname() 8 | PROJECT_NAME = 'dogscats' 9 | PROJECT_PATH = '/bigguy/data/' + PROJECT_NAME 10 | PROJECT_TYPE = c.SEGMENTATION 11 | IMG_INPUT_FORMATS = [c.JPG] 12 | IMG_TARGET_FORMATS = [c.BCOLZ] #segmentation or generative 13 | IMG_DATASET_TYPES = [c.TRAIN, c.TEST] 14 | METADATA_PATH = os.path.join(PROJECT_PATH, 'metadata.csv') 15 | PATHS = init_project.init_paths(PROJECT_PATH, IMG_DATASET_TYPES, 16 | IMG_INPUT_FORMATS, IMG_TARGET_FORMATS) 17 | 18 | # AWS Config 19 | AWS_ACCESS_KEY = os.getenv('KAGGLE_AWS_ACCESS_KEY', 'dummy') 20 | AWS_SECRET_KEY = os.getenv('KAGGLE_AWS_SECRET_ACCESS_KEY', 'dummy') 21 | AWS_REGION = 'us-west-1' 22 | AWS_SES_REGION = 'us-west-2' 23 | ES_ENDPOINT = 'search-kagglecarvana-s7dnklyyz6sm2zald6umybeuau.us-west-1.es.amazonaws.com' 24 | ES_PORT = 80 25 | KIBANA_URL = 'https://search-kagglecarvana-s7dnklyyz6sm2zald6umybeuau.us-west-1.es.amazonaws.com/_plugin/kibana' 26 | TIMEZONE = 'US/Pacific' 27 | 28 | # External Resources 29 | # If True, you must setup an S3 bucket, ES Instance, and SES address 30 | S3_ENABLED = bool(os.getenv('KAGGLE_S3_ENABLED', False)) 31 | ES_ENABLED = bool(os.getenv('KAGGLE_ES_ENABLED', False)) 32 | EMAIL_ENABLED = bool(os.getenv('KAGGLE_SES_ENABLED', False)) 33 | 34 | 35 | # Email Notifications 36 | ADMIN_EMAIL = 'bfortuner@gmail.com' 37 | USER_EMAIL = 'bfortuner@gmail.com' 38 | -------------------------------------------------------------------------------- /constants.py: -------------------------------------------------------------------------------- 1 | # Project Types 2 | CLASSIFICATION = 'classification' 3 | SEGMENTATION = 'segmentation' 4 | PROJECT_TYPES = [CLASSIFICATION, SEGMENTATION] 5 | 6 | # Datasets 7 | TRAIN = 'trn' 8 | VAL = 'val' 9 | TEST = 'tst' 10 | FULL = 'full' 11 | DSETS = [TRAIN, VAL, TEST, FULL] 12 | 13 | # Transforms 14 | JOINT = 'joint' 15 | UPSAMPLE = 'upsample' 16 | TARGET = 'target' 17 | TENSOR = 'tensor' 18 | MASK = 'mask' 19 | 20 | # File 21 | JPG = 'jpg' 22 | TIF = 'tif' 23 | PNG = 'png' 24 | GIF = 'gif' 25 | BCOLZ = 'bc' 26 | JPG_EXT = '.'+JPG 27 | TIF_EXT = '.'+TIF 28 | PNG_EXT = '.'+PNG 29 | GIF_EXT = '.'+GIF 30 | BCOLZ_EXT = '.'+BCOLZ 31 | IMG_EXTS = [JPG_EXT, TIF_EXT, PNG_EXT, GIF_EXT, BCOLZ_EXT] 32 | CHECKPOINT_EXT = '.th' 33 | EXPERIMENT_CONFIG_FILE_EXT = '.json' 34 | EXPERIMENT_CONFIG_FNAME = 'config.json' 35 | EXPERIMENT_HISTORY_FILE_EXT = '.csv' 36 | EXP_FILE_EXT = '.zip' 37 | PRED_FILE_EXT = '.bc' 38 | SUBMISSION_FILE_EXT = '.csv' 39 | ENSEMBLE_FILE_EXT = '.bc' 40 | DSET_FOLD_FILE_EXT = '.json' 41 | MODEL_EXT = '.mdl' 42 | WEIGHTS_EXT = '.th' 43 | OPTIM_EXT = '.th' 44 | 45 | # Postfix 46 | INPUT_POSTFIX = JPG_EXT 47 | TARGET_POSTFIX = '_mask'+GIF_EXT 48 | 49 | # Metrics 50 | LOSS = 'Loss' 51 | SCORE = 'Score' 52 | ACCURACY = 'Accuracy' 53 | F2_SCORE = 'F2' 54 | ENSEMBLE_F2 = 'EnsembleF2' 55 | DICE_SCORE = 'Dice' 56 | MEAN = 'mean' 57 | GMEAN = 'gmean' 58 | VOTE = 'vote' 59 | STD_DEV = 'std' 60 | ENSEMBLE_METHODS = [MEAN, GMEAN] 61 | 62 | # File Regex 63 | WEIGHTS_FNAME_REGEX = r'weights-(\d+)\.pth$' 64 | OPTIM_FNAME_REGEX = r'optim-(\d+)\.pth$' 65 | WEIGHTS_OPTIM_FNAME_REGEX = r'(weights|optim)-(\d+)\.th$' 66 | LATEST_WEIGHTS_FNAME = 'latest_weights.th' 67 | LATEST_OPTIM_FNAME = 'latest_optim.th' 68 | LATEST = 'latest' 69 | 70 | 71 | # Predictions 72 | SINGLE_MODEL_PRED = 'single-basic' 73 | SINGLE_MODEL_TTA_PRED = 'single-tta' 74 | PREDICTION_TYPES = [SINGLE_MODEL_PRED, SINGLE_MODEL_TTA_PRED] 75 | SINGLE_MODEL_ENSEMBLE = 'single-ens' 76 | SINGLE_MODEL_TTA_ENSEMBLE = 'single-ens-tta' 77 | ENSEMBLE_TYPES = [SINGLE_MODEL_ENSEMBLE, SINGLE_MODEL_TTA_ENSEMBLE] 78 | DEFAULT_BLOCK_NAME = 'preds' 79 | 80 | # Ensembles 81 | MEGA_ENSEMBLE = 'mega-ens' 82 | MEGA_ENSEMBLE_TYPES = [MEGA_ENSEMBLE] 83 | 84 | 85 | # Experiments 86 | INITIALIZED = 'INITIALIZED' 87 | RESUMED = 'RESUMED' 88 | COMPLETED = 'COMPLETED' 89 | IN_PROGRESS = 'IN_PROGRESS' 90 | FAILED = 'FAILED' 91 | MAX_PATIENCE_EXCEEDED = 'MAX_PATIENCE_EXCEEDED' 92 | EXPERIMENT_STATUSES = [INITIALIZED, RESUMED, COMPLETED, 93 | IN_PROGRESS, FAILED, MAX_PATIENCE_EXCEEDED] 94 | EXP_ID_FIELD = 'exp_id' 95 | ES_EXP_KEY_FIELD = 'key' 96 | LATEST_WEIGHTS_FNAME = 'latest_weights{:s}'.format(WEIGHTS_EXT) 97 | LATEST_OPTIM_FNAME = 'latest_optim{:s}'.format(OPTIM_EXT) 98 | MODEL_FNAME = 'model{:s}'.format(MODEL_EXT) 99 | OPTIM_FNAME = 'optim{:s}'.format(OPTIM_EXT) 100 | 101 | 102 | # Data Aug 103 | IMAGENET_MEAN = [0.485, 0.456, 0.406] 104 | IMAGENET_STD = [0.229, 0.224, 0.225] 105 | -------------------------------------------------------------------------------- /datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/datasets/__init__.py -------------------------------------------------------------------------------- /datasets/data_aug.py: -------------------------------------------------------------------------------- 1 | import math 2 | import random 3 | from PIL import Image, ImageFilter 4 | import cv2 5 | import numpy as np 6 | 7 | import torch 8 | import torchsample 9 | import torchvision 10 | from torch.utils.data import DataLoader 11 | from torch.utils.data import TensorDataset 12 | 13 | 14 | #http://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_geometric_transformations/py_geometric_transformations.html 15 | 16 | IMAGENET_MEAN = [0.485, 0.456, 0.406] 17 | IMAGENET_STD = [0.229, 0.224, 0.225] 18 | 19 | IMAGENET_NORMALIZE = torchvision.transforms.Normalize( 20 | mean=IMAGENET_MEAN, 21 | std=IMAGENET_STD 22 | ) 23 | 24 | def get_data_aug_summary(transforms): 25 | data_aug = [] 26 | for r in transforms.transforms: 27 | data_aug.append((str(r.__class__.__name__), r.__dict__)) 28 | return data_aug 29 | 30 | 31 | def get_basic_transform(scale, normalize=None): 32 | data_aug = [ 33 | torchvision.transforms.Scale(scale), 34 | torchvision.transforms.ToTensor() 35 | ] 36 | if normalize is not None: 37 | data_aug.append(normalize) 38 | return torchvision.transforms.Compose(data_aug) 39 | 40 | 41 | def get_single_pil_transform(scale, augmentation, normalize=None): 42 | data_aug = [ 43 | torchvision.transforms.Scale(scale), 44 | augmentation, 45 | torchvision.transforms.ToTensor() 46 | ] 47 | if normalize is not None: 48 | data_aug.append(normalize) 49 | return torchvision.transforms.Compose(data_aug) 50 | 51 | 52 | def get_single_tensor_transform(scale, augmentation, normalize=None): 53 | data_aug = [ 54 | torchvision.transforms.Scale(scale), 55 | torchvision.transforms.ToTensor(), 56 | augmentation 57 | ] 58 | if normalize is not None: 59 | data_aug.append(normalize) 60 | return torchvision.transforms.Compose(data_aug) 61 | 62 | 63 | class RandomRotate90(object): 64 | def __init__(self, p=0.75): 65 | self.p = p 66 | 67 | def __call__(self, *inputs): 68 | outputs = [] 69 | for idx, input_ in enumerate(inputs): 70 | input_ = random_rotate_90(input_, self.p) 71 | outputs.append(input_) 72 | return outputs if idx > 1 else outputs[0] 73 | 74 | 75 | class BinaryMask(object): 76 | def __init__(self, thresholds): 77 | self.thresholds = thresholds 78 | 79 | def __call__(self, *inputs): 80 | outputs = [] 81 | for idx, input_ in enumerate(inputs): 82 | input_[input_ >= self.thresholds] = 1.0 83 | input_[input_ < self.thresholds] = 0.0 84 | outputs.append(input_) 85 | return outputs if idx > 1 else outputs[0] 86 | 87 | 88 | class Slice1D(object): 89 | def __init__(self, dim=0, slice_idx=0): 90 | self.dim = dim 91 | self.slice_idx = slice_idx 92 | 93 | def __call__(self, *inputs): 94 | outputs = [] 95 | for idx, input_ in enumerate(inputs): 96 | input_ = torch.unsqueeze(input_[self.slice_idx,:,:], dim=self.dim) 97 | outputs.append(input_) 98 | return outputs if idx > 1 else outputs[0] 99 | 100 | 101 | class RandomHueSaturation(object): 102 | def __init__(self, hue_shift=(-180, 180), sat_shift=(-255, 255), 103 | val_shift=(-255, 255), u=0.5): 104 | self.hue_shift = hue_shift 105 | self.sat_shift = sat_shift 106 | self.val_shift = val_shift 107 | self.u = u 108 | 109 | def __call__(self, *inputs): 110 | outputs = [] 111 | for idx, input_ in enumerate(inputs): 112 | input_ = random_hue_saturation(input_, self.hue_shift, 113 | self.sat_shift, self.val_shift, self.u) 114 | outputs.append(input_) 115 | return outputs if idx > 1 else outputs[0] 116 | 117 | 118 | class RandomShiftScaleRotate(object): 119 | def __init__(self, shift=(-0.0625,0.0625), scale=(-0.1,0.1), 120 | rotate=(-45,45), aspect=(0,0), u=0.5): 121 | self.shift = shift 122 | self.scale = scale 123 | self.rotate = rotate 124 | self.aspect = aspect 125 | self.border_mode = cv2.BORDER_CONSTANT 126 | self.u = u 127 | 128 | def __call__(self, input_, target): 129 | input_, target = random_shift_scale_rot(input_, target, self.shift, 130 | self.scale, self.rotate, self.aspect, self.border_mode, self.u) 131 | return [input_, target] 132 | 133 | 134 | def random_rotate_90(pil_img, p=1.0): 135 | if random.random() < p: 136 | angle=random.randint(1,3)*90 137 | if angle == 90: 138 | pil_img = pil_img.rotate(90) 139 | elif angle == 180: 140 | pil_img = pil_img.rotate(180) 141 | elif angle == 270: 142 | pil_img = pil_img.rotate(270) 143 | return pil_img 144 | 145 | 146 | def random_hue_saturation(image, hue_shift=(-180, 180), sat_shift=(-255, 255), 147 | val_shift=(-255, 255), u=0.5): 148 | image = np.array(image) 149 | if np.random.random() < u: 150 | image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) 151 | h, s, v = cv2.split(image) 152 | hue_shift = np.random.uniform(hue_shift[0], hue_shift[1]) 153 | h = cv2.add(h, hue_shift) 154 | sat_shift = np.random.uniform(sat_shift[0], sat_shift[1]) 155 | s = cv2.add(s, sat_shift) 156 | val_shift = np.random.uniform(val_shift[0], val_shift[1]) 157 | v = cv2.add(v, val_shift) 158 | image = cv2.merge((h, s, v)) 159 | image = cv2.cvtColor(image, cv2.COLOR_HSV2BGR) 160 | 161 | return Image.fromarray(image) 162 | 163 | 164 | def random_shift_scale_rot(image, label, shift_limit=(-0.0625,0.0625), 165 | scale_limit=(-0.1,0.1), rotate_limit=(-45,45), aspect_limit = (0,0), 166 | borderMode=cv2.BORDER_CONSTANT, u=0.5): 167 | image = image.numpy().transpose(1,2,0) 168 | label = label.numpy().squeeze() 169 | if random.random() < u: 170 | height,width,channel = image.shape 171 | 172 | angle = random.uniform(rotate_limit[0],rotate_limit[1]) #degree 173 | scale = random.uniform(1+scale_limit[0],1+scale_limit[1]) 174 | aspect = random.uniform(1+aspect_limit[0],1+aspect_limit[1]) 175 | sx = scale*aspect/(aspect**0.5) 176 | sy = scale /(aspect**0.5) 177 | dx = round(random.uniform(shift_limit[0],shift_limit[1])*width ) 178 | dy = round(random.uniform(shift_limit[0],shift_limit[1])*height) 179 | 180 | cc = math.cos(angle/180*math.pi)*(sx) 181 | ss = math.sin(angle/180*math.pi)*(sy) 182 | rotate_matrix = np.array([ [cc,-ss], [ss,cc] ]) 183 | 184 | box0 = np.array([ [0,0], [width,0], [width,height], [0,height], ]) 185 | box1 = box0 - np.array([width/2,height/2]) 186 | box1 = np.dot(box1,rotate_matrix.T) + np.array([width/2+dx,height/2+dy]) 187 | box0 = box0.astype(np.float32) 188 | box1 = box1.astype(np.float32) 189 | mat = cv2.getPerspectiveTransform(box0,box1) 190 | image = cv2.warpPerspective(image, mat, (width,height), 191 | flags=cv2.INTER_LINEAR,borderMode=borderMode,borderValue=(0,0,0,)) 192 | #cv2.BORDER_CONSTANT, borderValue = (0, 0, 0)) #cv2.BORDER_REFLECT_101 193 | 194 | box0 = np.array([ [0,0], [width,0], [width,height], [0,height], ]) 195 | box1 = box0 - np.array([width/2,height/2]) 196 | box1 = np.dot(box1,rotate_matrix.T) + np.array([width/2+dx,height/2+dy]) 197 | box0 = box0.astype(np.float32) 198 | box1 = box1.astype(np.float32) 199 | mat = cv2.getPerspectiveTransform(box0,box1) 200 | label = cv2.warpPerspective(label, mat, (width,height), 201 | flags=cv2.INTER_LINEAR,borderMode=borderMode,borderValue=(0,0,0,)) 202 | #cv2.BORDER_CONSTANT, borderValue = (0, 0, 0)) #cv2.BORDER_REFLECT_101 203 | image = torch.from_numpy(image.transpose(2,0,1)) 204 | label = np.expand_dims(label, 0) 205 | label = torch.from_numpy(label)#.transpose(2,0,1)) 206 | return image,label 207 | 208 | 209 | blurTransform = torchvision.transforms.Lambda( 210 | lambda img: img.filter(ImageFilter.GaussianBlur(1.5))) -------------------------------------------------------------------------------- /datasets/data_folds.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import numpy as np 4 | 5 | import utils.files 6 | import constants as c 7 | 8 | 9 | def make_bag(fpaths, targets): 10 | bag_fpaths = [] 11 | bag_targets = [] 12 | for i in range(len(fpaths)): 13 | idx = random.randint(1,len(fpaths)-1) 14 | bag_fpaths.append(fpaths[idx]) 15 | bag_targets.append(targets[idx]) 16 | return bag_fpaths, np.array(bag_targets) 17 | 18 | 19 | def verify_bag(trn_fpaths_bag): 20 | trn_dict = {} 21 | for f in trn_fpaths_bag: 22 | if f in trn_dict: 23 | trn_dict[f] += 1 24 | else: 25 | trn_dict[f] = 1 26 | return max(trn_dict.values()) 27 | 28 | 29 | def make_fold(name, trn_path, tst_path, folds_dir, 30 | val_size, shuffle=True): 31 | _, trn_fnames = utils.files.get_paths_to_files( 32 | trn_path, file_ext=c.JPG, sort=True, strip_ext=True) 33 | _, tst_fnames = utils.files.get_paths_to_files( 34 | tst_path, file_ext=c.JPG, sort=True, strip_ext=True) 35 | 36 | if shuffle: 37 | random.shuffle(trn_fnames) 38 | 39 | fold = { 40 | c.TRAIN: trn_fnames[:-val_size], 41 | c.VAL: trn_fnames[-val_size:], 42 | c.TEST: tst_fnames 43 | } 44 | 45 | fpath = os.path.join(folds_dir, name + c.DSET_FOLD_FILE_EXT) 46 | utils.files.save_json(fpath, fold) 47 | return fold 48 | 49 | 50 | def load_data_fold(folds_dir, name): 51 | fpath = os.path.join(folds_dir, name + c.DSET_FOLD_FILE_EXT) 52 | return utils.files.load_json(fpath) 53 | 54 | 55 | def get_fpaths_from_fold(fold, dset, dset_path, postfix=''): 56 | fnames = fold[dset] 57 | fpaths = [os.path.join(dset_path, f+postfix) for f in fnames] 58 | return fpaths 59 | 60 | 61 | def get_targets_from_fold(fold, dset, lookup): 62 | img_names = [f.split('.')[0] for f in fold[dset]] 63 | targets = [] 64 | for img in img_names: 65 | targets.append(lookup[img]) 66 | return np.array(targets) 67 | 68 | 69 | def get_fpaths_targets_from_fold(fold, dset, dset_path, lookup): 70 | fpaths = get_fpaths_from_fold(fold, dset, dset_path) 71 | targets = get_fpaths_from_fold(fold, dset, lookup) 72 | return fpaths, targets 73 | -------------------------------------------------------------------------------- /datasets/data_loaders.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch.utils.data 3 | from torch.utils.data import DataLoader 4 | from config import * 5 | import utils.imgs as img_utils 6 | 7 | 8 | 9 | class MixDataLoader(): 10 | """ 11 | Combines batches from two data loaders. 12 | Useful for pseudolabeling. 13 | """ 14 | def __init__(self, dl1, dl2): 15 | self.dl1 = dl1 16 | self.dl2 = dl2 17 | self.dl1_iter = iter(dl1) 18 | self.dl2_iter = iter(dl2) 19 | self.n = len(dl1) 20 | self.cur = 0 21 | 22 | def _reset(self): 23 | self.cur = 0 24 | 25 | def _cat_lst(self, fn1, fn2): 26 | return fn1 + fn2 27 | 28 | def _cat_tns(self, t1, t2): 29 | return torch.cat([t1, t2]) 30 | 31 | def __next__(self): 32 | x1,y1,f1 = next(self.dl1_iter) 33 | x2,y2,f2 = next(self.dl2_iter) 34 | while self.cur < self.n: 35 | self.cur += 1 36 | return (self._cat_tns(x1,x2), self._cat_tns(y1,y2), 37 | self._cat_lst(f1,f2)) 38 | 39 | def __iter__(self): 40 | self.cur = 0 41 | self.dl1_iter = iter(self.dl1) 42 | self.dl2_iter = iter(self.dl2) 43 | return self 44 | 45 | def __len__(self): 46 | return self.n 47 | 48 | 49 | def get_batch(dataset, batch_size, shuffle=False): 50 | dataloader = DataLoader( 51 | dataset, batch_size=batch_size, shuffle=shuffle) 52 | inputs, targets, img_paths = next(iter(dataloader)) 53 | return inputs, targets, img_paths 54 | 55 | 56 | def get_data_loader(dset, batch_size, shuffle=False, 57 | n_workers=1, pin_memory=False): 58 | return DataLoader(dset, batch_size, shuffle=shuffle, 59 | pin_memory=pin_memory, num_workers=n_workers) 60 | -------------------------------------------------------------------------------- /datasets/data_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import numpy as np 4 | import utils 5 | from glob import glob 6 | from PIL import Image 7 | from skimage import io 8 | import torch 9 | 10 | import config as cfg 11 | import constants as c 12 | from datasets import metadata 13 | 14 | 15 | 16 | def pil_loader(path): 17 | return Image.open(path).convert('RGB') 18 | 19 | 20 | def tensor_loader(path): 21 | return torch.load(path) 22 | 23 | 24 | def numpy_loader(path): 25 | return np.load(path) 26 | 27 | 28 | def io_loader(path): 29 | return io.imread(path) 30 | 31 | 32 | def tif_loader(path): 33 | return calibrate_image(io.imread(path)[:,:,(2,1,0,3)]) 34 | 35 | 36 | def calibrate_image(rgb_image, ref_stds, ref_means): 37 | res = rgb_image.astype('float32') 38 | return np.clip((res - np.mean(res,axis=(0,1))) / np.std(res,axis=(0,1)) 39 | * ref_stds + ref_means,0,255).astype('uint8') 40 | 41 | 42 | def get_inputs_targets(fpaths, dframe): 43 | ## REFACTOR 44 | inputs = [] 45 | targets = [] 46 | for fpath in fpaths: 47 | # Refactor 48 | name, tags = metadata.get_img_name_and_tags(METADATA_DF, fpath) 49 | inputs.append(img_utils.load_img_as_arr(fpath)) 50 | targets.append(meta.get_one_hots_by_name(name, dframe)) 51 | return np.array(inputs), np.array(targets) -------------------------------------------------------------------------------- /datasets/datasets.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from . import data_utils 3 | 4 | 5 | loaders = { 6 | 'pil': data_utils.pil_loader, 7 | 'tns': data_utils.tensor_loader, 8 | 'npy': data_utils.numpy_loader, 9 | 'tif': data_utils.tif_loader, 10 | 'io': data_utils.io_loader 11 | } 12 | 13 | 14 | class FileDataset(torch.utils.data.Dataset): 15 | def __init__(self, fpaths, 16 | img_loader='pil', 17 | targets=None, 18 | transform=None, 19 | target_transform=None): 20 | self.fpaths = fpaths 21 | self.loader = self._get_loader(img_loader) 22 | self.targets = targets 23 | self.transform = transform 24 | self.target_transform = target_transform 25 | 26 | def _get_loader(self, loader_type): 27 | return loaders[loader_type] 28 | 29 | def _get_target(self, index): 30 | if self.targets is None: 31 | return torch.FloatTensor(1) 32 | target = self.targets[index] 33 | if self.target_transform is not None: 34 | return self.target_transform(target) 35 | return torch.FloatTensor(target) 36 | 37 | def _get_input(self, index): 38 | img_path = self.fpaths[index] 39 | img = self.loader(img_path) 40 | if self.transform is not None: 41 | img = self.transform(img) 42 | return img 43 | 44 | def __getitem__(self, index): 45 | input_ = self._get_input(index) 46 | target = self._get_target(index) 47 | img_path = self.fpaths[index] 48 | return input_, target, img_path 49 | 50 | def __len__(self): 51 | return len(self.fpaths) 52 | 53 | 54 | class MultiInputDataset(FileDataset): 55 | def __init__(self, fpaths, 56 | img_loader='pil', #'tns', 'npy' 57 | targets=None, 58 | other_inputs=None, 59 | transform=None, 60 | target_transform=None): 61 | super().__init__(fpaths, img_loader, targets, 62 | transform, target_transform) 63 | self.other_inputs = other_inputs 64 | 65 | def _get_other_input(self, index): 66 | other_input = self.other_inputs[index] 67 | return other_input 68 | 69 | def __getitem__(self, index): 70 | input_ = self._get_input(index) 71 | target = self._get_target(index) 72 | other_input = self._get_other_input(index) 73 | img_path = self.fpaths[index] 74 | return input_, target, other_input, img_path 75 | 76 | 77 | class MultiTargetDataset(FileDataset): 78 | def __init__(self, fpaths, 79 | img_loader='pil', 80 | targets=None, 81 | other_targets=None, 82 | transform=None, 83 | target_transform=None): 84 | super().__init__(fpaths, img_loader, targets, 85 | transform, target_transform) 86 | self.other_targets = other_targets 87 | 88 | def _get_other_target(self, index): 89 | if self.other_targets is None: 90 | return torch.FloatTensor(1) 91 | other_target = self.other_targets[index] 92 | return torch.FloatTensor(other_target) 93 | 94 | def __getitem__(self, index): 95 | input_ = self._get_input(index) 96 | target = self._get_target(index) 97 | other_target = self._get_other_target(index) 98 | img_path = self.fpaths[index] 99 | return input_, target, other_target, img_path 100 | 101 | 102 | class ImageTargetDataset(torch.utils.data.Dataset): 103 | def __init__(self, input_fpaths, 104 | target_fpaths, 105 | input_loader='pil', 106 | target_loader='pil', 107 | input_transform=None, 108 | target_transform=None, 109 | joint_transform=None): 110 | self.input_fpaths = input_fpaths 111 | self.target_fpaths = target_fpaths 112 | self.input_loader = loaders[input_loader] 113 | self.target_loader = loaders[target_loader] 114 | self.input_transform = input_transform 115 | self.target_transform = target_transform 116 | self.joint_transform = joint_transform 117 | 118 | def _get_target(self, index): 119 | if self.target_fpaths is None: 120 | return torch.FloatTensor(1), "" 121 | img_path = self.target_fpaths[index] 122 | img = self.target_loader(img_path) 123 | if self.target_transform is not None: 124 | img = self.target_transform(img) 125 | return img, img_path 126 | 127 | def _get_input(self, index): 128 | img_path = self.input_fpaths[index] 129 | img = self.input_loader(img_path) 130 | if self.input_transform is not None: 131 | img = self.input_transform(img) 132 | return img, img_path 133 | 134 | def __getitem__(self, index): 135 | input_, inp_path = self._get_input(index) 136 | target, tar_path = self._get_target(index) 137 | if self.joint_transform is not None: 138 | input_, target = self.joint_transform(input_, target) 139 | return input_, target, inp_path, tar_path 140 | 141 | def __len__(self): 142 | return len(self.input_fpaths) 143 | -------------------------------------------------------------------------------- /datasets/metadata.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | import constants as c 5 | 6 | 7 | def get_metadata_df(fpath): 8 | return pd.read_csv(fpath, header=0, names=['id','labels']) 9 | 10 | 11 | def get_labels_to_idxs(label_names): 12 | return {v:k for k,v in enumerate(label_names)} 13 | 14 | 15 | def get_idxs_to_labels(label_names): 16 | return {k:v for k,v in enumerate(label_names)} 17 | 18 | 19 | def convert_tags_to_one_hots(tags, label_names, delim=' '): 20 | label_to_idx = get_labels_to_idxs(label_names) 21 | idxs = [label_to_idx[o] for o in tags.split(delim)] 22 | onehot = np.zeros((len(label_names),), dtype=np.float32) 23 | onehot[idxs] = 1 24 | return onehot 25 | 26 | 27 | def get_one_hots_array(meta_fpath, label_names): 28 | meta_df = get_metadata_df(meta_fpath) 29 | onehots = np.zeros( (0, len(label_names)) ) 30 | for _, row in meta_df.iterrows(): 31 | onehot = convert_tags_to_one_hots(row[1], label_names) 32 | onehots = np.append(onehots, np.array([onehot]), axis=0) 33 | return onehots 34 | 35 | 36 | def get_one_hots_from_fold(meta_fpath, fold, dset, label_names): 37 | meta_df = get_metadata_df(meta_fpath) 38 | onehots = np.zeros( (0, len(label_names)) ) 39 | for _, name in enumerate(fold[dset]): 40 | tags = meta_df[meta_df['id'] == name]['labels'].values[0] 41 | onehot = convert_tags_to_one_hots(tags, label_names) 42 | onehots = np.append(onehots, np.array([onehot]), axis=0) 43 | return onehots 44 | 45 | 46 | def get_label_idx_by_name(label, label_names): 47 | return label_names.index(label) 48 | 49 | 50 | def get_tags_from_preds(preds, label_names): 51 | tags = [] 52 | for _, pred in enumerate(preds): 53 | tag_str = ' '.join(convert_one_hot_to_tags(pred, label_names)) 54 | tags.append(tag_str) 55 | return tags 56 | 57 | 58 | def convert_one_hot_to_tags(onehot, label_names): 59 | tags = [] 60 | for idx, val in enumerate(onehot): 61 | if val == 1: 62 | tags.append(label_names[idx]) 63 | return tags -------------------------------------------------------------------------------- /docs/email.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/docs/email.png -------------------------------------------------------------------------------- /docs/kibana1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/docs/kibana1.png -------------------------------------------------------------------------------- /docs/kibana2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/docs/kibana2.png -------------------------------------------------------------------------------- /docs/visdom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/docs/visdom.png -------------------------------------------------------------------------------- /docs/visdom2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/docs/visdom2.png -------------------------------------------------------------------------------- /ensembles/__init__.py: -------------------------------------------------------------------------------- 1 | from .ens_utils import * 2 | import os -------------------------------------------------------------------------------- /ensembles/ens_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import random 4 | import numpy as np 5 | import pandas as pd 6 | import shutil 7 | 8 | import config as cfg 9 | import constants as c 10 | import utils 11 | import predictions 12 | 13 | 14 | 15 | def get_ensemble_fpath(basename, dset): 16 | fname = '{:s}_{:s}_{:s}'.format(basename, 'ens', dset + c.PRED_FILE_EXT) 17 | return os.path.join(cfg.PATHS['predictions'], fname) 18 | 19 | 20 | def get_ensemble_meta(name, fpaths): 21 | preds = [predictions.load_pred(f) for f in fpaths] 22 | meta = preds[0].attrs['meta'].copy() 23 | meta['name'] = name 24 | meta['members'] = {p.attrs['meta']['name']:p.attrs['meta'] for p in preds} 25 | print("members", list(meta['members'].keys())) 26 | return meta 27 | 28 | 29 | def ens_prediction_files(ens_fpath, pred_fpaths, block_size=1, 30 | method=c.MEAN, meta=None): 31 | preds = [predictions.load_pred(f) for f in pred_fpaths] 32 | n_inputs = preds[0].shape[0] 33 | if os.path.exists(ens_fpath): 34 | print('Ens file exists. Overwriting') 35 | time.sleep(2) 36 | shutil.rmtree(ens_fpath) 37 | 38 | i = 0 39 | start = time.time() 40 | while i < n_inputs: 41 | pred_block = np.array([p[i:i+block_size] for p in preds]) 42 | ens_block = predictions.ensemble_with_method(pred_block, method) 43 | if i == 0: 44 | ens_pred = predictions.save_pred(ens_fpath, ens_block, meta) 45 | else: 46 | ens_pred = predictions.append_to_pred(ens_pred, ens_block) 47 | i += block_size 48 | print(utils.logger.get_time_msg(start)) 49 | return ens_fpath 50 | 51 | 52 | def build_scores(loss, score): 53 | return { 54 | c.LOSS: loss, 55 | c.SCORE: score 56 | } 57 | 58 | 59 | def build_metadata(labels, scores, thresholds, pred_type, dset): 60 | return { 61 | 'label_names': labels, 62 | 'scores': scores, 63 | 'thresholds': thresholds, 64 | 'pred_type': pred_type, 65 | 'dset': dset, 66 | 'created': time.strftime("%m/%d/%Y %H:%M:%S", time.localtime()) 67 | } -------------------------------------------------------------------------------- /ensembles/ensemble.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pprint 3 | import utils.files 4 | import copy 5 | import constants as c 6 | from predictions.prediction import Prediction 7 | 8 | 9 | 10 | class MegaEnsemblePrediction(Prediction): 11 | """ 12 | Prediction combining multiple experiments, models and epochs 13 | """ 14 | def __init__(self, name, pred_type, fpath, thresholds, 15 | label_names, val_score, val_probs, val_preds, 16 | test_probs, test_preds, created, sub_preds, 17 | ens_method, all_val_probs, all_test_probs): 18 | super().__init__(name, pred_type, fpath, thresholds, 19 | label_names, val_score, val_probs, val_preds, 20 | test_probs, test_preds, tta=None, created=created, 21 | other=None) 22 | 23 | self.sub_preds = self.get_sub_pred_docs(sub_preds) 24 | self.ens_method = ens_method 25 | self.all_val_probs = all_val_probs 26 | self.all_test_probs = all_test_probs 27 | 28 | def get_sub_pred_docs(self, sub_preds): 29 | docs = [] 30 | for pred in sub_preds: 31 | docs.append(pred.to_doc(include_exp=False)) 32 | return docs 33 | 34 | def to_doc(self): 35 | d = copy.deepcopy(self.__dict__) 36 | d['key'] = self.get_id() 37 | d['pred_id'] = self.get_id() 38 | d['display_name'] = self.get_display_name() 39 | d['preds'] = self.sub_preds 40 | del d['val_probs'] 41 | del d['val_preds'] 42 | del d['test_probs'] 43 | del d['test_preds'] 44 | del d['all_val_probs'] 45 | del d['all_test_probs'] 46 | return d 47 | 48 | 49 | -------------------------------------------------------------------------------- /experiments/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/experiments/__init__.py -------------------------------------------------------------------------------- /experiments/exp_builder.py: -------------------------------------------------------------------------------- 1 | from glob import glob 2 | import pandas as pd 3 | from .experiment import Experiment 4 | 5 | 6 | 7 | def prune_experiments(exp_dir, exp_names): 8 | # Delete weights except for best weight in `n_bins` 9 | for name in exp_names: 10 | exp = Experiment(name, exp_dir) 11 | exp.review(verbose=False) 12 | exp.auto_prune(n_bins=5, metric_name='Loss', func=max) 13 | 14 | 15 | def build_exp_summary_dict(exp): 16 | config = exp.config 17 | history = exp.history 18 | dict_ = { 19 | 'name': config.name, 20 | 'exp_id': config.get_id(), 21 | 'created': config.created, 22 | 'fold': config.data['dset_fold'], 23 | 'model_name' : config.model_name, 24 | 'threshold' : config.training['threshold'], 25 | 'model_name' : config.model['name'], 26 | 'optim' : config.optimizer['name'], 27 | 'optim_params' : str(config.optimizer['params']), 28 | 'lr_adjuster' : config.lr_adjuster['name'], 29 | 'lr_adjuster_params' : str(config.lr_adjuster['params']), 30 | 'criterion': config.criterion['name'], 31 | 'transforms' : ', '.join([t[0] for t in config.transforms]), 32 | ### initial lr, img_scale, rescale, total_epochs 33 | 'transforms' : ', '.join([t[0] for t in config.transforms]), 34 | 'init_lr':config.training['initial_lr'], 35 | 'wdecay':config.training['weight_decay'], 36 | 'batch': config.training['batch_size'], 37 | 'img_scl':config.data['img_scale'], 38 | 'img_rescl': config.data['img_rescale'], 39 | 'nepochs':config.training['n_epochs'], 40 | } 41 | for name in config.metrics: 42 | dict_[name+'Epoch'] = history.best_metrics[name]['epoch'] 43 | dict_[name+'Val'] = history.best_metrics[name]['value'] 44 | return dict_ 45 | 46 | 47 | def build_exps_df_from_dir(exps_dir): 48 | exp_names = glob(exps_dir+'/*/') 49 | summaries = [] 50 | for name in exp_names: 51 | exp = Experiment(name, exps_dir) 52 | exp.review(verbose=False) 53 | exp_summary = build_exp_summary_dict(exp) 54 | summaries.append(exp_summary) 55 | return pd.DataFrame(summaries) 56 | 57 | 58 | def upload_experiments(exp_dir): 59 | exp_paths = glob(exp_dir+'/*/') 60 | for path in exp_paths: 61 | name = path.strip('/').split('/')[-1] 62 | exp = Experiment(name, exp_dir) 63 | exp.review(verbose=False) 64 | exp.save() 65 | 66 | 67 | def upload_experiments(exp_dir): 68 | exp_paths = glob(exp_dir+'/*/') 69 | for path in exp_paths: 70 | name = path.strip('/').split('/')[-1] 71 | exp = Experiment(name, exp_dir) 72 | exp.review(verbose=False) 73 | exp.save() 74 | -------------------------------------------------------------------------------- /experiments/exp_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import pprint 4 | import logging 5 | import time 6 | import copy 7 | from datetime import datetime 8 | 9 | import pandas as pd 10 | 11 | import config 12 | import constants as c 13 | import utils.files 14 | import utils.general 15 | from clients import s3_client, es_client 16 | 17 | 18 | 19 | 20 | class ExperimentConfig(): 21 | def __init__(self, name, parent_dir, created, metrics, aux_metrics, 22 | model, optimizer, lr_adjuster, criterion, transforms, 23 | visualizers, training, data, hardware, other, progress=None): 24 | self.name = name 25 | self.parent_dir = parent_dir 26 | self.fpath = os.path.join(parent_dir, name, 27 | c.EXPERIMENT_CONFIG_FNAME) 28 | self.created = created 29 | self.metrics = metrics 30 | self.aux_metrics = aux_metrics 31 | self.visualizers = visualizers 32 | self.model = model 33 | self.optimizer = optimizer 34 | self.lr_adjuster = lr_adjuster 35 | self.criterion = criterion 36 | self.transforms = transforms 37 | self.data = data 38 | self.training = training 39 | self.hardware = hardware 40 | self.other = other 41 | self.progress = {} if progress is None else progress 42 | self.model_name = self.model['name'] 43 | 44 | def get_id(self): 45 | return self.name.split('-id')[-1] 46 | 47 | def get_display_name(self): 48 | return self.name.split('-id')[0] 49 | 50 | def summary(self, include_model=True): 51 | d = dict(self.__dict__) 52 | del d['model'] 53 | print(json.dumps(d, indent=4, ensure_ascii=False)) 54 | if include_model: 55 | print(self.model['layers']) 56 | 57 | def save(self, s3=config.S3_ENABLED, es=config.ES_ENABLED): 58 | dict_ = self.__dict__ 59 | utils.files.save_json(self.fpath, dict_) 60 | if s3: 61 | s3_client.upload_experiment_config(self.fpath, self.name) 62 | if es: 63 | es_client.upload_experiment_config(self) 64 | 65 | def to_dict(self): 66 | return self.__dict__ 67 | 68 | def to_json(self): 69 | return json.dumps(self.to_dict(), indent=4, ensure_ascii=False) 70 | 71 | def to_html(self): 72 | dict_ = self.to_dict() 73 | html = utils.general.dict_to_html_ul(dict_) 74 | return html 75 | 76 | def to_doc(self): 77 | # Changes to self.__dict__ also change instance variables?? 78 | doc = copy.deepcopy(self.to_dict()) 79 | doc[c.EXP_ID_FIELD] = self.get_id() 80 | doc[c.ES_EXP_KEY_FIELD] = self.get_id() 81 | doc['display_name'] = self.get_display_name() 82 | doc['transforms'] = str(doc['transforms']) 83 | del doc['model'] 84 | return doc 85 | 86 | 87 | ## Helpers 88 | 89 | def fetch_external_config(exp_name): 90 | str_ = s3_client.fetch_experiment_config(exp_name) 91 | dict_ = json.loads(str_) 92 | return load_config_from_json(dict_) 93 | 94 | 95 | def load_config_from_file(fpath): 96 | dict_ = utils.files.load_json(fpath) 97 | return load_config_from_json(dict_) 98 | 99 | 100 | def load_config_from_json(dict_): 101 | return ExperimentConfig( 102 | name=dict_['name'], 103 | parent_dir=dict_['parent_dir'], 104 | created=dict_['created'], 105 | metrics=dict_['metrics'], 106 | aux_metrics=dict_['aux_metrics'], 107 | visualizers=dict_['visualizers'], 108 | model=dict_['model'], 109 | optimizer=dict_['optimizer'], 110 | lr_adjuster=dict_['lr_adjuster'], 111 | criterion=dict_['criterion'], 112 | transforms=dict_['transforms'], 113 | data=dict_['data'], 114 | training=dict_['training'], 115 | hardware=dict_['hardware'], 116 | other=dict_['other'], 117 | progress=dict_['progress']) 118 | 119 | 120 | def create_config_from_dict(config): 121 | metrics_config = get_metrics_config(config['metrics']) 122 | aux_metrics_config = get_aux_metrics_config(config['aux_metrics']) 123 | visualizers_config = get_visualizers_config(config['visualizers']) 124 | transforms_config = get_transforms_config(config['transforms']) 125 | model_config = get_model_config(config['model']) 126 | optim_config = get_optim_config(config['optimizer']) 127 | lr_adjuster_config = get_lr_config(config['lr_adjuster']) 128 | criterion_config = get_criterion_config(config['criterion']) 129 | return ExperimentConfig( 130 | name=config['name'], 131 | parent_dir=config['parent_dir'], 132 | created=time.strftime("%m/%d/%Y %H:%M:%S", time.localtime()), 133 | metrics=metrics_config, 134 | aux_metrics=aux_metrics_config, 135 | visualizers=visualizers_config, 136 | model=model_config, 137 | optimizer=optim_config, 138 | lr_adjuster=lr_adjuster_config, 139 | criterion=criterion_config, 140 | transforms=transforms_config, 141 | data=config['data'], 142 | training=get_training_config(config['training']), 143 | hardware=config['hardware'], 144 | other=config['other']) 145 | 146 | 147 | def remove_large_items(dict_): 148 | max_len = 100 149 | new_dict = {} 150 | for k,v in dict_.items(): 151 | if isinstance(v, list) and len(v) > max_len: 152 | pass 153 | elif isinstance(v, dict): 154 | if len(v.items()) < max_len: 155 | new_dict[k] = str(v.items()) 156 | else: 157 | assert not isinstance(v, dict) 158 | new_dict[k] = v 159 | return new_dict 160 | 161 | 162 | def get_training_config(train_config): 163 | return remove_large_items(train_config) 164 | 165 | 166 | def get_model_config(model): 167 | name = utils.general.get_class_name(model) 168 | layers = str(model) 169 | return { 170 | 'name': name, 171 | 'layers': layers 172 | } 173 | 174 | 175 | def get_optim_config(optim): 176 | name = utils.general.get_class_name(optim) 177 | params = optim.param_groups[0] 178 | params = remove_large_items(dict(params)) 179 | if 'params' in params: 180 | del params['params'] 181 | return { 182 | 'name': name, 183 | 'params': params 184 | } 185 | 186 | 187 | def get_lr_config(lr_adjuster): 188 | name = utils.general.get_class_name(lr_adjuster) 189 | params = dict(vars(lr_adjuster)) 190 | params = remove_large_items(params) 191 | return { 192 | 'name': name, 193 | 'params': params 194 | } 195 | 196 | 197 | def get_criterion_config(criterion): 198 | name = utils.general.get_class_name(criterion) 199 | return { 200 | 'name': name 201 | } 202 | 203 | 204 | def get_transforms_config(transforms): 205 | data_aug = [] 206 | for r in transforms.transforms: 207 | data_aug.append((str(r.__class__.__name__), 208 | str(r.__dict__))) 209 | return data_aug 210 | 211 | 212 | def get_visualizers_config(visualizers): 213 | return [v.classname for v in visualizers] 214 | 215 | 216 | def get_metrics_config(metrics): 217 | return [m.name for m in metrics] 218 | 219 | 220 | def get_aux_metrics_config(aux_metrics): 221 | return [m.__dict__ for m in aux_metrics] 222 | 223 | 224 | -------------------------------------------------------------------------------- /experiments/exp_history.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from os.path import join 4 | from pathlib import Path 5 | import matplotlib as mpl 6 | import matplotlib.pyplot as plt 7 | import numpy as np 8 | import utils.files 9 | import pandas as pd 10 | from io import StringIO 11 | mpl.use('Agg') 12 | plt.style.use('bmh') 13 | 14 | import config as cfg 15 | import constants as c 16 | from clients import s3_client, es_client 17 | 18 | 19 | 20 | class ExperimentHistory(): 21 | 22 | def __init__(self, exp_name, history_dir, metrics=None, aux_metrics=None): 23 | self.exp_name = exp_name 24 | self.history_dir = history_dir 25 | self.train_history_fpath = join(self.history_dir, c.TRAIN+'.csv') 26 | self.val_history_fpath = join(self.history_dir, c.VAL+'.csv') 27 | self.aux_metrics_fpath = join(self.history_dir, 'aux_metrics.csv') 28 | self.summary_fpath = join(self.history_dir, exp_name+'.csv') 29 | self.metrics = metrics 30 | self.aux_metrics = aux_metrics 31 | self.metrics_history = None 32 | self.best_metrics = {} 33 | 34 | def init(self): 35 | self.init_metrics() 36 | self.init_history_files() 37 | 38 | def resume(self, fetch=False): 39 | self.init_metrics() 40 | if fetch: 41 | self.load_from_external() 42 | else: 43 | self.load_from_files() 44 | self.update_best_metrics() 45 | 46 | def init_history_files(self): 47 | Path(self.train_history_fpath).touch() 48 | Path(self.val_history_fpath).touch() 49 | Path(self.aux_metrics_fpath).touch() 50 | 51 | def init_metrics(self): 52 | histories = {} 53 | for metric in self.metrics: 54 | histories[metric.name] = { 55 | c.TRAIN: [], 56 | c.VAL: [] 57 | } 58 | for aux_metric in self.aux_metrics: 59 | histories[aux_metric.name] = [] 60 | self.metrics_history = histories 61 | 62 | def save(self, config, s3=cfg.S3_ENABLED, es=cfg.S3_ENABLED): 63 | df = pd.DataFrame() 64 | for metric in self.metrics: 65 | trn_data = self.metrics_history[metric.name][c.TRAIN] 66 | val_data = self.metrics_history[metric.name][c.VAL] 67 | df[c.TRAIN+'_'+metric.name] = trn_data 68 | df[c.VAL+'_'+metric.name] = val_data 69 | 70 | for aux_metric in self.aux_metrics: 71 | df[aux_metric.name] = self.metrics_history[aux_metric.name] 72 | 73 | epochs = pd.Series([i for i in range(1,len(trn_data)+1)]) 74 | df.insert(0, 'Epoch', epochs) 75 | df.to_csv(self.summary_fpath, index=False) 76 | 77 | if s3: 78 | s3_client.upload_experiment_history(self.summary_fpath, 79 | self.exp_name) 80 | if es: 81 | es_client.upload_experiment_history(config, self) 82 | 83 | def load_from_files(self): 84 | self.load_history_from_file(c.TRAIN) 85 | self.load_history_from_file(c.VAL) 86 | self.load_aux_metrics_from_file() 87 | 88 | def load_from_external(self): 89 | df = self.fetch_dataframe() 90 | for metric in self.metrics: 91 | for dset in [c.TRAIN, c.VAL]: 92 | data = df[dset+'_'+metric.name].tolist() 93 | self.metrics_history[metric.name][dset] = data 94 | for aux_metric in self.aux_metrics: 95 | data = df[aux_metric.name].tolist() 96 | self.metrics_history[aux_metric.name] = data 97 | 98 | def get_dataframe(self): 99 | if os.path.isfile(self.summary_fpath): 100 | return self.load_dataframe_from_file() 101 | return self.fetch_dataframe() 102 | 103 | def fetch_dataframe(self): 104 | csv_str = s3_client.fetch_experiment_history(self.exp_name) 105 | df = pd.DataFrame 106 | data = StringIO(csv_str) 107 | return pd.read_csv(data, sep=",") 108 | 109 | def load_dataframe_from_file(self): 110 | df = pd.read_csv(self.summary_fpath, sep=',') 111 | return df 112 | 113 | def save_metric(self, dset_type, values_dict, epoch): 114 | values_arr = [] 115 | for metric in self.metrics: 116 | value = values_dict[metric.name] 117 | self.metrics_history[metric.name][dset_type].append(value) 118 | values_arr.append(value) 119 | fpath = join(self.history_dir, dset_type+'.csv') 120 | self.append_history_to_file(fpath, values_arr, epoch) 121 | 122 | def load_history_from_file(self, dset_type): 123 | fpath = join(self.history_dir, dset_type+'.csv') 124 | data = np.loadtxt(fpath, delimiter=',').reshape( 125 | -1, len(self.metrics)+1) 126 | for i in range(len(self.metrics)): 127 | self.metrics_history[self.metrics[i].name][dset_type] = data[:,i+1].tolist() 128 | 129 | def append_history_to_file(self, fpath, values, epoch): 130 | # Restricts decimals to 6 places!!! 131 | formatted_vals = ["{:.6f}".format(v) for v in values] 132 | line = ','.join(formatted_vals) 133 | with open(fpath, 'a') as f: 134 | f.write('{},{}\n'.format(epoch, line)) 135 | 136 | def update_best_metrics(self): 137 | best_metrics = {} 138 | for metric in self.metrics: 139 | metric_history = self.metrics_history[metric.name][c.VAL] 140 | best_epoch, best_value = metric.get_best_epoch( 141 | metric_history) 142 | best_metrics[metric.name] = { 143 | 'epoch':best_epoch, 144 | 'value':best_value 145 | } 146 | self.best_metrics = best_metrics 147 | 148 | def load_aux_metrics_from_file(self): 149 | data = np.loadtxt(self.aux_metrics_fpath, delimiter=',').reshape( 150 | -1, len(self.aux_metrics)+1) 151 | for i in range(len(self.aux_metrics)): 152 | self.metrics_history[self.aux_metrics[i].name] = data[:,i+1].tolist() 153 | 154 | def save_aux_metrics(self, values, epoch): 155 | for i in range(len(self.aux_metrics)): 156 | self.metrics_history[self.aux_metrics[i].name].append(values[i]) 157 | self.append_history_to_file(self.aux_metrics_fpath, values, epoch) 158 | 159 | def get_dset_arr(self, dset): 160 | data = [] 161 | for metric in self.metrics: 162 | data.append(self.metrics_history[metric.name][dset]) 163 | epochs = [i+1 for i in range(len(data[0]))] 164 | data.insert(0,epochs) 165 | arr = np.array(data) 166 | return arr.T 167 | 168 | def plot(self, save=False): 169 | trn_data = self.get_dset_arr(c.TRAIN) 170 | val_data = self.get_dset_arr(c.VAL) 171 | metrics_idx = [i+1 for i in range(len(self.metrics))] 172 | trn_args = np.split(trn_data, metrics_idx, axis=1) 173 | val_args = np.split(val_data, metrics_idx, axis=1) 174 | 175 | metric_fpaths = [] 176 | for i in range(len(self.metrics)): 177 | metric_trn_data = trn_data[:,i+1] #skip epoch 178 | metric_val_data = val_data[:,i+1] 179 | 180 | fig, ax = plt.subplots(1, 1, figsize=(6, 5)) 181 | plt.plot(trn_args[0], metric_trn_data, label='Train') 182 | plt.plot(val_args[0], metric_val_data, label='Validation') 183 | plt.title(self.metrics[i].name) 184 | plt.xlabel('Epoch') 185 | plt.ylabel(self.metrics[i].name) 186 | plt.legend() 187 | ax.set_yscale('log') 188 | 189 | if save: 190 | metric_fpath = join(self.history_dir, 191 | self.metrics[i].name+'.png') 192 | metric_fpaths.append(metric_fpath) 193 | plt.savefig(metric_fpath) 194 | 195 | # Combined View 196 | if save: 197 | all_metrics_fpath = join(self.history_dir, 'all_metrics.png') 198 | metric_fpaths.append(all_metrics_fpath) 199 | os.system('convert +append ' + ' '.join(metric_fpaths)) 200 | 201 | plt.show() 202 | 203 | def to_doc(self, config): 204 | df = self.get_dataframe() 205 | df[c.EXP_ID_FIELD] = config.get_id() 206 | df[c.ES_EXP_KEY_FIELD] = df['Epoch'].map(str) + '_' + config.get_id() 207 | df['name'] = config.get_display_name() 208 | df['user'] = config.hardware['hostname'] 209 | df['criterion'] = config.criterion['name'] 210 | df['optim'] = config.optimizer['name'] 211 | df['init_lr'] = config.training['initial_lr'] 212 | df['wd'] = config.training['weight_decay'] 213 | df['bs'] = config.training['batch_size'] 214 | df['imsz'] = config.data['img_rescale'] 215 | df['model_name'] = config.model_name 216 | df['lr_adjuster'] = config.lr_adjuster['name'] 217 | df['threshold'] = config.training['threshold'] 218 | return json.loads(df.to_json(orient='records')) 219 | 220 | 221 | ### TODO 222 | def get_history_summary(self, epoch, early_stop_metric): 223 | msg = ['Epoch: %d' % epoch] 224 | for dset in [c.TRAIN, c.VAL]: 225 | dset_msg = dset.capitalize() + ' - ' 226 | for metric in self.metrics: 227 | value = self.metrics_history[metric.name][dset][-1] 228 | dset_msg += '{:s}: {:.3f} '.format(metric.name, value) 229 | msg.append(dset_msg) 230 | 231 | best_epoch = self.best_metrics[early_stop_metric]['epoch'] 232 | best_epoch_value = self.best_metrics[early_stop_metric]['value'] 233 | best_metric_msg = 'Best val {:s}: Epoch {:d} - {:.3f}'.format( 234 | early_stop_metric, best_epoch, best_epoch_value) 235 | msg.append(best_metric_msg) 236 | 237 | return '\n'.join(msg) 238 | 239 | -------------------------------------------------------------------------------- /experiments/exp_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import math 3 | import shutil 4 | from glob import glob 5 | 6 | import config as cfg 7 | import constants as c 8 | import numpy as np 9 | import utils.general as gen_utils 10 | import utils.files 11 | import models.utils 12 | from clients import s3_client, es_client 13 | 14 | 15 | 16 | def cleanup_experiments(exp_dir): 17 | exp_paths = glob(exp_dir+'/*/') 18 | for path in exp_paths: 19 | config_path = os.path.join(path, c.EXPERIMENT_CONFIG_FNAME) 20 | if not os.path.isfile(config_path): 21 | shutil.rmtree(path) 22 | 23 | 24 | def delete_experiment(exp_name, exp_dir, local=True, s3=False, es=False): 25 | if local: 26 | pattern = os.path.join(exp_dir, exp_name) 27 | exp_path_list = glob(pattern) 28 | if len(exp_path_list) > 0: 29 | for p in exp_path_list: 30 | print("Deleting local exp") 31 | shutil.rmtree(p) 32 | else: 33 | print("Local copy of exp not found!") 34 | if s3: 35 | print("Deleting S3 document") 36 | s3_client.delete_experiment(exp_name) 37 | if es: 38 | print("ES delete not implemented") 39 | es_client.delete_experiment_by_field(field='exp_name', value=exp_name) 40 | 41 | 42 | def prune(weights_dir, keep_epochs): 43 | prune_weights_and_optims(weights_dir, keep_epochs) 44 | 45 | 46 | def auto_prune(exp, n_bins=5, metric_name=c.LOSS, func=min): 47 | best_epochs = get_best_epochs(exp, metric_name, n_bins, func) 48 | print(best_epochs) 49 | prune(exp.weights_dir, best_epochs) 50 | 51 | 52 | def get_best_epochs(exp, metric_name, n_bins=5, func=max, end_epoch=10000): 53 | metric_arr = exp.history.metrics_history[metric_name][c.VAL][:end_epoch+1] 54 | idx, _ = get_best_values_in_bins(metric_arr, n_bins, func) 55 | return [i+1 for i in idx] #epoch starts at 1 56 | 57 | 58 | def get_best_values_in_bins(arr, n_bins, func): 59 | bucket_size = math.ceil(len(arr)/n_bins) 60 | if isinstance(arr, list): 61 | arr = np.array(arr) 62 | best_idxfunc = np.argmax if func is max else np.argmin 63 | best_valfunc = np.amax if func is max else np.amin 64 | best_idx, best_vals = [], [] 65 | for i in range(0, len(arr), bucket_size): 66 | best_idx.append(i+best_idxfunc(arr[i:i+bucket_size])) 67 | best_vals.append(best_valfunc(arr[i:i+bucket_size])) 68 | return best_idx, best_vals 69 | 70 | 71 | def prune_weights_and_optims(weights_dir, keep_epochs): 72 | matches, fpaths = utils.files.get_matching_files_in_dir( 73 | weights_dir, c.WEIGHTS_OPTIM_FNAME_REGEX) 74 | print(matches) 75 | for i in range(len(matches)): 76 | epoch = int(matches[i].group(2)) 77 | if epoch not in keep_epochs: 78 | os.remove(fpaths[i]) 79 | 80 | 81 | def get_weights_fpaths(weights_dir): 82 | return utils.files.get_matching_files_in_dir( 83 | weights_dir, c.WEIGHTS_FNAME_REGEX)[1] 84 | 85 | 86 | def get_weight_epochs_from_fpaths(fpaths): 87 | epochs = [] 88 | found_latest = False 89 | for path_ in fpaths: 90 | ## FIX THIS override 91 | if 'latest' not in path_: 92 | epochs.append(int(path_.strip(c.WEIGHTS_EXT).split('-')[-1])) 93 | else: 94 | found_latest = True 95 | epochs.sort() 96 | if found_latest: 97 | epochs.insert(0,'latest') 98 | return epochs 99 | 100 | 101 | def get_weight_fpaths_by_epoch(weights_dir, epochs): 102 | matches, fpaths = utils.files.get_matching_files_in_dir( 103 | weights_dir, c.WEIGHTS_FNAME_REGEX) 104 | weight_fpaths = [] 105 | for i in range(len(matches)): 106 | epoch = int(matches[i].group(1)) 107 | if epoch in epochs: 108 | weight_fpaths.append(fpaths[i]) 109 | return weight_fpaths 110 | 111 | 112 | def get_optim_fpaths_by_epoch(optims_dir, keep_epochs): 113 | matches, fpaths = utils.files.get_matching_files_in_dir( 114 | optims_dir, c.OPTIM_FNAME_REGEX) 115 | weight_fpaths = [] 116 | for i in range(len(matches)): 117 | epoch = int(matches[i].group(1)) 118 | if epoch in keep_epochs: 119 | weight_fpaths.append(fpaths[i]) 120 | return weight_fpaths 121 | 122 | 123 | def get_weights_fname(epoch): 124 | if epoch is None: 125 | return c.LATEST_WEIGHTS_FNAME 126 | return 'weights-%d%s' % (epoch, c.WEIGHTS_EXT) 127 | 128 | 129 | def get_optim_fname(epoch): 130 | if epoch is None: 131 | return c.LATEST_OPTIM_FNAME 132 | return 'optim-%d%s' % (epoch, c.OPTIM_EXT) 133 | 134 | 135 | def load_weights_by_exp_and_epoch(model, exp_name, epoch='latest'): 136 | if epoch is None or epoch == 'latest': 137 | weights_fname = c.LATEST_WEIGHTS_FNAME 138 | else: 139 | weights_fname = 'weights-{:d}.th'.format(epoch) 140 | fpath = os.path.join(cfg.PATHS['experiments'], exp_name, 'weights', weights_fname) 141 | models.utils.load_weights(model, fpath) 142 | 143 | 144 | def download_experiment(dest_dir, exp_name): 145 | fpath = os.path.join(dest_dir, exp_name + c.EXP_FILE_EXT) 146 | s3_client.download_experiment(fpath, exp_name) 147 | utils.files.unzipdir(fpath, dest_dir) 148 | 149 | 150 | def upload_experiment(parent_dir, exp_name): 151 | print(('Uploading experiment {:s}. ' 152 | 'This may take a while..').format(exp_name)) 153 | exp_path = os.path.join(parent_dir, exp_name) 154 | exp_copy_path = exp_path+'_copy' 155 | exp_copy_archive_path = os.path.join(exp_copy_path, exp_name) 156 | archive_path = exp_path + c.EXP_FILE_EXT 157 | shutil.copytree(exp_path, exp_copy_archive_path) 158 | print('Archiving..') 159 | utils.files.zipdir(exp_copy_path, archive_path) 160 | shutil.rmtree(exp_copy_path) 161 | print('Uploading..') 162 | s3_client.upload_experiment(archive_path, exp_name) 163 | os.remove(archive_path) 164 | print('Upload complete!') 165 | 166 | 167 | def generate_display_name(base_name, *args): 168 | unique_id = gen_utils.gen_unique_id() 169 | return base_name+'-'.join(args[0])+'-id'+unique_id 170 | 171 | 172 | def get_id_from_name(exp_name): 173 | return exp_name.split('-id')[-1] 174 | 175 | 176 | def get_transforms_config(transforms): 177 | data_aug = [] 178 | for r in transforms.transforms: 179 | data_aug.append((str(r.__class__.__name__), 180 | r.__dict__)) 181 | return data_aug -------------------------------------------------------------------------------- /experiments/experiment.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import time 4 | import logging 5 | from os.path import join 6 | 7 | import config as cfg 8 | import constants as c 9 | from metrics import metric_builder 10 | from visualizers import vis_utils 11 | from notifications import emailer 12 | import utils.logger 13 | import training 14 | import models.utils 15 | 16 | from .exp_history import ExperimentHistory 17 | from . import exp_utils 18 | from . import exp_config 19 | 20 | 21 | 22 | 23 | class Experiment(): 24 | def __init__(self, name, parent_dir): 25 | self.name = name 26 | self.parent_dir = parent_dir 27 | self.root = join(parent_dir, name) 28 | self.weights_dir = join(self.root, 'weights') 29 | self.results_dir = join(self.root, 'results') 30 | self.history_dir = join(self.root, 'history') 31 | self.config_fpath = join(self.root, c.EXPERIMENT_CONFIG_FNAME) 32 | self.model_fpath = join(self.root, c.MODEL_FNAME) 33 | self.optim_fpath = join(self.root, c.OPTIM_FNAME) 34 | 35 | # Initialized/loaded later 36 | self.config = None 37 | self.logger = None 38 | self.history = None 39 | self.model = None 40 | self.optim = None 41 | self.max_patience = None 42 | self.early_stop_metric = None 43 | self.epoch = 0 44 | self.best_epoch = 1 45 | self.best_epoch_value = None 46 | self.visualizers = [] 47 | self.metrics = [] 48 | self.aux_metrics = [] 49 | self.best_metrics = None 50 | 51 | def init(self, config_dict): 52 | self.config = exp_config.create_config_from_dict(config_dict) 53 | self.config.progress['status'] = c.INITIALIZED 54 | self.metrics = config_dict['metrics'] 55 | self.aux_metrics = config_dict['aux_metrics'] 56 | self.model = config_dict['model'] 57 | self.optim = config_dict['optimizer'] 58 | self.visualizers = config_dict['visualizers'] 59 | self.max_patience = self.config.training['max_patience'] 60 | self.early_stop_metric = self.config.training['early_stop_metric'] 61 | self.history = ExperimentHistory(self.name, self.history_dir, 62 | self.metrics, self.aux_metrics) 63 | self.init_dirs() 64 | self.history.init() 65 | self.init_logger() 66 | self.init_visualizers() 67 | self.save_components() 68 | self.model.logger = self.logger 69 | 70 | def resume(self, epoch=None, verbose=False): 71 | self.init_logger() 72 | self.log("Resuming existing experiment") 73 | self.config = exp_config.load_config_from_file(self.config_fpath) 74 | self.config.progress['status'] = c.RESUMED 75 | self.load(verbose) 76 | self.init_visualizers() 77 | self.load_components(epoch) 78 | self.model.logger = self.logger 79 | 80 | def review(self, download=False, verbose=True): 81 | self.init_logger() 82 | if download: 83 | self.config = exp_config.fetch_external_config(self.name) 84 | else: 85 | self.config = exp_config.load_config_from_file(self.config_fpath) 86 | self.load(verbose=verbose) 87 | 88 | def init_visualizers(self): 89 | for v in self.visualizers: 90 | v.init(self.config) 91 | 92 | def init_dirs(self): 93 | os.makedirs(self.weights_dir) 94 | os.makedirs(self.history_dir) 95 | os.makedirs(self.results_dir) 96 | 97 | def init_logger(self, log_level=logging.INFO): 98 | self.logger = utils.logger.get_logger( 99 | self.root, 'logger', ch_log_level=log_level, 100 | fh_log_level=log_level) 101 | 102 | def load(self, verbose=False): 103 | self.metrics = metric_builder.get_metrics_from_config(self.config) 104 | self.aux_metrics = metric_builder.get_aux_metrics_from_config(self.config) 105 | self.visualizers = vis_utils.get_visualizers_from_config(self.config) 106 | self.history = ExperimentHistory(self.name, self.history_dir, 107 | self.metrics, self.aux_metrics) 108 | self.history.resume() 109 | self.max_patience = self.config.training['max_patience'] 110 | self.early_stop_metric = self.config.training['early_stop_metric'] 111 | self.epoch = self.config.progress['epoch'] 112 | self.best_metrics = self.config.progress['best_metrics'] 113 | self.best_epoch = self.best_metrics[self.early_stop_metric]['epoch'] 114 | self.best_epoch_value = self.best_metrics[self.early_stop_metric]['value'] 115 | if verbose: self.config.summary(self.logger) 116 | 117 | def save(self, s3=cfg.S3_ENABLED, es=cfg.ES_ENABLED): 118 | self.config.save(s3, es) 119 | self.history.save(self.config, s3, es) 120 | 121 | def upload(self): 122 | exp_utils.upload_experiment(self.parent_dir, self.name) 123 | 124 | def load_components(self, epoch): 125 | self.model = models.utils.load_model(self.model_fpath) 126 | self.optim = training.load_optim(self.optim_fpath) 127 | self.load_model_state(epoch) 128 | self.load_optim_state(epoch) 129 | 130 | def save_components(self): 131 | models.utils.save_model(self.model.cpu(), self.model_fpath) 132 | training.save_optim(self.optim, self.optim_fpath) 133 | self.model = self.model.cuda() 134 | 135 | def log(self, msg): 136 | self.logger.info(msg) 137 | 138 | def update_visualizers(self, msg=None): 139 | for v in self.visualizers: 140 | v.update(self.config, self.history, msg) 141 | 142 | def update_progress(self): 143 | best = self.history.best_metrics 144 | self.best_epoch = best[self.early_stop_metric]['epoch'] 145 | self.best_epoch_value = best[self.early_stop_metric]['value'] 146 | self.config.progress['epoch'] = self.epoch 147 | self.config.progress['best_metrics'] = best 148 | 149 | def get_weights_fpath(self, epoch=None): 150 | fname = exp_utils.get_weights_fname(epoch) 151 | return join(self.weights_dir, fname) 152 | 153 | def get_optim_fpath(self, epoch=None): 154 | fname = exp_utils.get_optim_fname(epoch) 155 | return join(self.weights_dir, fname) 156 | 157 | def save_model_state(self, save_now=False): 158 | models.utils.save_weights(self.model, self.get_weights_fpath(), 159 | epoch=self.epoch, name=self.name) 160 | if (save_now or self.epoch 161 | % self.config.training['save_weights_cadence'] == 0): 162 | fpath = self.get_weights_fpath(self.epoch) 163 | shutil.copyfile(self.get_weights_fpath(), fpath) 164 | 165 | def load_model_state(self, epoch=None): 166 | fpath = self.get_weights_fpath(epoch) 167 | models.utils.load_weights(self.model, fpath) 168 | 169 | def save_optim_state(self, save_now=False): 170 | training.save_optim_params(self.optim, self.get_optim_fpath(), 171 | epoch=self.epoch, name=self.name) 172 | if (save_now or self.epoch 173 | % self.config.training['save_weights_cadence'] == 0): 174 | fpath = self.get_optim_fpath(self.epoch) 175 | shutil.copyfile(self.get_optim_fpath(), fpath) 176 | 177 | def load_optim_state(self, epoch=None): 178 | fpath = self.get_optim_fpath(epoch) 179 | training.load_optim_params(self.optim, fpath) 180 | 181 | def train(self, trainer, trn_loader, val_loader, n_epochs=None): 182 | start_epoch = self.epoch + 1 # Epochs start at 1 183 | self.config.progress['status'] = c.IN_PROGRESS 184 | self.config.progress['status_msg'] = 'Experiment in progress' 185 | 186 | if n_epochs is None: 187 | end_epoch = self.config.training['n_epochs'] + 1 188 | else: 189 | end_epoch = start_epoch + n_epochs 190 | try: 191 | for epoch in range(start_epoch, end_epoch): 192 | 193 | ### Adjust Lr ### 194 | lr_params = {'best_iter' : self.best_epoch} 195 | if trainer.lr_adjuster.iteration_type == 'epoch': 196 | trainer.lr_adjuster.adjust(self.optim, epoch, lr_params) 197 | current_lr = trainer.lr_adjuster.get_learning_rate(self.optim) 198 | 199 | ### Train ### 200 | trn_start_time = time.time() 201 | trn_metrics = trainer.train(self.model, trn_loader, 202 | self.config.training['threshold'], epoch, self.metrics) 203 | trn_msg = training.log_trn_msg(self.logger, trn_start_time, 204 | trn_metrics, current_lr, epoch) 205 | 206 | ### Test ### 207 | val_start_time = time.time() 208 | val_metrics = trainer.test(self.model, val_loader, 209 | self.config.training['threshold'], self.metrics) 210 | val_msg = training.log_val_msg(self.logger, val_start_time, 211 | val_metrics, current_lr) 212 | 213 | sys_mem = training.log_memory('') 214 | 215 | ### Save Metrics ### 216 | aux_metrics = [current_lr, sys_mem] 217 | self.history.save_metric(c.TRAIN, trn_metrics, epoch) 218 | self.history.save_metric(c.VAL, val_metrics, epoch) 219 | self.history.save_aux_metrics(aux_metrics, epoch) 220 | self.history.update_best_metrics() 221 | 222 | ### Checkpoint ### 223 | self.epoch = epoch 224 | self.update_progress() 225 | self.save() 226 | self.save_model_state() 227 | self.save_optim_state() 228 | self.update_visualizers('\n'.join([trn_msg, val_msg])) 229 | 230 | ### Early Stopping ### 231 | if training.early_stop(epoch, self.best_epoch, self.max_patience): 232 | msg = "Early stopping at epoch %d since no better %s found since epoch %d at %.3f" % ( 233 | epoch, self.early_stop_metric, self.best_epoch, self.best_epoch_value) 234 | self.config.progress['status'] = c.MAX_PATIENCE_EXCEEDED 235 | self.config.progress['status_msg'] = msg 236 | break 237 | 238 | except Exception as e: 239 | self.config.progress['status'] = c.FAILED 240 | self.config.progress['status_msg'] = e 241 | raise Exception(e) 242 | finally: 243 | if self.config.progress['status'] == c.IN_PROGRESS: 244 | self.config.progress['status'] = c.COMPLETED 245 | self.config.progress['status_msg'] = 'Experiment Complete!' 246 | if cfg.EMAIL_ENABLED: 247 | emailer.send_experiment_status_email(self, cfg.USER_EMAIL) 248 | self.log(self.config.progress['status_msg']) 249 | 250 | 251 | -------------------------------------------------------------------------------- /init_project.py: -------------------------------------------------------------------------------- 1 | import os 2 | import config as cfg 3 | 4 | 5 | def init_paths(root, dset_types, input_img_exts, target_img_exts): 6 | paths = { 7 | 'project': root, 8 | 'experiments': os.path.join(root, 'experiments'), 9 | 'predictions': os.path.join(root, 'predictions'), 10 | 'submissions': os.path.join(root, 'submissions'), 11 | 'folds': os.path.join(root, 'folds') 12 | } 13 | for key in paths: 14 | os.makedirs(paths[key], exist_ok=True) 15 | 16 | paths['datasets'] = {} 17 | datasets_root = os.path.join(root, 'datasets') 18 | os.makedirs(datasets_root, exist_ok=True) 19 | make_dataset(paths, datasets_root, 'inputs', dset_types, input_img_exts) 20 | make_dataset(paths, datasets_root, 'targets', dset_types, target_img_exts) 21 | 22 | return paths 23 | 24 | 25 | def make_dataset(paths, datasets_root, name, dset_types, img_exts): 26 | root = os.path.join(datasets_root, name) 27 | os.makedirs(root, exist_ok=True) 28 | paths['datasets'][name] = {} 29 | 30 | for dset in dset_types: 31 | for img in img_exts: 32 | dir_name = dset+'_'+img 33 | dir_path = os.path.join(root, dir_name) 34 | os.makedirs(dir_path, exist_ok=True) 35 | paths['datasets'][name][dir_name] = dir_path 36 | 37 | 38 | if __name__ == '__main__': 39 | init_paths(cfg.PROJECT_PATH, cfg.IMG_DATASET_TYPES, 40 | cfg.IMG_INPUT_FORMATS, cfg.IMG_TARGET_FORMATS) -------------------------------------------------------------------------------- /metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from .metric_utils import * -------------------------------------------------------------------------------- /metrics/loss_functions.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from . import metric_utils 4 | 5 | 6 | class DiceLoss(): 7 | ''' 8 | http://campar.in.tum.de/pub/milletari2016Vnet/milletari2016Vnet.pdf 9 | https://github.com/faustomilletari/VNet/blob/master/pyLayer.py 10 | https://github.com/pytorch/pytorch/issues/1249 11 | ''' 12 | def __init__(self): 13 | self.__class__.__name__ = 'Dice' 14 | 15 | def __call__(self, output, target): 16 | return 1.0 - get_torch_dice_score(output, target) 17 | 18 | 19 | class DiceBCELoss(): 20 | def __init__(self, dice_weight=1.0): 21 | self.__class__.__name__ = 'DiceBCE' 22 | self.dice_weight = dice_weight 23 | self.bce_weight = 1.0 - dice_weight 24 | 25 | def __call__(self, output, target): 26 | bce = F.binary_cross_entropy(output, target) 27 | dice = 1 - get_torch_dice_score(output, target) 28 | return (dice * self.dice_weight) + (bce * self.bce_weight) 29 | 30 | 31 | class WeightedBCELoss(): 32 | def __init__(self, weights): 33 | self.weights = weights 34 | self.__class__.__name__ = 'WeightedBCE' 35 | 36 | def __call__(self, output, target): 37 | return F.binary_cross_entropy(output, target, self.weights) 38 | 39 | 40 | class KnowledgeDistillLoss(): 41 | def __init__(self, target_weight=0.25): 42 | self.__class__.__name__ = 'KnowledgeDistill' 43 | self.target_weight = target_weight 44 | 45 | def __call__(self, output, target, soft_target): 46 | target_loss = F.binary_cross_entropy(output, target) * self.target_weight 47 | soft_target_loss = F.binary_cross_entropy(output, soft_target) 48 | return target_loss + soft_target_loss 49 | 50 | 51 | class HuberLoss(): 52 | def __init__(self, c=0.5): 53 | self.c = c 54 | self.__class__.__name__ = 'Huber' 55 | 56 | def __call__(self, output, target): 57 | bce = F.binary_cross_entropy(output, target) 58 | return self.c**2 * (torch.sqrt(1 + (bce/self.c)**2) - 1) 59 | 60 | 61 | class SmoothF2Loss(): 62 | def __init__(self, c=10.0, f2_weight=0.2, bce_weight=1.0): 63 | self.__class__.__name__ = 'SmoothF2' 64 | self.c = c 65 | self.f2_weight = f2_weight 66 | self.bce_weight = bce_weight 67 | 68 | def __call__(self, output, target, thresholds): 69 | f2 = get_smooth_f2_score(output, target, thresholds, self.c) * self.f2_weight 70 | bce = F.binary_cross_entropy(output, target) * self.bce_weight 71 | return f2 + bce 72 | 73 | 74 | 75 | # Helpers / Shared Methods 76 | 77 | def get_torch_dice_score(outputs, targets): 78 | eps = 1e-7 79 | batch_size = outputs.size()[0] 80 | outputs = outputs.view(batch_size, -1) 81 | targets = targets.view(batch_size, -1) 82 | 83 | total = torch.sum(outputs, dim=1) + torch.sum(targets, dim=1) 84 | intersection = torch.sum(outputs * targets, dim=1).float() 85 | 86 | dice_score = (2.0 * intersection) / (total + eps) 87 | return torch.mean(dice_score) 88 | 89 | 90 | def sigmoid(z, c=1.0): 91 | return 1.0 / (1.0 + torch.exp(-c*z)) 92 | 93 | 94 | def get_smooth_f2_score(outputs, targets, thresholds, c=10.0): 95 | eps = 1e-9 96 | outputs = sigmoid(thresholds - outputs, c).float() 97 | tot_out_pos = torch.sum(outputs, dim=1) 98 | tot_tar_pos = torch.sum(targets, dim=1) 99 | TP = torch.sum(outputs * targets, dim=1) 100 | 101 | P = TP / (tot_out_pos + eps) 102 | R = TP / tot_tar_pos + eps 103 | F2 = 5.0 * (P*R / (4*P + R)) 104 | return torch.mean(F2) -------------------------------------------------------------------------------- /metrics/metric.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import operator 3 | import constants as c 4 | from . import metric_utils 5 | 6 | 7 | class Metric(): 8 | def __init__(self, name, minimize=True): 9 | self.name = name 10 | self.minimize = minimize 11 | 12 | def get_best_epoch(self, values): 13 | if self.minimize: 14 | idx, value = min(enumerate(values), 15 | key=operator.itemgetter(1)) 16 | else: 17 | idx, value = max(enumerate(values), 18 | key=operator.itemgetter(1)) 19 | epoch = idx + 1 # epochs start at 1 20 | return epoch, value 21 | 22 | def evaluate(self, loss, preds, probs, targets): 23 | pass 24 | 25 | def format(self, value): 26 | pass 27 | 28 | 29 | class AuxiliaryMetric(): 30 | def __init__(self, name, units): 31 | self.name = name 32 | self.units = units 33 | 34 | 35 | class Accuracy(Metric): 36 | def __init__(self): 37 | super().__init__(c.ACCURACY, minimize=False) 38 | 39 | def evaluate(self, loss, preds, probs, targets): 40 | return metric_utils.get_accuracy(preds, targets) 41 | 42 | def format(self, value): 43 | return value 44 | 45 | 46 | class Loss(Metric): 47 | def __init__(self): 48 | super().__init__(c.LOSS, minimize=True) 49 | 50 | def evaluate(self, loss, preds, probs, targets): 51 | return loss 52 | 53 | def format(self, value): 54 | return value 55 | 56 | 57 | class F2Score(Metric): 58 | def __init__(self, target_threshold=None): 59 | super().__init__(c.F2_SCORE, minimize=False) 60 | self.target_threshold = target_threshold # pseudo soft targets 61 | 62 | def evaluate(self, loss, preds, probs, targets): 63 | average = 'samples' if targets.shape[1] > 1 else 'binary' 64 | if self.target_threshold is not None: 65 | targets = targets > self.target_threshold 66 | 67 | return metric_utils.get_f2_score(preds, targets, average) 68 | 69 | def format(self, value): 70 | return value 71 | 72 | 73 | class DiceScore(Metric): 74 | def __init__(self): 75 | super().__init__(c.DICE_SCORE, minimize=False) 76 | 77 | def evaluate(self, loss, preds, probs, targets): 78 | return metric_utils.get_dice_score(preds, targets) 79 | 80 | def format(self, value): 81 | return value 82 | 83 | 84 | class EnsembleF2(Metric): 85 | def __init__(self, ens_probs, threshold): 86 | super().__init__('EnsembleF2', minimize=False) 87 | self.ens_probs = ens_probs 88 | self.threshold = threshold 89 | 90 | def evaluate(self, loss, preds, probs, targets): 91 | if probs.shape[0] != self.ens_probs.shape[1]: 92 | return .950 93 | average = 'samples' if targets.shape[1] > 1 else 'binary' 94 | probs = np.expand_dims(probs, 0) 95 | joined_probs = np.concatenate([self.ens_probs, probs]) 96 | joined_probs = np.mean(joined_probs, axis=0) 97 | preds = joined_probs > self.threshold 98 | return metric_utils.get_f2_score(preds, targets, average) 99 | 100 | def format(self, value): 101 | return value -------------------------------------------------------------------------------- /metrics/metric_builder.py: -------------------------------------------------------------------------------- 1 | import constants as c 2 | from . import metric 3 | 4 | 5 | SUPPORTED_METRICS = { 6 | c.ACCURACY: metric.Accuracy(), 7 | c.LOSS: metric.Loss(), 8 | c.F2_SCORE: metric.F2Score(), 9 | c.ENSEMBLE_F2: metric.EnsembleF2(None,None), 10 | c.DICE_SCORE: metric.DiceScore(), 11 | } 12 | SUPPORTED_AUX_METRICS = {} 13 | 14 | 15 | def get_metric_by_name(name): 16 | return SUPPORTED_METRICS[name] 17 | 18 | 19 | def get_metrics_from_config(config): 20 | primary_metrics = [] 21 | for m in config.metrics: 22 | new_metric = get_metric_by_name(m) 23 | primary_metrics.append(new_metric) 24 | return primary_metrics 25 | 26 | 27 | def get_aux_metrics_from_config(config): 28 | aux_metrics = [] 29 | for m in config.aux_metrics: 30 | new_metric = metric.AuxiliaryMetric(m['name'], m['units']) 31 | aux_metrics.append(new_metric) 32 | return aux_metrics 33 | -------------------------------------------------------------------------------- /metrics/metric_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib as mpl 3 | import matplotlib.pyplot as plt 4 | from sklearn.metrics import fbeta_score 5 | from sklearn import metrics as scipy_metrics 6 | import torch 7 | import torch.nn.functional as F 8 | from torch.autograd import Variable 9 | import warnings 10 | import constants as c 11 | import predictions 12 | 13 | 14 | def get_default_loss(probs, targets, **kwargs): 15 | return get_cross_entropy_loss(probs, targets) 16 | 17 | 18 | def get_default_score(preds, targets, avg='samples', **kwargs): 19 | return get_f2_score(preds, targets, avg) 20 | 21 | 22 | def get_metric_in_blocks(outputs, targets, block_size, metric): 23 | sum_ = 0 24 | n = 0 25 | i = 0 26 | while i < len(outputs): 27 | out_block = outputs[i:i+block_size] 28 | tar_block = targets[i:i+block_size] 29 | score = metric(out_block, tar_block) 30 | sum_ += len(out_block) * score 31 | n += len(out_block) 32 | i += block_size 33 | return sum_ / n 34 | 35 | 36 | def get_metrics_in_batches(model, loader, thresholds, metrics): 37 | model.eval() 38 | n_batches = len(loader) 39 | metric_totals = [0 for m in metrics] 40 | 41 | for data in loader: 42 | if len(data[1].size()) == 1: 43 | targets = data[1].float().view(-1, 1) 44 | inputs = Variable(data[0].cuda(async=True)) 45 | targets = Variable(data[1].cuda(async=True)) 46 | 47 | output = model(inputs) 48 | 49 | labels = targets.data.cpu().numpy() 50 | probs = output.data.cpu().numpy() 51 | preds = predictions.get_predictions(probs, thresholds) 52 | 53 | for i,m in enumerate(metrics): 54 | score = m(preds, labels) 55 | metric_totals[i] += score 56 | 57 | metric_totals = [m / n_batches for m in metric_totals] 58 | return metric_totals 59 | 60 | 61 | def get_accuracy(preds, targets): 62 | preds = preds.flatten() 63 | targets = targets.flatten() 64 | correct = np.sum(preds==targets) 65 | return correct / len(targets) 66 | 67 | 68 | def get_cross_entropy_loss(probs, targets): 69 | return F.binary_cross_entropy( 70 | Variable(torch.from_numpy(probs)), 71 | Variable(torch.from_numpy(targets).float())).data[0] 72 | 73 | 74 | def get_recall(preds, targets): 75 | return scipy_metrics.recall_score(targets.flatten(), preds.flatten()) 76 | 77 | 78 | def get_precision(preds, targets): 79 | return scipy_metrics.precision_score(targets.flatten(), preds.flatten()) 80 | 81 | 82 | def get_roc_score(probs, targets): 83 | return scipy_metrics.roc_auc_score(targets.flatten(), probs.flatten()) 84 | 85 | 86 | def get_dice_score(preds, targets): 87 | eps = 1e-7 88 | batch_size = preds.shape[0] 89 | preds = preds.reshape(batch_size, -1) 90 | targets = targets.reshape(batch_size, -1) 91 | 92 | total = preds.sum(1) + targets.sum(1) + eps 93 | intersection = (preds * targets).astype(float) 94 | score = 2. * intersection.sum(1) / total 95 | return np.mean(score) 96 | 97 | 98 | def get_f2_score(y_pred, y_true, average='samples'): 99 | y_pred, y_true, = np.array(y_pred), np.array(y_true) 100 | return fbeta_score(y_true, y_pred, beta=2, average=average) 101 | 102 | 103 | def find_f2score_threshold(probs, targets, average='samples', 104 | try_all=True, verbose=False, step=.01): 105 | best = 0 106 | best_score = -1 107 | totry = np.arange(0.1, 0.9, step) 108 | for t in totry: 109 | score = get_f2_score(probs, targets, t) 110 | if score > best_score: 111 | best_score = score 112 | best = t 113 | if verbose is True: 114 | print('Best score: ', round(best_score, 5), 115 | ' @ threshold =', round(best,4)) 116 | return round(best,6) 117 | 118 | -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/models/__init__.py -------------------------------------------------------------------------------- /models/builder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision.models 4 | import models.utils 5 | 6 | 7 | def get_fc(in_feat, n_classes, activation=None): 8 | layers = [ 9 | nn.Linear(in_features=in_feat, out_features=n_classes) 10 | ] 11 | if activation is not None: 12 | layers.append(activation) 13 | return nn.Sequential(*layers) 14 | 15 | 16 | def get_classifier(in_feat, n_classes, activation, p=0.5): 17 | layers = [ 18 | nn.BatchNorm1d(num_features=in_feat), 19 | nn.Dropout(p), 20 | nn.Linear(in_features=in_feat, out_features=n_classes), 21 | activation 22 | ] 23 | return nn.Sequential(*layers) 24 | 25 | 26 | def get_mlp_classifier(in_feat, out_feat, n_classes, activation, p=0.01, p2=0.5): 27 | layers = [ 28 | nn.BatchNorm1d(num_features=in_feat), 29 | nn.Dropout(p), 30 | nn.Linear(in_features=in_feat, out_features=out_feat), 31 | nn.ReLU(), 32 | nn.BatchNorm1d(num_features=out_feat), 33 | nn.Dropout(p2), 34 | nn.Linear(in_features=out_feat, out_features=n_classes), 35 | activation 36 | ] 37 | return nn.Sequential(*layers) 38 | 39 | 40 | def cut_model(model, cut): 41 | return nn.Sequential(*list(model.children())[:cut]) -------------------------------------------------------------------------------- /models/layers.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | def conv_relu(in_channels, out_channels, kernel_size=3, stride=1, 5 | padding=1, bias=True): 6 | return [ 7 | nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, 8 | stride=stride, padding=padding, bias=bias), 9 | nn.ReLU(inplace=True), 10 | ] 11 | 12 | def conv_bn_relu(in_channels, out_channels, kernel_size=3, stride=1, 13 | padding=1, bias=False): 14 | return [ 15 | nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, 16 | stride=stride, padding=padding, bias=bias), 17 | nn.BatchNorm2d(out_channels), 18 | nn.ReLU(inplace=True), 19 | ] 20 | 21 | def linear_bn_relu_drop(in_channels, out_channels, dropout=0.5, bias=False): 22 | layers = [ 23 | nn.Linear(in_channels, out_channels, bias=bias), 24 | nn.BatchNorm1d(out_channels), 25 | nn.ReLU(inplace=True) 26 | ] 27 | if dropout > 0: 28 | layers.append(nn.Dropout(dropout)) 29 | return layers 30 | 31 | 32 | -------------------------------------------------------------------------------- /models/resnet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision.models 4 | import models.utils 5 | 6 | 7 | class SimpleResnet(nn.Module): 8 | def __init__(self, resnet, classifier): 9 | super().__init__() 10 | self.__class__.__name__ = "SimpleResnet" 11 | self.resnet = resnet 12 | self.classifier = classifier 13 | 14 | def forward(self, x): 15 | x = self.resnet(x) 16 | x = x.view(x.size(0), -1) 17 | x = self.classifier(x) 18 | return x 19 | 20 | 21 | class ConcatResnet(nn.Module): 22 | def __init__(self, resnet, classifier): 23 | super().__init__() 24 | self.__class__.__name__ = 'ConcatResnet' 25 | self.resnet = resnet 26 | self.ap = nn.AdaptiveAvgPool2d((1,1)) 27 | self.mp = nn.AdaptiveMaxPool2d((1,1)) 28 | self.classifier = classifier 29 | 30 | def forward(self, x): 31 | x = self.resnet(x) 32 | x = torch.cat([self.mp(x), self.ap(x)], 1) 33 | x = x.view(x.size(0), -1) 34 | x = self.classifier(x) 35 | return x 36 | 37 | 38 | def get_resnet18(pretrained, n_freeze): 39 | resnet = torchvision.models.resnet18(pretrained) 40 | if n_freeze > 0: 41 | models.utils.freeze_layers(resnet, n_freeze) 42 | return resnet 43 | 44 | 45 | def get_resnet34(pretrained, n_freeze): 46 | resnet = torchvision.models.resnet34(pretrained) 47 | if n_freeze > 0: 48 | models.utils.freeze_layers(resnet, n_freeze) 49 | return resnet 50 | 51 | 52 | def get_resnet50(pretrained, n_freeze): 53 | resnet = torchvision.models.resnet50(pretrained) 54 | if n_freeze > 0: 55 | models.utils.freeze_layers(resnet, n_freeze) 56 | return resnet 57 | -------------------------------------------------------------------------------- /models/simplenet.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import models.layers as layers 3 | 4 | 5 | class SimpleNet(nn.Module): 6 | def __init__(self, in_feat, n_classes): 7 | super().__init__() 8 | self.features = nn.Sequential( 9 | *layers.conv_bn_relu(in_feat, 8, kernel_size=1, stride=1, padding=0, bias=False), 10 | *layers.conv_bn_relu(8, 32, kernel_size=3, stride=1, padding=1, bias=False), 11 | nn.MaxPool2d(kernel_size=2, stride=2), #size/2 12 | *layers.conv_bn_relu(32, 32, kernel_size=3, stride=1, padding=1, bias=False), 13 | nn.MaxPool2d(kernel_size=2, stride=2), #size/2 14 | *layers.conv_bn_relu(32, 64, kernel_size=3, stride=1, padding=1, bias=False), 15 | nn.MaxPool2d(kernel_size=2, stride=2), #size/2 16 | ) 17 | self.classifier = nn.Sequential( 18 | *layers.linear_bn_relu_drop(64, 512, dropout=0.0, bias=False), 19 | nn.Linear(512, n_classes, bias=False), 20 | nn.Sigmoid() 21 | ) 22 | 23 | def forward(self, x): 24 | x = self.features(x) 25 | x = x.view(x.size(0), -1) 26 | x = self.classifier(x) 27 | return x -------------------------------------------------------------------------------- /models/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def load_model(fpath, cuda=True): 5 | if cuda: 6 | return torch.load(fpath).cuda() 7 | return torch.load(fpath) 8 | 9 | 10 | def save_model(model, fpath): 11 | torch.save(model.cpu(), fpath) 12 | 13 | 14 | def load_weights(model, fpath): 15 | state = torch.load(fpath) 16 | model.load_state_dict(state['state_dict']) 17 | 18 | 19 | def save_weights(model, fpath, epoch=None, name=None): 20 | torch.save({ 21 | 'name': name, 22 | 'epoch': epoch, 23 | 'state_dict': model.state_dict() 24 | }, fpath) 25 | 26 | 27 | def freeze_layers(model, n_layers): 28 | i = 0 29 | for child in model.children(): 30 | if i >= n_layers: 31 | break 32 | print(i, "freezing", child) 33 | for param in child.parameters(): 34 | param.requires_grad = False 35 | i += 1 36 | 37 | 38 | def freeze_nested_layers(model, n_layers): 39 | i = 0 40 | for child in model.children(): 41 | for grandchild in child.children(): 42 | if isinstance(grandchild, torch.nn.modules.container.Sequential): 43 | for greatgrand in grandchild.children(): 44 | if i >= n_layers: 45 | break 46 | for param in greatgrand.parameters(): 47 | param.requires_grad = False 48 | print(i, "freezing", greatgrand) 49 | i += 1 50 | else: 51 | if i >= n_layers: 52 | break 53 | for param in grandchild.parameters(): 54 | param.requires_grad = False 55 | print(i, "freezing", grandchild) 56 | i += 1 57 | 58 | 59 | def init_nested_layers(module, conv_init, fc_init): 60 | for child in module.children(): 61 | if len(list(child.children())) > 0: 62 | init_nested_layers(child, conv_init, fc_init) 63 | else: 64 | init_weights(child, conv_init, fc_init) 65 | 66 | 67 | def init_weights(layer, conv_init, fc_init): 68 | if isinstance(layer, torch.nn.Conv2d): 69 | print("init", layer, "with", conv_init) 70 | conv_init(layer.weight) 71 | elif isinstance(layer, torch.nn.Linear): 72 | print("init", layer, "with", fc_init) 73 | fc_init(layer.weight) 74 | -------------------------------------------------------------------------------- /notifications/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfortuner/pytorch-federated-learning/99cc406a361f23a34aa4576c8781274fd0312158/notifications/__init__.py -------------------------------------------------------------------------------- /notifications/email_constants.py: -------------------------------------------------------------------------------- 1 | import config 2 | import constants as c 3 | 4 | 5 | WEBSITE_URL = config.KIBANA_URL 6 | ADMIN_EMAIL = config.ADMIN_EMAIL 7 | USER_EMAIL = config.USER_EMAIL 8 | EMAIL_CHARSET = 'UTF-8' 9 | 10 | HEADER="" 11 | FOOTER="" 12 | 13 | EXPERIMENT_STATUS_EMAIL_TEMPLATE=""" 14 |
Hello,
15 |Your experiment has ended.
16 |Name: %s
17 |Status: %s
18 |Status Msg: %s
19 | 20 |Experiment Results:
21 |%s
22 |Experiment Config:
23 |%s
24 |Thanks,
25 | Team