├── .gitignore ├── .travis.yml ├── CODEOWNERS ├── LICENSE ├── README.md ├── bert-distillation-multimetric ├── README.md ├── __init__.py ├── distilbert_data_model_loaders │ ├── __init__.py │ ├── load_squad_dataset.py │ ├── load_transfomer_model.py │ ├── load_transformer_from_scratch.py │ ├── load_transformer_pretrained.py │ └── load_transformer_semi_pretrained.py ├── distilbert_run_and_hpo_configurations │ ├── __init__.py │ ├── distilbert_squad_hpo_parameters.py │ └── distilbert_squad_run_parameters.py ├── logger.py ├── raytune_wrapper │ ├── __init__.py │ ├── ray_launch_config.yaml │ ├── ray_optimize_squad_distillation.py │ └── sigopt_ray_multimetric_wrapper_distilbert.py ├── run_squad_w_distillation_cli.py ├── sigopt_clients │ ├── __init__.py │ ├── sigopt_experiment_client.py │ └── sigopt_experiment_datatypes.py ├── sigopt_optimization_cli.py ├── sigopt_optimization_wrapper │ ├── __init__.py │ ├── runs_optimize_squad_distillation.py │ ├── sigopt_hyperparameter_definition.py │ ├── sigopt_multimetric_definition.py │ └── sigopt_runs_optimization_cycle.py ├── sigopt_ray_optimization_cli.py ├── squad_distillation_abstract_clis │ ├── __init__.py │ ├── a_optimizaton_run_squad_cli.py │ └── a_run_squad_w_distillation_cli.py └── squad_fine_tuning │ ├── __init__.py │ ├── a_squad_w_distillation.py │ ├── eval_squad_distillation.py │ ├── optimize_squad_distillation.py │ ├── set_seed_and_dist.py │ ├── squad_w_distillation.py │ ├── train_squad_distillation.py │ └── training_run_squad_distillation.py ├── classifier ├── README.md ├── classifier_tuner.py ├── constant.py ├── data │ └── .gitignore └── requirements.txt ├── dnn-tuning-nvidia-mxnet ├── .gitignore ├── README.md ├── cnn_text │ ├── __init__.py │ ├── dataset.py │ ├── evaluate.py │ ├── graph.py │ ├── hyperparams.json │ ├── long_hyperparams.json │ ├── model.py │ ├── objective.py │ └── preprocess.py ├── multi_run_experiments.py ├── multi_run_experiments_with_failures.py ├── mxnet-setup │ ├── install_mxnet_gpu_ubuntu.sh │ └── setup_gpu.sh ├── nvidia-smi.png └── run_experiments.py ├── estimated-training-time ├── How are My Hyperparameters Affecting My Training Time?.ipynb ├── README.md ├── metric-vs-time.png └── requirements.txt ├── get-started ├── README.md ├── sigopt_experiment_and_optimization_demo.ipynb └── sigopt_runs_demo.ipynb ├── java ├── .gitignore ├── README.md ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── example │ ├── App.java │ └── Result.java ├── metric-constraints-demo ├── GTSRB_preprocessing_augmentation.ipynb ├── README.md └── sigopt_metric_constraints_demo.ipynb ├── multimetric-timeseries ├── .gitignore ├── README.md ├── UCR_TS_Archive_2015 │ └── Adiac │ │ ├── Adiac_TEST │ │ └── Adiac_TRAIN ├── __init__.py ├── config.py ├── main.py └── train.py ├── optimizing-memn2n ├── LICENSE ├── README.md ├── end2end_mem_nn_tensorflow │ ├── memn2n │ │ ├── data_utils.py │ │ ├── joint.py │ │ ├── memn2n_components.py │ │ └── single.py │ ├── sigopt_memn2n_setup │ │ ├── random_search_parameters_config.py │ │ ├── sigopt_conditionals_parameters_config.py │ │ ├── sigopt_experiment_client.py │ │ ├── sigopt_hyperparameters_enum.py │ │ ├── sigopt_memn2n_experiment_setup.py │ │ └── sigopt_parameters_config.py │ └── sigopt_optimization_run.py └── virtual_env_setup │ ├── README.md │ ├── python3_memn2n_tf_env_cpu_requirements.txt │ └── python3_memn2n_tf_env_gpu_requirements.txt ├── other-languages ├── README.md ├── other_languages.py └── requirements.txt ├── stanford-augmented-image-classification ├── README.md ├── __init__.py ├── a_resnet_training_common_cli.py ├── a_stanford_car_dataset.py ├── a_stanford_data_processor.py ├── i_commandline_arguments.py ├── i_orchestrate_multitask.py ├── orchestrate_stanford_cars_augmentation_cli.py ├── orchestrate_stanford_cars_cli.py ├── orchestrate_yml_files │ ├── orchestrate_cluster_deploy_sample.yml │ ├── orchestrate_stanford_cars_augmented_tuning_config.yml │ └── orchestrate_stanford_cars_tuning_config.yml ├── resnet.py ├── resnet_stanford_cars_augmented_training.py ├── resnet_stanford_cars_training.py ├── stanford_augmented_data_processor.py ├── stanford_car_dataset.py ├── stanford_car_dataset_augmented.py ├── stanford_cars_augmentation_cli.py ├── stanford_cars_cli.py ├── stanford_cars_data_augmentation.py ├── stanford_cars_data_config.py └── stanford_data_processor.py ├── stanford-car-classification ├── README.md ├── orchestrate_cluster_deploy_sample.yml ├── orchestrate_experiment_wrapper_stanford_cars_cli.py ├── orchestrate_stanford_cars_tuning_config.yml ├── resnet.py ├── resnet_stanford_cars_cli.py ├── run_resnet_training_cli.py ├── stanford_cars.py └── stanford_cars_venv_requirements.txt ├── surprise-py-recsys ├── README.md └── surprise_recommender.ipynb ├── test ├── README ├── other_languages_tester ├── run_python └── with_new_experiment ├── text-classifier ├── README.md ├── python │ ├── .gitignore │ ├── README.md │ ├── SigOpt Text Classifier Walkthrough.ipynb │ ├── requirements.txt │ ├── sentiment_classifier.py │ └── setup_env.sh └── r │ ├── README.md │ └── sentiment_classifier.r ├── vision-nas-search-keras-cifar-ray ├── README.md └── mininas.ipynb ├── xgboost-integration-examples ├── README.md ├── requirements.txt ├── xgb-integration-py-class.ipynb └── xgb-integration-py-reg.ipynb ├── xgboost-py-classifier ├── README.md └── xgboost_py_classifier.ipynb └── xgboost-py-regression ├── README.md └── xgboost_py_regressor.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *~ 3 | *.swp 4 | *.pdf 5 | */venv 6 | */requirements-clean.txt 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | env/ 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | 57 | # Sphinx documentation 58 | docs/_build/ 59 | 60 | # PyBuilder 61 | target/ 62 | 63 | # Pycharm caching 64 | .idea 65 | 66 | # Jupyter notebooks 67 | .ipynb_checkpoints/ 68 | 69 | tags 70 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - "2.7" 5 | - "3.5" 6 | cache: 7 | pip: false 8 | 9 | addons: 10 | apt: 11 | packages: 12 | # numpy and scipy cause a lot of trouble and are expensive to install, so install 13 | # them once for all packages, and let the virtualenvs see this parent package 14 | - 'python-numpy' 15 | - 'python-scipy' 16 | 17 | install: 18 | - pip install sigopt 19 | - python -c "import numpy; print(numpy.__version__)" 20 | 21 | # Use the SIGOPT_API_TOKEN environment variable to set the API token. 22 | # Be careful to not expose it in the travis logs 23 | # If the example requires an existing experiment ID, use 24 | # ./test/with_new_experiment and it will replace NEW_EXPERIMENT_ID with 25 | # a newly created valid experiment 26 | script: 27 | - ./test/run_python classifier classifier_tuner.py --client-token $SIGOPT_API_TOKEN --num-sigopt-suggestions 2 --num-random-searches 1 --grid-search-width 1 28 | - ./test/with_new_experiment ./test/run_python other-languages other_languages.py --command "../test/other_languages_tester" --client_token $SIGOPT_API_TOKEN --experiment_id NEW_EXPERIMENT_ID 29 | - ./test/with_new_experiment ./test/run_python parallel parallel.py --client_token $SIGOPT_API_TOKEN --experiment_id NEW_EXPERIMENT_ID 30 | - cd java && mvn compile && mvn exec:java -Dexec.mainClass="com.example.App" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--client_token $SIGOPT_API_TOKEN" 31 | - cd spark/recommender_sys && sbt assembly 32 | 33 | branches: 34 | only: 35 | - master 36 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # default fallback for all files: 2 | * @Chris-Sigopt @tjs-intel 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 SigOpt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # Getting Started with SigOpt 4 | 5 | Welcome to the SigOpt Examples. These examples show you how to use [SigOpt](https://sigopt.com) for model tuning tasks in various machine learning environments. 6 | 7 | ## Requirements 8 | 9 | Most of these examples will run on any Linux or Mac OS X machine from the command line. Each example contains a README.md with specific setup instructions. 10 | 11 | ## Questions? 12 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 13 | 14 | ## API Reference 15 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 16 | 17 | ## About SigOpt 18 | 19 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 20 | 21 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 22 | 23 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API and client libraries (Python, R, Java) integrate into any existing ML workflow. 24 | 25 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 26 | 27 | [Visit our website](https://sigopt.com) to learn more! 28 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/bert-distillation-multimetric/__init__.py -------------------------------------------------------------------------------- /bert-distillation-multimetric/distilbert_data_model_loaders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/bert-distillation-multimetric/distilbert_data_model_loaders/__init__.py -------------------------------------------------------------------------------- /bert-distillation-multimetric/distilbert_data_model_loaders/load_transfomer_model.py: -------------------------------------------------------------------------------- 1 | from transformers import DistilBertConfig, DistilBertForQuestionAnswering, BertConfig, \ 2 | BertForQuestionAnswering, BertTokenizer, DistilBertTokenizer 3 | 4 | 5 | MODEL_CLASSES = { 6 | "bert": (BertConfig, BertForQuestionAnswering, BertTokenizer), 7 | "distilbert": (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer), 8 | } 9 | 10 | 11 | class LoadModel(object): 12 | 13 | def __init__(self, model_type): 14 | 15 | self.model_type = model_type 16 | 17 | assert self.model_type in MODEL_CLASSES.keys(), "model type has to be bert or distilbert" 18 | config_class, model_class, tokenizer_class = MODEL_CLASSES[self.model_type] 19 | 20 | self.config_class = config_class 21 | self.model_class = model_class 22 | self.tokenizer_class = tokenizer_class 23 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/distilbert_data_model_loaders/load_transformer_from_scratch.py: -------------------------------------------------------------------------------- 1 | from distilbert_data_model_loaders.load_transfomer_model import LoadModel 2 | import logging 3 | 4 | 5 | class LoadModelFromScratch(LoadModel): 6 | 7 | def __init__(self, model_type): 8 | super().__init__(model_type) 9 | 10 | def get_model(self, config): 11 | return self.model_class(config=config) 12 | 13 | def get_config(self, config_dict=dict()): 14 | # config_dict overwrites existing default values for given model_type 15 | return self.config_class.from_dict(config_dict) 16 | 17 | 18 | def get_model_from_scratch(model_type, config_dict): 19 | model_loader = LoadModelFromScratch(model_type=model_type) 20 | config = model_loader.get_config(config_dict) 21 | model = model_loader.get_model(config) 22 | return model_loader, model, config 23 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/distilbert_data_model_loaders/load_transformer_pretrained.py: -------------------------------------------------------------------------------- 1 | from distilbert_data_model_loaders.load_transfomer_model import LoadModel 2 | import logging 3 | 4 | 5 | class LoadPretrainedModel(LoadModel): 6 | 7 | def __init__(self, model_type, model_name_or_path, cache_dir): 8 | super().__init__(model_type) 9 | self.model_name_or_path = model_name_or_path 10 | self.cache_dir = cache_dir 11 | 12 | def get_pretrained_model(self, config): 13 | return self.model_class.from_pretrained( 14 | self.model_name_or_path, 15 | from_tf=bool(".ckpt" in self.model_name_or_path), 16 | config=config, 17 | cache_dir=self.cache_dir if self.cache_dir else None, 18 | ) 19 | 20 | def get_pretrained_config(self, config_name=None): 21 | return self.config_class.from_pretrained( 22 | config_name if config_name else self.model_name_or_path, 23 | cache_dir=self.cache_dir if self.cache_dir else None, 24 | ) 25 | 26 | def get_tokenizer(self, max_positional_embedding_length, tokenizer_name=None, do_lower=True): 27 | return self.tokenizer_class.from_pretrained( 28 | tokenizer_name if tokenizer_name else self.model_name_or_path, 29 | do_lower_case=do_lower, 30 | cache_dir=self.cache_dir if self.cache_dir else None, 31 | max_len=max_positional_embedding_length, 32 | ) 33 | 34 | 35 | def get_pretrained_model(model_type, model_name_or_path, cache_dir): 36 | logging.info("loading pretrained model with model type: {}, model name or path: {}, and cache_dir: {}" 37 | .format(model_type, model_name_or_path, cache_dir)) 38 | pretrained_loader = LoadPretrainedModel(model_type=model_type, 39 | model_name_or_path=model_name_or_path, 40 | cache_dir=cache_dir) 41 | pretrained_config = pretrained_loader.get_pretrained_config() 42 | pretrained_model = pretrained_loader.get_pretrained_model(config=pretrained_config) 43 | return pretrained_loader, pretrained_model, pretrained_config 44 | 45 | 46 | def get_pretrained_tokenizer(model_type, model_name_or_path, cache_dir, max_positional_embedding_length=512): 47 | logging.info("loading pretrained tokenizer with model type: {}, model name or path: {}, cache_dir: {}, and pos " 48 | "embedding length: {}".format(model_type, model_name_or_path, cache_dir, max_positional_embedding_length)) 49 | pretrained_loader = LoadPretrainedModel(model_type=model_type, 50 | model_name_or_path=model_name_or_path, 51 | cache_dir=cache_dir) 52 | pretrained_tokenizer = pretrained_loader.get_tokenizer(max_positional_embedding_length) 53 | return pretrained_tokenizer 54 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/distilbert_data_model_loaders/load_transformer_semi_pretrained.py: -------------------------------------------------------------------------------- 1 | from distilbert_data_model_loaders.load_transfomer_model import LoadModel 2 | import logging 3 | 4 | 5 | class LoadSemiPretrainedModel(LoadModel): 6 | 7 | def __init__(self, model_type, model_name_or_path, cache_dir): 8 | super().__init__(model_type) 9 | self.model_name_or_path = model_name_or_path 10 | self.cache_dir = cache_dir 11 | 12 | def get_pretrained_model(self, config): 13 | return self.model_class.from_pretrained( 14 | self.model_name_or_path, 15 | from_tf=bool(".ckpt" in self.model_name_or_path), 16 | config=config, 17 | cache_dir=self.cache_dir if self.cache_dir else None, 18 | ) 19 | 20 | def get_config(self, config_dict=dict()): 21 | # config_dict overwrites existing default values for given model_type 22 | return self.config_class.from_dict(config_dict) 23 | 24 | 25 | def get_semi_pretrained_model(model_type, model_name_or_path, cache_dir, config_dict): 26 | logging.info("Loading semi-pretrained model with model type: {} and model path: {}".format(model_type, model_name_or_path)) 27 | model_loader = LoadSemiPretrainedModel(model_type, model_name_or_path, cache_dir) 28 | config = model_loader.get_config(config_dict) 29 | model = model_loader.get_pretrained_model(config) 30 | return model_loader, model, config 31 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/distilbert_run_and_hpo_configurations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/bert-distillation-multimetric/distilbert_run_and_hpo_configurations/__init__.py -------------------------------------------------------------------------------- /bert-distillation-multimetric/distilbert_run_and_hpo_configurations/distilbert_squad_hpo_parameters.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class DistillationHyperparameter(Enum): 5 | ALPHA_CE = 'alpha_ce' 6 | ALPHA_SQUAD = 'alpha_squad' 7 | TEMPERATURE = 'temperature' 8 | 9 | 10 | class SGDHyperparameter(Enum): 11 | LEARNING_RATE = 'learning_rate' 12 | WEIGHT_DECAY = 'weight_decay' 13 | ADAM_EPSILON = 'adam_epsilon' 14 | MAX_GRAD_NORM = 'max_grad_norm' 15 | WARM_UP_STEPS = 'warm_up_steps' 16 | BETA_1 = 'beta_1' 17 | BETA_2 = 'beta_2' 18 | PER_COMPUTE_TRAIN_BATCH_SIZE = 'per_compute_train_batch_size' 19 | PER_COMPUTE_EVAL_BATCH_SIZE = 'per_compute_eval_batch_size' 20 | GRADIENT_ACCUMULATION_STEPS = 'gradient_accumulation_steps' 21 | 22 | 23 | class SquadArchitectureHyperparameter(Enum): 24 | MAX_SEQ_LENGTH = 'max_seq_length' # for generating features from squad 25 | DOC_STRIDE = 'doc_stride' # for generating features from squad 26 | MAX_QUERY_LENGTH = 'max_query_length' # for generating features from squad 27 | MAX_ANSWER_LENGTH = 'max_answer_length' # for calculating squad metrics 28 | 29 | 30 | class ArchitectureHyperparameter(Enum): 31 | PRUNING_SEED = "pruning_seed" #seed set for pruning 32 | N_LAYERS = 'n_layers' # DistilBert number of layers 33 | N_HEADS = 'n_heads' # DistilBert number of attention heads 34 | DIMENSION = 'dim' 35 | HIDDEN_DIMENSION = 'hidden_dim' # intermediate hidden layer dimension for ffn 36 | DROPOUT = "dropout" # used for embeddings and ffn 37 | ATTENTION_DROPOUT = 'attention_dropout' # dropout applied to attention heads 38 | INIT_RANGE = 'initializer_range' # weight initialization for student model 39 | QA_DROPOUT = 'qa_dropout' # dropout used for whole network for question answering 40 | 41 | 42 | fine_tuning_squad2_default_hyperparameters = { 43 | DistillationHyperparameter.ALPHA_CE.value: 0.5, 44 | DistillationHyperparameter.ALPHA_SQUAD.value: 0.5, 45 | DistillationHyperparameter.TEMPERATURE.value: 1.0, 46 | SquadArchitectureHyperparameter.MAX_SEQ_LENGTH.value: 384, 47 | SquadArchitectureHyperparameter.MAX_QUERY_LENGTH.value: 64, 48 | SquadArchitectureHyperparameter.MAX_ANSWER_LENGTH.value: 30, 49 | SGDHyperparameter.LEARNING_RATE.value: float(5e-5), 50 | SGDHyperparameter.WEIGHT_DECAY.value: float(0.0), 51 | SGDHyperparameter.ADAM_EPSILON.value: float(1e-8), 52 | SGDHyperparameter.MAX_GRAD_NORM.value: float(1.0), 53 | SquadArchitectureHyperparameter.DOC_STRIDE.value: 128, 54 | SGDHyperparameter.PER_COMPUTE_TRAIN_BATCH_SIZE.value: 8, 55 | SGDHyperparameter.PER_COMPUTE_EVAL_BATCH_SIZE.value: 8, 56 | SGDHyperparameter.GRADIENT_ACCUMULATION_STEPS.value: 1, 57 | SGDHyperparameter.WARM_UP_STEPS.value: 0, 58 | ArchitectureHyperparameter.N_HEADS.value: 12, 59 | ArchitectureHyperparameter.N_LAYERS.value: 6, 60 | ArchitectureHyperparameter.DIMENSION.value: 768, 61 | ArchitectureHyperparameter.HIDDEN_DIMENSION.value: 3072, 62 | ArchitectureHyperparameter.DROPOUT.value: 0.1, 63 | ArchitectureHyperparameter.ATTENTION_DROPOUT.value: 0.1, 64 | ArchitectureHyperparameter.INIT_RANGE.value: 0.02, 65 | ArchitectureHyperparameter.QA_DROPOUT.value: 0.1, 66 | } 67 | 68 | 69 | def get_default_hyperparameters(): 70 | return fine_tuning_squad2_default_hyperparameters 71 | 72 | 73 | def get_all_hyperparameter_names(): 74 | all_hyperparameter_names = list() 75 | for distill_parameter in DistillationHyperparameter: 76 | all_hyperparameter_names.append(distill_parameter.value) 77 | for sgd_parameter in SGDHyperparameter: 78 | all_hyperparameter_names.append(sgd_parameter.value) 79 | for squad_arch_parameter in SquadArchitectureHyperparameter: 80 | all_hyperparameter_names.append(squad_arch_parameter.value) 81 | for arch_hyperparameter in ArchitectureHyperparameter: 82 | all_hyperparameter_names.append(arch_hyperparameter.value) 83 | return all_hyperparameter_names 84 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/distilbert_run_and_hpo_configurations/distilbert_squad_run_parameters.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class RunParameters(Enum): 5 | MODEL_TYPE = 'model_type' 6 | MODEL_NAME_OR_PATH = 'model_name_or_path' 7 | OUTPUT_DIR = 'output_dir' 8 | TEACHER_TYPE = 'teacher_type' 9 | TEACHER_NAME_OR_PATH = 'teacher_name_or_path' 10 | TRAIN_FILE = 'train_file' 11 | PREDICT_FILE = 'predict_file' 12 | CONFIG_NAME = 'config_name' 13 | TOKENIZER_NAME = 'tokenizer_name' 14 | CACHE_DIR = 'cache_dir' 15 | VERSION_2 = 'version_2_with_negative' 16 | EVALUATE_DURING_TRAINING = 'evaluate_during_training' 17 | DO_LOWER_CASE = 'do_lower_case' 18 | SEED = 'seed' 19 | LOGGING_STEPS = 'logging_steps' 20 | DO_TRAIN = 'do_train' 21 | DO_EVAL = 'do_eval' 22 | SAVE_STEPS = 'save_steps' 23 | EVAL_ALL_CHECKPOINTS = 'eval_all_checkpoints' 24 | NO_CUDA = 'no_cuda' 25 | OVERWRITE_OUTPUT_DIR = 'overwrite_output_dir' 26 | OVERWRTIE_CACHE = 'overwrite_cache' 27 | LOCAL_RANK = 'local_rank' 28 | NUM_TRAIN_EPOCHS = 'num_train_epochs' 29 | MAX_STEPS = 'max_steps' 30 | N_BEST_SIZE = 'n_best_size' 31 | VERBOSE_LOGGING = 'verbose_logging' 32 | FP16_OPT_LEVEL = 'fp16_opt_level' 33 | SERVER_IP = 'server_ip' 34 | SERVER_PORT = 'server_port' 35 | FP_16 = 'use_bfloat16' 36 | LOAD_PRETRAINED_MODEL = "load_student_pretrained" 37 | LOAD_SEMI_PRETRAINED_MODEL = "load_student_semi_pretrained" 38 | CACHE_S3_BUCKET = "cache_s3_bucket" 39 | TRAIN_CACHE_S3_DIRECTORY = "train_cache_s3_directory" 40 | EVAL_CACHE_S3_DIRECTORY = "eval_cache_s3_directory" 41 | 42 | 43 | class OptimizationRunParameters(Enum): 44 | EXPERIMENT_NAME = "experiment_name" 45 | API_TOKEN = "api_token" 46 | USE_HPO_DEFAULT_RANGES = "use_hpo_default_ranges" 47 | PROJECT_NAME = "project_name" 48 | SIGOPT_EXPERIMENT_ID = "sigopt_experiment_id" 49 | SIGOPT_OBSERVATION_BUDGET = "sigopt_observation_budget" 50 | SIGOPT_RUN_DIRECTORY = "sigopt_run_directory" 51 | STORE_S3 = "store_s3" 52 | S3_BUCKET = "s3_bucket" 53 | 54 | 55 | class RayTuneRunParameters(Enum): 56 | MAX_CONCURRENT = "max_concurrent" 57 | PARALLEL = "parallel" 58 | NUM_CPU = "num_cpu" 59 | NUM_GPU = "num_gpu" 60 | RAY_ADDRESS = "ray_address" 61 | CLEAN_RAYTUNE_OUTPUT = "clean_raytune_output" 62 | RAY_OUTPUT_DIRECTORY = "raytune_output_directory" 63 | 64 | 65 | fine_tuning_squad2_default_run_parameters = { 66 | RunParameters.SEED.value: 42, 67 | RunParameters.LOGGING_STEPS.value: 1000, 68 | RunParameters.SAVE_STEPS.value: 1000, 69 | RunParameters.VERSION_2.value: True, 70 | RunParameters.EVALUATE_DURING_TRAINING.value: True, 71 | RunParameters.DO_LOWER_CASE.value: True, 72 | RunParameters.DO_EVAL.value: True, 73 | RunParameters.DO_TRAIN.value: True, 74 | RunParameters.EVAL_ALL_CHECKPOINTS.value: False, 75 | RunParameters.NO_CUDA.value: False, 76 | RunParameters.OVERWRITE_OUTPUT_DIR.value: False, 77 | RunParameters.OVERWRTIE_CACHE.value: False, 78 | RunParameters.LOCAL_RANK.value: -1, 79 | RunParameters.NUM_TRAIN_EPOCHS.value: 3, 80 | RunParameters.N_BEST_SIZE.value: 20, 81 | RunParameters.FP_16.value: False, 82 | RunParameters.FP16_OPT_LEVEL.value: '01', 83 | RunParameters.TRAIN_CACHE_S3_DIRECTORY.value: None, 84 | RunParameters.EVAL_CACHE_S3_DIRECTORY.value: None, 85 | RunParameters.CACHE_S3_BUCKET.value: None, 86 | } 87 | 88 | 89 | def get_default_run_parameters(): 90 | return fine_tuning_squad2_default_run_parameters 91 | 92 | 93 | def get_all_run_parameter_names(): 94 | all_run_parameter_names = list() 95 | for run_parameter in RunParameters: 96 | all_run_parameter_names.append(run_parameter.value) 97 | for opt_run_parameter in OptimizationRunParameters: 98 | all_run_parameter_names.append(opt_run_parameter.value) 99 | return all_run_parameter_names 100 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | logger = logging.getLogger(__name__) 5 | 6 | os.environ["AWS_DEFAULT_REGION"] = "us-west-2" 7 | 8 | logging.basicConfig( 9 | format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", 10 | datefmt="%m/%d/%Y %H:%M:%S", 11 | level=logging.INFO, 12 | ) 13 | logger.setLevel(logging.INFO) -------------------------------------------------------------------------------- /bert-distillation-multimetric/raytune_wrapper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/bert-distillation-multimetric/raytune_wrapper/__init__.py -------------------------------------------------------------------------------- /bert-distillation-multimetric/raytune_wrapper/ray_launch_config.yaml: -------------------------------------------------------------------------------- 1 | # An unique identifier for the head node and workers of this cluster. 2 | cluster_name: ray-cluster 3 | 4 | # The minimum number of workers nodes to launch in addition to the head 5 | # node. This number should be >= 0. 6 | min_workers: 0 7 | 8 | # The maximum number of workers nodes to launch in addition to the head 9 | # node. This takes precedence over min_workers. 10 | max_workers: 19 11 | 12 | # The initial number of worker nodes to launch in addition to the head 13 | # node. When the cluster is first brought up (or when it is refreshed with a 14 | # subsequent `ray up`) this number of nodes will be started. 15 | initial_workers: 19 16 | 17 | # Whether or not to autoscale aggressively. If this is enabled, if at any point 18 | # we would start more workers, we start at least enough to bring us to 19 | # initial_workers. 20 | autoscaling_mode: default 21 | 22 | # This executes all commands on all nodes in the docker container, 23 | # and opens all the necessary ports to support the Ray cluster. 24 | # Empty string means disabled. 25 | docker: 26 | image: "" # e.g., tensorflow/tensorflow:1.5.0-py3 27 | container_name: "" # e.g. ray_docker 28 | # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image 29 | # if no cached version is present. 30 | pull_before_run: True 31 | run_options: [] # Extra options to pass into "docker run" 32 | 33 | # Example of running a GPU head with CPU workers 34 | # head_image: "tensorflow/tensorflow:1.13.1-py3" 35 | # head_run_options: 36 | # - --runtime=nvidia 37 | 38 | # worker_image: "ubuntu:18.04" 39 | # worker_run_options: [] 40 | 41 | # The autoscaler will scale up the cluster to this target fraction of resource 42 | # usage. For example, if a cluster of 10 nodes is 100% busy and 43 | # target_utilization is 0.8, it would resize the cluster to 13. This fraction 44 | # can be decreased to increase the aggressiveness of upscaling. 45 | # This value must be less than 1.0 for scaling to happen. 46 | target_utilization_fraction: 0.8 47 | 48 | # If a node is idle for this many minutes, it will be removed. 49 | idle_timeout_minutes: 10 50 | 51 | # Cloud-provider specific configuration. 52 | provider: 53 | type: aws 54 | region: us-west-2 55 | # Availability zone(s), comma-separated, that nodes may be launched in. 56 | # Nodes are currently spread between zones by a round-robin approach, 57 | # however this implementation detail should not be relied upon. 58 | availability_zone: us-west-2a,us-west-2b 59 | cache_stopped_nodes: False 60 | 61 | # How Ray will authenticate with newly launched nodes. 62 | auth: 63 | ssh_user: ubuntu 64 | # By default Ray creates a new private keypair, but you can also use your own. 65 | # If you do so, make sure to also set "KeyName" in the head and worker node 66 | # configurations below. 67 | 68 | # Provider-specific config for the head node, e.g. instance type. By default 69 | # Ray will auto-configure unspecified fields such as SubnetId and KeyName. 70 | # For more documentation on available fields, see: 71 | # http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances 72 | head_node: 73 | InstanceType: p3.2xlarge 74 | ImageId: ami-0b5664a1c8c6b5a72 # custom deep learning image 75 | 76 | # Provider-specific config for worker nodes, e.g. instance type. By default 77 | # Ray will auto-configure unspecified fields such as SubnetId and KeyName. 78 | # For more documentation on available fields, see: 79 | # http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances 80 | worker_nodes: 81 | InstanceType: p3.2xlarge 82 | ImageId: ami-0b5664a1c8c6b5a72 # custom deep learning image 83 | 84 | # Additional options in the boto docs. 85 | IamInstanceProfile: 86 | Arn: # workers need full access to s3 87 | 88 | # Files or directories to copy to the head and worker nodes. The format is a 89 | # dictionary from REMOTE_PATH: LOCAL_PATH, e.g. 90 | file_mounts: {} 91 | 92 | # List of commands that will be run before `setup_commands`. If docker is 93 | # enabled, these commands will run outside the container and before docker 94 | # is setup. 95 | initialization_commands: [] 96 | 97 | # List of shell commands to run to set up nodes. 98 | setup_commands: [] 99 | 100 | # Custom commands that will be run on the head node after common setup. 101 | head_setup_commands: 102 | - sudo pip install boto3==1.4.8 # 1.4.8 adds InstanceMarketOptions 103 | 104 | # Custom commands that will be run on worker nodes after common setup. 105 | worker_setup_commands: [] 106 | 107 | # Command to start ray on the head node. You don't need to change this. 108 | head_start_ray_commands: 109 | - ray stop 110 | - ulimit -n 65536; ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml 111 | 112 | # Command to start ray on worker nodes. You don't need to change this. 113 | worker_start_ray_commands: 114 | - ray stop 115 | - ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 116 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/sigopt_clients/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/bert-distillation-multimetric/sigopt_clients/__init__.py -------------------------------------------------------------------------------- /bert-distillation-multimetric/sigopt_clients/sigopt_experiment_client.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | class SigOptExperiment: 5 | 6 | def __init__(self, connection): 7 | self.connection = connection 8 | 9 | def initialize_random_experiment(self, experiment_name, project_name, parameters_list, metrics_list, observation_budget, 10 | metadata, parallel_bandwidth=1): 11 | return self.initialize_experiment(experiment_name, project_name, parameters_list, list(), list(), metrics_list, 12 | observation_budget, metadata, "random", parallel_bandwidth) 13 | 14 | def initialize_bayesian_experiment(self, experiment_name, project_name, parameters_list, metrics_list, observation_budget, metadata, parallel_bandwidth): 15 | return self.initialize_experiment(experiment_name, project_name, parameters_list, list(), list(), metrics_list, 16 | observation_budget, 17 | metadata, "offline", parallel_bandwidth) 18 | 19 | def initialize_experiment(self, experiment_name, project_name, parameters_list, conditionals_list, 20 | linear_constraints_list, metrics_list, 21 | observation_budget, metadata, experiment_type, parallel_bandwidth=1): 22 | experiment = self.connection.experiments().create( 23 | name=experiment_name, 24 | project=project_name, 25 | # Define which parameters you would like to tune 26 | parameters=parameters_list, 27 | linear_constraints=linear_constraints_list, 28 | conditionals=conditionals_list, 29 | metrics=metrics_list, 30 | parallel_bandwidth=parallel_bandwidth, 31 | # Define an Observation Budget for your experiment 32 | observation_budget=observation_budget, 33 | metadata=metadata, 34 | type=experiment_type 35 | ) 36 | logging.info("Created experiment: https://sigopt.com/experiment/%s", experiment.id) 37 | 38 | return experiment 39 | 40 | def get_initialized_experiment(self, experiment_id): 41 | return self.connection.experiments(experiment_id).fetch() 42 | 43 | def get_suggestions(self, experiment): 44 | return self.connection.experiments(experiment.id).suggestions().create() 45 | 46 | def get_suggestions_meatadata(self, experiment, metadata_dict): 47 | return self.connection.experiments(experiment.id).suggestions().create(metadata=metadata_dict) 48 | 49 | def get_best_suggestions(self, experiment): 50 | return self.connection.experiments(experiment.id).best_assignments().fetch() 51 | 52 | def update_suggestion(self, experiment_id, suggestion_id, metadata_dict): 53 | return self.connection.experiments(experiment_id).suggestions(suggestion_id).update( 54 | metadata=metadata_dict 55 | ) 56 | 57 | def update_experiment(self, experiment, suggestion, evaluated_value): 58 | observation = self.connection.experiments(experiment.id).observations().create(suggestion=suggestion.id, value=evaluated_value) 59 | return self.connection.experiments(experiment.id).fetch(), observation 60 | 61 | def update_experiment_multimetric_metadata(self, experiment, suggestion, evaluated_value, metadata_dict, failed=False): 62 | logging.info("updating experiment %s with metadata %s", experiment.id, str(metadata_dict)) 63 | self.connection.experiments(experiment.id).observations().create(suggestion=suggestion.id, 64 | values=evaluated_value, 65 | failed=failed, 66 | metadata=metadata_dict) 67 | return self.connection.experiments(experiment.id).fetch() 68 | 69 | def update_experiment_multimetric(self, experiment, suggestion, evaluated_values, failed=False): 70 | self.connection.experiments(experiment.id).observations().create(suggestion=suggestion.id, 71 | values=evaluated_values, 72 | failed=failed) 73 | return self.connection.experiments(experiment.id).fetch() 74 | 75 | def create_experiment_metadata(self, experiment, metadata_dict): 76 | self.connection.experiments(experiment.id).observations().create(metadata=metadata_dict) 77 | return self.connection.experiments(experiment.id).fetch() 78 | 79 | def create_observation_metadata(self, experiment, observation, metadata_dict): 80 | updated_observation = self.connection.experiments(experiment.id).observations(observation.id).update(metadata=metadata_dict) 81 | return self.connection.experiments(experiment.id).fetch(), updated_observation 82 | 83 | def get_all_experiments(self): 84 | return self.connection.experiments().fetch() 85 | 86 | def get_all_observations(self, experiment): 87 | return self.connection.experiments(experiment.id).observations().fetch() 88 | 89 | def archive_experiment(self, experiment): 90 | logging.info("archiving experiment with id: %s", experiment.id) 91 | self.connection.experiments(experiment.id).delete() 92 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/sigopt_clients/sigopt_experiment_datatypes.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class SigOptDataTypes(Enum): 5 | CATEGORICAL = 'categorical' 6 | INT = 'int' 7 | DOUBLE = 'double' 8 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/sigopt_optimization_cli.py: -------------------------------------------------------------------------------- 1 | from sigopt_clients import sigopt_experiment_client 2 | from sigopt_optimization_wrapper import sigopt_hyperparameter_definition 3 | from sigopt import Connection 4 | from sigopt_optimization_wrapper import sigopt_runs_optimization_cycle 5 | from distilbert_run_and_hpo_configurations.distilbert_squad_run_parameters import OptimizationRunParameters, RunParameters 6 | from squad_distillation_abstract_clis.a_optimizaton_run_squad_cli import OptimizeDistilBertQuadCLI 7 | import logging 8 | import os 9 | 10 | logging.basicConfig( 11 | format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", 12 | datefmt="%m/%d/%Y %H:%M:%S", 13 | level=logging.INFO, 14 | ) 15 | 16 | SIGOPT_API_TOKEN = "SIGOPT_API_TOKEN" 17 | 18 | 19 | class SigOptDistilBertQuadCLI(OptimizeDistilBertQuadCLI): 20 | 21 | def __init__(self): 22 | super().__init__() 23 | 24 | def get_commandline_args(self): 25 | return self.define_run_commandline_args() 26 | 27 | 28 | if __name__ == "__main__": 29 | 30 | optimization_cli = SigOptDistilBertQuadCLI() 31 | 32 | args = optimization_cli.get_commandline_args().parse_args() 33 | args_dict = vars(args) 34 | logging.info("running optimization experiment with arguments: {}".format(args_dict)) 35 | 36 | # set environment variable for SigOpt API Token 37 | logging.info("Setting SigOpt API Token") 38 | os.environ[SIGOPT_API_TOKEN] = args_dict[OptimizationRunParameters.API_TOKEN.value] 39 | 40 | sigopt_client_connection = Connection(client_token=args_dict[OptimizationRunParameters.API_TOKEN.value]) 41 | sigopt_client = sigopt_experiment_client.SigOptExperiment(connection=sigopt_client_connection) 42 | 43 | logging.info("tracking flag turned on, will execute optimization with runs tracking") 44 | sigopt_cycle = sigopt_runs_optimization_cycle.SigOptCycle(sigopt_experiment_client=sigopt_client, 45 | observation_budget=args_dict[ 46 | OptimizationRunParameters.SIGOPT_OBSERVATION_BUDGET.value], 47 | project_name=args_dict[ 48 | OptimizationRunParameters.PROJECT_NAME.value]) 49 | 50 | run_params_dict = dict() 51 | for default_param_key, default_param_value in args_dict.items(): 52 | if type(default_param_value) == bool: 53 | if default_param_value is True: 54 | run_params_dict[default_param_key] = "True" 55 | else: 56 | run_params_dict[default_param_key] = "False" 57 | else: 58 | if default_param_key is not OptimizationRunParameters.API_TOKEN.value: 59 | run_params_dict[default_param_key] = default_param_value 60 | 61 | if args_dict[OptimizationRunParameters.USE_HPO_DEFAULT_RANGES.value] is True: 62 | if args_dict[OptimizationRunParameters.SIGOPT_EXPERIMENT_ID.value] is not None: 63 | sigopt_experiment = sigopt_client.get_initialized_experiment( 64 | args_dict[OptimizationRunParameters.SIGOPT_EXPERIMENT_ID.value]) 65 | else: 66 | sigopt_experiment = sigopt_cycle.create_sigopt_experiment( 67 | experiment_name=args_dict[OptimizationRunParameters.EXPERIMENT_NAME.value], 68 | hyperparameter_definition=sigopt_hyperparameter_definition.get_sigopt_hyperparameter_list(), 69 | metadata=run_params_dict) 70 | logging.debug("SigOpt Experiment: {}".format(sigopt_experiment)) 71 | # set output directory 72 | args_dict[OptimizationRunParameters.SIGOPT_RUN_DIRECTORY.value] = os.path.join(args_dict[RunParameters.OUTPUT_DIR.value], 73 | str(sigopt_experiment.id)) 74 | logging.info("Run outputs can be found: {}".format(args_dict[OptimizationRunParameters.SIGOPT_RUN_DIRECTORY.value])) 75 | sigopt_experiment = sigopt_cycle.run_optimization_cycle(sigopt_experiment, 76 | run_parameters=args_dict, 77 | ) 78 | else: 79 | raise RuntimeError("Currently does not support custom HPO ranges. Change class file to accomodate.") 80 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/sigopt_optimization_wrapper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/bert-distillation-multimetric/sigopt_optimization_wrapper/__init__.py -------------------------------------------------------------------------------- /bert-distillation-multimetric/sigopt_optimization_wrapper/runs_optimize_squad_distillation.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sigopt 3 | from distilbert_run_and_hpo_configurations.distilbert_squad_run_parameters import OptimizationRunParameters 4 | from squad_fine_tuning.optimize_squad_distillation import OptimizeSquadDistillation 5 | 6 | 7 | class RunsOptimizeSquadDistillation(OptimizeSquadDistillation): 8 | 9 | def __init__(self, args_dict): 10 | super().__init__(args_dict) 11 | 12 | 13 | def main(args_dict, config_dict, sigopt_experiment_id, suggestion_id): 14 | logging.info("arguments passed to distillation training: {}".format(args_dict)) 15 | logging.info("configuration passed to distillation training: {}".format(config_dict)) 16 | 17 | runs_optimize_squad_distillation = RunsOptimizeSquadDistillation(args_dict=args_dict) 18 | parameter_values, args_dict, run_training_squad_distillation, model = runs_optimize_squad_distillation.setup_run( 19 | suggestion_id, args_dict, config_dict) 20 | 21 | all_parameters = dict() 22 | all_parameters.update(args_dict) 23 | all_parameters.update(parameter_values) 24 | 25 | with sigopt.create_run(name="Distillation Run_experiment_{}_suggestion_{}".format(sigopt_experiment_id, 26 | suggestion_id)) as run: 27 | run.log_dataset("SQUAD 2.0") 28 | run.log_model("DistilBert for question answering") 29 | run.log_metadata("suggestion_id", suggestion_id) 30 | run.log_metadata("experiment_id", sigopt_experiment_id) 31 | failed, error_str, evaluated_values, results, model = runs_optimize_squad_distillation.try_distillation_tuning( 32 | run_training_squad_distillation, 33 | all_parameters, 34 | model, 35 | run) 36 | if failed is True: 37 | run.log_failure() 38 | run.log_metadata(key="error_str", value=error_str) 39 | else: 40 | run.log_checkpoint(results) 41 | for evaluated_metric in evaluated_values: 42 | run.log_metric(evaluated_metric["name"], evaluated_metric["value"]) 43 | 44 | return model, evaluated_values, failed, error_str 45 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/sigopt_optimization_wrapper/sigopt_hyperparameter_definition.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sigopt_clients import sigopt_experiment_datatypes 3 | from distilbert_run_and_hpo_configurations.distilbert_squad_hpo_parameters import ArchitectureHyperparameter,\ 4 | DistillationHyperparameter, SGDHyperparameter 5 | 6 | LOG_TRANSFORM_HYPERPARAMETERS = [SGDHyperparameter.LEARNING_RATE.value, SGDHyperparameter.ADAM_EPSILON.value, 7 | SGDHyperparameter.WEIGHT_DECAY.value] 8 | 9 | 10 | def get_sigopt_hyperparameter_list(): 11 | """Creates SigOpt formatted hyperparameter dictionary of enums and values above.""" 12 | 13 | parameters_list = [ 14 | dict(name=SGDHyperparameter.LEARNING_RATE.value, bounds=dict(min=np.log(2e-6), max=np.log(1e-1)), 15 | type=sigopt_experiment_datatypes.SigOptDataTypes.DOUBLE.value), 16 | 17 | dict(name=SGDHyperparameter.BETA_1.value, bounds=dict(min=0.7, max=0.9999), 18 | type=sigopt_experiment_datatypes.SigOptDataTypes.DOUBLE.value), 19 | 20 | dict(name=SGDHyperparameter.BETA_2.value, bounds=dict(min=0.7, max=0.9999), 21 | type=sigopt_experiment_datatypes.SigOptDataTypes.DOUBLE.value), 22 | 23 | dict(name=SGDHyperparameter.ADAM_EPSILON.value, bounds=dict(min=np.log(1e-10), max=np.log(1e-5)), 24 | type=sigopt_experiment_datatypes.SigOptDataTypes.DOUBLE.value), 25 | 26 | dict(name=SGDHyperparameter.WEIGHT_DECAY.value, bounds=dict(min=np.log(1e-7), max=np.log(1e-2)), 27 | type=sigopt_experiment_datatypes.SigOptDataTypes.DOUBLE.value), 28 | 29 | dict(name=DistillationHyperparameter.ALPHA_CE.value, bounds=dict(min=0.0000001, max=1.0), 30 | type=sigopt_experiment_datatypes.SigOptDataTypes.DOUBLE.value), 31 | 32 | dict(name=DistillationHyperparameter.TEMPERATURE.value, bounds=dict(min=1, max=10), 33 | type=sigopt_experiment_datatypes.SigOptDataTypes.INT.value), 34 | 35 | dict(name=ArchitectureHyperparameter.INIT_RANGE.value, bounds=dict(min=0.0, max=1.0), 36 | type=sigopt_experiment_datatypes.SigOptDataTypes.DOUBLE.value), 37 | 38 | dict(name=ArchitectureHyperparameter.ATTENTION_DROPOUT.value, bounds=dict(min=0.0, max=1.0), 39 | type=sigopt_experiment_datatypes.SigOptDataTypes.DOUBLE.value), 40 | 41 | dict(name=ArchitectureHyperparameter.QA_DROPOUT.value, bounds=dict(min=0.0, max=1.0), 42 | type=sigopt_experiment_datatypes.SigOptDataTypes.DOUBLE.value), 43 | 44 | dict(name=SGDHyperparameter.WARM_UP_STEPS.value, bounds=dict(min=0, max=100), 45 | type=sigopt_experiment_datatypes.SigOptDataTypes.INT.value), 46 | 47 | dict(name=ArchitectureHyperparameter.N_LAYERS.value, bounds=dict(min=1, max=20), 48 | type=sigopt_experiment_datatypes.SigOptDataTypes.INT.value), 49 | 50 | dict(name=ArchitectureHyperparameter.N_HEADS.value, bounds=dict(min=1, max=12), 51 | type=sigopt_experiment_datatypes.SigOptDataTypes.INT.value), 52 | 53 | dict(name=ArchitectureHyperparameter.PRUNING_SEED.value, bounds=dict(min=1, max=100), 54 | type=sigopt_experiment_datatypes.SigOptDataTypes.INT.value), 55 | 56 | dict(name=ArchitectureHyperparameter.DROPOUT.value, bounds=dict(min=0.0, max=1.0), 57 | type=sigopt_experiment_datatypes.SigOptDataTypes.DOUBLE.value), 58 | 59 | dict(name=SGDHyperparameter.PER_COMPUTE_TRAIN_BATCH_SIZE.value, bounds=dict(min=4, max=32), 60 | type=sigopt_experiment_datatypes.SigOptDataTypes.INT.value), 61 | 62 | dict(name=SGDHyperparameter.PER_COMPUTE_EVAL_BATCH_SIZE.value, bounds=dict(min=4, max=32), 63 | type=sigopt_experiment_datatypes.SigOptDataTypes.INT.value), 64 | ] 65 | 66 | return parameters_list 67 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/sigopt_optimization_wrapper/sigopt_multimetric_definition.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | EXACT_THRESHOLD = 50 4 | 5 | 6 | class MetricObjectives(Enum): 7 | MAXIMIZE = 'maximize' 8 | MINIMIZE = 'minimize' 9 | 10 | 11 | class MetricStrategy(Enum): 12 | OPTIMIZE = 'optimize' 13 | STORE = 'store' 14 | 15 | 16 | class ResultsAttributes(Enum): 17 | EXACT = "exact" 18 | F1 = "f1" 19 | NUM_PARAMETERS = "num_parameters" 20 | INFERENCE_TIME = "inference_time" 21 | 22 | 23 | def get_metrics_list(): 24 | metrics = list() 25 | metrics.append(dict(name=ResultsAttributes.EXACT.value, objective="maximize", strategy="optimize", threshold=EXACT_THRESHOLD)) 26 | metrics.append(dict(name=ResultsAttributes.NUM_PARAMETERS.value, objective="minimize", strategy="optimize")) 27 | metrics.append(dict(name=ResultsAttributes.F1.value, objective="maximize", strategy="store")) 28 | metrics.append(dict(name=ResultsAttributes.INFERENCE_TIME.value, objective="minimize", strategy="store")) 29 | return metrics 30 | 31 | 32 | def get_metric_names(): 33 | return [result_attribute.value for result_attribute in ResultsAttributes] 34 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/sigopt_optimization_wrapper/sigopt_runs_optimization_cycle.py: -------------------------------------------------------------------------------- 1 | from distilbert_run_and_hpo_configurations.distilbert_squad_hpo_parameters import DistillationHyperparameter 2 | import logging 3 | from sigopt_optimization_wrapper.sigopt_multimetric_definition import get_metrics_list 4 | from sigopt_optimization_wrapper import runs_optimize_squad_distillation 5 | 6 | logging.basicConfig( 7 | format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", 8 | datefmt="%m/%d/%Y %H:%M:%S", 9 | level=logging.INFO, 10 | ) 11 | 12 | 13 | class SigOptCycle(object): 14 | 15 | RUNS_NAME = "Distillation Run" 16 | 17 | def __init__(self, sigopt_experiment_client, observation_budget, project_name): 18 | self.sigopt_experiment_client = sigopt_experiment_client 19 | self.observation_budget = observation_budget 20 | self.project_name = project_name 21 | 22 | def create_sigopt_experiment(self, experiment_name, hyperparameter_definition, metadata): 23 | sigopt_experiment = self.sigopt_experiment_client.initialize_bayesian_experiment( 24 | experiment_name=experiment_name, 25 | project_name=self.project_name, 26 | parameters_list=hyperparameter_definition, 27 | metrics_list=get_metrics_list(), 28 | observation_budget=self.observation_budget, 29 | metadata=metadata, 30 | parallel_bandwidth=1, 31 | ) 32 | return sigopt_experiment 33 | 34 | def run_optimization_cycle(self, sigopt_experiment, run_parameters): 35 | while sigopt_experiment.progress.observation_count < sigopt_experiment.observation_budget: 36 | logging.info("On observation count {} of {}".format(sigopt_experiment.progress.observation_count, 37 | sigopt_experiment.observation_budget)) 38 | # get suggestion 39 | suggestion = self.sigopt_experiment_client.get_suggestions(sigopt_experiment) 40 | # run fine-tuning DistilBert on Squad 2.0 41 | parameter_values = suggestion.assignments 42 | # setting flag to track runs 43 | logging.info("running distillation training process on SQUAD 2.0") 44 | model, evaluated_values, failed, error_str = runs_optimize_squad_distillation.main( 45 | run_parameters, 46 | config_dict=parameter_values, 47 | sigopt_experiment_id=sigopt_experiment.id, 48 | suggestion_id=suggestion.id) 49 | 50 | # update experiment with metric values 51 | sigopt_experiment = self.sigopt_experiment_client.update_experiment_multimetric_metadata(experiment=sigopt_experiment, 52 | suggestion=suggestion, 53 | evaluated_value=evaluated_values, 54 | metadata_dict={ 55 | "error_string": 56 | error_str, 57 | DistillationHyperparameter.ALPHA_SQUAD.value: 58 | parameter_values[DistillationHyperparameter.ALPHA_SQUAD.value], 59 | }, 60 | failed=failed) 61 | 62 | return sigopt_experiment 63 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/squad_distillation_abstract_clis/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/bert-distillation-multimetric/squad_distillation_abstract_clis/__init__.py -------------------------------------------------------------------------------- /bert-distillation-multimetric/squad_distillation_abstract_clis/a_optimizaton_run_squad_cli.py: -------------------------------------------------------------------------------- 1 | from squad_distillation_abstract_clis.a_run_squad_w_distillation_cli import ARunDistilBertSquadCLI 2 | from distilbert_run_and_hpo_configurations.distilbert_squad_run_parameters import OptimizationRunParameters 3 | import logging 4 | 5 | logging.basicConfig( 6 | format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", 7 | datefmt="%m/%d/%Y %H:%M:%S", 8 | level=logging.INFO, 9 | ) 10 | 11 | SIGOPT_API_TOKEN = "SIGOPT_API_TOKEN" 12 | 13 | 14 | class OptimizeDistilBertQuadCLI(ARunDistilBertSquadCLI): 15 | 16 | def __init__(self): 17 | super().__init__() 18 | 19 | def define_run_commandline_args(self): 20 | parser = super().define_run_commandline_args() 21 | parser = super().define_common_hpo_commandline_args(parser) 22 | 23 | parser.add_argument(super().parser_prefix + OptimizationRunParameters.EXPERIMENT_NAME.value, type=str, 24 | required=False, 25 | help="SigOpt " 26 | "experiment name") 27 | parser.add_argument(super().parser_prefix + OptimizationRunParameters.PROJECT_NAME.value, type=str, 28 | required=False, 29 | help="SigOpt project name") 30 | parser.add_argument(super().parser_prefix + OptimizationRunParameters.USE_HPO_DEFAULT_RANGES.value, 31 | action="store_true", 32 | help="If flagged, " 33 | "will use default " 34 | "hyperparameter values.") 35 | parser.add_argument(super().parser_prefix + OptimizationRunParameters.API_TOKEN.value, type=str, required=True, 36 | help="SigOpt API Token to use") 37 | 38 | parser.add_argument(super().parser_prefix + OptimizationRunParameters.SIGOPT_EXPERIMENT_ID.value, type=int, 39 | required=False, default=None, 40 | help="If provided, will load SigOpt experiment of given ID.") 41 | 42 | parser.add_argument(super().parser_prefix + OptimizationRunParameters.STORE_S3.value, action="store_true", 43 | help="Flag to store runs on s3") 44 | 45 | parser.add_argument(super().parser_prefix + OptimizationRunParameters.S3_BUCKET.value, type=str, 46 | required=False, 47 | help="s3 bucket name to " 48 | "store run outputs.") 49 | parser.add_argument(super().parser_prefix + OptimizationRunParameters.SIGOPT_OBSERVATION_BUDGET.value, type=int, 50 | required=True, 51 | help="Number of observations for SigOpt") 52 | return parser 53 | 54 | def get_commandline_args(self): 55 | parser = self.define_run_commandline_args() 56 | return parser 57 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/squad_fine_tuning/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/bert-distillation-multimetric/squad_fine_tuning/__init__.py -------------------------------------------------------------------------------- /bert-distillation-multimetric/squad_fine_tuning/set_seed_and_dist.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | 4 | import numpy as np 5 | import torch 6 | 7 | from logger import logger 8 | from distilbert_run_and_hpo_configurations.distilbert_squad_run_parameters import RunParameters 9 | 10 | N_GPU = "n_gpu" 11 | DEVICE = "device" 12 | 13 | 14 | def set_seed(args_dict): 15 | logging.info("setting seed: {}".format(args_dict[RunParameters.SEED.value])) 16 | random.seed(args_dict[RunParameters.SEED.value]) 17 | np.random.seed(args_dict[RunParameters.SEED.value]) 18 | torch.manual_seed(args_dict[RunParameters.SEED.value]) 19 | if args_dict[N_GPU] > 0: 20 | torch.cuda.manual_seed_all(args_dict[RunParameters.SEED.value]) 21 | 22 | 23 | def setup_dist_training(args_dict): 24 | logging.info("setting device") 25 | if args_dict[RunParameters.LOCAL_RANK.value] == -1 or args_dict[RunParameters.NO_CUDA.value]: 26 | device = torch.device( 27 | "cuda" if torch.cuda.is_available() and not args_dict[RunParameters.NO_CUDA.value] else "cpu") 28 | logging.info("Device used is: {}".format(device)) 29 | args_dict[N_GPU] = torch.cuda.device_count() 30 | else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs 31 | torch.cuda.set_device(args_dict[RunParameters.LOCAL_RANK.value]) 32 | device = torch.device("cuda", args_dict[RunParameters.LOCAL_RANK.value]) 33 | torch.distributed.init_process_group(backend="nccl") 34 | args_dict[N_GPU] = 1 35 | args_dict[DEVICE] = device 36 | logger.warning( 37 | "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", 38 | args_dict[RunParameters.LOCAL_RANK.value], 39 | device, 40 | args_dict[N_GPU], 41 | bool(args_dict[RunParameters.LOCAL_RANK.value] != -1), 42 | args_dict[RunParameters.FP_16.value], 43 | ) 44 | return args_dict 45 | -------------------------------------------------------------------------------- /bert-distillation-multimetric/squad_fine_tuning/squad_w_distillation.py: -------------------------------------------------------------------------------- 1 | from squad_fine_tuning.a_squad_w_distillation import ARunSquadDistillation 2 | import logging 3 | from logger import logger 4 | from distilbert_run_and_hpo_configurations.distilbert_squad_run_parameters import RunParameters 5 | from squad_fine_tuning.set_seed_and_dist import setup_dist_training, set_seed, DEVICE 6 | from squad_fine_tuning.training_run_squad_distillation import RunTrainingSquadDistillation 7 | from squad_fine_tuning.a_squad_w_distillation import separate_config 8 | 9 | 10 | class RunSquadDistillation(ARunSquadDistillation): 11 | 12 | def __init__(self, args_dict): 13 | super().__init__(args_dict) 14 | 15 | def run_training(self, run_training_squad_distillation, all_parameters, model): 16 | model, tokenizer = run_training_squad_distillation.run_trainining_and_save(all_parameters, model, 17 | sigopt_run=None) 18 | return model, tokenizer 19 | 20 | def run_eval(self, run_training_squad_distillation, all_parameters): 21 | results, eval_times = run_training_squad_distillation.run_eval(all_parameters) 22 | return results, eval_times 23 | 24 | 25 | def main(args_dict): 26 | 27 | logging.info("arguments passed to distillation training: {}".format(args_dict)) 28 | 29 | args_dict, config_dict = separate_config(all_args_dict=args_dict) 30 | run_squad_distillation = RunSquadDistillation(args_dict=args_dict) 31 | run_squad_distillation.set_parameter_values(config_dict) 32 | run_squad_distillation.run_checks() 33 | 34 | # Setup CUDA, GPU & distributed training 35 | args_dict = setup_dist_training(args_dict) 36 | 37 | # Set seed 38 | set_seed(args_dict) 39 | 40 | # Load pretrained teacherm model, student model, and tokenizer 41 | teacher = run_squad_distillation.get_teacher() 42 | tokenizer = run_squad_distillation.get_tokenizer() 43 | student_loader, model, student_config = run_squad_distillation.get_student() 44 | 45 | model.to(args_dict[DEVICE]) 46 | 47 | run_training_squad_distillation = RunTrainingSquadDistillation(model_class=student_loader.model_class, 48 | tokenizer=tokenizer, 49 | tokenizer_class=student_loader.tokenizer_class, 50 | teacher=teacher) 51 | all_parameters = dict() 52 | all_parameters.update(args_dict) 53 | all_parameters.update(config_dict) 54 | 55 | if args_dict[RunParameters.DO_TRAIN.value] is True: 56 | logging.info("running training and saving checkpoints") 57 | model, tokenizer = run_squad_distillation.run_training(run_training_squad_distillation, all_parameters, 58 | model) 59 | 60 | logging.info("running evaluation") 61 | results, eval_time = run_squad_distillation.run_eval(run_training_squad_distillation, all_parameters) 62 | logger.info("Results: {}".format(results)) 63 | 64 | return model, results, eval_time 65 | -------------------------------------------------------------------------------- /classifier/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # Classifier Tuning 4 | 5 | Machine learning classifier hyperparameter optimization example. 6 | 7 | More details about this example can be found in [the associated blog post](https://sigopt.com/blog/tuning-machine-learning-models/). 8 | 9 | ## Setup 10 | 1. (Optional) Log in to your SigOpt account at [https://app.sigopt.com](https://app.sigopt.com) 11 | 2. (Optional) Find your API Token on the [API tokens page](https://app.sigopt.com/tokens). 12 | 3. Install requirements `pip install -r requirements.txt` 13 | 14 | If you don't provide your SigOpt API Token then this demo will automatically default to using SigOptlite, a in-memory tool which provides a more limited set of capabilities than the full SigOpt service but which is sufficient for this demo. 15 | 16 | ## Run 17 | 18 | Run default example using small sklearn dataset and Gradient Boosting Classifier. 19 | 20 | ```bash 21 | python classifier_tuner.py --client-token API_TOKEN 22 | ``` 23 | 24 | Run using connect-4 dataset (this takes a long time) and Support Vector Classfier 25 | 26 | ```bash 27 | python classifier_tuner.py --classifier-type SVC --dataset-name connect-4 --test-set-size 7557 --client-token API_TOKEN 28 | ``` 29 | 30 | ## Questions? 31 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 32 | 33 | ## API Reference 34 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 35 | 36 | ## About SigOpt 37 | 38 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 39 | 40 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 41 | 42 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API and client libraries (Python, R, Java) integrate into any existing ML workflow. 43 | 44 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 45 | 46 | [Visit our website](https://sigopt.com) to learn more! 47 | -------------------------------------------------------------------------------- /classifier/constant.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | # Parameter search constants 4 | NUM_SIGOPT_SUGGESTIONS = 40 5 | GRID_SEARCH_WIDTH = 4 6 | NUM_RANDOM_SEARCHES = 192 7 | 8 | Dataset = namedtuple( 9 | "Dataset", 10 | [ 11 | "X_train", 12 | "y_train", 13 | "X_test", 14 | "y_test", 15 | ], 16 | ) 17 | 18 | # ML hyperparameter spaces 19 | # Email contact@sigopt.com if you need help 20 | # Examples at http://sigopt.com/docs 21 | 22 | SVC_PARAMS = [ 23 | { 24 | "bounds": { 25 | "max": 10.0, 26 | "min": 0.01, 27 | }, 28 | "name": "C", 29 | "type": "double", 30 | }, 31 | { 32 | "bounds": { 33 | "max": 1.0, 34 | "min": 0.0001, 35 | }, 36 | "name": "gamma", 37 | "type": "double", 38 | }, 39 | { 40 | "type": "categorical", 41 | "name": "kernel", 42 | "categorical_values": [ 43 | {"name": "rbf"}, 44 | {"name": "poly"}, 45 | {"name": "sigmoid"}, 46 | ], 47 | }, 48 | ] 49 | 50 | GBC_PARAMS = [ 51 | { 52 | "bounds": { 53 | "max": 1.0, 54 | "min": 0.01, 55 | }, 56 | "name": "learning_rate", 57 | "type": "double", 58 | }, 59 | { 60 | "bounds": { 61 | "max": 500, 62 | "min": 20, 63 | }, 64 | "name": "n_estimators", 65 | "type": "int", 66 | }, 67 | { 68 | "bounds": { 69 | "max": 5, 70 | "min": 2, 71 | }, 72 | "name": "min_samples_split", 73 | "type": "int", 74 | }, 75 | { 76 | "bounds": { 77 | "max": 3, 78 | "min": 1, 79 | }, 80 | "name": "min_samples_leaf", 81 | "type": "int", 82 | }, 83 | ] 84 | 85 | RFC_PARAMS = [ 86 | { 87 | "bounds": { 88 | "max": 3, 89 | "min": 1, 90 | }, 91 | "name": "min_samples_leaf", 92 | "type": "int", 93 | }, 94 | { 95 | "bounds": { 96 | "max": 20, 97 | "min": 3, 98 | }, 99 | "name": "n_estimators", 100 | "type": "int", 101 | }, 102 | { 103 | "bounds": { 104 | "max": 4, 105 | "min": 1, 106 | }, 107 | "name": "min_samples_split", 108 | "type": "int", 109 | }, 110 | { 111 | "bounds": { 112 | "max": 1.0, 113 | "min": 0.1, 114 | }, 115 | "name": "max_features", 116 | "type": "double", 117 | }, 118 | ] 119 | 120 | CLASSIFIER_TYPE_TO_PARAMS = { 121 | "GBC": GBC_PARAMS, 122 | "SVC": SVC_PARAMS, 123 | "RFC": RFC_PARAMS, 124 | } 125 | -------------------------------------------------------------------------------- /classifier/data/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory, except this file. 2 | * 3 | !.gitignore 4 | -------------------------------------------------------------------------------- /classifier/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scikit-learn 3 | scipy 4 | sigopt 5 | sigopt[lite] 6 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/dnn-tuning-nvidia-mxnet/.gitignore -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # Using SigOpt, NVIDIA, and MXNet to Tune Machine Learning Pipelines 4 | 5 | Learn more at the associated blog post: [Fast CNN Tuning with AWS GPU Instances and SigOpt](https://aws.amazon.com/blogs/ai/fast-cnn-tuning-with-aws-gpu-instances-and-sigopt/). 6 | 7 | _Limited Access: For those interested in replicating this blog post, let us know so that we can provide a SigOpt account for you beyond your free trial!_ 8 | 9 | 10 | ## AWS Setup 11 | 12 | This section will get the following up and running: 13 | 14 | > - OS: Ubuntu 16.04 15 | > - DNN Library: MXNet 0.9.3 16 | > - GPU Driver: CUDA 8.0, cuDNN 5.1 17 | > - GPU: NVIDIA K80 GPU 18 | > - Server: Amazon EC2’s P2 instances 19 | 20 | 1. Sign up for [Amazon Web Services](https://aws.amazon.com/). 21 | 22 | 2. This example is made for the US East (N. Virginia) region by selecting it from the dropdown menu on the top-right of the [EC2 Dashboard](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1). 23 | 24 | 3. Create Key pair in the region you'll be spinning up your instance. 25 | - _Tip: To make it easier to ssh or scp using this keypair, add it to the authentication agent on your machine: `ssh-add /path/to/key.pem`_ 26 | 27 | 4. Pick an instance type and image to be using. This example uses the `p2.xlarge` instance type, though really any [AWS EC2 P2](https://aws.amazon.com/ec2/instance-types/p2/) instance would work. 28 | 29 | 5. Get amazon instance up and running with sufficient storage. One simple way to do this is using and installing the [AWS CLI](https://aws.amazon.com/cli/) and entering the command below. 30 | 31 | - The community image SigOpt made and used to generate this example is _ami-193e860f_. (This image is based on [Ubuntu Cloud's Amazon EC2 AMI Locator](https://cloud-images.ubuntu.com/locator/ec2/) using image id _ami-2757f631_.) 32 | 33 | - ``` 34 | aws ec2 run-instances \ 35 | --image-id ami-193e860f \ 36 | --instance-type p2.xlarge \ 37 | --key-name \ 38 | --ebs-optimized \ 39 | --block-device-mapping \ 40 | "[ { \"DeviceName\": \"/dev/sda1\", \"Ebs\": { \"VolumeSize\": 32 } } ]" 41 | 42 | 6. Wait about a minute to access and reboot your instance. 43 | 44 | - If not operating on a private network, determine the _Public DNS_ of your instance. Otherwise, stick with a private hostname. 45 | - `ssh ubuntu@` 46 | - A system restart seems to be requested by the OS most times when instantiating this image. `sudo reboot` to be safe. 47 | 48 | 7. Verify the NVIDIA GPU is up and running by entering `nvidia-smi`. You should see the status of the GPU driver as indicated below. 49 | 50 | ![nvidia-smi](nvidia-smi.png "proper output") 51 | 52 | ## Replicate Blog Post 53 | 54 | 1. Sign up for a SigOpt account through [our website](https://sigopt.com) or [AWS Marketplace](https://aws.amazon.com/marketplace/pp/B075Y4YKJK). 55 | 56 | 8. Copy this repository over to your new MXNet + NVIDIA Ubuntu instance! 57 | - `scp -r dnn-tuning-nvidia-mxnet/ ubuntu@:/home/ubuntu` 58 | 59 | 9. ssh back into your machine, install the SigOpt Python client: 60 | - `sudo pip install sigopt` 61 | 62 | 10. Add an API key as an environmental variable (get this from the [API tokens page](http://www.sigopt.com/tokens)): 63 | - `export SIGOPT_API_TOKEN=` 64 | 65 | 11. Run the example! 66 | - `cd dnn-tuning-nvidia-mxnet` 67 | - `python run_experiments.py` 68 | - `python run_experiments.py --with-architecture` 69 | - _Protip: Use nohup so you don't have to stay logged in._ 70 | - `nohup python run_experiments.py &> logs-no-architecture.out &` 71 | - `nohup python run_experiments.py --with-architecture &> logs-with-architecture.out &` 72 | 73 | 12. Check out your [experiment dashboard](http://www.sigopt.com/experiments) and view your experiment progress from anywhere! 74 | - As a reminder, we run useful analytics so you may start introspecting how your choice of hyperparameters impact your objective function. 75 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/cnn_text/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/dnn-tuning-nvidia-mxnet/cnn_text/__init__.py -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/cnn_text/dataset.py: -------------------------------------------------------------------------------- 1 | """ 2 | Loads and preprocessed data for the MR dataset. 3 | Returns input vectors, labels, vocabulary, and inverse vocabulary. 4 | """ 5 | from collections import namedtuple 6 | 7 | import numpy as np 8 | 9 | from preprocess import (load_data_and_labels, pad_sentences, 10 | build_vocab, build_input_data) 11 | 12 | Data = namedtuple('Data', ['x_train', 'y_train', 'x_dev', 13 | 'y_dev', 'vocab_size', 'sentence_size']) 14 | 15 | def get_data(): 16 | # Load and preprocess data 17 | sentences, labels = load_data_and_labels() 18 | sentences_padded = pad_sentences(sentences) 19 | vocabulary, vocabulary_inv = build_vocab(sentences_padded) 20 | x, y = build_input_data(sentences_padded, labels, vocabulary) 21 | 22 | vocab_size = len(vocabulary) 23 | 24 | # randomly shuffle data 25 | np.random.seed(10) 26 | shuffle_indices = np.random.permutation(np.arange(len(y))) 27 | x_shuffled = x[shuffle_indices] 28 | y_shuffled = y[shuffle_indices] 29 | 30 | # split train/dev set 31 | # there are a total of 10662 labeled examples to train on 32 | x_train, x_dev = x_shuffled[:-1000], x_shuffled[-1000:] 33 | y_train, y_dev = y_shuffled[:-1000], y_shuffled[-1000:] 34 | 35 | sentence_size = x_train.shape[1] 36 | 37 | print 'Train/Dev split: %d/%d' % (len(y_train), len(y_dev)) 38 | print 'train shape:', x_train.shape 39 | print 'dev shape:', x_dev.shape 40 | print 'vocab_size', vocab_size 41 | print 'sentence max words', sentence_size 42 | 43 | return Data(x_train, y_train, x_dev, y_dev, vocab_size, sentence_size) 44 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/cnn_text/evaluate.py: -------------------------------------------------------------------------------- 1 | import sys, time 2 | import time 3 | 4 | from collections import namedtuple 5 | 6 | import mxnet as mx 7 | import numpy as np 8 | 9 | def evaluate_model(cnn_model, batch_size, 10 | max_grad_norm, learning_rate, 11 | epoch, x_train, y_train, x_dev, 12 | y_dev): 13 | ''' 14 | Train the cnn_model using back prop. 15 | ''' 16 | optimizer='rmsprop' 17 | 18 | print 'optimizer', optimizer 19 | print 'maximum gradient', max_grad_norm 20 | print 'learning rate (step size)', learning_rate 21 | print 'epochs to train for', epoch 22 | 23 | # create optimizer 24 | opt = mx.optimizer.create(optimizer) 25 | opt.lr = learning_rate 26 | 27 | updater = mx.optimizer.get_updater(opt) 28 | 29 | # create logging output 30 | logs = sys.stderr 31 | 32 | # For each training epoch 33 | for iteration in range(epoch): 34 | tic = time.time() 35 | num_correct = 0 36 | num_total = 0 37 | 38 | # Over each batch of training data 39 | for begin in range(0, x_train.shape[0], batch_size): 40 | batchX = x_train[begin:begin+batch_size] 41 | batchY = y_train[begin:begin+batch_size] 42 | if batchX.shape[0] != batch_size: 43 | continue 44 | 45 | cnn_model.data[:] = batchX 46 | cnn_model.label[:] = batchY 47 | 48 | # forward 49 | cnn_model.cnn_exec.forward(is_train=True) 50 | 51 | # backward 52 | cnn_model.cnn_exec.backward() 53 | 54 | # eval on training data 55 | num_correct += sum(batchY == np.argmax(cnn_model.cnn_exec.outputs[0].asnumpy(), axis=1)) 56 | num_total += len(batchY) 57 | 58 | # update weights 59 | norm = 0 60 | for idx, weight, grad, name in cnn_model.param_blocks: 61 | grad /= batch_size 62 | l2_norm = mx.nd.norm(grad).asscalar() 63 | norm += l2_norm * l2_norm 64 | 65 | norm = np.sqrt(norm) 66 | for idx, weight, grad, name in cnn_model.param_blocks: 67 | if norm > max_grad_norm: 68 | grad *= (max_grad_norm / norm) 69 | 70 | updater(idx, grad, weight) 71 | 72 | # reset gradient to zero 73 | grad[:] = 0.0 74 | 75 | # Decay learning rate for this epoch to ensure we are not "overshooting" optima 76 | if iteration % 50 == 0 and iteration > 0: 77 | opt.lr *= 0.5 78 | print >> logs, 'reset learning rate to %g' % opt.lr 79 | 80 | # End of training loop for this epoch 81 | toc = time.time() 82 | train_time = toc - tic 83 | train_acc = num_correct * 100 / float(num_total) 84 | 85 | # Evaluate model after this epoch on dev (test) set 86 | num_correct = 0 87 | num_total = 0 88 | 89 | # For each test batch 90 | for begin in range(0, x_dev.shape[0], batch_size): 91 | batchX = x_dev[begin:begin+batch_size] 92 | batchY = y_dev[begin:begin+batch_size] 93 | 94 | if batchX.shape[0] != batch_size: 95 | continue 96 | 97 | cnn_model.data[:] = batchX 98 | cnn_model.cnn_exec.forward(is_train=False) 99 | 100 | num_correct += sum(batchY == np.argmax(cnn_model.cnn_exec.outputs[0].asnumpy(), axis=1)) 101 | num_total += len(batchY) 102 | 103 | dev_acc = num_correct * 100 / float(num_total) 104 | print >> logs, 'Iter [%d] Train: Time: %.3fs, Training Accuracy: %.3f \ 105 | --- Dev Accuracy thus far: %.3f' % (iteration, train_time, train_acc, dev_acc) 106 | return dev_acc 107 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/cnn_text/graph.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import mxnet as mx 4 | 5 | def build_compute_graph(vocab_size, num_embed, sentence_size, 6 | batch_size, dropout, filter_list, num_filter): 7 | ''' 8 | Define place holders for network inputs and outputs 9 | ''' 10 | 11 | input_x = mx.sym.Variable('data') # placeholder for input data 12 | input_y = mx.sym.Variable('softmax_label') # placeholder for output label 13 | 14 | ''' 15 | Define the first network layer (embedding) 16 | ''' 17 | embed_layer = mx.sym.Embedding(data=input_x, 18 | input_dim=vocab_size, 19 | output_dim=num_embed, 20 | name='vocab_embed') 21 | 22 | # reshape embedded data for next layer 23 | conv_input = mx.sym.Reshape(data=embed_layer, 24 | target_shape=(batch_size, 1, 25 | sentence_size, 26 | num_embed)) 27 | 28 | # create convolution + (max) pooling layer for each filter operation 29 | pooled_outputs = [] 30 | for i, filter_size in enumerate(filter_list): 31 | convi = mx.sym.Convolution(data=conv_input, 32 | kernel=(filter_size, num_embed), 33 | num_filter=num_filter) 34 | relui = mx.sym.Activation(data=convi, act_type='relu') 35 | pooli = mx.sym.Pooling(data=relui, 36 | pool_type='max', 37 | kernel=(sentence_size - filter_size + 1, 1), 38 | stride=(1,1)) 39 | pooled_outputs.append(pooli) 40 | 41 | # combine all pooled outputs 42 | total_filters = num_filter * len(filter_list) 43 | concat = mx.sym.Concat(*pooled_outputs, dim=1) 44 | 45 | # reshape for next layer 46 | h_pool = mx.sym.Reshape(data=concat, 47 | target_shape=(batch_size, total_filters)) 48 | 49 | # dropout layer 50 | if dropout > 0.0: 51 | h_drop = mx.sym.Dropout(data=h_pool, p=dropout) 52 | else: 53 | h_drop = h_pool 54 | 55 | # fully connected layer 56 | num_label=2 57 | 58 | cls_weight = mx.sym.Variable('cls_weight') 59 | cls_bias = mx.sym.Variable('cls_bias') 60 | 61 | fc = mx.sym.FullyConnected(data=h_drop, 62 | weight=cls_weight, 63 | bias=cls_bias, 64 | num_hidden=num_label) 65 | 66 | # softmax output 67 | sm = mx.sym.SoftmaxOutput(data=fc, 68 | label=input_y, 69 | name='softmax') 70 | 71 | # set CNN pointer to the "back" of the network 72 | cnn = sm 73 | 74 | return cnn 75 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/cnn_text/hyperparams.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type": "double", 4 | "name": "dropout_rate", 5 | "bounds": { 6 | "max": 0.9, 7 | "min": 0.1 8 | } 9 | }, 10 | { 11 | "type": "double", 12 | "name": "log_learning_rate", 13 | "bounds": { 14 | "max": 0.0, 15 | "min": -18.420680743952367 16 | } 17 | }, 18 | { 19 | "type": "double", 20 | "name": "max_grad_norm", 21 | "bounds": { 22 | "max": 10, 23 | "min": 1 24 | } 25 | }, 26 | { 27 | "type": "int", 28 | "name": "epochs", 29 | "bounds": { 30 | "max": 100, 31 | "min": 5 32 | } 33 | }, 34 | { 35 | "type": "int", 36 | "bounds": { 37 | "max": 100, 38 | "min": 10 39 | }, 40 | "name": "batch_size" 41 | }, 42 | { 43 | "type": "int", 44 | "bounds": { 45 | "max": 500, 46 | "min": 100 47 | }, 48 | "name": "embed_dim" 49 | } 50 | ] 51 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/cnn_text/long_hyperparams.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type": "double", 4 | "name": "dropout_rate", 5 | "bounds": { 6 | "max": 0.9, 7 | "min": 0.1 8 | } 9 | }, 10 | { 11 | "type": "double", 12 | "name": "log_learning_rate", 13 | "bounds": { 14 | "max": 0.0, 15 | "min": -18.420680743952367 16 | } 17 | }, 18 | { 19 | "type": "double", 20 | "name": "max_grad_norm", 21 | "bounds": { 22 | "max": 10, 23 | "min": 1 24 | } 25 | }, 26 | { 27 | "type": "int", 28 | "name": "epochs", 29 | "bounds": { 30 | "max": 100, 31 | "min": 5 32 | } 33 | }, 34 | { 35 | "type": "int", 36 | "name": "filter_size_1", 37 | "bounds": { 38 | "max": 7, 39 | "min": 1 40 | } 41 | }, 42 | { 43 | "type": "int", 44 | "name": "filter_size_2", 45 | "bounds": { 46 | "max": 7, 47 | "min": 1 48 | } 49 | }, 50 | { 51 | "type": "int", 52 | "name": "filter_size_3", 53 | "bounds": { 54 | "max": 7, 55 | "min": 1 56 | } 57 | }, 58 | { 59 | "type": "int", 60 | "name": "num_feature_maps", 61 | "bounds": { 62 | "max": 200, 63 | "min": 10 64 | } 65 | }, 66 | { 67 | "type": "int", 68 | "bounds": { 69 | "max": 100, 70 | "min": 10 71 | }, 72 | "name": "batch_size" 73 | }, 74 | { 75 | "type": "int", 76 | "bounds": { 77 | "max": 500, 78 | "min": 100 79 | }, 80 | "name": "embed_dim" 81 | } 82 | ] 83 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/cnn_text/model.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | import mxnet as mx 4 | 5 | # Define the structure of our CNN Model (as a named tuple) 6 | CNNModel = namedtuple("CNNModel", ['cnn_exec', 'symbol', 'data', 'label', 'param_blocks']) 7 | 8 | 9 | def get_cnn_model(cnn, batch_size, sentence_size): 10 | # Define what device to train/test on 11 | ctx = mx.gpu(0) 12 | # If you have no GPU on your machine change this to 13 | # ctx=mx.cpu(0) 14 | 15 | arg_names = cnn.list_arguments() 16 | 17 | input_shapes = {} 18 | input_shapes['data'] = (batch_size, sentence_size) 19 | 20 | arg_shape, out_shape, aux_shape = cnn.infer_shape(**input_shapes) 21 | arg_arrays = [mx.nd.zeros(s, ctx) for s in arg_shape] 22 | args_grad = {} 23 | for shape, name in zip(arg_shape, arg_names): 24 | if name in ['softmax_label', 'data']: # input, output 25 | continue 26 | args_grad[name] = mx.nd.zeros(shape, ctx) 27 | 28 | cnn_exec = cnn.bind(ctx=ctx, args=arg_arrays, args_grad=args_grad, grad_req='add') 29 | 30 | param_blocks = [] 31 | arg_dict = dict(zip(arg_names, cnn_exec.arg_arrays)) 32 | initializer = mx.initializer.Uniform(0.1) 33 | for i, name in enumerate(arg_names): 34 | if name in ['softmax_label', 'data']: # input, output 35 | continue 36 | initializer(name, arg_dict[name]) 37 | 38 | param_blocks.append( (i, arg_dict[name], args_grad[name], name) ) 39 | 40 | out_dict = dict(zip(cnn.list_outputs(), cnn_exec.outputs)) 41 | 42 | data = cnn_exec.arg_dict['data'] 43 | label = cnn_exec.arg_dict['softmax_label'] 44 | 45 | cnn_model= CNNModel(cnn_exec=cnn_exec, 46 | symbol=cnn, data=data, 47 | label=label, param_blocks=param_blocks) 48 | 49 | return cnn_model 50 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/cnn_text/objective.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from cnn_text.graph import build_compute_graph 4 | from cnn_text.model import get_cnn_model 5 | from cnn_text.evaluate import evaluate_model 6 | 7 | def calculate_objective(assignments, data, with_architecture=False): 8 | """Return final value to return to SigOpt API""" 9 | if with_architecture: 10 | f1 = assignments['filter_size_1'] 11 | f2 = assignments['filter_size_2'] 12 | f3 = assignments['filter_size_3'] 13 | filter_list = [f1, f2, f3] 14 | num_filter = assignments['num_feature_maps'] 15 | else: 16 | filter_list = [3,4,5] 17 | num_filter = 100 18 | cnn = build_compute_graph(vocab_size=data.vocab_size, 19 | num_embed=assignments['embed_dim'], 20 | sentence_size=data.sentence_size, 21 | batch_size=assignments['batch_size'], 22 | dropout=assignments['dropout_rate'], 23 | filter_list=filter_list, 24 | num_filter=num_filter 25 | ) 26 | cnn_model = get_cnn_model(cnn=cnn, 27 | batch_size=assignments['batch_size'], 28 | sentence_size=data.sentence_size, 29 | ) 30 | dev_acc = evaluate_model(cnn_model=cnn_model, 31 | batch_size=assignments['batch_size'], 32 | max_grad_norm=assignments['max_grad_norm'], 33 | learning_rate=np.exp(assignments['log_learning_rate']), 34 | epoch=assignments['epochs'], 35 | x_train=data.x_train, 36 | y_train=data.y_train, 37 | x_dev=data.x_dev, 38 | y_dev=data.y_dev) 39 | return dev_acc 40 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/cnn_text/preprocess.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | import numpy as np 3 | import re 4 | import itertools 5 | from collections import Counter 6 | 7 | def clean_str(string): 8 | """ 9 | Tokenization/string cleaning for all datasets except for SST. 10 | Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py 11 | """ 12 | string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string) 13 | string = re.sub(r"\'s", " \'s", string) 14 | string = re.sub(r"\'ve", " \'ve", string) 15 | string = re.sub(r"n\'t", " n\'t", string) 16 | string = re.sub(r"\'re", " \'re", string) 17 | string = re.sub(r"\'d", " \'d", string) 18 | string = re.sub(r"\'ll", " \'ll", string) 19 | string = re.sub(r",", " , ", string) 20 | string = re.sub(r"!", " ! ", string) 21 | string = re.sub(r"\(", " \( ", string) 22 | string = re.sub(r"\)", " \) ", string) 23 | string = re.sub(r"\?", " \? ", string) 24 | string = re.sub(r"\s{2,}", " ", string) 25 | return string.strip().lower() 26 | 27 | def load_data_and_labels(): 28 | """ 29 | Loads MR polarity data from files, splits the data into words and generates labels. 30 | Returns split sentences and labels. 31 | """ 32 | # Pull sentences with positive sentiment 33 | pos_file = urllib2.urlopen('https://raw.githubusercontent.com/yoonkim/CNN_sentence/master/rt-polarity.pos') 34 | 35 | # Pull sentences with negative sentiment 36 | neg_file = urllib2.urlopen('https://raw.githubusercontent.com/yoonkim/CNN_sentence/master/rt-polarity.neg') 37 | 38 | # Load data from files 39 | positive_examples = list(pos_file.readlines()) 40 | positive_examples = [s.strip() for s in positive_examples] 41 | negative_examples = list(neg_file.readlines()) 42 | negative_examples = [s.strip() for s in negative_examples] 43 | # Split by words 44 | x_text = positive_examples + negative_examples 45 | x_text = [clean_str(sent) for sent in x_text] 46 | x_text = [s.split(" ") for s in x_text] 47 | # Generate labels 48 | positive_labels = [1 for _ in positive_examples] 49 | negative_labels = [0 for _ in negative_examples] 50 | y = np.concatenate([positive_labels, negative_labels], 0) 51 | return [x_text, y] 52 | 53 | 54 | def pad_sentences(sentences, padding_word=""): 55 | """ 56 | Pads all sentences to the same length. The length is defined by the longest sentence. 57 | Returns padded sentences. 58 | """ 59 | sequence_length = max(len(x) for x in sentences) 60 | padded_sentences = [] 61 | for i in range(len(sentences)): 62 | sentence = sentences[i] 63 | num_padding = sequence_length - len(sentence) 64 | new_sentence = sentence + [padding_word] * num_padding 65 | padded_sentences.append(new_sentence) 66 | return padded_sentences 67 | 68 | 69 | def build_vocab(sentences): 70 | """ 71 | Builds a vocabulary mapping from word to index based on the sentences. 72 | Returns vocabulary mapping and inverse vocabulary mapping. 73 | """ 74 | # Build vocabulary 75 | word_counts = Counter(itertools.chain(*sentences)) 76 | # Mapping from index to word 77 | vocabulary_inv = [x[0] for x in word_counts.most_common()] 78 | # Mapping from word to index 79 | vocabulary = {x: i for i, x in enumerate(vocabulary_inv)} 80 | return [vocabulary, vocabulary_inv] 81 | 82 | 83 | def build_input_data(sentences, labels, vocabulary): 84 | """ 85 | Maps sentences and labels to vectors based on a vocabulary. 86 | """ 87 | x = np.array([[vocabulary[word] for word in sentence] for sentence in sentences]) 88 | y = np.array(labels) 89 | return [x, y] 90 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/multi_run_experiments.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import argparse 4 | import time 5 | 6 | from sigopt import Connection 7 | 8 | from cnn_text.dataset import get_data 9 | from cnn_text.objective import calculate_objective 10 | 11 | parser = argparse.ArgumentParser(description='cnn text classification with SigOpt') 12 | parser.add_argument('--with-architecture', default=False, action='store_true') 13 | parser.add_argument('--experiment-id', type=int) 14 | 15 | args = parser.parse_args() 16 | with_architecture = args.with_architecture 17 | experiment_id = args.experiment_id 18 | 19 | # Instantiate Connection Object 20 | SIGOPT_API_TOKEN = os.environ['SIGOPT_API_TOKEN'] 21 | conn = Connection(client_token=SIGOPT_API_TOKEN) 22 | 23 | # Get experiment object 24 | if experiment_id is None: 25 | 26 | # Get hyperparameters 27 | if with_architecture: 28 | exp_name = 'Multicriteria GPU-powered Sentiment Analysis (SGD + Architecture)' 29 | param_filepath = 'cnn_text/long_hyperparams.json' 30 | else: 31 | exp_name = 'Multicriteria GPU-powered Sentiment Analysis (SGD Only)' 32 | param_filepath='cnn_text/hyperparams.json' 33 | 34 | with open(param_filepath) as f: 35 | hyperparams = f.read() 36 | hyperparams = json.loads(hyperparams) 37 | 38 | experiment = conn.experiments().create( 39 | name=exp_name, 40 | parameters=hyperparams, 41 | observation_budget=30 * len(hyperparams), 42 | metrics=[ 43 | {'name': 'accuracy', 'objective': 'maximize'}, 44 | {'name': 'train_time', 'objective': 'minimize'}, 45 | ], 46 | ) 47 | 48 | print("Created experiment: https://sigopt.com/experiment/" + experiment.id) 49 | else: 50 | experiment = conn.experiments(experiment_id).fetch() 51 | 52 | # Optimization Loop 53 | data = get_data() 54 | 55 | def create_observation_dict(suggestion): 56 | start = time.time() 57 | accuracy = calculate_objective( 58 | suggestion.assignments, 59 | data, 60 | with_architecture=with_architecture, 61 | ) 62 | end = time.time() 63 | return { 64 | 'suggestion': suggestion.id, 65 | 'values': [ 66 | {'name': 'accuracy', 'value': accuracy}, 67 | {'name': 'train_time', 'value': end - start}, 68 | ], 69 | } 70 | 71 | for _ in range(experiment.observation_budget): 72 | suggestion = conn.experiments(experiment.id).suggestions().create() 73 | observation = conn.experiments(experiment.id).observations().create(**create_observation_dict(suggestion)) 74 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/multi_run_experiments_with_failures.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import argparse 4 | import time 5 | 6 | from sigopt import Connection 7 | 8 | from cnn_text.dataset import get_data 9 | from cnn_text.objective import calculate_objective 10 | 11 | parser = argparse.ArgumentParser(description='cnn text classification with SigOpt') 12 | parser.add_argument('--with-architecture', default=False, action='store_true') 13 | parser.add_argument('--experiment-id', type=int) 14 | 15 | args = parser.parse_args() 16 | with_architecture = args.with_architecture 17 | experiment_id = args.experiment_id 18 | 19 | # Instantiate Connection Object 20 | SIGOPT_API_TOKEN = os.environ['SIGOPT_API_TOKEN'] 21 | conn = Connection(client_token=SIGOPT_API_TOKEN) 22 | 23 | # Get experiment object 24 | if experiment_id is None: 25 | 26 | # Get hyperparameters 27 | if with_architecture: 28 | exp_name = 'Multicriteria[failures] GPU-powered Sentiment Analysis (SGD + Architecture)' 29 | param_filepath = 'cnn_text/long_hyperparams.json' 30 | else: 31 | exp_name = 'Multicriteria[failures] GPU-powered Sentiment Analysis (SGD Only)' 32 | param_filepath='cnn_text/hyperparams.json' 33 | 34 | with open(param_filepath) as f: 35 | hyperparams = f.read() 36 | hyperparams = json.loads(hyperparams) 37 | 38 | experiment = conn.experiments().create( 39 | name=exp_name, 40 | parameters=hyperparams, 41 | observation_budget=30 * len(hyperparams), 42 | metrics=[ 43 | {'name': 'accuracy', 'objective': 'maximize'}, 44 | {'name': 'train_time', 'objective': 'minimize'}, 45 | ], 46 | ) 47 | 48 | print("Created experiment: https://sigopt.com/experiment/" + experiment.id) 49 | else: 50 | experiment = conn.experiments(experiment_id).fetch() 51 | 52 | # Optimization Loop 53 | data = get_data() 54 | 55 | def create_observation_dict(suggestion): 56 | start = time.time() 57 | accuracy = calculate_objective( 58 | suggestion.assignments, 59 | data, 60 | with_architecture=with_architecture, 61 | ) 62 | end = time.time() 63 | 64 | failed = True 65 | values = None 66 | duration = end - start 67 | if accuracy > 75 and duration < 250: 68 | values = [ 69 | {'name': 'accuracy', 'value': accuracy}, 70 | {'name': 'train_time', 'value': duration}, 71 | ] 72 | failed = False 73 | return { 74 | 'suggestion': suggestion.id, 75 | 'values': values, 76 | 'failed': failed, 77 | } 78 | 79 | for _ in range(experiment.observation_budget): 80 | suggestion = conn.experiments(experiment.id).suggestions().create() 81 | observation = conn.experiments(experiment.id).observations().create(**create_observation_dict(suggestion)) 82 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/mxnet-setup/install_mxnet_gpu_ubuntu.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ###################################################################### 3 | # This script installs MXNet for Python along with all required dependencies on a Amazon Linux Machine. 4 | ###################################################################### 5 | set -ex 6 | 7 | sudo apt-get update -y 8 | sudo apt-get install -y git 9 | 10 | # Clone mxnet repository. In terminal, run the commands WITHOUT "sudo" 11 | git clone https://github.com/dmlc/mxnet.git ~/mxnet --recursive 12 | # If building with GPU, add configurations to config.mk file: 13 | cd ~/mxnet 14 | cp make/config.mk . 15 | echo "USE_CUDA=1" >>config.mk 16 | echo "USE_CUDA_PATH=/usr/local/cuda" >>config.mk 17 | echo "USE_CUDNN=1" >>config.mk 18 | # Install MXNet for Python with all required dependencies 19 | cd ~/mxnet/setup-utils 20 | bash install-mxnet-ubuntu-python.sh 21 | # We have added MXNet Python package path in your ~/.bashrc. 22 | # Run the following command to refresh environment variables. 23 | source ~/.bashrc 24 | # Install MXNet Python package 25 | #!/usr/bin/env bash 26 | 27 | ###################################################################### 28 | # This script installs MXNet for Python along with all required dependencies on a Ubuntu Machine. 29 | # Tested on Ubuntu 16.04 + distro. 30 | ###################################################################### 31 | 32 | MXNET_HOME="$HOME/mxnet/" 33 | echo "MXNet root folder: $MXNET_HOME" 34 | 35 | echo "Installing build-essential, libatlas-base-dev, libopencv-dev, pip, graphviz ..." 36 | sudo apt-get update 37 | sudo apt-get install -y build-essential libatlas-base-dev libopencv-dev 38 | 39 | echo "Building MXNet core. This can take few minutes..." 40 | cd "$MXNET_HOME" 41 | make -j$(nproc) 42 | 43 | echo "Installing Python setuptools..." 44 | sudo apt-get install -y python-setuptools python-pip 45 | sudo pip install --upgrade pip 46 | 47 | echo "Installing Numpy..." 48 | sudo pip install numpy 49 | 50 | echo "Installing Python package for MXNet..." 51 | cd python; sudo python setup.py install 52 | 53 | echo "Adding MXNet path to your ~/.bashrc file" 54 | echo "export PYTHONPATH=$MXNET_HOME/python:$PYTHONPATH" >> ~/.bashrc 55 | source ~/.bashrc 56 | 57 | echo "Install Graphviz for plotting MXNet network graph..." 58 | sudo pip install graphviz 59 | 60 | echo "Installing Jupyter notebook..." 61 | sudo pip install jupyter 62 | 63 | echo "Done! MXNet for Python installation is complete. Go ahead and explore MXNet with Python :-)" 64 | Contact GitHub API Training Shop Blog About 65 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/mxnet-setup/setup_gpu.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ###################################################################### 3 | # This script installs MXNet for Python along with all required dependencies on a Amazon Linux Machine. 4 | ###################################################################### 5 | set -ex 6 | ######### 7 | # aws ec2 run-instances --image-id ami-2757f631 --instance-type p2.8xlarge --key-name useast1 --ebs-optimized --block-device-mapping "[ { \"DeviceName\": \"/dev/sda1\", \"Ebs\": { \"VolumeSize\": 32 } } ]" 8 | # ssh ubuntu@ 9 | wget https://developer.nvidia.com/compute/cuda/8.0/Prod2/local_installers/cuda-repo-ubuntu1604-8-0-local-ga2_8.0.61-1_amd64-deb 10 | sudo dpkg -i cuda-repo-ubuntu1604-8-0-local-ga2_8.0.61-1_amd64-deb 11 | sudo apt-get update -y 12 | sudo apt-get install -y cuda 13 | #sudo apt install -y awscli 14 | #aws configure 15 | aws s3 cp s3://blog-nvidia-mxnet/cudnn-8.0-linux-x64-v5.1.tgz . 16 | #cudaNN 17 | tar xvzf cudnn-8.0-linux-x64-v5.1.tgz 18 | sudo cp -P cuda/include/cudnn.h /usr/local/cuda/include 19 | sudo cp -P cuda/lib64/libcudnn* /usr/local/cuda/lib64 20 | sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* 21 | sudo ldconfig 22 | ########## 23 | -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/nvidia-smi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/dnn-tuning-nvidia-mxnet/nvidia-smi.png -------------------------------------------------------------------------------- /dnn-tuning-nvidia-mxnet/run_experiments.py: -------------------------------------------------------------------------------- 1 | import os, json, argparse 2 | 3 | from sigopt import Connection 4 | 5 | from cnn_text.dataset import get_data 6 | from cnn_text.objective import calculate_objective 7 | 8 | parser = argparse.ArgumentParser(description='cnn text classification with SigOpt') 9 | parser.add_argument('--with-architecture', default=False, action='store_true') 10 | parser.add_argument('--experiment-id', type=int) 11 | 12 | args = parser.parse_args() 13 | with_architecture = args.with_architecture 14 | experiment_id = args.experiment_id 15 | 16 | # Instantiate Connection Object 17 | SIGOPT_API_TOKEN = os.environ['SIGOPT_API_TOKEN'] 18 | conn = Connection(client_token=SIGOPT_API_TOKEN) 19 | 20 | # Get experiment object 21 | if experiment_id is None: 22 | 23 | # Get hyperparameters 24 | if with_architecture: 25 | exp_name = 'GPU-powered Sentiment Analysis (SGD + Architecture)' 26 | param_filepath = 'cnn_text/long_hyperparams.json' 27 | else: 28 | exp_name = 'GPU-powered Sentiment Analysis (SGD Only)' 29 | param_filepath='cnn_text/hyperparams.json' 30 | 31 | with open(param_filepath) as f: 32 | hyperparams = f.read() 33 | hyperparams = json.loads(hyperparams) 34 | 35 | experiment = conn.experiments().create( 36 | name=exp_name, 37 | metrics=[dict(name='dev_acc', objective='maximize')], 38 | parameters=hyperparams, 39 | observation_budget=40*len(hyperparams)) 40 | 41 | print("Created experiment: https://sigopt.com/experiment/" + experiment.id) 42 | else: 43 | experiment = conn.experiments(experiment_id).fetch() 44 | 45 | # Optimization Loop 46 | data = get_data() 47 | for _ in range(experiment.observation_budget): 48 | suggestion = conn.experiments(experiment.id).suggestions().create() 49 | dev_acc = calculate_objective(suggestion.assignments, data, 50 | with_architecture=with_architecture) 51 | observation = conn.experiments(experiment.id).observations().create( 52 | values=[dict(name="dev_acc", value=dev_acc)], 53 | suggestion=suggestion.id) 54 | -------------------------------------------------------------------------------- /estimated-training-time/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # How are My Hyperparameters Affecting My Training Time? 4 | 5 | Example using SigOpt to explore approximated how long it takes to train your models. 6 | 7 | ![alt tag](metric-vs-time.png) 8 | 9 | ## Setup 10 | 11 | 1. Log in to your SigOpt account at [https://app.sigopt.com](https://app.sigopt.com) 12 | 2. Find your API Token on the [API tokens page](https://app.sigopt.com/tokens) 13 | 3. Recommended: Set the environment variable `SIGOPT_API_TOKEN` to your API token. Alternative: insert your api token into the jupyter notebook. 14 | 4. Clone the repo and install dependencies: 15 | 16 | ``` 17 | git clone https://github.com/sigopt/sigopt-examples.git 18 | cd sigopt-examples/estimated-training-time/ 19 | pip install -r requirements.txt 20 | ``` 21 | 22 | ## Run 23 | 24 | We recommend using [Jupyter](http://jupyter.readthedocs.org/en/latest/install.html) to walk through this example. Run Jupyter: 25 | 26 | ``` 27 | jupyter notebook 28 | ``` 29 | 30 | Then open [`How are My Hyperparameter Affecting My Training Time?.ipynb`](https://github.com/sigopt/sigopt-examples/blob/master/estimated-training-time/How%20are%20My%20Hyperparameters%20Affecting%20My%20Training%20Time%3F.ipynb) in the browser window that pops up. 31 | 32 | ## Questions? 33 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 34 | 35 | ## API Reference 36 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 37 | 38 | ## About SigOpt 39 | 40 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 41 | 42 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 43 | 44 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API and client libraries (Python, R, Java) integrate into any existing ML workflow. 45 | 46 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 47 | 48 | [Visit our website](https://sigopt.com) to learn more! 49 | -------------------------------------------------------------------------------- /estimated-training-time/metric-vs-time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/estimated-training-time/metric-vs-time.png -------------------------------------------------------------------------------- /estimated-training-time/requirements.txt: -------------------------------------------------------------------------------- 1 | jupyter>=1.0.0 2 | plotly>=1.9.0 3 | sigopt>=1.6.4 4 | -------------------------------------------------------------------------------- /get-started/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # Getting Started with SigOpt 4 | 5 | These notebook examples use SigOpt to track and optimize an XGBoost classifier model on the sklearn wine dataset in Python. 6 | 7 | ## Google Colab 8 | 9 | 1. Open the SigOpt Runs demo in https://colab.research.google.com/github/sigopt/sigopt-examples/blob/master/get-started/sigopt_runs_demo.ipynb 10 | 2. Open the SigOpt Experiment and Optimization demo in https://colab.research.google.com/github/sigopt/sigopt-examples/blob/master/get-started/sigopt_experiment_and_optimization_demo.ipynb 11 | 12 | ## Jupyter Setup 13 | 14 | 1. `git clone https:https://docs.sigopt.com/ai-module-api-references/get_started//github.com/sigopt/sigopt-examples.git` 15 | 2. `cd sigopt-examples/get-started` 16 | 3. Run `jupyter lab` in that directory and open sigopt_runs_demo.ipynb or sigopt_experiment_and_optimization_demo.ipynb in the web interface 17 | 4. Add your SigOpt [API token](https://docs.sigopt.com/ai-module-api-references/get_started) when prompted after running the `!sigopt config` command 18 | 5. Run all cells or step through the notebook 19 | 20 | ## Questions? 21 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 22 | 23 | ## API Reference 24 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 25 | 26 | ## About SigOpt 27 | 28 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 29 | 30 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 31 | 32 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API, Python, and R libraries integrate into any existing ML workflow. 33 | 34 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 35 | 36 | [Visit our website](https://sigopt.com) to learn more! 37 | -------------------------------------------------------------------------------- /java/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | 3 | *.class 4 | 5 | # Mobile Tools for Java (J2ME) 6 | .mtj.tmp/ 7 | 8 | # Package Files # 9 | *.jar 10 | *.war 11 | *.ear 12 | 13 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 14 | hs_err_pid* 15 | -------------------------------------------------------------------------------- /java/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | Use SigOpt's [Java API Client](https://docs.sigopt.com/core-module-api-references/get_started) to tune the [Franke function](http://www.sfu.ca/~ssurjano/franke2d.html) 4 | 5 | # Installation 6 | You will need [maven](https://maven.apache.org/) to compile and run this example. 7 | On OS X, you can install maven with Homebrew by running 8 | 9 | ```bash 10 | brew install maven 11 | ``` 12 | 13 | # Compile 14 | 15 | ```bash 16 | mvn compile 17 | ``` 18 | 19 | # Run 20 | Insert your SigOpt API Token into the example below 21 | 22 | ```bash 23 | mvn exec:java -Dexec.mainClass="com.example.App" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--client_token SIGOPT_API_TOKEN" 24 | ``` 25 | 26 | ## Questions? 27 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 28 | 29 | ## API Reference 30 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 31 | ## About SigOpt 32 | 33 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 34 | 35 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 36 | 37 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API and client libraries (Python, R, Java) integrate into any existing ML workflow. 38 | 39 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 40 | 41 | [Visit our website](https://sigopt.com) to learn more! 42 | -------------------------------------------------------------------------------- /java/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.example 5 | my-app 6 | jar 7 | https://jcenter.bintray.com 8 | 1.0-SNAPSHOT 9 | my-app 10 | 11 | 12 | junit 13 | junit 14 | 4.13.1 15 | test 16 | 17 | 18 | com.sigopt 19 | sigopt-java 20 | 3.1.0 21 | 22 | 23 | 24 | 25 | main 26 | 27 | 28 | 29 | org.codehaus.mojo 30 | exec-maven-plugin 31 | 1.1.1 32 | 33 | 34 | test 35 | 36 | java 37 | 38 | 39 | 40 | 41 | com.example.App 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 1.7 50 | 1.7 51 | 52 | 53 | -------------------------------------------------------------------------------- /java/src/main/java/com/example/App.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import java.util.Arrays; 4 | import java.util.Map; 5 | 6 | import com.example.Result; 7 | 8 | import com.sigopt.Sigopt; 9 | import com.sigopt.example.Franke; 10 | import com.sigopt.exception.APIException; 11 | import com.sigopt.model.Assignments; 12 | import com.sigopt.model.Bounds; 13 | import com.sigopt.model.Experiment; 14 | import com.sigopt.model.Observation; 15 | import com.sigopt.model.Parameter; 16 | import com.sigopt.model.Suggestion; 17 | 18 | 19 | public class App 20 | { 21 | public static void main(String[] args) throws APIException, InterruptedException 22 | { 23 | String clientId = null; 24 | for (int i = 0; i < args.length; i++) { 25 | if (args[i].equals("--client_token") && i < args.length - 1) { 26 | Sigopt.clientToken = args[i+1]; 27 | } 28 | } 29 | 30 | Experiment experiment = Experiment.create( 31 | new Experiment.Builder() 32 | .name("Franke Optimization (Java)") 33 | .parameters(Arrays.asList( 34 | new Parameter.Builder() 35 | .name("x") 36 | .type("double") 37 | .bounds(new Bounds(0.0, 1.0)) 38 | .build(), 39 | new Parameter.Builder() 40 | .name("y") 41 | .type("double") 42 | .bounds(new Bounds(0.0, 1.0)) 43 | .build() 44 | )) 45 | .observationBudget(20) 46 | .build()) 47 | .call(); 48 | System.out.println("Created experiment: https://sigopt.com/experiment/" + experiment.getId()); 49 | 50 | // Run the Optimization Loop between 10x - 20x the number of parameters 51 | for (int i = 0; i < experiment.getObservationBudget(); i++) { 52 | Suggestion suggestion = experiment.suggestions().create().call(); 53 | Result result = App.evaluateModel(suggestion.getAssignments()); 54 | Observation observation = experiment.observations().create() 55 | .data(new Observation.Builder() 56 | .suggestion(suggestion.getId()) 57 | .value(result.value) 58 | .valueStddev(result.stddev) 59 | .build()) 60 | .call(); 61 | } 62 | } 63 | 64 | // Evaluate your model with the suggested parameter assignments 65 | // Franke function - http://www.sfu.ca/~ssurjano/franke2d.html 66 | public static Result evaluateModel(Assignments assignments) throws InterruptedException 67 | { 68 | double x = assignments.getDouble("x"); 69 | double y = assignments.getDouble("y"); 70 | double result = Franke.evaluate(x, y); 71 | return new Result(result, null); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /java/src/main/java/com/example/Result.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | class Result 4 | { 5 | Double value; 6 | Double stddev; 7 | 8 | public Result(Double value, Double stddev) 9 | { 10 | this.value = value; 11 | this.stddev = stddev; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /metric-constraints-demo/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # Metric Constraints Demo with Tensorflow/Keras MicronNet 4 | 5 | This example showcases the [Metric Constraints](https://docs.sigopt.com/advanced_experimentation/metric_constraints) feature in SigOpt, as described in this [blog post](https://sigopt.com/blog/metric-constraints-demo/). We use the Metric Constraints feature to optimize for the top-1 accuracy of a CNN with a constraint of the size of the network. We demonstrate this feature using the German Traffic Signs Dataset (GTSRB). 6 | 7 | The CNN model is inspired by the [*MicronNet*](https://arxiv.org/abs/1804.00497) model. 8 | ## Run in Google Colab 9 | 10 | To run the notebooks in Google Colab, click the `Open in Colab` button in each notebook. 11 | 12 | * The `GTSRB_preprocessing_augmentation.ipynb` notebook preprocesses the dataset. 13 | * The `sigopt_metric_constraints_demo.ipynb` notebook runs the SigOpt experiment of tuning the neural network. 14 | 15 | ## Run locally 16 | 17 | Alternatively, you can also run these notebooks locally in Jupyter notebooks. You will need to install the following dependencies. Note that this example uses the Tensorflow2/Keras API. 18 | 19 | ``` 20 | matplotlib 21 | numpy 22 | scikit-image 23 | sigopt 24 | tensorflow 25 | ``` 26 | 27 | ## Questions? 28 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 29 | 30 | ## API Reference 31 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 32 | 33 | ## About SigOpt 34 | 35 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 36 | 37 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 38 | 39 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API, Python, and R libraries integrate into any existing ML workflow. 40 | 41 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 42 | 43 | [Visit our website](https://sigopt.com) to learn more! 44 | -------------------------------------------------------------------------------- /multimetric-timeseries/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | 3 | *.secret 4 | /UCR_TS_Archive_2015/* 5 | !/UCR_TS_Archive_2015/Adiac 6 | -------------------------------------------------------------------------------- /multimetric-timeseries/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # Using SigOpt to tune Time Series Classifiers with Keras and Tensorflow 4 | 5 | 6 | ## Summary 7 | 8 | The data that comes with this example is sequence data extracted from diatoms. We encourage interested readers to replicate this blog using other datasets see how _optimal hyperparameter settings are data dependent_. There are over 80 different sequential datasets in the [UCR Time Series Classification Archive](http://www.cs.ucr.edu/~eamonn/time_series_data/) that can be used to replicate this example. Learn more at the associated blog post: [Deep Learning Hyperparameter Optimization with Competing Objectives](https://devblogs.nvidia.com/parallelforall/sigopt-deep-learning-hyperparameter-optimization/). 9 | 10 | In this example, SigOpt _carves out an efficient [Pareto frontier](https://en.wikipedia.org/wiki/Pareto_efficiency)_, simultaneously looking for hyperparameter configurations which yield fast inference times and high classifier accuracy. 11 | 12 | > _Limited Access: For those interested in replicating this blog post, let us know so that we can provide a SigOpt account for you beyond your free trial!_ 13 | 14 | 15 | ## EC2 Setup 16 | 17 | This example is made for the US East (N. Virginia) region. Follow the instructions from [here](../dnn-tuning-nvidia-mxnet) to get your AWS EC2 GPU-enabled instance up and running with the following specifications: 18 | 19 | > - OS: Ubuntu 16.04 20 | > - GPU Driver: CUDA 8.0, cuDNN 5.1 21 | > - GPU: NVIDIA K80 GPU 22 | > - Server: Amazon EC2’s P2 instances 23 | > - DNN Library: Keras 2.0.5 on Tensorflow 1.2.0 24 | 25 | ## Instructions to Replicate Blog Post 26 | 27 | > Post an issue or email us if you have any questions. 28 | 29 | 1. Log in to your SigOpt account at [https://app.sigopt.com](https://app.sigopt.com) 30 | 31 | 2. Copy this folder over to your EC2 instance. 32 | 33 | - `scp -r multimetric-timeseries/ ubuntu@:/home/ubuntu` 34 | 35 | 3. From your EC2 instance, install SigOpt, Keras, Tensorflow and Pandas. 36 | 37 | - `sudo pip install sigopt tensorflow-gpu pandas` 38 | 39 | 4. Create a file named _sigopt.secret_ and the following line, entering your API key (see the [API tokens page](http://www.sigopt.com/tokens)): 40 | 41 | - `{"SIGOPT_API_TOKEN":""}` 42 | 43 | 5. Navigate to the folder: 44 | 45 | - `cd multimetric-timeseries` 46 | 47 | 7. Run the example! 48 | 49 | - `python main.py` (This can take a while, you may want to run it with `nohup` or `tmux`.) 50 | 51 | 8. Check out your [experiment dashboard](http://www.sigopt.com/experiments) to view your experiment progress! 52 | -------------------------------------------------------------------------------- /multimetric-timeseries/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/multimetric-timeseries/__init__.py -------------------------------------------------------------------------------- /multimetric-timeseries/config.py: -------------------------------------------------------------------------------- 1 | import os, json 2 | 3 | import numpy as np 4 | 5 | THIS_DIR = os.path.dirname(__file__) 6 | 7 | 8 | with open(os.path.join(THIS_DIR, 'sigopt.secret')) as f: 9 | config = f.read() 10 | SIGOPT_API_TOKEN = json.loads(config)['SIGOPT_API_TOKEN'] 11 | 12 | PARAMETERS = [ 13 | dict(name='batch_size', type='int', bounds={'min': 8,'max': 32}), 14 | dict(name='conv_1_num_filters', type='int', bounds={'min': 32,'max': 256}), 15 | dict(name='conv_1_filter_size', type='int', bounds={'min': 2,'max': 10}), 16 | dict(name='conv_2_num_filters', type='int', bounds={'min': 32,'max': 256}), 17 | dict(name='conv_2_filter_size', type='int', bounds={'min': 2,'max': 10}), 18 | dict(name='conv_3_num_filters', type='int', bounds={'min': 32,'max': 256}), 19 | dict(name='conv_3_filter_size', type='int', bounds={'min': 2,'max': 10}), 20 | dict(name='log_lr', type='double', bounds={'min': np.log(1e-10),'max': np.log(1)}), 21 | dict(name='log_beta_1', type='double', bounds={'min': np.log(1e-2),'max': np.log(.5)}), 22 | dict(name='log_beta_2', type='double', bounds={'min': np.log(1e-6),'max': np.log(.5)}), 23 | dict(name='log_epsilon', type='double', bounds={'min': np.log(1e-10),'max': np.log(1e-6)}), 24 | dict(name='log_decay', type='double', bounds={'min': np.log(1e-10),'max': np.log(1e-1)}), 25 | ] 26 | 27 | DATASET = 'Adiac' 28 | DATASET_FOLDER = os.path.join('UCR_TS_Archive_2015', DATASET) 29 | DATASET_FILE = DATASET 30 | DATASET_PATH = os.path.join(THIS_DIR, DATASET_FOLDER, DATASET_FILE) 31 | 32 | EXPERIMENT_NAME = 'multimetric time series accuracy vs. inference time' 33 | 34 | PROJECT_NAME = 'sigopt-examples' 35 | 36 | METRIC_1 = 'val_acc' 37 | METRIC_2 = 'inference_time' 38 | 39 | METRICS = [ 40 | {'name': METRIC_1, 'objective': 'maximize'}, 41 | {'name': METRIC_2, 'objective': 'maximize'}, 42 | ] 43 | 44 | NB_EPOCHS = 500 45 | 46 | OBSERVATION_BUDGET = len(PARAMETERS) * 20 * 2 47 | -------------------------------------------------------------------------------- /multimetric-timeseries/main.py: -------------------------------------------------------------------------------- 1 | from multiprocessing import Process, Queue 2 | 3 | from sigopt import Connection 4 | 5 | from config import (SIGOPT_API_TOKEN, PARAMETERS, EXPERIMENT_NAME, 6 | PROJECT_NAME, METRICS, OBSERVATION_BUDGET, DATASET_PATH) 7 | 8 | from train import prepare_data, evaluate_assignments 9 | 10 | conn = Connection(client_token=SIGOPT_API_TOKEN) 11 | 12 | experiment = conn.experiments().create( 13 | name=EXPERIMENT_NAME, 14 | parameters=PARAMETERS, 15 | metrics = METRICS, 16 | observation_budget = OBSERVATION_BUDGET 17 | ) 18 | 19 | nb_classes, x_train, Y_train, x_test, Y_test = prepare_data(DATASET_PATH) 20 | q = Queue() 21 | 22 | while experiment.progress.observation_count < experiment.observation_budget: 23 | 24 | suggestion = conn.experiments(experiment.id).suggestions().create() 25 | 26 | p = Process(target=evaluate_assignments, args=( 27 | q, experiment, suggestion, x_train, Y_train, 28 | x_test, Y_test, nb_classes) 29 | ) 30 | p.start() 31 | p.join() 32 | metrics, metadata = q.get() 33 | 34 | conn.experiments(experiment.id).observations().create( 35 | suggestion=suggestion.id, 36 | values = metrics, 37 | metadata=metadata 38 | ) 39 | 40 | experiment = conn.experiments(experiment.id).fetch() 41 | -------------------------------------------------------------------------------- /optimizing-memn2n/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 SigOpt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /optimizing-memn2n/end2end_mem_nn_tensorflow/memn2n/data_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import numpy as np 4 | 5 | 6 | def load_task(data_dir, task_id, only_supporting=False): 7 | '''Load the nth task. There are 20 tasks in total. 8 | 9 | Returns a tuple containing the training and testing data for the task. 10 | ''' 11 | assert task_id > 0 and task_id < 21 12 | 13 | files = os.listdir(data_dir) 14 | files = [os.path.join(data_dir, f) for f in files] 15 | s = 'qa{}_'.format(task_id) 16 | train_file = [f for f in files if s in f and 'train' in f][0] 17 | test_file = [f for f in files if s in f and 'test' in f][0] 18 | train_data = get_stories(train_file, only_supporting) 19 | test_data = get_stories(test_file, only_supporting) 20 | return train_data, test_data 21 | 22 | def tokenize(sent): 23 | '''Return the tokens of a sentence including punctuation. 24 | >>> tokenize('Bob dropped the apple. Where is the apple?') 25 | ['Bob', 'dropped', 'the', 'apple', '.', 'Where', 'is', 'the', 'apple', '?'] 26 | ''' 27 | tokens = re.split('(\W+)?', sent) 28 | tokens = list(filter(lambda item: item is not None, tokens)) 29 | return [x.strip() if len(x) >=1 else x for x in tokens] 30 | 31 | 32 | def parse_stories(lines, only_supporting=False): 33 | '''Parse stories provided in the bAbI tasks format 34 | If only_supporting is true, only the sentences that support the answer are kept. 35 | ''' 36 | data = [] 37 | story = [] 38 | for line in lines: 39 | line = str.lower(line) 40 | nid, line = line.split(' ', 1) 41 | nid = int(nid) 42 | if nid == 1: 43 | story = [] 44 | if '\t' in line: # question 45 | q, a, supporting = line.split('\t') 46 | q = tokenize(q) 47 | #a = tokenize(a) 48 | # answer is one vocab word even if it's actually multiple words 49 | a = [a] 50 | substory = None 51 | 52 | # remove question marks 53 | if q[-1] == "?": 54 | q = q[:-1] 55 | 56 | if only_supporting: 57 | # Only select the related substory 58 | supporting = map(int, supporting.split()) 59 | substory = [story[i - 1] for i in supporting] 60 | else: 61 | # Provide all the substories 62 | substory = [x for x in story if x] 63 | 64 | data.append((substory, q, a)) 65 | story.append('') 66 | else: # regular sentence 67 | # remove periods 68 | sent = tokenize(line) 69 | if sent[-1] == ".": 70 | sent = sent[:-1] 71 | story.append(sent) 72 | return data 73 | 74 | 75 | def get_stories(f, only_supporting=False): 76 | '''Given a file name, read the file, retrieve the stories, and then convert the sentences into a single story. 77 | If max_length is supplied, any stories longer than max_length tokens will be discarded. 78 | ''' 79 | with open(f) as f: 80 | return parse_stories(f.readlines(), only_supporting=only_supporting) 81 | 82 | def vectorize_data(data, word_idx, sentence_size, memory_size): 83 | """ 84 | Vectorize stories and queries. 85 | 86 | If a sentence length < sentence_size, the sentence will be padded with 0's. 87 | 88 | If a story length < memory_size, the story will be padded with empty memories. 89 | Empty memories are 1-D arrays of length sentence_size filled with 0's. 90 | 91 | The answer array is returned as a one-hot encoding. 92 | """ 93 | S = [] 94 | Q = [] 95 | A = [] 96 | for story, query, answer in data: 97 | ss = [] 98 | for i, sentence in enumerate(story, 1): 99 | ls = max(0, sentence_size - len(sentence)) 100 | ss.append([word_idx[w] for w in sentence] + [0] * ls) 101 | 102 | # take only the most recent sentences that fit in memory 103 | ss = ss[::-1][:memory_size][::-1] 104 | 105 | # Make the last word of each sentence the time 'word' which 106 | # corresponds to vector of lookup table 107 | for i in range(len(ss)): 108 | ss[i][-1] = len(word_idx) - memory_size - i + len(ss) 109 | 110 | # pad to memory_size 111 | lm = max(0, memory_size - len(ss)) 112 | for _ in range(lm): 113 | ss.append([0] * sentence_size) 114 | 115 | lq = max(0, sentence_size - len(query)) 116 | q = [word_idx[w] for w in query] + [0] * lq 117 | 118 | y = np.zeros(len(word_idx) + 1) # 0 is reserved for nil word 119 | for a in answer: 120 | y[word_idx[a]] = 1 121 | 122 | S.append(ss) 123 | Q.append(q) 124 | A.append(y) 125 | return np.array(S), np.array(Q), np.array(A) 126 | -------------------------------------------------------------------------------- /optimizing-memn2n/end2end_mem_nn_tensorflow/sigopt_memn2n_setup/random_search_parameters_config.py: -------------------------------------------------------------------------------- 1 | from sigopt_memn2n_setup.sigopt_hyperparameters_enum import ParametersList 2 | from sigopt_memn2n_setup.sigopt_hyperparameters_enum import SGDOptimizer 3 | import numpy as np 4 | 5 | parameters_list = [dict(name=ParametersList.LEARNING_RATE.value, bounds=dict(min=np.log(10e-7), max=np.log(1)), type="double"), 6 | dict(name=ParametersList.MOMENTUM.value, bounds=dict(min=0.1, max=0.99), type="double"), 7 | dict(name=ParametersList.NESTEROV.value, type="categorical", categorical_values=[dict(name="true"), dict(name="false")]), 8 | dict(name=ParametersList.BETA_1.value, bounds=dict(min=0.8, max=0.99), type="double"), 9 | dict(name=ParametersList.BETA_2.value, bounds=dict(min=0.95, max=0.9999), type="double"), 10 | dict(name=ParametersList.EPSILON.value, bounds=dict(min=1e-9, max=1e-5), type="double"), 11 | dict(name=ParametersList.DECAY_RATE.value, bounds=dict(min=0.9, max=0.99), type="double"), 12 | dict(name=ParametersList.WORD_EMBEDDING.value, bounds=dict(min=10, max=100), type="int"), 13 | dict(name=ParametersList.MEMORY_SIZE.value, bounds=dict(min=1, max=50), type="int"), 14 | dict(name=ParametersList.HOP_SIZE.value, bounds=dict(min=1, max=3), type="int"), 15 | dict(name=ParametersList.OPTIMIZER.value, type="categorical", 16 | categorical_values=[dict(name=SGDOptimizer.ADAGRAD.value), dict(name=SGDOptimizer.GRADIENT_DESCENT_MOMENTUM.value), 17 | dict(name=SGDOptimizer.RMSPROP.value), dict(name=SGDOptimizer.ADAM.value), 18 | dict(name=SGDOptimizer.ADADELTA.value)]) 19 | ] 20 | metrics_list = [dict(name="accuracy", objective="maximize")] 21 | 22 | conditionals_list = [] 23 | 24 | experiment_type = "random" 25 | -------------------------------------------------------------------------------- /optimizing-memn2n/end2end_mem_nn_tensorflow/sigopt_memn2n_setup/sigopt_conditionals_parameters_config.py: -------------------------------------------------------------------------------- 1 | from sigopt_memn2n_setup.sigopt_hyperparameters_enum import ParametersList 2 | from sigopt_memn2n_setup.sigopt_hyperparameters_enum import SGDOptimizer 3 | import numpy as np 4 | 5 | 6 | conditionals_list = [dict(name=ParametersList.OPTIMIZER.value, values=[SGDOptimizer.ADAM.value, SGDOptimizer.ADAGRAD.value, SGDOptimizer.GRADIENT_DESCENT_MOMENTUM.value, SGDOptimizer.RMSPROP.value, SGDOptimizer.ADADELTA.value])] 7 | 8 | parameters_list = [dict(name=ParametersList.LEARNING_RATE.value, bounds=dict(min=np.log(10e-7), max=np.log(1)), type="double", 9 | conditions=dict(optimizer=[SGDOptimizer.ADAM.value, SGDOptimizer.ADAGRAD.value, SGDOptimizer.RMSPROP.value, SGDOptimizer.GRADIENT_DESCENT_MOMENTUM.value, SGDOptimizer.ADADELTA.value])), 10 | dict(name=ParametersList.MOMENTUM.value, bounds=dict(min=0.1, max=0.99), type="double", 11 | conditions=dict(optimizer=[SGDOptimizer.GRADIENT_DESCENT_MOMENTUM.value, SGDOptimizer.RMSPROP.value])), 12 | dict(name=ParametersList.NESTEROV.value, type="categorical", categorical_values=[dict(name="true"), dict(name="false")], 13 | conditions=dict(optimizer=[SGDOptimizer.GRADIENT_DESCENT_MOMENTUM.value])), 14 | dict(name=ParametersList.BETA_1.value, bounds=dict(min=0.8, max=0.99), type="double", 15 | conditions=dict(optimizer=[SGDOptimizer.ADAM.value])), 16 | dict(name=ParametersList.BETA_2.value, bounds=dict(min=0.95, max=0.9999), type="double", 17 | conditions=dict(optimizer=[SGDOptimizer.ADAM.value])), 18 | dict(name=ParametersList.EPSILON.value, bounds=dict(min=1e-9, max=1e-5), type="double", 19 | conditions=dict(optimizer=[SGDOptimizer.RMSPROP.value, SGDOptimizer.ADADELTA.value, SGDOptimizer.ADAM.value])), 20 | dict(name=ParametersList.DECAY_RATE.value, bounds=dict(min=0.9, max=0.99), type="double", 21 | conditions=dict(optimizer=[SGDOptimizer.RMSPROP.value, SGDOptimizer.ADADELTA.value])), 22 | dict(name=ParametersList.WORD_EMBEDDING.value, bounds=dict(min=10, max=100), type="int"), 23 | dict(name=ParametersList.MEMORY_SIZE.value, bounds=dict(min=1, max=50), type="int"), 24 | dict(name=ParametersList.HOP_SIZE.value, bounds=dict(min=1, max=3), type="int") 25 | ] 26 | metrics_list = [dict(name="accuracy", objective="maximize")] 27 | 28 | experiment_type = "offline" 29 | -------------------------------------------------------------------------------- /optimizing-memn2n/end2end_mem_nn_tensorflow/sigopt_memn2n_setup/sigopt_experiment_client.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | class SigOptExperiment: 4 | 5 | def __init__(self, connection): 6 | self.connection = connection 7 | 8 | def initialize_random_experiment(self, experiment_name, parameters_list, metrics_list, observation_budget, metadata): 9 | return self.initialize_bayesian_experiment(experiment_name, parameters_list, None, metrics_list, observation_budget, metadata, "random") 10 | 11 | def initialize_bayesian_experiment(self, experiment_name, parameters_list, metrics_list, observation_budget, metadata): 12 | return self.initialize_experiment(experiment_name, parameters_list, None, metrics_list, observation_budget, metadata, "offline") 13 | 14 | def initialize_experiment(self, experiment_name, parameters_list, conditionals_list, metrics_list, observation_budget, metadata, experiment_type): 15 | experiment = self.connection.experiments().create( 16 | name=experiment_name, 17 | # Define which parameters you would like to tune 18 | parameters=parameters_list, 19 | conditionals=conditionals_list, 20 | metrics=metrics_list, 21 | parallel_bandwidth=1, 22 | # Define an Observation Budget for your experiment 23 | observation_budget=observation_budget, 24 | metadata=metadata, 25 | type=experiment_type 26 | ) 27 | logging.info("Created experiment: https://sigopt.com/experiment/%s", experiment.id) 28 | 29 | return experiment 30 | 31 | def get_initialized_experiment(self, experiment_id): 32 | return self.connection.experiments(experiment_id).fetch() 33 | 34 | def get_suggestions(self, experiment): 35 | return self.connection.experiments(experiment.id).suggestions().create() 36 | 37 | def get_best_suggestions(self, experiment): 38 | return self.connection.experiments(experiment.id).best_assignments().fetch() 39 | 40 | def update_experiment(self, experiment, suggestion, evaluated_value): 41 | observation = self.connection.experiments(experiment.id).observations().create(suggestion=suggestion.id, value=evaluated_value) 42 | return self.connection.experiments(experiment.id).fetch(), observation 43 | 44 | def update_experiment_metadata(self, experiment, suggestion, evaluated_value, metadata_dict): 45 | logging.info("updating experiment %s with metadata %s", experiment.id, str(metadata_dict)) 46 | observation = self.connection.experiments(experiment.id).observations().create(suggestion=suggestion.id, value=evaluated_value, metadata=metadata_dict) 47 | return self.connection.experiments(experiment.id).fetch(), observation 48 | 49 | def update_experiment_multimetric(self, experiment, suggestion, evaluated_value): 50 | self.connection.experiments(experiment.id).observations().create(suggestion=suggestion.id, values=evaluated_value) 51 | return self.connection.experiments(experiment.id).fetch() 52 | 53 | def create_experiment_metadata(self, experiment, metadata_dict): 54 | self.connection.experiments(experiment.id).observations().create(metadata=metadata_dict) 55 | return self.connection.experiments(experiment.id).fetch() 56 | 57 | def create_observation_metadata(self, experiment, observation, metadata_dict): 58 | updated_observation = self.connection.experiments(experiment.id).observations(observation.id).update(metadata=metadata_dict) 59 | return self.connection.experiments(experiment.id).fetch(), updated_observation 60 | 61 | def get_all_experiments(self): 62 | return self.connection.experiments().fetch() 63 | 64 | def get_all_observations(self, experiment): 65 | return self.connection.experiments(experiment.id).observations().fetch() 66 | 67 | def archive_experiment(self, experiment): 68 | logging.info("archiving experiment with id: %s", experiment.id) 69 | self.connection.experiments(experiment.id).delete() 70 | -------------------------------------------------------------------------------- /optimizing-memn2n/end2end_mem_nn_tensorflow/sigopt_memn2n_setup/sigopt_hyperparameters_enum.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | import tensorflow as tf 3 | 4 | 5 | class ParametersList(Enum): 6 | WORD_EMBEDDING = "word_embedding_size" 7 | MEMORY_SIZE = "memory_size" 8 | HOP_SIZE = "hop_size" 9 | DECAY_RATE = "decay_rate" 10 | EPSILON = "epsilon" 11 | BETA_1 = "beta_1" 12 | BETA_2 = "beta_2" 13 | NESTEROV = "nesterov" 14 | MOMENTUM = "momentum" 15 | LEARNING_RATE = "learning_rate" 16 | OPTIMIZER = "optimizer" 17 | 18 | 19 | class SGDOptimizer(Enum): 20 | ADAM = "Adam" 21 | ADAGRAD = "Adagrad" 22 | GRADIENT_DESCENT_MOMENTUM = "GradientDescentMomentum" 23 | RMSPROP = "RMSProp" 24 | ADADELTA = "Adadelta" 25 | 26 | 27 | ## mapping from string to tensorflow object 28 | optimizer_mapping = {SGDOptimizer.ADAM.value: tf.compat.v1.train.AdamOptimizer, SGDOptimizer.ADAGRAD.value: tf.compat.v1.train.AdagradOptimizer, 29 | SGDOptimizer.GRADIENT_DESCENT_MOMENTUM.value: tf.compat.v1.train.MomentumOptimizer, 30 | SGDOptimizer.RMSPROP.value: tf.compat.v1.train.RMSPropOptimizer} -------------------------------------------------------------------------------- /optimizing-memn2n/end2end_mem_nn_tensorflow/sigopt_memn2n_setup/sigopt_parameters_config.py: -------------------------------------------------------------------------------- 1 | from sigopt_memn2n_setup.sigopt_hyperparameters_enum import ParametersList 2 | from sigopt_memn2n_setup.sigopt_hyperparameters_enum import SGDOptimizer 3 | import numpy as np 4 | 5 | parameters_list = [dict(name=ParametersList.LEARNING_RATE.value, bounds=dict(min=np.log(10e-7), max=np.log(1)), type="double"), 6 | dict(name=ParametersList.MOMENTUM.value, bounds=dict(min=0.1, max=0.99), type="double"), 7 | dict(name=ParametersList.NESTEROV.value, type="categorical", categorical_values=[dict(name="true"), dict(name="false")]), 8 | dict(name=ParametersList.BETA_1.value, bounds=dict(min=0.8, max=0.99), type="double"), 9 | dict(name=ParametersList.BETA_2.value, bounds=dict(min=0.95, max=0.9999), type="double"), 10 | dict(name=ParametersList.EPSILON.value, bounds=dict(min=1e-9, max=1e-5), type="double"), 11 | dict(name=ParametersList.DECAY_RATE.value, bounds=dict(min=0.9, max=0.99), type="double"), 12 | dict(name=ParametersList.WORD_EMBEDDING.value, bounds=dict(min=10, max=100), type="int"), 13 | dict(name=ParametersList.MEMORY_SIZE.value, bounds=dict(min=1, max=50), type="int"), 14 | dict(name=ParametersList.HOP_SIZE.value, bounds=dict(min=1, max=3), type="int"), 15 | dict(name=ParametersList.OPTIMIZER.value, type="categorical", 16 | categorical_values=[dict(name=SGDOptimizer.ADAGRAD.value), dict(name=SGDOptimizer.GRADIENT_DESCENT_MOMENTUM.value), 17 | dict(name=SGDOptimizer.RMSPROP.value), dict(name=SGDOptimizer.ADAM.value), 18 | dict(name=SGDOptimizer.ADADELTA.value)]) 19 | ] 20 | metrics_list = [dict(name="accuracy", objective="maximize")] 21 | 22 | conditionals_list = [] 23 | 24 | experiment_type = "offline" 25 | -------------------------------------------------------------------------------- /optimizing-memn2n/end2end_mem_nn_tensorflow/sigopt_optimization_run.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from memn2n import joint, single 3 | import logging 4 | import sys 5 | from absl import app 6 | from absl import flags 7 | 8 | 9 | # specify output file for logs 10 | flags.DEFINE_string("log_file", "memn2n_optimization_run.log", 11 | "file path. Default will output file: memn2n_optimization_run.log in working directory") 12 | 13 | # specify to run singly or jointly trained experiments 14 | flags.DEFINE_boolean("run_single_exp", None, "running a single experiment with task defined. False by default.") 15 | flags.DEFINE_boolean("run_joint_exp", None, "running joint experiment. True by default.") 16 | 17 | # for joint training only 18 | flags.DEFINE_string("sigopt_calc_accuracy_tasks", "5,17,19", 19 | "Tasks used to measure optimizaiton progress in increasing order. Default 5, 17, 19.") 20 | 21 | # for single training only 22 | flags.DEFINE_integer("task_id", None, "bAbI task id, 1 <= id <= 20") 23 | 24 | flags.DEFINE_integer("evaluation_interval", None, "equal to number of epochs") 25 | 26 | # common flags for memn2n experiments 27 | flags.DEFINE_float("max_grad_norm", 40.0, "Clip gradients to this norm. Default 40") 28 | flags.DEFINE_integer("batch_size", 32, "Batch size for training. Default 32") 29 | flags.DEFINE_integer("epochs", 60, "Number of epochs to train for. Default 60.") 30 | flags.DEFINE_integer("random_state", None, "Random state seed. Default none") 31 | flags.DEFINE_string("data_dir", "data/tasks_1-20_v1-2/en/", 32 | "Directory containing bAbI tasks. Default data/tasks_1-20_v1-2/en/") 33 | 34 | # SigOpt experiment flags 35 | flags.DEFINE_integer("sigopt_observation_budget", None, "Define the observation budget for the SigOpt Experiment") 36 | flags.DEFINE_string("sigopt_connection_token", None, "SigOpt API token") 37 | flags.DEFINE_string("sigopt_experiment_name", None, "Experiment name") 38 | flags.DEFINE_string("sigopt_experiment_id", None, "Existing experiment id. If not none, will be used in experiment.") 39 | flags.DEFINE_string("experiment_type", None, "Must be: random, sigopt, or conditionals") 40 | 41 | 42 | FLAGS=flags.FLAGS 43 | FLAGS(sys.argv) 44 | logging.basicConfig(filename=FLAGS.log_file, format='%(asctime)s %(message)s', level=logging.INFO) 45 | 46 | assert (FLAGS.run_single_exp is True and FLAGS.run_joint_exp is None) or (FLAGS.run_single_exp is None and 47 | FLAGS.run_joint_exp is True) 48 | 49 | # set evaluation interval to number of epochs 50 | FLAGS.evaluation_interval = FLAGS.epochs 51 | 52 | if FLAGS.run_joint_exp: 53 | joint.run_memn2n_joint_training(tensorflow_commandline_flags=FLAGS) 54 | 55 | if FLAGS.run_single_exp: 56 | single.run_memn2n_single_training(tensorflow_commandline_flags=FLAGS) 57 | -------------------------------------------------------------------------------- /optimizing-memn2n/virtual_env_setup/README.md: -------------------------------------------------------------------------------- 1 | # Instructions to create a python virtualenv for running this end to end memory network implementation. 2 | 3 | ### Install pip3 4 | 5 | #### For mac: 6 | 7 | `brew install pip3` 8 | 9 | #### For Ubuntu: 10 | 11 | `sudo apt-get install python3-pip` 12 | 13 | ### Install virtualenv 14 | 15 | `pip3 install virtualenv` 16 | 17 | ### Create a new virtualenv 18 | 19 | `virtualenv -p $(which python3) ` 20 | 21 | activate: 22 | 23 | `source /bin/activate` 24 | 25 | example: 26 | 27 | `virtualenv -p $(which python3) ./memn2n_tf_python3_venv` 28 | `source ./memn2n_tf_python3_venv/bin/activate` 29 | 30 | ### Install requirements 31 | 32 | `pip3 install -r ` 33 | 34 | #### For GPU compatible environment: 35 | 36 | `pip install -r [.]/python3_memn2n_tf_env_gpu_requirements.txt` 37 | 38 | #### For CPU compatible environment: 39 | 40 | `pip install -r [.]/python3_memn2n_tf_env_cpu_requirements.txt` 41 | 42 | ### Activate virtualenv 43 | 44 | `source /bin/activate` 45 | 46 | example: 47 | 48 | `source ./memn2n_tf_python3_venv/bin/activate` 49 | 50 | To deactivate: 51 | 52 | `deactivate` 53 | -------------------------------------------------------------------------------- /optimizing-memn2n/virtual_env_setup/python3_memn2n_tf_env_cpu_requirements.txt: -------------------------------------------------------------------------------- 1 | Werkzeug==3.0.1 2 | absl-py==1.3.0 3 | certifi==2023.7.22 4 | gast==0.4.0 5 | grpcio==1.53.2 6 | h5py==3.7.0 7 | idna==3.7 8 | numpy==1.23.5 9 | protobuf==3.19.6 10 | python-dateutil==2.8.2 11 | requests==2.31.0 12 | scikit-learn==1.2.0 13 | scipy==1.10.0 14 | sigopt==8.6.3 15 | tensorboard==2.10.1 16 | tensorflow-estimator==2.10.0 17 | tensorflow==2.11.1 18 | termcolor==1.1.0 19 | urllib3==1.26.18 20 | -------------------------------------------------------------------------------- /optimizing-memn2n/virtual_env_setup/python3_memn2n_tf_env_gpu_requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.7.1 2 | astor==0.7.1 3 | bleach==3.3.0 4 | certifi==2023.7.22 5 | chardet==3.0.4 6 | gast==0.2.2 7 | grpcio==1.53.2 8 | h5py==2.9.0 9 | html5lib==0.999999999 10 | idna==3.7 11 | Keras-Applications==1.0.7 12 | Keras-Preprocessing==1.0.9 13 | Markdown==3.1 14 | mock==2.0.0 15 | numpy==1.22.0 16 | pandas==0.23.4 17 | pbr==5.2.0 18 | protobuf==3.18.3 19 | python-dateutil==2.7.5 20 | pytz==2018.9 21 | requests==2.31.0 22 | scikit-learn==0.17.1 23 | scipy==1.10.0 24 | sigopt==3.6.0 25 | six==1.10.0 26 | tensorboard==1.13.1 27 | tensorflow-gpu>=1.15.4 28 | tensorflow-estimator==1.13.0 29 | termcolor==1.1.0 30 | urllib3==1.26.18 31 | Werkzeug==3.0.1 32 | -------------------------------------------------------------------------------- /other-languages/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # SigOpt with Other Languages 4 | Our `other_languages` example is one way to use SigOpt when your metric evaluation function is in a language other than python. All you need to do is create an executable file that accepts parameters as command line arguments, and then create an experiment with the same parameter names as the executable. The executable file should accept the suggested parameters at the command line, evaluate the metric, and print out a float. 5 | 6 | ## Setup 7 | 1. Log in to your SigOpt account at [https://app.sigopt.com](https://app.sigopt.com) 8 | 2. Find your API Token on the [API tokens page](https://app.sigopt.com/tokens). 9 | 3. `export CLIENT_TOKEN=` 10 | 4. Install the SigOpt python client `pip install sigopt` 11 | 12 | ## Example Usage 13 | ``` 14 | python other_languages.py --command='' --experiment_id=EXPERIMENT_ID --client_token=$CLIENT_TOKEN 15 | ``` 16 | The above command will run the following sub process to evaluate your metric, automatially requesting the suggsetion beforehand and reporting the observation afterwards: 17 | ``` 18 | --x=SUGGESTION_FOR_X 19 | ``` 20 | 21 | Feel free to use, or modify for your own needs! 22 | 23 | ## Questions? 24 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 25 | 26 | ## API Reference 27 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 28 | 29 | ## About SigOpt 30 | 31 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 32 | 33 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 34 | 35 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API, Python, and R libraries integrate into any existing ML workflow. 36 | 37 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 38 | 39 | [Visit our website](https://sigopt.com) to learn more! 40 | -------------------------------------------------------------------------------- /other-languages/other_languages.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | from subprocess import PIPE, Popen 4 | import sys 5 | 6 | from sigopt import Connection 7 | 8 | class SubProcessEvaluator(object): 9 | def __init__(self, command): 10 | self.command = command 11 | 12 | # Take a suggestion from sigopt and evaluate your function 13 | # Sends command line arguments to your executable file with the same names as the 14 | # parameters of your experiment. Expected output is one line containing a float that 15 | # is your function evaluated at the suggested assignments. 16 | # For example, if your command is './test' and you have one double parameter with suggested 17 | # value 11.05, this script will run 18 | # ./test --x=11.05 19 | def evaluate_metric(self, assignments): 20 | arguments = [ 21 | '--{}={}'.format(param_name, assignment) 22 | for param_name, assignment 23 | in assignments.to_json().iteritems() 24 | ] 25 | process = Popen(self.command.split() + arguments, stdout=PIPE, stderr=PIPE) 26 | (stdoutdata,stderrdata) = process.communicate() 27 | sys.stderr.write(stderrdata) 28 | return float(stdoutdata.strip()) 29 | 30 | 31 | if __name__ == '__main__': 32 | parser = argparse.ArgumentParser() 33 | parser.add_argument( 34 | '--command', 35 | required=True, 36 | help="The command to run the function whose parameters you would " 37 | "like to optimize. Should accept parameters as command line argument and output only the evaluated metric at the " 38 | "suggested point." 39 | ) 40 | parser.add_argument( 41 | '--experiment_id', 42 | required=True, 43 | help="The parameters of this experiment should be the " 44 | "same type and name of the command line arguments to your executable file." 45 | ) 46 | parser.add_argument('--client_token', required=True, help="Find your CLIENT_TOKEN at https://sigopt.com/tokens") 47 | the_args = parser.parse_args() 48 | 49 | connection = Connection(client_token=the_args.client_token) 50 | experiment = connection.experiments(the_args.experiment_id).fetch() 51 | connection.experiments(the_args.experiment_id).suggestions().delete(state="open") 52 | evaluator = SubProcessEvaluator(the_args.command) 53 | 54 | # In a loop: receive a suggestion, evaluate the metric, report an observation 55 | for _ in range(experiment.observation_budget or 10): 56 | suggestion = connection.experiments(experiment.id).suggestions().create() 57 | print('Evaluating at suggested assignments: {0}'.format(suggestion.assignments)) 58 | value = evaluator.evaluate_metric(suggestion.assignments) 59 | print('Reporting observation of value: {0}'.format(value)) 60 | connection.experiments(experiment.id).observations().create( 61 | suggestion=suggestion.id, 62 | values=[dict(name="optimized_metric", value=value)], 63 | ) 64 | -------------------------------------------------------------------------------- /other-languages/requirements.txt: -------------------------------------------------------------------------------- 1 | sigopt>=1.6.4 2 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sigopt/sigopt-examples/6e9a35fbc17ba3fabf2d7a3da262e136cf8bb296/stanford-augmented-image-classification/__init__.py -------------------------------------------------------------------------------- /stanford-augmented-image-classification/a_stanford_car_dataset.py: -------------------------------------------------------------------------------- 1 | from scipy.io import loadmat 2 | import numpy as np 3 | from torch.utils.data import Dataset 4 | from stanford_cars_data_config import CarCommonDatasetAttributes 5 | import logging 6 | logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) 7 | 8 | 9 | class AStanfordCarDataset(Dataset): 10 | 11 | TRANSFORMED_IMAGE = 'train' 12 | LABEL = 'label' 13 | 14 | def __init__(self, data_matrix, path_images, transforms, path_human_readable_labels): 15 | self.data_matrix = data_matrix 16 | self.transforms = transforms 17 | self.path_images = path_images 18 | self.path_human_readable_labels = path_human_readable_labels 19 | self.human_readable_labels = None 20 | self.load_human_readable_labels() 21 | 22 | def load_human_readable_labels(self): 23 | self.human_readable_labels = loadmat(self.path_human_readable_labels)['class_names'] 24 | 25 | def get_label_unique_count(self): 26 | return np.unique(self.data_matrix[CarCommonDatasetAttributes.LABEL], return_counts=True) 27 | 28 | def get_class_distribution(self): 29 | return self.get_label_unique_count()[1]/len(self.data_matrix) 30 | 31 | def __len__(self): 32 | pass 33 | 34 | def __getitem__(self, item): 35 | pass 36 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/a_stanford_data_processor.py: -------------------------------------------------------------------------------- 1 | 2 | class AStanfordDataProcessor(object): 3 | 4 | def __init__(self, path_images, transforms, path_human_readable_labels): 5 | self.path_images = path_images 6 | self.transforms = transforms 7 | self.path_human_readable_labels = path_human_readable_labels 8 | 9 | def preprocess_data(self, path_to_matdata, validation_percentage, data_subset): 10 | pass 11 | 12 | def get_data_generator(self, data_matrix): 13 | pass 14 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/i_commandline_arguments.py: -------------------------------------------------------------------------------- 1 | from a_resnet_training_common_cli import Hyperparameters 2 | from stanford_cars_augmentation_cli import AugmentHyperparameters, AugmentCLI 3 | 4 | 5 | def generate_cli_hpo(parser): 6 | """Adding Hyperparameters to CLI arguments""" 7 | parser.add_argument("--" + Hyperparameters.SCEDULER_RATE.value, dest=Hyperparameters.SCEDULER_RATE.value, 8 | type=float, 9 | help="number of epochs to wait before annealing learning rate", required=True) 10 | parser.add_argument("--" + Hyperparameters.LEARNING_RATE.value, dest=Hyperparameters.LEARNING_RATE.value, 11 | type=float, 12 | help="learning rate to use", required=True) 13 | parser.add_argument("--" + Hyperparameters.BATCH_SIZE.value, dest=Hyperparameters.BATCH_SIZE.value, type=int, 14 | help="batch size to use", required=True) 15 | parser.add_argument("--" + Hyperparameters.LEARNING_RATE_SCHEDULER.value, 16 | dest=Hyperparameters.LEARNING_RATE_SCHEDULER.value, type=float, 17 | help="annealing schedule rate to use. multiplied to learning rate", required=True) 18 | parser.add_argument("--" + Hyperparameters.WEIGHT_DECAY.value, dest=Hyperparameters.WEIGHT_DECAY.value, type=float, 19 | help="weight decay to use", required=True) 20 | parser.add_argument("--" + Hyperparameters.MOMENTUM.value, dest=Hyperparameters.MOMENTUM.value, type=float, 21 | help="momentum to use", required=True) 22 | parser.add_argument("--" + Hyperparameters.NESTEROV.value, dest=Hyperparameters.NESTEROV.value, action='store_true', 23 | help="use Nesterov") 24 | parser.add_argument("--" + "no-" + Hyperparameters.NESTEROV.value, dest=Hyperparameters.NESTEROV.value, 25 | action='store_false', 26 | help="do not use Nesterov") 27 | return parser 28 | 29 | 30 | def generate_cli_hpo_augment(parser): 31 | parser.add_argument("--" + AugmentHyperparameters.BRIGHTNESS.value, dest=AugmentHyperparameters.BRIGHTNESS.value, 32 | type=float, 33 | help="brightness factor. recommended range 0 - 9", required=True, default=3.2907) 34 | parser.add_argument("--" + AugmentHyperparameters.CONTRAST.value, dest=AugmentHyperparameters.CONTRAST.value, 35 | type=float, 36 | help="contrast factor. recommended range 0-100", required=True, default=56.793) 37 | parser.add_argument("--" + AugmentHyperparameters.HUE.value, dest=AugmentHyperparameters.HUE.value, 38 | type=float, 39 | help="hue factor. recommend range -0.5 - 0.5", required=True, default=-0.01286) 40 | parser.add_argument("--" + AugmentHyperparameters.SATURATION.value, dest=AugmentHyperparameters.SATURATION.value, 41 | type=float, 42 | help="saturation factor. recommended range 0-100", required=True, default=2.36640) 43 | return parser 44 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/i_orchestrate_multitask.py: -------------------------------------------------------------------------------- 1 | import sigopt 2 | import logging 3 | from enum import Enum 4 | 5 | 6 | class Multitask(Enum): 7 | full = 1.0 8 | medium = 0.50 9 | small = 0.10 10 | 11 | 12 | def get_assignments(*args): 13 | orchestrate_assignments = dict() 14 | for hyperparameter_enum_class in args: 15 | for hyperparameter_enum in hyperparameter_enum_class: 16 | orchestrate_assignments[hyperparameter_enum.value] = sigopt.get_parameter( 17 | hyperparameter_enum.value) 18 | 19 | logging.info("sigopt assignments being used: {}".format(orchestrate_assignments)) 20 | 21 | task = sigopt.get_task() 22 | percentage_epochs = Multitask[task.name].value 23 | 24 | return orchestrate_assignments, percentage_epochs 25 | 26 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/orchestrate_stanford_cars_augmentation_cli.py: -------------------------------------------------------------------------------- 1 | from stanford_cars_augmentation_cli import StanfordCarsAugmentationCLI, AugmentHyperparameters, CLI 2 | from a_resnet_training_common_cli import Hyperparameters 3 | import math 4 | from i_orchestrate_multitask import get_assignments 5 | 6 | 7 | class OrchestrateAugmentationCLI(StanfordCarsAugmentationCLI): 8 | 9 | def __init__(self): 10 | super().__init__() 11 | 12 | def get_run_arguments(self, parsed_cli_arguments): 13 | orchestrate_assignments, percentage_epochs = get_assignments(Hyperparameters, AugmentHyperparameters) 14 | num_epochs = math.ceil(parsed_cli_arguments[CLI.EPOCHS.value]*percentage_epochs) 15 | return orchestrate_assignments, num_epochs 16 | 17 | 18 | if __name__ == "__main__": 19 | orchestrate_augmentation_cli = OrchestrateAugmentationCLI() 20 | orchestrate_augmentation_cli.run_all() 21 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/orchestrate_stanford_cars_cli.py: -------------------------------------------------------------------------------- 1 | import math 2 | from stanford_cars_cli import CLI, StanfordCarsCLI 3 | from a_resnet_training_common_cli import Hyperparameters 4 | from i_orchestrate_multitask import get_assignments 5 | 6 | 7 | class OrchestrateCLI(StanfordCarsCLI): 8 | def __init__(self): 9 | super().__init__() 10 | 11 | def get_run_arguments(self, parsed_cli_arguments): 12 | orchestrate_assignments, percentage_epochs = get_assignments(Hyperparameters) 13 | num_epochs = math.ceil(parsed_cli_arguments[CLI.EPOCHS.value] * percentage_epochs) 14 | return orchestrate_assignments, num_epochs 15 | 16 | 17 | if __name__ == "__main__": 18 | orchestrate_cli = OrchestrateCLI() 19 | orchestrate_cli.run_all() 20 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/orchestrate_yml_files/orchestrate_cluster_deploy_sample.yml: -------------------------------------------------------------------------------- 1 | # cluster.yml 2 | 3 | provider: aws 4 | 5 | # We have provided a name that is short and descriptive 6 | cluster_name: stanford-cars-run-gpu-cluster 7 | 8 | # Your cluster config can have CPU nodes, GPU nodes, or both. 9 | # The configuration of your nodes is defined in the sections below. 10 | 11 | 12 | # # (Optional) Define GPU compute here 13 | gpu: 14 | # # AWS GPU-enabled instance type 15 | # # This can be any p* instance type 16 | instance_type: p2.xlarge 17 | max_nodes: 2 18 | min_nodes: 2 19 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/orchestrate_yml_files/orchestrate_stanford_cars_augmented_tuning_config.yml: -------------------------------------------------------------------------------- 1 | # The model.py file is written to run in python3.6, so we specify that language here 2 | language: python3.6 3 | 4 | # Select the framework to use for your model 5 | # We want to setup our environment manually so we just choose python 6 | framework: cuda 7 | 8 | # Choose the number of GPUs that your model will use 9 | # We don't need any GPUs for this example, so we'll leave this commented out 10 | resources_per_model: 11 | gpus: 1 12 | 13 | # Choose a descriptive name for your model 14 | name: Stanford Cars Augmentation and Fine Tuning with CUDA and Orchestrate 15 | 16 | # Here, we install the requirements 17 | install: 18 | - pip install -r stanford_cars_venv_requirements.txt 19 | 20 | # Here, we run the model 21 | run: python orchestrate_stanford_cars_augmentation_cli.py --path_images ./stanford-car-classification/data/ --path_data ./stanford-car-classification/data/cars_annos.mat --path_labels ./stanford-car-classification/data/cars_meta.mat --model ResNet18 --epochs 35 --validation_frequency 10 --data_subset 1.0 --number_of_classes 196 --no-freeze_weights --multiplier 1 --store_to_s3 --s3_bucket_name --probability 1.0 22 | # Here, we define optimization details 23 | optimization: 24 | # Use this name when logging a metric with: 25 | # sigopt.log_metric() 26 | # Every experiment needs at least one named metric. 27 | metrics: 28 | - name: accuracy 29 | 30 | # Parameter that are defined here are available to 31 | # sigopt.get_parameter() 32 | parameters: 33 | - name: learning_rate 34 | type: double 35 | bounds: 36 | min: -9.21 37 | max: 0.0 38 | - name: learning_rate_scheduler 39 | type: double 40 | bounds: 41 | min: 0.00 42 | max: 0.99 43 | - name: batch_size 44 | type: int 45 | bounds: 46 | min: 4 47 | max: 8 48 | - name: nesterov 49 | type: categorical 50 | categorical_values: ["True", "False"] 51 | - name: weight_decay 52 | type: double 53 | bounds: 54 | min: -11.3306 55 | max: 0.00 56 | - name: momentum 57 | type: double 58 | bounds: 59 | min: 0.001 60 | max: 0.90 61 | - name: scheduler_rate 62 | type: int 63 | bounds: 64 | min: 0 65 | max: 20 66 | - name: brightness 67 | type: double 68 | bounds: 69 | min: 0 70 | max: 10 71 | - name: contrast 72 | type: double 73 | bounds: 74 | min: 0 75 | max: 100 76 | - name: saturation 77 | type: double 78 | bounds: 79 | min: 0 80 | max: 100 81 | - name: hue 82 | type: double 83 | bounds: 84 | min: -0.5 85 | max: 0.5 86 | tasks: 87 | - name: full 88 | cost: 1.00 89 | - name: medium 90 | cost: 0.50 91 | - name: small 92 | cost: 0.10 93 | 94 | # Our exaxmple cluster has two machines, so we have enough compute power 95 | # to execute two models in parallel. 96 | parallel_bandwidth: 20 97 | 98 | # We want to evaluate our model on sixty different sets of hyperparameters 99 | observation_budget: 220 100 | 101 | # SigOpt Orchestrate creates a container for your model. Since we're using an AWS 102 | # cluster, it's easy to securely store the model in the Amazon Elastic Container Registry. 103 | # Chose a descriptive and unique name for each new experiment configuration file. 104 | image: orchestrate/stanford-cars-augmented-fine-tuning 105 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/orchestrate_yml_files/orchestrate_stanford_cars_tuning_config.yml: -------------------------------------------------------------------------------- 1 | # The model.py file is written to run in python3.6, so we specify that language here 2 | language: python3.6 3 | 4 | # Select the framework to use for your model 5 | # We want to setup our environment manually so we just choose python 6 | framework: cuda 7 | 8 | # Choose the number of GPUs that your model will use 9 | # We don't need any GPUs for this example, so we'll leave this commented out 10 | resources_per_model: 11 | gpus: 1 12 | 13 | # Choose a descriptive name for your model 14 | name: Stanford Cars Orchestrate CUDA Full Tuning 15 | 16 | # Here, we install the requirements 17 | install: 18 | - pip install -r stanford_cars_venv_requirements.txt 19 | 20 | # Here, we run the model 21 | run: python orchestrate_stanford_cars_cli.py --path_images ./stanford-car-classification/data/ --path_data ./stanford-car-classification/data/cars_annos.mat --path_labels ./stanford-car-classification/data/cars_meta.mat --model ResNet18 --epochs 35 --validation_frequency 10 --data_subset 1.0 --number_of_classes 196 --no-freeze_weights 22 | # Here, we define optimization details 23 | optimization: 24 | # Use this name when logging a metric with: 25 | # sigopt.log_metric() 26 | # Every experiment needs at least one named metric. 27 | metrics: 28 | - name: accuracy 29 | 30 | # Parameter that are defined here are available to 31 | # sigopt.get_parameter() 32 | parameters: 33 | - name: learning_rate 34 | type: double 35 | bounds: 36 | min: -9.21 37 | max: 0.0 38 | - name: learning_rate_scheduler 39 | type: double 40 | bounds: 41 | min: 0.00 42 | max: 0.99 43 | - name: batch_size 44 | type: int 45 | bounds: 46 | min: 4 47 | max: 8 48 | - name: nesterov 49 | type: categorical 50 | categorical_values: ["True", "False"] 51 | - name: weight_decay 52 | type: double 53 | bounds: 54 | min: -11.3306 55 | max: 0.00 56 | - name: momentum 57 | type: double 58 | bounds: 59 | min: 0.001 60 | max: 0.90 61 | - name: scheduler_rate 62 | type: int 63 | bounds: 64 | min: 0 65 | max: 20 66 | tasks: 67 | - name: full 68 | cost: 1.00 69 | - name: medium 70 | cost: 0.50 71 | - name: small 72 | cost: 0.10 73 | 74 | # Our exaxmple cluster has two machines, so we have enough compute power 75 | # to execute two models in parallel. 76 | parallel_bandwidth: 20 77 | 78 | # We want to evaluate our model on sixty different sets of hyperparameters 79 | observation_budget: 220 80 | 81 | # SigOpt Orchestrate creates a container for your model. Since we're using an AWS 82 | # cluster, it's easy to securely store the model in the Amazon Elastic Container Registry. 83 | # Chose a descriptive and unique name for each new experiment configuration file. 84 | image: orchestrate/stanford-cars-fine-tuning-2 85 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/resnet_stanford_cars_augmented_training.py: -------------------------------------------------------------------------------- 1 | from stanford_cars_augmentation_cli import StanfordCarsAugmentationCLI, CLI 2 | from i_commandline_arguments import generate_cli_hpo_augment, generate_cli_hpo 3 | 4 | 5 | class ResNetAugmentedTraining(StanfordCarsAugmentationCLI): 6 | 7 | def __init__(self): 8 | super().__init__() 9 | 10 | def arg_parse(self): 11 | parser = super().arg_parse() 12 | parser = generate_cli_hpo(parser) 13 | parser = generate_cli_hpo_augment(parser) 14 | return parser 15 | 16 | def get_run_arguments(self, parsed_cli_dict): 17 | return parsed_cli_dict, parsed_cli_dict[CLI.EPOCHS.value] 18 | 19 | 20 | if __name__ == "__main__": 21 | resnet_augmented_training = ResNetAugmentedTraining() 22 | resnet_augmented_training.run_all() 23 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/resnet_stanford_cars_training.py: -------------------------------------------------------------------------------- 1 | from stanford_cars_cli import CLI, StanfordCarsCLI 2 | from i_commandline_arguments import generate_cli_hpo 3 | 4 | 5 | class ResNetTraining(StanfordCarsCLI): 6 | 7 | def __init__(self): 8 | super().__init__() 9 | 10 | def arg_parse(self): 11 | parser = super().arg_parse() 12 | parser = generate_cli_hpo(parser) 13 | return parser 14 | 15 | def get_run_arguments(self, parsed_cli_dict): 16 | return parsed_cli_dict, parsed_cli_dict[CLI.EPOCHS.value] 17 | 18 | 19 | if __name__ == "__main__": 20 | resnet_training = ResNetTraining() 21 | resnet_training.run_all() 22 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/stanford_augmented_data_processor.py: -------------------------------------------------------------------------------- 1 | from a_stanford_data_processor import AStanfordDataProcessor 2 | from stanford_car_dataset_augmented import StanfordCarAugmentedDataset 3 | from stanford_car_dataset_augmented import preprocess_data 4 | from stanford_cars_data_augmentation import StanfordCarsDataAugmentation 5 | 6 | 7 | class StanfordAugmentedDataProcessor(AStanfordDataProcessor): 8 | 9 | def __init__(self, path_images, transforms, path_human_readable_labels): 10 | super().__init__(path_images, transforms, path_human_readable_labels) 11 | 12 | def augment_data(self, 13 | path_to_data_mat, 14 | augmentation_multiple, 15 | s3_bucket_name, 16 | store_to_disk, 17 | store_to_s3, 18 | probability, 19 | brightness, 20 | contrast, 21 | saturation, 22 | hue 23 | ): 24 | stanford_cars_data_augmentation = StanfordCarsDataAugmentation(path_to_data_mat=path_to_data_mat, 25 | path_images=self.path_images) 26 | augmented_data_directory = stanford_cars_data_augmentation.augment_data( 27 | s3_bucket_name=s3_bucket_name, 28 | store_to_disk=store_to_disk, 29 | store_to_s3=store_to_s3, 30 | augmentation_multiple=augmentation_multiple, 31 | probability=probability, 32 | brightness=brightness, 33 | contrast=contrast, 34 | saturation=saturation, 35 | hue=hue) 36 | return stanford_cars_data_augmentation.get_all_data_matrix(), augmented_data_directory 37 | 38 | def preprocess_data(self, augmented_mat_data, validation_percentage, data_subset): 39 | training_struct, validation_struct, unique_labels = preprocess_data(augmented_mat_data, 40 | validation_percentage, 41 | data_subset) 42 | return training_struct, validation_struct, unique_labels 43 | 44 | def get_data_generator(self, data_matrix): 45 | return StanfordCarAugmentedDataset(data_matrix=data_matrix, 46 | path_images=self.path_images, 47 | transforms=self.transforms, 48 | path_human_readable_labels=self.path_human_readable_labels) 49 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/stanford_car_dataset.py: -------------------------------------------------------------------------------- 1 | from scipy.io import loadmat 2 | import numpy as np 3 | import torchvision 4 | import os 5 | from PIL import Image 6 | import math 7 | from a_stanford_car_dataset import AStanfordCarDataset 8 | 9 | import logging 10 | 11 | from stanford_cars_data_config import CarDatasetAttributes 12 | 13 | logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) 14 | 15 | 16 | class StanfordCarDataset(AStanfordCarDataset): 17 | 18 | def __len__(self): 19 | return self.data_matrix.size 20 | 21 | def __getitem__(self, item): 22 | data_point = self.data_matrix[item] 23 | image_path = os.path.join(self.path_images, data_point[CarDatasetAttributes.REL_IMAGE_PATH][0][0]) 24 | # shifting labels from 1-index to 0-index 25 | label = data_point[CarDatasetAttributes.LABEL][0][0][0] - 1 26 | logging.debug("image: %s is a %s with label %s", image_path, self.human_readable_labels[0, label], label) 27 | image = Image.open(image_path) 28 | if len(np.array(image).shape) < 3: 29 | image = image.convert("RGB") 30 | composed_transforms = torchvision.transforms.Compose(self.transforms) 31 | return {AStanfordCarDataset.TRANSFORMED_IMAGE: composed_transforms(image), AStanfordCarDataset.LABEL: label} 32 | 33 | 34 | def preprocess_data(path_to_matdata, validation_percentage, data_subset): 35 | 36 | logging.info("preprocessing data") 37 | data_struct = loadmat(path_to_matdata) 38 | annotations = data_struct[CarDatasetAttributes.ANNOTATIONS] 39 | annotations_labels = annotations[CarDatasetAttributes.LABEL] 40 | 41 | validation_struct = np.array([]) 42 | training_struct = np.array([]) 43 | 44 | unique_labels = np.unique(annotations_labels) 45 | 46 | for label in unique_labels: 47 | class_label = label[0][0] 48 | class_struct = annotations[annotations[CarDatasetAttributes.LABEL] == class_label] 49 | class_struct = np.reshape(class_struct, (class_struct.shape[0], 1)) 50 | np.random.shuffle(class_struct) 51 | 52 | #subset data 53 | class_struct = class_struct[:math.ceil(class_struct.shape[0] * data_subset)] 54 | 55 | # split data into training and validation 56 | class_struct_shape = class_struct.shape 57 | validation_split = math.floor(class_struct_shape[0] * validation_percentage) 58 | validation_data_points = class_struct[:validation_split] 59 | training_data_points = class_struct[validation_split:] 60 | 61 | if validation_struct.shape[0] == 0: 62 | validation_struct = validation_data_points 63 | else: 64 | validation_struct = np.append(validation_struct, validation_data_points) 65 | 66 | if training_struct.shape[0] == 0: 67 | training_struct = training_data_points 68 | else: 69 | training_struct = np.append(training_struct, training_data_points) 70 | 71 | # shuffle training and validation data 72 | validation_struct = np.reshape(validation_struct, (validation_struct.shape[0],1)) 73 | np.random.shuffle(validation_struct) 74 | 75 | training_struct = np.reshape(training_struct, (training_struct.shape[0],1)) 76 | np.random.shuffle(training_struct) 77 | 78 | return training_struct, validation_struct, unique_labels 79 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/stanford_car_dataset_augmented.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torchvision 3 | import os 4 | from PIL import Image 5 | import math 6 | from a_stanford_car_dataset import AStanfordCarDataset 7 | from stanford_cars_data_config import CarAugmentedDatasetAttributes 8 | 9 | import logging 10 | logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) 11 | 12 | 13 | class StanfordCarAugmentedDataset(AStanfordCarDataset): 14 | 15 | def __len__(self): 16 | return self.data_matrix.size 17 | 18 | def __getitem__(self, item): 19 | data_point = self.data_matrix[item] 20 | image_path = os.path.join(self.path_images, data_point[ 21 | CarAugmentedDatasetAttributes.REL_IMAGE_PATH]) 22 | # shifting labels from 1-index to 0-index 23 | label = data_point[CarAugmentedDatasetAttributes.LABEL] - 1 24 | logging.debug("image: %s is a %s with label %s", image_path, self.human_readable_labels[0, label], label) 25 | image = Image.open(image_path) 26 | if len(np.array(image).shape) < 3: 27 | image = image.convert("RGB") 28 | composed_transforms = torchvision.transforms.Compose(self.transforms) 29 | return {AStanfordCarDataset.TRANSFORMED_IMAGE: composed_transforms(image), AStanfordCarDataset.LABEL: label} 30 | 31 | 32 | def preprocess_data(augmented_matdata, validation_percentage, data_subset): 33 | 34 | logging.info("preprocessing data") 35 | 36 | validation_struct = np.array([]) 37 | training_struct = np.array([]) 38 | 39 | unique_labels = np.unique(augmented_matdata[CarAugmentedDatasetAttributes.LABEL]) 40 | 41 | for label in unique_labels: 42 | class_label = label 43 | class_struct = augmented_matdata[augmented_matdata[CarAugmentedDatasetAttributes.LABEL] == class_label] 44 | class_struct = np.sort(class_struct, order=CarAugmentedDatasetAttributes.IS_AUGMENT) 45 | 46 | #subset data 47 | class_struct = class_struct[:math.ceil(class_struct.shape[0] * data_subset)] 48 | 49 | # split data into training and validation 50 | class_struct_shape = class_struct.shape 51 | validation_split = math.floor(class_struct_shape[0] * validation_percentage) 52 | validation_data_points = class_struct[:validation_split] 53 | training_data_points = class_struct[validation_split:] 54 | 55 | # shuffle validation and training datasets for current label 56 | np.random.shuffle(validation_data_points) 57 | np.random.shuffle(training_data_points) 58 | 59 | if validation_struct.shape[0] == 0: 60 | validation_struct = validation_data_points 61 | else: 62 | validation_struct = np.append(validation_struct, validation_data_points) 63 | 64 | if training_struct.shape[0] == 0: 65 | training_struct = training_data_points 66 | else: 67 | training_struct = np.append(training_struct, training_data_points) 68 | 69 | # shuffle all training and validation data 70 | np.random.shuffle(validation_struct) 71 | np.random.shuffle(training_struct) 72 | 73 | return training_struct, validation_struct, unique_labels 74 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/stanford_cars_cli.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from stanford_car_dataset import StanfordCarDataset, preprocess_data 4 | from a_resnet_training_common_cli import AStanfordCarsCLI, CLI 5 | 6 | 7 | class StanfordCarsCLI(AStanfordCarsCLI): 8 | 9 | def __init__(self): 10 | super().__init__() 11 | 12 | def load_datasets(self, parsed_cli_arguments): 13 | logging.info("loading and preprocessing data") 14 | 15 | training_struct, validation_struct, unique_labels = preprocess_data( 16 | os.path.abspath(parsed_cli_arguments[CLI.DATA.value]), validation_percentage=0.20, 17 | data_subset=parsed_cli_arguments[CLI.DATA_SUBSET.value]) 18 | 19 | training_set = StanfordCarDataset(data_matrix=training_struct, 20 | path_images=os.path.abspath(parsed_cli_arguments[CLI.IMAGES.value]), 21 | transforms=AStanfordCarsCLI.IMAGE_TRANSFORMS, 22 | path_human_readable_labels=os.path.abspath(parsed_cli_arguments[ 23 | CLI.LABELS.value])) 24 | 25 | validation_set = StanfordCarDataset(data_matrix=validation_struct, 26 | path_images=os.path.abspath(parsed_cli_arguments[CLI.IMAGES.value]), 27 | transforms=AStanfordCarsCLI.IMAGE_TRANSFORMS, 28 | path_human_readable_labels=os.path.abspath(parsed_cli_arguments[ 29 | CLI.LABELS.value])) 30 | 31 | assert len(training_set.get_label_unique_count()[0]) == parsed_cli_arguments[CLI.NUM_CLASSES.value] 32 | 33 | return training_set, validation_set, os.path.abspath(parsed_cli_arguments[CLI.IMAGES.value]) 34 | 35 | def run_all(self): 36 | arg_parse = self.arg_parse() 37 | parsed_cli = arg_parse.parse_args() 38 | parsed_cli_dict = parsed_cli.__dict__ 39 | logging.debug("command line arguments: %s", parsed_cli_dict) 40 | training_data, validation_data, images_directory = self.load_datasets(parsed_cli_dict) 41 | parameter_arguments, num_epochs = self.get_run_arguments(parsed_cli_dict) 42 | parameter_arguments.update(parsed_cli_dict) 43 | self.run(parameter_arguments, num_epochs, training_data, validation_data) 44 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/stanford_cars_data_config.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | import logging 3 | logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) 4 | 5 | 6 | class CarCommonDatasetAttributes(object): 7 | LABEL = 'class' 8 | REL_IMAGE_PATH = 'relative_im_path' 9 | ANNOTATIONS = 'annotations' 10 | 11 | 12 | class CarDatasetAttributes(CarCommonDatasetAttributes): 13 | BBOX_X1 = 'bbox_x1' 14 | BBOX_X2 = 'bbox_x2' 15 | BBOX_Y1 = 'bbox_y1' 16 | BBOX_Y2 = 'bbox_y2' 17 | TEST = 'test' 18 | 19 | 20 | class CarAugmentedDatasetAttributes(CarCommonDatasetAttributes): 21 | IS_AUGMENT = 'is_augment' 22 | -------------------------------------------------------------------------------- /stanford-augmented-image-classification/stanford_data_processor.py: -------------------------------------------------------------------------------- 1 | from a_stanford_data_processor import AStanfordDataProcessor 2 | from stanford_car_dataset import StanfordCarDataset 3 | from stanford_car_dataset import preprocess_data 4 | 5 | 6 | class StanfordDataProcessor(AStanfordDataProcessor): 7 | 8 | def __init__(self, path_images, transforms, path_human_readable_labels): 9 | super().__init__(path_images, transforms, path_human_readable_labels) 10 | 11 | def preprocess_data(self, path_to_matdata, validation_percentage, data_subset): 12 | training_struct, validation_struct, unique_labels = preprocess_data(path_to_matdata, 13 | validation_percentage, 14 | data_subset) 15 | return training_struct, validation_struct, unique_labels 16 | 17 | def get_data_generator(self, data_matrix): 18 | return StanfordCarDataset(data_matrix=data_matrix, 19 | path_images=self.path_images, 20 | transforms=self.transforms, 21 | path_human_readable_labels=self.path_human_readable_labels) 22 | -------------------------------------------------------------------------------- /stanford-car-classification/orchestrate_cluster_deploy_sample.yml: -------------------------------------------------------------------------------- 1 | # cluster.yml 2 | 3 | provider: aws 4 | 5 | # We have provided a name that is short and descriptive 6 | cluster_name: stanford-cars-run-gpu-cluster 7 | 8 | # Your cluster config can have CPU nodes, GPU nodes, or both. 9 | # The configuration of your nodes is defined in the sections below. 10 | 11 | 12 | # # (Optional) Define GPU compute here 13 | gpu: 14 | # # AWS GPU-enabled instance type 15 | # # This can be any p* instance type 16 | instance_type: p2.xlarge 17 | max_nodes: 2 18 | min_nodes: 2 19 | -------------------------------------------------------------------------------- /stanford-car-classification/orchestrate_experiment_wrapper_stanford_cars_cli.py: -------------------------------------------------------------------------------- 1 | from resnet import get_pretrained_resnet 2 | import logging 3 | from enum import Enum 4 | from torch.utils.data import DataLoader 5 | import torch 6 | from resnet import PalmNet 7 | import sigopt 8 | import numpy as np 9 | import math 10 | from resnet_stanford_cars_cli import StanfordCarsCLI, Hyperparameters, CLI 11 | 12 | 13 | class OrchestrateCLI(StanfordCarsCLI): 14 | def __init__(self): 15 | super().__init__() 16 | 17 | def run(self, parsed_cli_arguments, training_data, validation_data): 18 | class Multitask(Enum): 19 | full = parsed_cli_arguments[CLI.EPOCHS.value] 20 | medium = math.ceil(parsed_cli_arguments[CLI.EPOCHS.value] * 0.50) 21 | small = math.ceil(parsed_cli_arguments[CLI.EPOCHS.value] * 0.10) 22 | 23 | logging.info("loading pretrained model and establishing model characteristics") 24 | 25 | logging.debug("sigopt assignments being used: %s", 26 | {hyperparameter.value: sigopt.get_parameter(hyperparameter.value) for hyperparameter in 27 | Hyperparameters}) 28 | 29 | resnet_pretrained_model = get_pretrained_resnet(parsed_cli_arguments[CLI.FREEZE_WEIGHTS.value], 30 | parsed_cli_arguments[CLI.NUM_CLASSES.value], 31 | parsed_cli_arguments[CLI.MODEL.value]) 32 | cross_entropy_loss = torch.nn.CrossEntropyLoss() 33 | 34 | sgd_optimizer = torch.optim.SGD(resnet_pretrained_model.parameters(), 35 | lr=np.exp(sigopt.get_parameter(Hyperparameters.LEARNING_RATE.value)), 36 | momentum=sigopt.get_parameter(Hyperparameters.MOMENTUM.value), 37 | weight_decay=np.exp( 38 | sigopt.get_parameter(Hyperparameters.WEIGHT_DECAY.value)), 39 | nesterov=sigopt.get_parameter(Hyperparameters.NESTEROV.value)) 40 | learning_rate_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(sgd_optimizer, mode='min', 41 | factor=sigopt.get_parameter( 42 | Hyperparameters.LEARNING_RATE_SCHEDULER.value), 43 | patience=sigopt.get_parameter( 44 | Hyperparameters.SCEDULER_RATE.value), 45 | verbose=True) 46 | 47 | task = sigopt.get_task() 48 | num_epochs = Multitask[task.name].value 49 | logging.info("task assignment for multitask run: %s at cost %f", task.name, task.cost) 50 | logging.info("task corresponds to %d epochs", num_epochs) 51 | 52 | logging.info("training model") 53 | palm_net = PalmNet(epochs=num_epochs, gd_optimizer=sgd_optimizer, model=resnet_pretrained_model, 54 | loss_function=cross_entropy_loss, 55 | learning_rate_scheduler=learning_rate_scheduler, 56 | validation_frequency=parsed_cli_arguments[CLI.VALIDATION_FREQUENCY.value], 57 | torch_checkpoint_location=parsed_cli_arguments[CLI.CHECKPOINT.value], 58 | model_checkpointing=parsed_cli_arguments[CLI.CHECKPOINT_FREQUENCY.value]) 59 | 60 | trained_model, validation_metric = palm_net.train_model(training_data=DataLoader(training_data, 61 | batch_size=2 ** ( 62 | sigopt.get_parameter( 63 | Hyperparameters.BATCH_SIZE.value)), 64 | shuffle=True), 65 | validation_data=DataLoader(validation_data, 66 | batch_size=2 ** ( 67 | sigopt.get_parameter( 68 | Hyperparameters.BATCH_SIZE.value)), 69 | shuffle=True), 70 | number_of_labels=parsed_cli_arguments[ 71 | CLI.NUM_CLASSES.value]) 72 | 73 | return trained_model, validation_metric 74 | 75 | 76 | if __name__ == "__main__": 77 | orchestrate_cli = OrchestrateCLI() 78 | orchestrate_cli.run_all() 79 | -------------------------------------------------------------------------------- /stanford-car-classification/orchestrate_stanford_cars_tuning_config.yml: -------------------------------------------------------------------------------- 1 | # The model.py file is written to run in python3.6, so we specify that language here 2 | language: python3.6 3 | 4 | # Select the framework to use for your model 5 | # We want to setup our environment manually so we just choose python 6 | framework: cuda 7 | 8 | # Choose the number of GPUs that your model will use 9 | # We don't need any GPUs for this example, so we'll leave this commented out 10 | resources_per_model: 11 | gpus: 1 12 | 13 | # Choose a descriptive name for your model 14 | name: Stanford Cars Orchestrate CUDA Full Tuning 15 | 16 | # Here, we install the requirements 17 | install: 18 | - pip install orchestrate 19 | - pip install sklearn 20 | - pip install matplotlib 21 | - pip install torch torchvision 22 | - pip install botocore 23 | - pip install retrying 24 | 25 | # Here, we run the model 26 | run: python orchestrate_experiment_wrapper_stanford_cars_cli.py --path_images ./data/ --path_data ./data/cars_annos.mat --path_labels ./data/cars_meta.mat --model ResNet18 --epochs 4 --validation_frequency 10 --data_subset 1.0 --number_of_classes 196 --freeze_weights 27 | 28 | # Here, we define optimization details 29 | optimization: 30 | # Use this name when logging a metric with: 31 | # sigopt.log_metric() 32 | # Every experiment needs at least one named metric. 33 | metrics: 34 | - name: accuracy 35 | 36 | # Parameter that are defined here are available to 37 | # sigopt.get_parameter() 38 | parameters: 39 | - name: learning_rate 40 | type: double 41 | bounds: 42 | min: -9.21 43 | max: 0.0 44 | - name: learning_rate_scheduler 45 | type: double 46 | bounds: 47 | min: 0.00 48 | max: 0.99 49 | - name: batch_size 50 | type: int 51 | bounds: 52 | min: 4 53 | max: 8 54 | - name: nesterov 55 | type: categorical 56 | categorical_values: ["True", "False"] 57 | - name: weight_decay 58 | type: double 59 | bounds: 60 | min: -11.3306 61 | max: 0.00 62 | - name: momentum 63 | type: double 64 | bounds: 65 | min: 0.001 66 | max: 0.90 67 | - name: scheduler_rate 68 | type: int 69 | bounds: 70 | min: 0 71 | max: 20 72 | tasks: 73 | - name: full 74 | cost: 1.00 75 | - name: medium 76 | cost: 0.50 77 | - name: small 78 | cost: 0.10 79 | 80 | # Our exaxmple cluster has two machines, so we have enough compute power 81 | # to execute two models in parallel. 82 | parallel_bandwidth: 20 83 | 84 | # We want to evaluate our model on sixty different sets of hyperparameters 85 | observation_budget: 220 86 | 87 | # SigOpt Orchestrate creates a container for your model. Since we're using an AWS 88 | # cluster, it's easy to securely store the model in the Amazon Elastic Container Registry. 89 | # Chose a descriptive and unique name for each new experiment configuration file. 90 | image: orchestrate/stanford-cars-fine-tuning-2 91 | -------------------------------------------------------------------------------- /stanford-car-classification/stanford_cars_venv_requirements.txt: -------------------------------------------------------------------------------- 1 | adal==1.2.2 2 | awscli==1.16.14 3 | boto3==1.9.4 4 | botocore==1.12.4 5 | cachetools==4.0.0 6 | cement==2.10.12 7 | certifi==2023.7.22 8 | cffi==1.14.0 9 | chardet==3.0.4 10 | colorama==0.3.9 11 | cryptography==42.0.4 12 | cycler==0.10.0 13 | docker==3.4.1 14 | docker-pycreds==0.4.0 15 | docutils==0.16 16 | google-auth==1.12.0 17 | idna==3.7 18 | jmespath==0.9.5 19 | joblib==1.2.0 20 | kiwisolver==1.1.0 21 | kubernetes==8.0.0 22 | matplotlib==3.2.1 23 | numpy==1.22.0 24 | oauthlib==3.1.0 25 | orchestrate==0.8.2 26 | Pillow==10.3.0 27 | pyasn1==0.4.8 28 | pyasn1-modules==0.2.8 29 | pycparser==2.20 30 | PyJWT==2.4.0 31 | pyparsing==2.4.6 32 | pystache==0.5.4 33 | python-dateutil==2.8.1 34 | PyYAML==5.4 35 | requests==2.31.0 36 | requests-oauthlib==1.3.0 37 | rsa==4.7 38 | s3transfer==0.1.13 39 | scikit-learn==0.22.2.post1 40 | scipy==1.10.0 41 | sigopt==7.3.0 42 | six==1.11.0 43 | sklearn==0.0 44 | torch>=1.13.0 45 | torchvision>=0.14.0 46 | urllib3==1.26.18 47 | websocket-client==0.57.0 48 | -------------------------------------------------------------------------------- /surprise-py-recsys/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # Surprise Lib (scikit-surprise) Recommendation System Python Jupyter Notebook Model Tuning 4 | 5 | This example uses SigOpt to tune a Surprise Lib recommender model based on the SVD approach that influence the winners of the Netflix prize in the 2009-20012 time frame. 6 | 7 | ## Jupyter Setup 8 | 9 | 1. `git clone https://github.com/sigopt/sigopt-examples.git` 10 | 2. `cd sigopt-examples/surprise-py-recsys` 11 | 3. Run `jupyter lab` in that directory and open surprise_recommender.ipynb in the web interface 12 | 4. Add your SigOpt [API token](https://docs.sigopt.com/core-module-api-references/get_started) in the Jupyter cell where you see `YOUR_API_TOKEN_HERE` 13 | 5. Run all cells or step through the notebook 14 | 15 | 16 | ## Optimize 17 | 18 | Once the SigOpt optimization loop is initiated, you can track the progress on your [experiment dashboard](https://sigopt.com/experiments). 19 | 20 | ## Questions? 21 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 22 | 23 | ## API Reference 24 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 25 | 26 | ## About SigOpt 27 | 28 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 29 | 30 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 31 | 32 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API, Python, and R libraries integrate into any existing ML workflow. 33 | 34 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 35 | 36 | [Visit our website](https://sigopt.com) to learn more! 37 | -------------------------------------------------------------------------------- /test/README: -------------------------------------------------------------------------------- 1 | These is code for testing the SigOpt examples to make sure they work. 2 | 3 | You probably don't need to look at these files directly - check out 4 | the individual examples in the base directory of this package. 5 | -------------------------------------------------------------------------------- /test/other_languages_tester: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo 1.0 4 | -------------------------------------------------------------------------------- /test/run_python: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | TEST_DIR="$1" 6 | shift 7 | 8 | TEST_FILE="$1" 9 | shift 10 | 11 | cd "$TEST_DIR" 12 | # We allow system site packages to make installing numpy/scipy less burdensome 13 | virtualenv venv --system-site-packages 14 | source venv/bin/activate 15 | if [ -f requirements.txt ]; then 16 | cat requirements.txt | grep sigopt # sigopt python package must be included 17 | set +e 18 | cat requirements.txt | grep -v numpy | grep -v scipy >requirements-clean.txt 19 | set -e 20 | pip install -r requirements-clean.txt 21 | fi 22 | exec python "$TEST_FILE" "$@" 23 | -------------------------------------------------------------------------------- /test/with_new_experiment: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import print_function 4 | 5 | import argparse 6 | import os 7 | import sigopt 8 | import subprocess 9 | import sys 10 | 11 | if __name__ == "__main__": 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument('rest', nargs=argparse.REMAINDER) 14 | args = parser.parse_args().rest 15 | 16 | conn = sigopt.Connection(client_token=os.environ['SIGOPT_API_TOKEN']) 17 | e = conn.experiments().create( 18 | name='sigopt-examples test experiment', 19 | metrics=[dict(name='score', objective='maximize')], 20 | parameters=[ 21 | {'type': 'double', 'name': 'd', 'bounds': {'min': 10, 'max': 20}}, 22 | {'type': 'int', 'name': 'i', 'bounds': {'min': 1, 'max': 100}}, 23 | {'type': 'categorical', 'name': 'c', 'categorical_values': [{'name': 'a'}, {'name': 'b'}]}, 24 | ], 25 | ) 26 | args = [(e.id if a == 'NEW_EXPERIMENT_ID' else a) for a in args] 27 | p = subprocess.Popen( 28 | args=args, 29 | stdout=subprocess.PIPE, 30 | stderr=subprocess.PIPE, 31 | env=os.environ, 32 | ) 33 | out, err = p.communicate() 34 | print(out) 35 | print(err, file=sys.stderr) 36 | sys.exit(p.wait()) 37 | -------------------------------------------------------------------------------- /text-classifier/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # Text Classifier Tuning 4 | 5 | Example using SigOpt to tune logistic regression model for text sentiment classification. This example is offered in Python and R. 6 | 7 | More details about this example can be found in [the associated blog post](https://sigopt.com/blog/automatically-tuning-text-classifiers/). 8 | 9 | ## [Python](python) 10 | * Python API Client 11 | * Notebook Version of Python API Client 12 | * SigOpt + scikit-learn Integration 13 | 14 | ## [R](r) 15 | * R API Client 16 | 17 | ## Questions? 18 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 19 | 20 | ## API Reference 21 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 22 | 23 | ## About SigOpt 24 | 25 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 26 | 27 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 28 | 29 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API, Python, and R libraries integrate into any existing ML workflow. 30 | 31 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 32 | 33 | [Visit our website](https://sigopt.com) to learn more! 34 | -------------------------------------------------------------------------------- /text-classifier/python/.gitignore: -------------------------------------------------------------------------------- 1 | POSITIVE_LIST.json 2 | NEGATIVE_LIST.json 3 | -------------------------------------------------------------------------------- /text-classifier/python/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # Text Classifier Tuning Python Example 4 | 5 | Example using SigOpt and Python to tune logistic regression model for text sentiment classification. 6 | 7 | More details about this example can be found in [the associated blog post](https://sigopt.com/blog/automatically-tuning-text-classifiers/). 8 | 9 | ## Setup 10 | 11 | 1. Log in to your SigOpt account at [https://app.sigopt.com](https://app.sigopt.com) 12 | 2. Find your API Token on the [API tokens page](https://app.sigopt.com/tokens) and set it 13 | as the `SIGOPT_API_TOKEN` environment variable. 14 | 3. `git clone https://github.com/sigopt/sigopt-examples.git` 15 | 4. `cd sigopt-examples/text-classifier/python` 16 | 5. `sudo ./setup_env.sh` 17 | 18 | ## Run 19 | 20 | We recommend using [Jupyter](http://jupyter.readthedocs.org/en/latest/install.html) to walk through this example. Start Jupyter (run `jupyter notebook`), then open: 21 | 22 | [`SigOpt Text Classifier Walkthrough.ipynb`](https://github.com/sigopt/sigopt-examples/blob/master/text-classifier/SigOpt%20Text%20Classifier%20Walkthrough.ipynb) 23 | 24 | The classifier tuning can also be run without Jupyter with this command: 25 | 26 | ``` 27 | nohup python sentiment_classifier.py & 28 | ``` 29 | 30 | ## Optimize 31 | 32 | Once the text classifier model tuning loop is running, you can track the progress on your [experiment dashboard](https://sigopt.com/experiments). 33 | 34 | ## Questions? 35 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 36 | 37 | ## API Reference 38 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 39 | 40 | ## About SigOpt 41 | 42 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 43 | 44 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 45 | 46 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API, Python, and R libraries integrate into any existing ML workflow. 47 | 48 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 49 | 50 | [Visit our website](https://sigopt.com) to learn more! 51 | -------------------------------------------------------------------------------- /text-classifier/python/requirements.txt: -------------------------------------------------------------------------------- 1 | jupyter>=1.0.0 2 | matplotlib 3 | numpy==1.23.5 4 | scipy==1.10.0 5 | scikit-learn==1.2.0 6 | sigopt>=8.6.3 7 | -------------------------------------------------------------------------------- /text-classifier/python/sentiment_classifier.py: -------------------------------------------------------------------------------- 1 | import json, math, numpy 2 | import sigopt 3 | from sklearn.feature_extraction.text import CountVectorizer 4 | from sklearn.linear_model import SGDClassifier 5 | from sklearn import model_selection 6 | 7 | #load text training data 8 | POSITIVE_TEXT = json.load(open("POSITIVE_list.json")) 9 | NEGATIVE_TEXT = json.load(open("NEGATIVE_list.json")) 10 | 11 | # optimization metric : see blogpost https://sigopt.com/blog/automatically-tuning-text-classifiers/ 12 | def sentiment_metric(POS_TEXT, NEG_TEXT, params): 13 | min_ngram = params['min_n_gram'] 14 | max_ngram = min_ngram + params['n_gram_offset'] 15 | min_doc_freq = math.exp(params['log_min_df']) 16 | max_doc_freq = min_doc_freq + params['df_offset'] 17 | vectorizer = CountVectorizer(min_df=min_doc_freq, max_df=max_doc_freq, 18 | ngram_range=(min_ngram, max_ngram)) 19 | X = vectorizer.fit_transform(POS_TEXT+NEG_TEXT) 20 | y = [1]*len(POS_TEXT) + [-1]*len(NEG_TEXT) 21 | clf = SGDClassifier(loss='log_loss', penalty='elasticnet', 22 | alpha=math.exp(params['log_reg_coef']), l1_ratio=params['l1_coef']) 23 | cv = model_selection.ShuffleSplit(n_splits=5, test_size=0.3, random_state=0) 24 | cv_scores = model_selection.cross_val_score(clf, X, y, cv=cv) 25 | return numpy.mean(cv_scores) 26 | 27 | conn = sigopt.Connection() 28 | experiment = conn.experiments().create( 29 | name='Sentiment LR Classifier (Python)', 30 | metrics=[dict(name='cv_scores', objective='maximize')], 31 | parameters=[ 32 | { 'name':'l1_coef', 'type': 'double', 'bounds': { 'min': 0, 'max': 1.0 }}, 33 | { 'name':'log_reg_coef', 'type': 'double', 'bounds': { 'min': math.log(0.000001), 'max': math.log(100.0) }}, 34 | { 'name':'min_n_gram', 'type': 'int', 'bounds': { 'min': 1, 'max': 2 }}, 35 | { 'name':'n_gram_offset','type': 'int', 'bounds': { 'min': 0, 'max': 2 }}, 36 | { 'name':'log_min_df', 'type': 'double', 'bounds': { 'min': math.log(0.00000001), 'max': math.log(0.1) }}, 37 | { 'name':'df_offset', 'type': 'double', 'bounds': { 'min': 0.01, 'max': 0.25 }} 38 | ], 39 | observation_budget=60, 40 | ) 41 | 42 | # run experimentation loop 43 | for _ in range(experiment.observation_budget): 44 | suggestion = conn.experiments(experiment.id).suggestions().create() 45 | opt_metric = sentiment_metric(POSITIVE_TEXT, NEGATIVE_TEXT, suggestion.assignments) 46 | conn.experiments(experiment.id).observations().create( 47 | suggestion=suggestion.id, 48 | value=opt_metric, 49 | ) # track progress on your experiment : https://sigopt.com/experiments 50 | -------------------------------------------------------------------------------- /text-classifier/python/setup_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | OS_NAME="$(uname -s)" 3 | if [ "$OS_NAME" == 'Darwin' ]; then 4 | brew install gfortran 5 | sudo easy_install pip 6 | else 7 | # assuming using ubuntu 8 | sudo apt-get update 9 | sudo apt-get -y install python-pip python-dev libopenblas-dev liblapack-dev gfortran 10 | fi 11 | sudo pip install -r requirements.txt 12 | curl https://public.sigopt.com/NEGATIVE_list.json -O 13 | curl https://public.sigopt.com/POSITIVE_list.json -O 14 | -------------------------------------------------------------------------------- /text-classifier/r/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # Text Classifier Tuning R Example 4 | 5 | Example using SigOpt and R to tune logistic regression model for text sentiment classification. 6 | 7 | More details about this example can be found in [the associated blog post](https://sigopt.com/blog/automatically-tuning-text-classifiers/). 8 | 9 | ## Setup 10 | 11 | 1. Log in to your SigOpt account at [https://app.sigopt.com](https://app.sigopt.com) 12 | 2. Find your API Token on the [API tokens page](https://app.sigopt.com/tokens) and add it to line 12 of `sentiment_classifier.r`. 13 | 3. `git clone https://github.com/sigopt/sigopt-examples.git` 14 | 4. Execute the R script in R Studio, or in the terminal: 15 | 16 | ``` 17 | cd sigopt-examples/text-classifier/r 18 | RScript sentiment_classifier.R 19 | ``` 20 | 21 | ## Optimize 22 | 23 | Once the text classifier model tuning loop is running, you can track the progress on your [experiment dashboard](https://sigopt.com/experiments). 24 | 25 | ## Questions? 26 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 27 | 28 | ## API Reference 29 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 30 | 31 | ## About SigOpt 32 | 33 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 34 | 35 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 36 | 37 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API, Python, and R libraries integrate into any existing ML workflow. 38 | 39 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 40 | 41 | [Visit our website](https://sigopt.com) to learn more! 42 | -------------------------------------------------------------------------------- /text-classifier/r/sentiment_classifier.r: -------------------------------------------------------------------------------- 1 | # Use SigOpt to tune a text classifier in R 2 | # Learn more about SigOpt's R Client: 3 | # https://sigopt.com/docs/overview/r 4 | 5 | install.packages("devtools", repos = "http://cran.us.r-project.org") 6 | library(devtools) 7 | install_github("sigopt/SigOptR") 8 | library(SigOptR) 9 | install.packages("rjson", repos = "http://cran.us.r-project.org") 10 | library(rjson) 11 | install.packages("text2vec", repos = "http://cran.us.r-project.org") 12 | library(text2vec) 13 | install.packages("glmnet", repos = "http://cran.us.r-project.org") 14 | library(glmnet) 15 | 16 | # Learn more about authenticating the SigOpt API: 17 | # https://sigopt.com/docs/overview/authentication 18 | Sys.setenv(SIGOPT_API_TOKEN="YOUR_API_TOKEN_HERE") 19 | 20 | # load text training data 21 | POSITIVE_TEXT <- fromJSON(file="https://public.sigopt.com/POSITIVE_list.json") 22 | NEGATIVE_TEXT <- fromJSON(file="https://public.sigopt.com/NEGATIVE_list.json") 23 | 24 | # optimization metric : see blogpost https://sigopt.com/blog/automatically-tuning-text-classifiers/ 25 | sentiment_metric <- function(POS_TEXT, NEG_TEXT, params) { 26 | min_ngram <- params$min_n_gram 27 | max_ngram <- min_ngram + params$n_gram_offset 28 | min_doc_freq <- exp(params$log_min_df) 29 | max_doc_freq <- min_doc_freq + params$df_offset 30 | text = c(POSITIVE_TEXT, NEGATIVE_TEXT) 31 | 32 | # Construct a matrix representation of the text 33 | it <- itoken(text, tolower, word_tokenizer) 34 | vocab <- create_vocabulary(it, ngram = c(min_ngram, max_ngram)) 35 | pruned_vocab = prune_vocabulary(vocab, doc_proportion_min = min_doc_freq, doc_proportion_max = max_doc_freq) 36 | it <- itoken(text, tolower, word_tokenizer) 37 | vectorizer <- vocab_vectorizer(pruned_vocab) 38 | 39 | X <- create_dtm(it, vectorizer) 40 | y <- c(rep(1, length(POSITIVE_TEXT)), rep(0, length(NEGATIVE_TEXT))) 41 | 42 | # Perform cross-validation 43 | num_folds = 5 44 | cv_accuracies = c() 45 | for(i in 1:num_folds) { 46 | cv_split <- sample(2, nrow(X), replace=TRUE, prob=c(0.7, 0.3)) 47 | X_train <- X[cv_split==1, ] 48 | y_train <- y[cv_split==1] 49 | X_valid <- X[cv_split==2, ] 50 | y_valid <- y[cv_split==2] 51 | 52 | fit <- glmnet(X_train, y_train, family = "binomial", lambda=exp(params$log_reg_coef), alpha=params$l1_coef) 53 | prediction <-predict(fit, X_valid, type="class") 54 | 55 | correct_predictions <- prediction == y_valid 56 | accuracy <- sum(correct_predictions)/nrow(X_valid) 57 | cv_accuracies <- c(accuracy, cv_accuracies) 58 | } 59 | 60 | return(c(mean(cv_accuracies), sd(cv_accuracies))) 61 | } 62 | 63 | experiment <- create_experiment(list( 64 | name="Sentiment LR Classifier (R)", 65 | parameters=list( 66 | list(name="l1_coef", type="double", bounds=list(min=0, max=1.0)), 67 | list(name="log_reg_coef", type="double", bounds=list(min=log(0.000001), max=log(100.0))), 68 | list(name="min_n_gram", type="int", bounds=list(min=1, max=2)), 69 | list(name="n_gram_offset", type="int", bounds=list(min=0, max=2)), 70 | list(name="log_min_df", type="double", bounds=list(min=log(0.00000001), max=log(0.1))), 71 | list(name="df_offset", type="double", bounds=list(min=0.01, max=0.25)) 72 | ), 73 | # Run the Optimization Loop between 10x - 20x the number of parameters 74 | observation_budget=60 75 | )) 76 | 77 | # run experimentation loop 78 | for(i in 1:experiment$observation_budget) { 79 | suggestion <- create_suggestion(experiment$id) 80 | 81 | opt_metric <- sentiment_metric(POSITIVE_TEXT, NEGATIVE_TEXT, suggestion$assignments) 82 | 83 | create_observation(experiment$id, list( 84 | suggestion=suggestion$id, 85 | value=opt_metric[1], 86 | value_stddev=opt_metric[2] 87 | )) 88 | } # track progress on your experiment : https://sigopt.com/experiments 89 | -------------------------------------------------------------------------------- /vision-nas-search-keras-cifar-ray/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # Neural Architecture Search to Train a Computer Vision Classifier in Keras on CIFAR10 Data using Ray Tune and SigOpt 4 | 5 | Open In Colab 6 | 7 | This example uses SigOpt to tune Neural Architecture search, using a Keras image classifier. Ray Tune is used to orchestrate training, and uses its built in interface to the SigOpt API. 8 | 9 | Its contributors are: 10 | 11 | [Ellick Chan](https://github.com/ellick-chan), Head, Intel AI Academy University Relations and Research 12 | [Barrett Williams](https://github.com/brwillia), Product Marketing Lead, SigOpt 13 | 14 | ## Jupyter Setup 15 | 16 | 1. `git clone https://github.com/sigopt/sigopt-examples.git` 17 | 2. `cd sigopt-examples/vision-nas-search-keras-cifar-ray` 18 | 3. Run `jupyter lab` in that directory and open mininas.ipynb in the web interface 19 | 4. Add your SigOpt [API token](https://docs.sigopt.com/core-module-api-references/get_started) in the Jupyter cell where you see `YOUR_SIGOPT_API_KEY_HERE` 20 | 5. Run all cells or step through the notebook 21 | 22 | ## Optimize 23 | 24 | Once the SigOpt optimization loop is initiated, you can track the progress on your [experiment dashboard](https://sigopt.com/experiments). 25 | 26 | ## Questions? 27 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 28 | 29 | ## API Reference 30 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 31 | 32 | ## About SigOpt 33 | 34 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 35 | 36 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 37 | 38 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API, Python, and R libraries integrate into any existing ML workflow. 39 | 40 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 41 | 42 | [Visit our website](https://sigopt.com) to learn more, and be sure to [sign up for a free account](https://app.sigopt.com/signup) if you haven't already. 43 | 44 | Open In Colab 45 | -------------------------------------------------------------------------------- /xgboost-integration-examples/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # Using SigOpt's XGBoost Integration for Fraud Classification 4 | 5 | This tutorial uses SigOpt's [XGBoost integration](https://docs.sigopt.com/ai-module-api-references/xgboost) to optimize a fraud classifier. You will need a [SigOpt account](https://sigopt.com/signup) and SigOpt [API token](https://app.sigopt.com/tokens/info), as well as a Kaggle account and Kaggle API token in order to run this code. 6 | 7 | ## Jupyter Setup 8 | 9 | 1. `git clone https://github.com/sigopt/sigopt-examples.git` 10 | 2. `cd sigopt-examples/xgboost-integration-examples` 11 | 3. `pip install -r requirements.txt` if you do not have all required packages installed 12 | 4. Run `jupyter notebook` in that directory and open `xgb-integration-py-class.ipynb` or `xgb-integration-py-reg.ipynb` in the web interface 13 | 5. Run all cells or step through the notebook 14 | 15 | Alternatively, you can also run these notebook in Google Colab by clicking the `Run in Colab` button. 16 | 17 | ## Optimize 18 | 19 | Check the progress of your SigOpt runs and experiments on your [experiment dashboard](https://sigopt.com/experiments). 20 | 21 | ## Questions? 22 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 23 | 24 | ## API Reference 25 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 26 | 27 | ## About SigOpt 28 | 29 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 30 | 31 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 32 | 33 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API, Python, and R libraries integrate into any existing ML workflow. 34 | 35 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 36 | 37 | [Visit our website](https://sigopt.com) to learn more and sign up today! 38 | -------------------------------------------------------------------------------- /xgboost-integration-examples/requirements.txt: -------------------------------------------------------------------------------- 1 | sigopt[xgboost] 2 | xgboost==1.5.2 3 | scikit-learn==1.1.1 4 | opendatasets==0.1.22 5 | pandas==1.4.2 6 | -------------------------------------------------------------------------------- /xgboost-py-classifier/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # XGBoost Supervised Python Jupyter Notebook Classifier Tuning 4 | 5 | This example uses SigOpt to tune an XGBoost classifier on the Wisconsin breast cancer dataset in Python. 6 | 7 | ## Jupyter Setup 8 | 9 | 1. `git clone https://github.com/sigopt/sigopt-examples.git` 10 | 2. `cd sigopt-examples/xgboost-py-classifier` 11 | 3. Run `jupyter lab` in that directory and open xgboost_py_classifier.ipynb in the web interface 12 | 4. Add your SigOpt [API token](https://docs.sigopt.com/core-module-api-references/get_started) in the Jupyter cell where you see `YOUR_API_TOKEN_HERE` 13 | 5. Run all cells or step through the notebook 14 | 15 | ## Optimize 16 | 17 | Once the SigOpt optimization loop is initiated, you can track the progress on your [experiment dashboard](https://sigopt.com/experiments). 18 | 19 | ## Questions? 20 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 21 | 22 | ## API Reference 23 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 24 | 25 | ## About SigOpt 26 | 27 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 28 | 29 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 30 | 31 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API, Python, and R libraries integrate into any existing ML workflow. 32 | 33 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 34 | 35 | [Visit our website](https://sigopt.com) to learn more! 36 | -------------------------------------------------------------------------------- /xgboost-py-regression/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://sigopt.com/static/img/SigOpt_logo_horiz.png?raw=true)](https://sigopt.com) 2 | 3 | # XGBoost Supervised Python Jupyter Notebook Regression Tuning 4 | 5 | This example uses SigOpt to tune an XGBoost regression on scikit-learn's California housing dataset in Python. 6 | 7 | ## Jupyter Setup 8 | 9 | 1. `git clone https://github.com/sigopt/sigopt-examples.git` 10 | 2. `cd sigopt-examples/xgboost-py-regression` 11 | 3. Run `jupyter lab` in that directory and open xgboost_py_regressor.ipynb in the web interface 12 | 4. Add your SigOpt [API token](https://docs.sigopt.com/core-module-api-references/get_started) in the Jupyter cell where you see `YOUR_API_TOKEN_HERE` 13 | 5. Run all cells or step through the notebook 14 | 15 | ## Optimize 16 | 17 | Once the SigOpt optimization loop is initiated, you can track the progress on your [experiment dashboard](https://sigopt.com/experiments). 18 | 19 | ## Questions? 20 | Visit the [SigOpt Community page](https://community.sigopt.com) and leave your questions. 21 | 22 | ## API Reference 23 | To implement SigOpt for your use case, feel free to use or extend the code in this repository. Our [API](https://docs.sigopt.com) can bolt on top of any complex model or process and guide it to its optimal configuration in as few iterations as possible. 24 | 25 | ## About SigOpt 26 | 27 | With SigOpt, data scientists and machine learning engineers can build better models with less trial and error. 28 | 29 | Machine learning models depend on hyperparameters that trade off bias/variance and other key outcomes. SigOpt provides Bayesian hyperparameter optimization using an ensemble of the latest research. 30 | 31 | SigOpt can tune any machine learning model, including popular techniques like gradient boosting, deep neural networks, and support vector machines. SigOpt’s REST API, Python, and R libraries integrate into any existing ML workflow. 32 | 33 | SigOpt augments your existing model training pipeline, suggesting parameter configurations to maximize any online or offline objective, such as AUC ROC, model accuracy, or revenue. You only send SigOpt your metadata, not the underlying training data or model. 34 | 35 | [Visit our website](https://sigopt.com) to learn more! 36 | --------------------------------------------------------------------------------