├── setup.cfg ├── .gitignore ├── __init__.py ├── .images ├── figure_1.png ├── figure_2-1.png └── figure_2-2.png ├── hyperengine ├── impl │ ├── __init__.py │ └── tensorflow │ │ ├── __init__.py │ │ ├── tf_util.py │ │ ├── tensorflow_model_io.py │ │ ├── tensorflow_solver.py │ │ └── tensorflow_runner.py ├── examples │ ├── __init__.py │ ├── 1_1_hello_word_with_mnist.py │ ├── 1_6_optimizing_regression.py │ ├── 3_1_lstm_mnist.py │ ├── 1_4_fine_tuning_saved_model.py │ ├── README.md │ ├── 1_3_saving_best_models_mnist.py │ ├── 1_2_getting_started_with_tuning.py │ ├── 1_5_learning_curve_prediction.py │ ├── 2_1_cnn_mnist.py │ ├── 2_2_cnn_cifar.py │ ├── 4_1_word2vec_embedding.py │ ├── 2_3_all_conv_net_cifar.py │ ├── 3_2_rnn_sms_spam_detector.py │ └── common.py ├── tests │ ├── __init__.py │ ├── data_set_test.py │ ├── named_dict_test.py │ ├── curve_predictor_test.py │ ├── spec_test.py │ ├── strategy_test.py │ └── curve-test-data.xjson ├── __init__.py ├── model │ ├── __init__.py │ ├── base_runner.py │ ├── hyper_tuner.py │ ├── model_io.py │ ├── data_set.py │ ├── curve_predictor.py │ └── base_solver.py ├── spec │ ├── __init__.py │ ├── sugar.py │ ├── parsed_spec.py │ └── nodes.py ├── bayesian │ ├── __init__.py │ ├── sampler.py │ ├── maximizer.py │ ├── kernel.py │ ├── utility.py │ └── strategy.py └── base │ ├── __init__.py │ ├── logging.py │ ├── named_dict.py │ ├── base_io.py │ └── util.py ├── .idea ├── vcs.xml ├── other.xml ├── dictionaries │ └── maxim.xml ├── misc.xml ├── inspectionProfiles │ ├── profiles_settings.xml │ └── Project_Default.xml ├── modules.xml └── hyper-engine.iml ├── .travis.yml ├── setup.py ├── README.rst └── LICENSE /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.rst 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | dist/* 4 | .cache/* 5 | workspace.xml 6 | temp-* 7 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'maxim' 4 | -------------------------------------------------------------------------------- /.images/figure_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxim5/hyper-engine/HEAD/.images/figure_1.png -------------------------------------------------------------------------------- /.images/figure_2-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxim5/hyper-engine/HEAD/.images/figure_2-1.png -------------------------------------------------------------------------------- /.images/figure_2-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxim5/hyper-engine/HEAD/.images/figure_2-2.png -------------------------------------------------------------------------------- /hyperengine/impl/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'maxim' 4 | -------------------------------------------------------------------------------- /hyperengine/examples/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'maxim' 4 | -------------------------------------------------------------------------------- /hyperengine/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'maxim' 4 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/other.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | -------------------------------------------------------------------------------- /.idea/dictionaries/maxim.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | hyperengine 5 | theano 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /hyperengine/impl/tensorflow/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'maxim' 4 | 5 | from .tensorflow_model_io import TensorflowModelIO 6 | from .tensorflow_runner import TensorflowRunner 7 | from .tensorflow_solver import TensorflowSolver 8 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /hyperengine/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'maxim' 4 | 5 | from .bayesian import * 6 | from .model import * 7 | 8 | try: 9 | from .impl.tensorflow import * 10 | except ImportError: 11 | pass 12 | 13 | from . import base as util 14 | from . import spec 15 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | - "3.5" 5 | - "3.6" 6 | 7 | install: 8 | - pip install six==1.11.0 numpy==1.14.0 scipy==1.0.0 tensorflow==1.5.0 9 | 10 | script: 11 | - cd hyperengine/tests 12 | - python -m pytest curve_predictor_test.py data_set_test.py named_dict_test.py spec_test.py 13 | 14 | notifications: 15 | email: false 16 | -------------------------------------------------------------------------------- /hyperengine/model/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'maxim' 4 | 5 | from .base_runner import BaseRunner 6 | from .base_solver import BaseSolver 7 | from .curve_predictor import BaseCurvePredictor, LinearCurvePredictor 8 | from .data_set import Data, DataSet, DataProvider, IterableDataProvider, merge_data_sets 9 | from .hyper_tuner import HyperTuner 10 | from .model_io import ModelIO 11 | -------------------------------------------------------------------------------- /hyperengine/spec/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'maxim' 4 | 5 | from .sugar import wrap, choice, uniform, normal, merge, random_bit, random_bool, random_int, \ 6 | min_ as min, max_ as max, \ 7 | exp, expm1, frexp, ldexp, \ 8 | sqrt, pow, \ 9 | log, log1p, log10, \ 10 | new 11 | from .parsed_spec import ParsedSpec, get_instance 12 | -------------------------------------------------------------------------------- /.idea/hyper-engine.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 11 | -------------------------------------------------------------------------------- /hyperengine/bayesian/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'maxim' 4 | 5 | from .kernel import BaseKernel, RadialBasisFunction 6 | from .maximizer import BaseUtilityMaximizer, MonteCarloUtilityMaximizer 7 | from .sampler import BaseSampler, DefaultSampler 8 | from .strategy import BaseStrategy, BaseBayesianStrategy, BayesianStrategy, BayesianPortfolioStrategy 9 | from .utility import BaseUtility, BaseGaussianUtility, \ 10 | ProbabilityOfImprovement, ExpectedImprovement, UpperConfidenceBound, RandomPoint 11 | -------------------------------------------------------------------------------- /hyperengine/base/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'maxim' 4 | 5 | from .logging import log, set_silence, set_verbose, is_info_logged, is_debug_logged, \ 6 | debug, info, warn, vlog, vlog2, vlog3, log_at_level 7 | from .util import smart_str, str_to_dict, zip_longest, deep_update, mini_batch, \ 8 | random_id, safe_concat, call, slice_dict, as_function, as_numeric_function, download_if_needed 9 | from .named_dict import NamedDict 10 | from .base_io import BaseIO, DefaultIO, Serializable 11 | -------------------------------------------------------------------------------- /hyperengine/impl/tensorflow/tf_util.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import functools 7 | from tensorflow.python.client import device_lib 8 | 9 | def graph_vars(graph): 10 | for n in graph.as_graph_def().node: 11 | element = graph.as_graph_element(n.name) 12 | if element.type == 'Variable' or element.type == 'VariableV2': 13 | yield element 14 | 15 | def get_total_dim(element): 16 | # See TensorShapeProto 17 | shape = element.get_attr('shape') 18 | dims = shape.dim 19 | return functools.reduce(lambda x, y: x * y, [dim.size for dim in dims], 1) 20 | 21 | def is_gpu_available(): 22 | local_devices = device_lib.list_local_devices() 23 | return len([x for x in local_devices if x.device_type == 'GPU']) > 0 24 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'maxim' 4 | 5 | import os 6 | from setuptools import setup, find_packages 7 | 8 | def read(file_): 9 | return open(os.path.join(os.path.dirname(__file__), file_)).read() 10 | 11 | setup( 12 | name = 'hyperengine', 13 | version = '0.1.1', 14 | author = 'Maxim Podkolzine', 15 | author_email = 'maxim.podkolzine@gmail.com', 16 | description = 'Python library for Bayesian hyper-parameters optimization', 17 | license = 'Apache 2.0', 18 | keywords = 'machine learning, hyper-parameters, model selection, bayesian optimization', 19 | url = 'https://github.com/maxim5/hyper-engine', 20 | packages = find_packages(exclude=['*tests*']), 21 | long_description = read('README.rst'), 22 | install_requires = ['numpy', 'scipy', 'six'], 23 | extras_require = { 24 | 'tf': ['tensorflow'], 25 | }, 26 | ) 27 | -------------------------------------------------------------------------------- /hyperengine/base/logging.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import print_function 5 | __author__ = 'maxim' 6 | 7 | 8 | LOG_LEVEL = 1 9 | 10 | def log(*msg): 11 | import datetime 12 | print('[%s]' % datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), ' '.join([str(it) for it in msg])) 13 | 14 | def set_silence(): 15 | global LOG_LEVEL 16 | LOG_LEVEL = 10 17 | 18 | def set_verbose(level=1): 19 | global LOG_LEVEL 20 | LOG_LEVEL = -level 21 | 22 | def is_debug_logged(): 23 | return LOG_LEVEL <= 0 24 | 25 | def is_info_logged(): 26 | return LOG_LEVEL <= 1 27 | 28 | def debug(*msg): 29 | log_at_level(0, *msg) 30 | 31 | def info(*msg): 32 | log_at_level(1, *msg) 33 | 34 | def warn(*msg): 35 | log_at_level(2, *msg) 36 | 37 | def vlog(*msg): 38 | log_at_level(-1, *msg) 39 | 40 | def vlog2(*msg): 41 | log_at_level(-2, *msg) 42 | 43 | def vlog3(*msg): 44 | log_at_level(-3, *msg) 45 | 46 | def log_at_level(level, *msg): 47 | if level >= LOG_LEVEL: 48 | log(*msg) 49 | -------------------------------------------------------------------------------- /hyperengine/impl/tensorflow/tensorflow_model_io.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import os 7 | 8 | import tensorflow as tf 9 | 10 | from ...base import * 11 | from hyperengine.model import ModelIO 12 | 13 | 14 | class TensorflowModelIO(ModelIO): 15 | def __init__(self, **params): 16 | super(TensorflowModelIO, self).__init__(**params) 17 | self.saver = tf.train.Saver(defer_build=True) 18 | 19 | def save_session(self, session, directory=None): 20 | directory = directory or self.save_dir 21 | if not ModelIO._prepare(directory): 22 | return 23 | 24 | destination = os.path.join(self.save_dir, 'session.data') 25 | self.saver.build() 26 | self.saver.save(session, destination) 27 | debug('Session saved to %s' % destination) 28 | 29 | def load_session(self, session, directory, log_level): 30 | if not directory: 31 | return 32 | 33 | directory = os.path.abspath(directory) 34 | session_file = os.path.join(directory, 'session.data') 35 | if os.path.exists(session_file) or os.path.exists(session_file + '.meta'): 36 | self.saver.build() 37 | self.saver.restore(session, session_file) 38 | log_at_level(log_level, 'Loaded session from %s' % session_file) 39 | -------------------------------------------------------------------------------- /hyperengine/base/named_dict.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | 7 | from .util import smart_str 8 | 9 | # Inspired by https://stackoverflow.com/questions/1305532/convert-python-dict-to-object 10 | class NamedDict(object): 11 | def __init__(self, input): 12 | for k, v in input.items(): 13 | if isinstance(v, (list, tuple)): 14 | setattr(self, k, [NamedDict(x) if isinstance(x, dict) else x for x in v]) 15 | else: 16 | setattr(self, k, NamedDict(v) if isinstance(v, dict) else v) 17 | 18 | def keys(self): 19 | return self.__dict__.keys() 20 | 21 | def items(self): 22 | return self.__dict__.items() 23 | 24 | def values(self): 25 | return self.__dict__.values() 26 | 27 | def get(self, key, default=None): 28 | return self.__dict__.get(key, default) 29 | 30 | def __getitem__(self, key): 31 | return self.__dict__[key] 32 | 33 | def __getattribute__(self, key): 34 | if (key in ['get', 'keys', 'items', 'values']) or (key.startswith('__') and key.endswith('__')): 35 | return object.__getattribute__(self, key) 36 | return self.__dict__.get(key, None) 37 | 38 | def __contains__(self, key): 39 | return key in self.__dict__ 40 | 41 | def __repr__(self): 42 | return smart_str(self.__dict__) 43 | -------------------------------------------------------------------------------- /hyperengine/bayesian/sampler.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import numpy as np 7 | from six.moves import xrange 8 | 9 | class BaseSampler(object): 10 | """ 11 | Represents a helper class that allows to sample from high-dimensional spaces, usually unit hypercube. 12 | """ 13 | def sample(self, size): 14 | """ 15 | Returns a sample of the specified `size` (numpy array). 16 | Note that `size` defines just one dimension of the result. 17 | Other dimensions (usually hypercube) are fixed for the particular sampler instance. 18 | """ 19 | raise NotImplementedError() 20 | 21 | 22 | class DefaultSampler(BaseSampler): 23 | """ 24 | Represents the default implementation of the sampler. 25 | """ 26 | 27 | def __init__(self): 28 | self._random_functions = [] 29 | 30 | def add(self, func): 31 | assert callable(func) 32 | self._random_functions.append(func) 33 | 34 | def add_uniform(self, size): 35 | func = lambda : np.random.uniform(0, 1, size=(size,)) 36 | self._random_functions.append(func) 37 | 38 | def sample(self, size): 39 | result = [] 40 | for i in xrange(size): 41 | item = [func() for func in self._random_functions] 42 | result.append(np.array(item).flatten()) 43 | return np.array(result) 44 | -------------------------------------------------------------------------------- /hyperengine/model/base_runner.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | 7 | class BaseRunner(object): 8 | """ 9 | The runner represents a connecting layer between the solver and the machine learning model. 10 | Responsible for communicating with the model with a data batch: prepare, train, evaluate. 11 | """ 12 | 13 | def build_model(self): 14 | """ 15 | Builds and prepares a model. Method is not expected to return anything. 16 | """ 17 | raise NotImplementedError() 18 | 19 | def init(self, **kwargs): 20 | """ 21 | Runs the model initializer. 22 | """ 23 | raise NotImplementedError() 24 | 25 | def run_batch(self, batch_x, batch_y): 26 | """ 27 | Runs the training iteration for a batch of data. Method is not expected to return anything. 28 | """ 29 | raise NotImplementedError() 30 | 31 | def evaluate(self, batch_x, batch_y): 32 | """ 33 | Evaluates the test result for a batch of data. Method should return the dictionary that contains 34 | one (or all) of the following: 35 | - batch accuracy (key 'accuracy') 36 | - associated loss (key 'loss') 37 | - any other computed data (key 'data') 38 | """ 39 | raise NotImplementedError() 40 | 41 | def model_size(self): 42 | """ 43 | Returns the model size. 44 | """ 45 | raise NotImplementedError() 46 | -------------------------------------------------------------------------------- /hyperengine/bayesian/maximizer.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import numpy as np 7 | 8 | from ..base import * 9 | 10 | 11 | class BaseUtilityMaximizer(object): 12 | """ 13 | Represents a helper class that computes the max value of an utility method 14 | over a certain domain, usually unit N-dimensional hypercube. 15 | """ 16 | def __init__(self, utility): 17 | super(BaseUtilityMaximizer, self).__init__() 18 | self._utility = utility 19 | 20 | def compute_max_point(self): 21 | """ 22 | Returns the point that reaches the maximum value. 23 | """ 24 | raise NotImplementedError() 25 | 26 | 27 | class MonteCarloUtilityMaximizer(BaseUtilityMaximizer): 28 | """ 29 | A simple implementation that applies Monte-Carlo method to pick the maximum. 30 | """ 31 | 32 | def __init__(self, utility, sampler, **params): 33 | super(MonteCarloUtilityMaximizer, self).__init__(utility) 34 | self._sampler = sampler 35 | self._batch_size = params.get('batch_size', 100000) 36 | self._tweak_probability = params.get('tweak_probability', 0) 37 | self._flip_probability = params.get('flip_probability', 0) 38 | 39 | def compute_max_point(self): 40 | batch = self._sampler.sample(size=self._batch_size) 41 | values = self._utility.compute_values(batch) 42 | i = np.argmax(values) 43 | debug('Max utility point:', batch[i]) 44 | vlog('Max utility value:', values[i]) 45 | point = self._tweak_randomly(batch[i], batch[0]) 46 | return point 47 | 48 | def _tweak_randomly(self, optimal, point): 49 | if np.random.uniform() < self._tweak_probability: 50 | debug('Tweaking optimal point with probability:', self._flip_probability) 51 | p = self._flip_probability or min(2.0 / optimal.shape[0], 0.5) 52 | mask = np.random.choice([False, True], size=optimal.shape, p=[1-p, p]) 53 | optimal[mask] = point[mask] 54 | debug('Selected point after random tweaking:', optimal) 55 | return optimal 56 | -------------------------------------------------------------------------------- /hyperengine/bayesian/kernel.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | 7 | import numpy as np 8 | from scipy.spatial.distance import pdist, squareform, cdist 9 | 10 | 11 | class BaseKernel(object): 12 | """ 13 | Represents a kernel function See https://en.wikipedia.org/wiki/Positive-definite_kernel for details. 14 | All methods operate with a batch of points. 15 | """ 16 | 17 | def id(self, batch_x): 18 | """ 19 | Computes the kernel function of a point (a batch of points) with itself. 20 | Standard kernels satisfy `k(x, x) = 1`, but a derived kernel may compute it differently. 21 | Note that the result vector equals to the diagonal of `compute(batch_x)` 22 | """ 23 | raise NotImplementedError() 24 | 25 | def compute(self, batch_x, batch_y=None): 26 | """ 27 | Computes the kernel matrix, consisting of kernel values for each pair of points. 28 | If `batch_y is None`, the result is a square matrix computed from `batch_x`. 29 | If `batch_y is not None`, the result is computed from both `batch_x` and `batch_y`. 30 | """ 31 | raise NotImplementedError() 32 | 33 | 34 | class RadialBasisFunction(BaseKernel): 35 | """ 36 | Implements RBF kernel function. 37 | See https://en.wikipedia.org/wiki/Radial_basis_function_kernel 38 | """ 39 | 40 | def __init__(self, gamma=0.5, **params): 41 | self._gamma = gamma 42 | self._params = params 43 | self._params.setdefault('metric', 'sqeuclidean') 44 | 45 | def id(self, batch_x): 46 | return np.ones(batch_x.shape[:1]) 47 | 48 | def compute(self, batch_x, batch_y=None): 49 | if len(batch_x.shape) == 1: 50 | batch_x = batch_x.reshape((1, -1)) 51 | if batch_y is not None and len(batch_y.shape) == 1: 52 | batch_y = batch_y.reshape((1, -1)) 53 | 54 | if batch_y is None: 55 | dist = squareform(pdist(batch_x, **self._params)) 56 | else: 57 | dist = cdist(batch_x, batch_y, **self._params) 58 | return np.exp(-self._gamma * dist) 59 | -------------------------------------------------------------------------------- /hyperengine/model/hyper_tuner.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import numpy as np 7 | import time 8 | 9 | from ..base import * 10 | from ..spec import ParsedSpec 11 | from ..bayesian.sampler import DefaultSampler 12 | from ..bayesian.strategy import BayesianStrategy, BayesianPortfolioStrategy 13 | 14 | 15 | strategies = { 16 | 'bayesian': lambda sampler, params: BayesianStrategy(sampler, **params), 17 | 'portfolio': lambda sampler, params: BayesianPortfolioStrategy(sampler, **params), 18 | } 19 | 20 | 21 | class HyperTuner(object): 22 | def __init__(self, hyper_params_spec, solver_generator, **strategy_params): 23 | self._solver_generator = solver_generator 24 | 25 | self._parsed = ParsedSpec(hyper_params_spec) 26 | info('Spec size:', self._parsed.size()) 27 | 28 | sampler = DefaultSampler() 29 | sampler.add_uniform(self._parsed.size()) 30 | 31 | strategy_gen = as_function(strategy_params.get('strategy', 'bayesian'), presets=strategies) 32 | self._strategy = strategy_gen(sampler, strategy_params) 33 | 34 | self._timeout = strategy_params.get('timeout', 0) 35 | self._max_points = strategy_params.get('max_points', None) 36 | 37 | def tune(self): 38 | info('Start hyper tuner') 39 | 40 | while True: 41 | if self._max_points is not None and len(self._strategy.values) >= self._max_points: 42 | info('Maximum points reached: max_points=%d. Aborting hyper tuner' % self._max_points) 43 | break 44 | 45 | point = self._strategy.next_proposal() 46 | hyper_params = self._parsed.instantiate(point) 47 | solver = self._solver_generator(hyper_params) 48 | 49 | accuracy = solver.train() 50 | 51 | previous_max = np.max(self._strategy.values) if len(self._strategy.values) > 0 else -np.inf 52 | self._strategy.add_point(point, accuracy) 53 | index = len(self._strategy.values) 54 | 55 | marker = '!' if accuracy > previous_max else ' ' 56 | info('%s [%d] accuracy=%.4f, params: %s' % (marker, index, accuracy, smart_str(hyper_params))) 57 | info('Current top-%d:' % min(len(self._strategy.values), 5)) 58 | for value in sorted(self._strategy.values, reverse=True)[:5]: 59 | info(' accuracy=%.4f' % value) 60 | 61 | if self._timeout: 62 | time.sleep(self._timeout) 63 | 64 | solver.terminate() 65 | -------------------------------------------------------------------------------- /hyperengine/model/model_io.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import os 7 | 8 | from ..base import * 9 | 10 | 11 | class ModelIO(BaseIO): 12 | def __init__(self, **params): 13 | super(ModelIO, self).__init__(**params) 14 | self.data_saver = params.get('data_saver') 15 | 16 | def save_results(self, results, directory=None): 17 | directory = directory or self.save_dir 18 | if not ModelIO._prepare(directory): 19 | return 20 | 21 | destination = os.path.join(directory, 'results.xjson') 22 | with open(destination, 'w') as file_: 23 | file_.write(smart_str(results)) 24 | debug('Results saved to %s' % destination) 25 | 26 | def load_results(self, directory, log_level): 27 | if directory is None: 28 | return 29 | 30 | destination = os.path.join(directory, 'results.xjson') 31 | if os.path.exists(destination): 32 | results = ModelIO._load_dict(destination) 33 | log_at_level(log_level, 'Loaded results: %s from %s' % (smart_str(results), destination)) 34 | return results 35 | 36 | def save_hyper_params(self, hyper_params, directory=None): 37 | directory = directory or self.save_dir 38 | if not ModelIO._prepare(directory): 39 | return 40 | 41 | destination = os.path.join(directory, 'hyper_params.xjson') 42 | with open(destination, 'w') as file_: 43 | file_.write(smart_str(hyper_params)) 44 | debug('Hyper parameters saved to %s' % destination) 45 | 46 | def load_hyper_params(self, directory=None): 47 | directory = directory or self.load_dir 48 | if directory is None: 49 | return 50 | 51 | hyper_params = ModelIO._load_dict(os.path.join(directory, 'hyper_params.xjson')) 52 | if hyper_params: 53 | info('Loaded hyper-params: %s' % smart_str(hyper_params)) 54 | return hyper_params 55 | 56 | def save_data(self, data, directory=None): 57 | directory = directory or self.save_dir 58 | if self.data_saver is None or data is None or not ModelIO._prepare(directory): 59 | return 60 | 61 | destination = os.path.join(directory, 'misclassified') 62 | actual_destination = call(self.data_saver, data, destination) 63 | if actual_destination: 64 | debug('Misclassified data saved to %s' % actual_destination) 65 | else: 66 | warn('Data saver can not be not called or returns None') 67 | -------------------------------------------------------------------------------- /hyperengine/examples/1_1_hello_word_with_mnist.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """A hello-world example. All hyper-parameters are hard-coded as constants. 5 | In order to run this model, it's only necessary to name the tensors and ops 6 | so that hyper-engine could find them. By convention, hyper-engine looks up the following: 7 | - input 8 | - label 9 | - mode (optional) 10 | - minimize 11 | - loss (optional) 12 | - accuracy (optional) 13 | These are the defaults and they are customizable. 14 | The whole integration is creation and start of the solver. 15 | """ 16 | 17 | __author__ = 'maxim' 18 | 19 | import tensorflow as tf 20 | from tensorflow.examples.tutorials.mnist import input_data 21 | 22 | import hyperengine as hype 23 | 24 | x = tf.placeholder(tf.float32, [None, 28, 28, 1], name='input') 25 | y = tf.placeholder(tf.float32, [None, 10], name='label') 26 | 27 | conv1 = tf.layers.conv2d(x, filters=32, kernel_size=[3, 3], padding='same', activation=tf.nn.relu) 28 | pool1 = tf.layers.max_pooling2d(conv1, pool_size=[2, 2], strides=[2, 2]) 29 | 30 | conv2 = tf.layers.conv2d(pool1, filters=64, kernel_size=[3, 3], padding='same', activation=tf.nn.relu) 31 | pool2 = tf.layers.max_pooling2d(conv2, pool_size=[2, 2], strides=[2, 2]) 32 | 33 | flat = tf.reshape(pool2, [-1, pool2.shape[1] * pool2.shape[2] * pool2.shape[3]]) 34 | dense = tf.layers.dense(inputs=flat, units=1024, activation=tf.nn.relu) 35 | logits = tf.layers.dense(inputs=dense, units=10) 36 | 37 | loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y), name='loss') 38 | optimizer = tf.train.AdamOptimizer(learning_rate=0.001) 39 | train_op = optimizer.minimize(loss_op, name='minimize') 40 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), tf.float32), name='accuracy') 41 | 42 | tf_data_sets = input_data.read_data_sets('temp-mnist/data', one_hot=True) 43 | convert = lambda data_set: hype.DataSet(data_set.images.reshape((-1, 28, 28, 1)), data_set.labels) 44 | data = hype.Data(train=convert(tf_data_sets.train), 45 | validation=convert(tf_data_sets.validation), 46 | test=convert(tf_data_sets.test)) 47 | 48 | solver_params = { 49 | 'batch_size': 1000, 50 | 'eval_batch_size': 2500, 51 | 'epochs': 10, 52 | 'evaluate_test': True, 53 | 'eval_flexible': False, 54 | } 55 | solver = hype.TensorflowSolver(data=data, **solver_params) 56 | solver.train() 57 | -------------------------------------------------------------------------------- /hyperengine/examples/1_6_optimizing_regression.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import tensorflow as tf 7 | 8 | import hyperengine as hype 9 | from common import get_wine_data 10 | 11 | def dnn_model(params): 12 | x = tf.placeholder(shape=[None, 11], dtype=tf.float32, name='input') 13 | y = tf.placeholder(shape=[None], dtype=tf.float32, name='label') 14 | mode = tf.placeholder(tf.string, name='mode') 15 | training = tf.equal(mode, 'train') 16 | 17 | layer = tf.layers.batch_normalization(x, training=training) if params.batch_norm else x 18 | for hidden_size in params.hidden_layers: 19 | layer = tf.layers.dense(layer, units=hidden_size, activation=tf.nn.relu) 20 | layer = tf.layers.dropout(layer, params.dropout, training=training) if params.dropout else layer 21 | predictions = tf.layers.dense(layer, units=1) 22 | 23 | loss = tf.reduce_mean((predictions - y) ** 2, name='loss') 24 | optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate) 25 | optimizer.minimize(loss, name='minimize') 26 | 27 | # Hyper-tuner is generally designed for classification, so it tries to maximize the accuracy. 28 | # But we have a regression problem. A simple possible metric to emulate the accuracy would be: 29 | # 30 | # tf.negative(loss, name='accuracy') 31 | # 32 | # But we'll use this one: 33 | tf.reduce_mean(tf.cast(tf.abs(predictions - y) < 0.5, tf.float32), name='accuracy') 34 | 35 | x_train, y_train, x_test, y_test, x_val, y_val = get_wine_data(path='temp-wine/data') 36 | data = hype.Data(train=hype.DataSet(x_train, y_train), 37 | validation=hype.DataSet(x_val, y_val), 38 | test=hype.DataSet(x_test, y_test)) 39 | 40 | def solver_generator(params): 41 | solver_params = { 42 | 'batch_size': 1000, 43 | 'eval_batch_size': 1000, 44 | 'epochs': 20, 45 | 'evaluate_test': True, 46 | 'eval_flexible': True, 47 | } 48 | dnn_model(params) 49 | solver = hype.TensorflowSolver(data=data, hyper_params=params, **solver_params) 50 | return solver 51 | 52 | 53 | hyper_params_spec = hype.spec.new( 54 | batch_norm = hype.spec.random_bool(), 55 | hidden_layers = [hype.spec.choice(range(8, 33))], 56 | dropout = hype.spec.uniform(0.5, 1.0), 57 | learning_rate = 10**hype.spec.uniform(-1, -3), 58 | ) 59 | strategy_params = { 60 | 'io_load_dir': 'temp-wine/example-1-6', 61 | 'io_save_dir': 'temp-wine/example-1-6', 62 | } 63 | 64 | tuner = hype.HyperTuner(hyper_params_spec, solver_generator, **strategy_params) 65 | tuner.tune() 66 | -------------------------------------------------------------------------------- /hyperengine/examples/3_1_lstm_mnist.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import tensorflow as tf 7 | 8 | import hyperengine as hype 9 | 10 | 11 | def rnn_model(params): 12 | x = tf.placeholder(tf.float32, [None, 28, 28], name='input') 13 | y = tf.placeholder(tf.int32, [None], name='label') 14 | 15 | lstm_cells = [tf.nn.rnn_cell.BasicLSTMCell(num_units=layer) for layer in params.lstm.layers] 16 | multi_cell = tf.nn.rnn_cell.MultiRNNCell(lstm_cells) 17 | outputs, states = tf.nn.dynamic_rnn(multi_cell, x, dtype=tf.float32) 18 | 19 | # Here, `states` holds the final states of 3 layers, `states[-1]` is the state of the last layer. 20 | # Hence the name: `h` state of the top layer (short-term state) 21 | top_layer_h_state = states[-1][1] 22 | logits = tf.layers.dense(top_layer_h_state, 10) 23 | xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits) 24 | loss = tf.reduce_mean(xentropy, name='loss') 25 | optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate) 26 | optimizer.minimize(loss, name='minimize') 27 | correct = tf.nn.in_top_k(logits, y, 1) 28 | tf.reduce_mean(tf.cast(correct, tf.float32), name='accuracy') 29 | 30 | from tensorflow.examples.tutorials.mnist import input_data 31 | tf_data_sets = input_data.read_data_sets('temp-mnist/data', one_hot=False) 32 | convert = lambda data_set: hype.DataSet(data_set.images.reshape((-1, 28, 28)), data_set.labels) 33 | data = hype.Data(train=convert(tf_data_sets.train), 34 | validation=convert(tf_data_sets.validation), 35 | test=convert(tf_data_sets.test)) 36 | 37 | def solver_generator(params): 38 | rnn_model(params) 39 | 40 | solver_params = { 41 | 'batch_size': 1000, 42 | 'eval_batch_size': 2500, 43 | 'epochs': 10, 44 | 'evaluate_test': True, 45 | 'eval_flexible': False, 46 | 'save_dir': 'temp-mnist/model-zoo/example-3-1-{date}-{random_id}', 47 | 'save_accuracy_limit': 0.99, 48 | } 49 | solver = hype.TensorflowSolver(data=data, hyper_params=params, **solver_params) 50 | return solver 51 | 52 | 53 | hyper_params_spec = hype.spec.new( 54 | learning_rate = 10**hype.spec.uniform(-2, -3), 55 | lstm = hype.spec.new( 56 | layers = [hype.spec.choice([128, 160, 256]), 57 | hype.spec.choice([128, 160, 256]), 58 | hype.spec.choice([128, 160, 256])] 59 | ) 60 | ) 61 | strategy_params = { 62 | 'io_load_dir': 'temp-mnist/example-3-1', 63 | 'io_save_dir': 'temp-mnist/example-3-1', 64 | } 65 | 66 | tuner = hype.HyperTuner(hyper_params_spec, solver_generator, **strategy_params) 67 | tuner.tune() 68 | -------------------------------------------------------------------------------- /hyperengine/impl/tensorflow/tensorflow_solver.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import tensorflow as tf 7 | 8 | from ...base import * 9 | from ...model import BaseSolver 10 | from .tensorflow_model_io import TensorflowModelIO 11 | from .tensorflow_runner import TensorflowRunner 12 | from .tf_util import is_gpu_available 13 | 14 | 15 | class TensorflowSolver(BaseSolver): 16 | def __init__(self, data, model=None, hyper_params=None, augmentation=None, model_io=None, reducer='max', **params): 17 | if isinstance(model, TensorflowRunner): 18 | runner = model 19 | else: 20 | runner = TensorflowRunner(model) 21 | 22 | self._session = None 23 | self._model_io = model_io if model_io is not None else TensorflowModelIO(**params) 24 | self._save_accuracy_limit = params.get('save_accuracy_limit', 0) 25 | 26 | params['eval_flexible'] = params.get('eval_flexible', True) and is_gpu_available() 27 | super(TensorflowSolver, self).__init__(runner, data, hyper_params, augmentation, reducer, **params) 28 | 29 | def create_session(self): 30 | self._session = tf.Session(graph=self._runner.graph()) 31 | return self._session 32 | 33 | def init_session(self): 34 | self._runner.init(session=self._session) 35 | results = self._load(directory=self._model_io.load_dir, log_level=1) 36 | return results.get('validation_accuracy', 0) 37 | 38 | def terminate(self): 39 | self._runner.terminate() 40 | 41 | def on_best_accuracy(self, accuracy, eval_result): 42 | if accuracy >= self._save_accuracy_limit: 43 | self._model_io.save_results({'validation_accuracy': accuracy, 'model_size': self._runner.model_size()}) 44 | self._model_io.save_hyper_params(self._hyper_params) 45 | self._model_io.save_session(self._session) 46 | self._model_io.save_data(eval_result.get('data')) 47 | 48 | def _evaluate_test(self): 49 | if not self._eval_test: 50 | return 51 | 52 | if self._test_set is None: 53 | warn('Test set is not provided. Skip test evaluation') 54 | return 55 | 56 | # Load the best session if available before test evaluation 57 | current_results = self._load(directory=self._model_io.save_dir, log_level=0) 58 | eval_ = super(TensorflowSolver, self)._evaluate_test() 59 | if not current_results: 60 | return eval_ 61 | 62 | # Update the current results 63 | current_results['test_accuracy'] = eval_.get('accuracy', 0) 64 | self._model_io.save_results(current_results) 65 | return eval_ 66 | 67 | def _load(self, directory, log_level): 68 | self._model_io.load_session(self._session, directory, log_level) 69 | results = self._model_io.load_results(directory, log_level) 70 | return results or {} 71 | -------------------------------------------------------------------------------- /hyperengine/examples/1_4_fine_tuning_saved_model.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import os 7 | import sys 8 | import tensorflow as tf 9 | from tensorflow.examples.tutorials.mnist import input_data 10 | 11 | import hyperengine as hype 12 | 13 | assert len(sys.argv) == 2, 'Error: Mandatory cmd argument missing.\n\n' \ 14 | 'Usage: %s ' % (os.path.basename(__file__), ) 15 | model_path = sys.argv[1] 16 | 17 | def cnn_model(params): 18 | x = tf.placeholder(tf.float32, [None, 28, 28, 1], name='input') 19 | y = tf.placeholder(tf.float32, [None, 10], name='label') 20 | mode = tf.placeholder(tf.string, name='mode') 21 | 22 | conv1 = tf.layers.conv2d(x, filters=params.conv_filters_1, kernel_size=[3, 3], padding='same', activation=tf.nn.relu) 23 | pool1 = tf.layers.max_pooling2d(conv1, pool_size=[2, 2], strides=[2, 2]) 24 | 25 | conv2 = tf.layers.conv2d(pool1, filters=params.conv_filters_2, kernel_size=[3, 3], padding='same', activation=tf.nn.relu) 26 | pool2 = tf.layers.max_pooling2d(conv2, pool_size=[2, 2], strides=[2, 2]) 27 | 28 | flat = tf.reshape(pool2, [-1, pool2.shape[1] * pool2.shape[2] * pool2.shape[3]]) 29 | dense = tf.layers.dense(inputs=flat, units=params.dense_size, activation=tf.nn.relu) 30 | dense = tf.layers.dropout(dense, rate=params.dropout_rate, training=tf.equal(mode, 'train')) 31 | logits = tf.layers.dense(inputs=dense, units=10) 32 | 33 | loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y), name='loss') 34 | optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate) 35 | train_op = optimizer.minimize(loss_op, name='minimize') 36 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), tf.float32), name='accuracy') 37 | 38 | return locals() # to avoid GC 39 | 40 | tf_data_sets = input_data.read_data_sets('temp-mnist/data', one_hot=True) 41 | convert = lambda data_set: hype.DataSet(data_set.images.reshape((-1, 28, 28, 1)), data_set.labels) 42 | data = hype.Data(train=convert(tf_data_sets.train), 43 | validation=convert(tf_data_sets.validation), 44 | test=convert(tf_data_sets.test)) 45 | 46 | solver_params = { 47 | 'batch_size': 1000, 48 | 'eval_batch_size': 2500, 49 | 'epochs': 50, 50 | 'evaluate_test': True, 51 | 'eval_flexible': True, 52 | 'save_dir': model_path, 53 | 'load_dir': model_path, 54 | } 55 | 56 | model_io = hype.TensorflowModelIO(**solver_params) 57 | hyper_params = model_io.load_hyper_params() 58 | assert hyper_params, 'No model found in "%s"' % model_path 59 | hyper_params = hype.spec.new(hyper_params) # convert a dict to an object 60 | 61 | cnn_model(hyper_params) 62 | solver = hype.TensorflowSolver(data=data, hyper_params=hyper_params, **solver_params) 63 | solver.train() 64 | -------------------------------------------------------------------------------- /hyperengine/examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | All examples are complete runnable scripts and are self-contained to ease understanding 4 | (the only exception is dataset fetching that has been moved to [`common.py`](common.py)). 5 | 6 | #### 1. Getting Started 7 | - [**Hello World**](1_1_hello_word_with_mnist.py): 8 | a very simple example to run your existing model in HyperEngine environment. 9 | - [**Getting started with tuning**](1_2_getting_started_with_tuning.py): 10 | tuning a single parameter (`learning_rate`) to find the best value for 11 | a simple CNN. 12 | - [**Saving best models**](1_3_saving_best_models_mnist.py): 13 | tuning several hyper-parameters and saving the best CNN models on the disk. 14 | - [**Fine-tuning the saved model**](1_4_fine_tuning_saved_model.py): 15 | training the selected model further to squeeze the highest possible accuracy out of it. 16 | - [**Learning curve prediction**](1_5_learning_curve_prediction.py): 17 | optimizing the process with the learning curve prediction. 18 | - [**Regression problem**](1_6_optimizing_regression.py): HyperEngine is designed for classification problems, 19 | but with some tricks can be applied in regression settings as well. 20 | 21 | #### 2. Convolutional Neural Networks 22 | - [**Exploring CNN architectures to beat MNIST record**](2_1_cnn_mnist.py): 23 | exploring and tuning all possible variations of the CNN design to get 24 | to the [top accuracy](http://rodrigob.github.io/are_we_there_yet/build/classification_datasets_results.html#4d4e495354) 25 | for the MNIST dataset. 26 | - [**Exploring CNN architectures for CIFAR-10 dataset**](2_2_cnn_cifar.py): 27 | exploring different variations of the CNN design to get good accuracy for the CIFAR-10 dataset 28 | (state-of-the art accuracy would require a bit more efforts - in the next examples). 29 | - [**Exploring architectures of All Convolutional Nets for CIFAR-10 dataset**](2_3_all_conv_net_cifar.py): 30 | exploring different variations of the fully convolutional nets (FCNN), which achieve state-of-the-art accuracy 31 | with few parameters and computational costs. 32 | See ["Striving for Simplicity: The All Convolutional Net"](https://arxiv.org/abs/1412.6806) paper for details. 33 | 34 | #### 3. Recurrent Neural Networks 35 | - [**LSTM to classify MNIST digits**](3_1_lstm_mnist.py): 36 | recurrent neural networks can process images too. Let's see if it can get to 99% with right hyper-parameters. 37 | - [**RNN to detect SMS spam**](3_2_rnn_sms_spam_detector.py): 38 | solving a simple NLP problem with different types of RNN. 39 | This example shows how hyper-parameters can be used in data processing. 40 | 41 | #### 4. Natutal Language Processing 42 | - [**Word2Vec training**](4_1_word2vec_embedding.py): 43 | an example of custom data provider to train Word2Vec embedding vectors. In addition, it shows how to use the solver 44 | without accuracy metric and validation. 45 | -------------------------------------------------------------------------------- /hyperengine/examples/1_3_saving_best_models_mnist.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import tensorflow as tf 7 | from tensorflow.examples.tutorials.mnist import input_data 8 | 9 | import hyperengine as hype 10 | 11 | def cnn_model(params): 12 | x = tf.placeholder(tf.float32, [None, 28, 28, 1], name='input') 13 | y = tf.placeholder(tf.float32, [None, 10], name='label') 14 | mode = tf.placeholder(tf.string, name='mode') 15 | 16 | conv1 = tf.layers.conv2d(x, filters=params.conv_filters_1, kernel_size=[3, 3], padding='same', activation=tf.nn.relu) 17 | pool1 = tf.layers.max_pooling2d(conv1, pool_size=[2, 2], strides=[2, 2]) 18 | 19 | conv2 = tf.layers.conv2d(pool1, filters=params.conv_filters_2, kernel_size=[3, 3], padding='same', activation=tf.nn.relu) 20 | pool2 = tf.layers.max_pooling2d(conv2, pool_size=[2, 2], strides=[2, 2]) 21 | 22 | flat = tf.reshape(pool2, [-1, pool2.shape[1] * pool2.shape[2] * pool2.shape[3]]) 23 | dense = tf.layers.dense(inputs=flat, units=params.dense_size, activation=tf.nn.relu) 24 | dense = tf.layers.dropout(dense, rate=params.dropout_rate, training=tf.equal(mode, 'train')) 25 | logits = tf.layers.dense(inputs=dense, units=10) 26 | 27 | loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y), name='loss') 28 | optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate) 29 | train_op = optimizer.minimize(loss_op, name='minimize') 30 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), tf.float32), name='accuracy') 31 | 32 | return locals() # to avoid GC 33 | 34 | tf_data_sets = input_data.read_data_sets('temp-mnist/data', one_hot=True) 35 | convert = lambda data_set: hype.DataSet(data_set.images.reshape((-1, 28, 28, 1)), data_set.labels) 36 | data = hype.Data(train=convert(tf_data_sets.train), 37 | validation=convert(tf_data_sets.validation), 38 | test=convert(tf_data_sets.test)) 39 | 40 | def solver_generator(params): 41 | solver_params = { 42 | 'batch_size': 1000, 43 | 'eval_batch_size': 2500, 44 | 'epochs': 10, 45 | 'evaluate_test': True, 46 | 'eval_flexible': False, 47 | 'save_dir': 'temp-mnist/model-zoo/example-1-3-{date}-{random_id}', 48 | 'save_accuracy_limit': 0.9930, 49 | } 50 | cnn_model(params) 51 | solver = hype.TensorflowSolver(data=data, hyper_params=params, **solver_params) 52 | return solver 53 | 54 | 55 | hyper_params_spec = hype.spec.new( 56 | learning_rate = 10**hype.spec.uniform(-2, -3), 57 | conv_filters_1 = hype.spec.choice([20, 32, 48]), 58 | conv_filters_2 = hype.spec.choice([64, 96, 128]), 59 | dense_size = hype.spec.choice([256, 512, 768, 1024]), 60 | dropout_rate = hype.spec.uniform(0.5, 0.9), 61 | ) 62 | strategy_params = { 63 | 'io_load_dir': 'temp-mnist/example-1-3', 64 | 'io_save_dir': 'temp-mnist/example-1-3', 65 | } 66 | 67 | tuner = hype.HyperTuner(hyper_params_spec, solver_generator, **strategy_params) 68 | tuner.tune() 69 | -------------------------------------------------------------------------------- /hyperengine/examples/1_2_getting_started_with_tuning.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Optimizing a simple CNN model with one hyper-parameter (learning rate). 5 | 6 | The model itself isn't different from "Hello World Example", except for the hyper-parameter that 7 | is now taken from the input dictionary: `params['learning_rate']`. 8 | 9 | Also pay attention to `strategy_params` argument that dictates to save Bayesian optimization parameters 10 | automatically and continue from the last saved state on subsequent runs. 11 | """ 12 | 13 | __author__ = 'maxim' 14 | 15 | import tensorflow as tf 16 | from tensorflow.examples.tutorials.mnist import input_data 17 | 18 | import hyperengine as hype 19 | 20 | def cnn_model(params): 21 | x = tf.placeholder(tf.float32, [None, 28, 28, 1], name='input') 22 | y = tf.placeholder(tf.float32, [None, 10], name='label') 23 | mode = tf.placeholder(tf.string, name='mode') 24 | 25 | conv1 = tf.layers.conv2d(x, filters=32, kernel_size=[3, 3], padding='same', activation=tf.nn.relu) 26 | pool1 = tf.layers.max_pooling2d(conv1, pool_size=[2, 2], strides=[2, 2]) 27 | 28 | conv2 = tf.layers.conv2d(pool1, filters=64, kernel_size=[3, 3], padding='same', activation=tf.nn.relu) 29 | pool2 = tf.layers.max_pooling2d(conv2, pool_size=[2, 2], strides=[2, 2]) 30 | 31 | flat = tf.reshape(pool2, [-1, pool2.shape[1] * pool2.shape[2] * pool2.shape[3]]) 32 | dense = tf.layers.dense(inputs=flat, units=1024, activation=tf.nn.relu) 33 | dense = tf.layers.dropout(dense, rate=0.5, training=tf.equal(mode, 'train')) 34 | logits = tf.layers.dense(inputs=dense, units=10) 35 | 36 | loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y), name='loss') 37 | optimizer = tf.train.AdamOptimizer(learning_rate=params['learning_rate']) 38 | train_op = optimizer.minimize(loss_op, name='minimize') 39 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), tf.float32), name='accuracy') 40 | 41 | return locals() # to avoid GC 42 | 43 | tf_data_sets = input_data.read_data_sets('temp-mnist/data', one_hot=True) 44 | convert = lambda data_set: hype.DataSet(data_set.images.reshape((-1, 28, 28, 1)), data_set.labels) 45 | data = hype.Data(train=convert(tf_data_sets.train), 46 | validation=convert(tf_data_sets.validation), 47 | test=convert(tf_data_sets.test)) 48 | 49 | def solver_generator(params): 50 | solver_params = { 51 | 'batch_size': 1000, 52 | 'eval_batch_size': 2500, 53 | 'epochs': 10, 54 | 'evaluate_test': True, 55 | 'eval_flexible': False, 56 | } 57 | cnn_model(params) 58 | solver = hype.TensorflowSolver(data=data, hyper_params=params, **solver_params) 59 | return solver 60 | 61 | 62 | hyper_params_spec = { 63 | 'learning_rate': 10**hype.spec.uniform(-1, -3), 64 | } 65 | strategy_params = { 66 | 'io_load_dir': 'temp-mnist/example-1-2', 67 | 'io_save_dir': 'temp-mnist/example-1-2', 68 | } 69 | 70 | tuner = hype.HyperTuner(hyper_params_spec, solver_generator, **strategy_params) 71 | tuner.tune() 72 | -------------------------------------------------------------------------------- /hyperengine/tests/data_set_test.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import unittest 7 | 8 | import numpy as np 9 | 10 | from hyperengine import DataSet, merge_data_sets 11 | 12 | 13 | class DataSetTest(unittest.TestCase): 14 | @classmethod 15 | def setUpClass(cls): 16 | super(DataSetTest, cls).setUpClass() 17 | np.random.shuffle = lambda x : x 18 | 19 | def test_fit(self): 20 | arr = np.asanyarray([1, 2, 3, 4]) 21 | self.ds = DataSet(arr, arr) 22 | self.check_next_batch(2, [1, 2], 2, 2, 0, False) 23 | self.check_next_batch(2, [3, 4], 4, 0, 1, True) 24 | self.check_next_batch(2, [1, 2], 6, 2, 1, False) 25 | self.check_next_batch(2, [3, 4], 8, 0, 2, True) 26 | 27 | def test_fit2(self): 28 | arr = np.asanyarray([1, 2, 3, 4, 5, 6]) 29 | self.ds = DataSet(arr, arr) 30 | self.check_next_batch(2, [1, 2], 2, 2, 0, False) 31 | self.check_next_batch(2, [3, 4], 4, 4, 0, False) 32 | self.check_next_batch(2, [5, 6], 6, 0, 1, True) 33 | self.check_next_batch(2, [1, 2], 8, 2, 1, False) 34 | self.check_next_batch(2, [3, 4], 10, 4, 1, False) 35 | self.check_next_batch(2, [5, 6], 12, 0, 2, True) 36 | 37 | def test_does_not_fit(self): 38 | arr = np.asanyarray([1, 2, 3, 4, 5]) 39 | self.ds = DataSet(arr, arr) 40 | self.check_next_batch(2, [1, 2], 2, 2, 0, False) 41 | self.check_next_batch(2, [3, 4], 4, 4, 0, False) 42 | self.check_next_batch(2, [5, ], 5, 0, 1, True) 43 | self.check_next_batch(2, [1, 2], 7, 2, 1, False) 44 | self.check_next_batch(2, [3, 4], 9, 4, 1, False) 45 | self.check_next_batch(2, [5, ], 10, 0, 2, True) 46 | 47 | def test_too_small_batch(self): 48 | arr = np.asanyarray([1, 2, 3]) 49 | self.ds = DataSet(arr, arr) 50 | self.check_next_batch(4, [1, 2, 3], 3, 0, 1, True) 51 | self.check_next_batch(4, [1, 2, 3], 6, 0, 2, True) 52 | self.check_next_batch(4, [1, 2, 3], 9, 0, 3, True) 53 | 54 | def test_merge_data_sets(self): 55 | x_t = np.array([[[1, 2], [3, 4]], 56 | [[5, 6], [7, 8]]]).reshape((-1, 2, 2, 1)) 57 | y_t = np.array([0, 1]) 58 | ds_t = DataSet(x_t, y_t) 59 | 60 | x_v = np.array([[[0, 0], [0, 0]]]).reshape((-1, 2, 2, 1)) 61 | y_v = np.array([2]) 62 | ds_v = DataSet(x_v, y_v) 63 | 64 | ds = merge_data_sets(ds_t, ds_v) 65 | self.assertEqual((3, 2, 2, 1), ds.x.shape) 66 | self.assertEqual((3, ), ds.y.shape) 67 | 68 | self.assert_equal_arrays(ds_t.x[0,:,:,0], ds.x[0,:,:,0]) 69 | self.assert_equal_arrays(ds_t.x[1,:,:,0], ds.x[1,:,:,0]) 70 | self.assert_equal_arrays(ds_v.x[0,:,:,0], ds.x[2,:,:,0]) 71 | self.assert_equal_arrays([0, 1, 2], ds.y) 72 | 73 | def check_next_batch(self, batch_size, array, index, index_in_epoch, epochs_completed, just_completed): 74 | batch = self.ds.next_batch(batch_size) 75 | self.assertEquals(list(batch[0]), array) 76 | self.assertEquals(list(batch[1]), array) 77 | self.assertEquals(self.ds.index, index) 78 | self.assertEquals(self.ds.index_in_epoch, index_in_epoch) 79 | self.assertEquals(self.ds.epochs_completed, epochs_completed) 80 | self.assertEquals(self.ds.just_completed, just_completed) 81 | 82 | def assert_equal_arrays(self, array1, array2): 83 | self.assertTrue((array1 == array2).all()) 84 | 85 | if __name__ == '__main__': 86 | unittest.main() 87 | -------------------------------------------------------------------------------- /hyperengine/examples/1_5_learning_curve_prediction.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import tensorflow as tf 7 | from tensorflow.examples.tutorials.mnist import input_data 8 | 9 | import hyperengine as hype 10 | 11 | def cnn_model(params): 12 | x = tf.placeholder(tf.float32, [None, 28, 28, 1], name='input') 13 | y = tf.placeholder(tf.float32, [None, 10], name='label') 14 | mode = tf.placeholder(tf.string, name='mode') 15 | 16 | conv1 = tf.layers.conv2d(x, filters=params.conv_filters_1, kernel_size=[3, 3], padding='same', activation=tf.nn.relu) 17 | pool1 = tf.layers.max_pooling2d(conv1, pool_size=[2, 2], strides=[2, 2]) 18 | 19 | conv2 = tf.layers.conv2d(pool1, filters=params.conv_filters_2, kernel_size=[3, 3], padding='same', activation=tf.nn.relu) 20 | pool2 = tf.layers.max_pooling2d(conv2, pool_size=[2, 2], strides=[2, 2]) 21 | 22 | flat = tf.reshape(pool2, [-1, pool2.shape[1] * pool2.shape[2] * pool2.shape[3]]) 23 | dense = tf.layers.dense(inputs=flat, units=params.dense_size, activation=tf.nn.relu) 24 | dense = tf.layers.dropout(dense, rate=params.dropout_rate, training=tf.equal(mode, 'train')) 25 | logits = tf.layers.dense(inputs=dense, units=10) 26 | 27 | loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y), name='loss') 28 | optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate) 29 | train_op = optimizer.minimize(loss_op, name='minimize') 30 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), tf.float32), name='accuracy') 31 | 32 | return locals() # to avoid GC 33 | 34 | tf_data_sets = input_data.read_data_sets('temp-mnist/data', one_hot=True) 35 | convert = lambda data_set: hype.DataSet(data_set.images.reshape((-1, 28, 28, 1)), data_set.labels) 36 | data = hype.Data(train=convert(tf_data_sets.train), 37 | validation=convert(tf_data_sets.validation), 38 | test=convert(tf_data_sets.test)) 39 | 40 | curve_params = { 41 | 'burn_in': 20, 42 | 'min_input_size': 4, 43 | 'value_limit': 0.82, 44 | 'io_load_dir': 'temp-mnist/example-1-5', 45 | 'io_save_dir': 'temp-mnist/example-1-5', 46 | } 47 | curve_predictor = hype.LinearCurvePredictor(**curve_params) 48 | 49 | def solver_generator(params): 50 | solver_params = { 51 | 'batch_size': 1000, 52 | 'eval_batch_size': 2500, 53 | 'epochs': 10, 54 | 'stop_condition': curve_predictor.stop_condition(), 55 | 'reducer': curve_predictor.result_metric(), 56 | 'evaluate_test': True, 57 | 'eval_flexible': False, 58 | 'save_dir': 'temp-mnist/model-zoo/example-1-5-{date}-{random_id}', 59 | 'save_accuracy_limit': 0.9930, 60 | } 61 | cnn_model(params) 62 | solver = hype.TensorflowSolver(data=data, hyper_params=params, **solver_params) 63 | return solver 64 | 65 | 66 | hyper_params_spec = hype.spec.new( 67 | learning_rate = 10**hype.spec.uniform(-2, -3), 68 | conv_filters_1 = hype.spec.choice([20, 32, 48]), 69 | conv_filters_2 = hype.spec.choice([64, 96, 128]), 70 | dense_size = hype.spec.choice([256, 512, 768, 1024]), 71 | dropout_rate = hype.spec.uniform(0.5, 0.9), 72 | ) 73 | strategy_params = { 74 | 'io_load_dir': 'temp-mnist/example-1-5', 75 | 'io_save_dir': 'temp-mnist/example-1-5', 76 | } 77 | 78 | tuner = hype.HyperTuner(hyper_params_spec, solver_generator, **strategy_params) 79 | tuner.tune() 80 | -------------------------------------------------------------------------------- /hyperengine/base/base_io.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import datetime 7 | import os 8 | 9 | import numpy as np 10 | 11 | from .logging import * 12 | from .util import str_to_dict, smart_str, random_id 13 | 14 | 15 | class BaseIO(object): 16 | """ 17 | The base class responsible for I/O operations. 18 | """ 19 | 20 | def __init__(self, **params): 21 | self.load_dir = params.get('load_dir') 22 | self.save_dir = _format_path(params.get('save_dir')) 23 | 24 | @staticmethod 25 | def _prepare(directory): 26 | if directory is None: 27 | return False 28 | if not os.path.exists(directory): 29 | vlog('Creating a directory ', directory) 30 | os.makedirs(directory) 31 | return True 32 | 33 | @staticmethod 34 | def _load_dict(from_file): 35 | if not os.path.exists(from_file): 36 | debug('Cannot load a dict. File does not exist: ', from_file) 37 | return {} 38 | try: 39 | with open(from_file, 'r') as file_: 40 | line = file_.readline() 41 | return str_to_dict(line) 42 | except BaseException as e: 43 | warn('Cannot load a dict. Error: ', e.message) 44 | return {} 45 | 46 | 47 | class Serializable(object): 48 | """ 49 | Represents a serializable object: can export own state to the dictionary and import back. 50 | """ 51 | 52 | def import_from(self, data): 53 | """ 54 | Imports the object state from the object `data`, usually a dictionary. 55 | """ 56 | raise NotImplementedError() 57 | 58 | def _import_from_keys(self, data, keys, default_value): 59 | for key in keys: 60 | value = np.array(data.get(key, default_value)) 61 | setattr(self, '_%s' % key, value) 62 | 63 | def export_to(self): 64 | """ 65 | Exports the object to the dictionary. 66 | """ 67 | raise NotImplementedError() 68 | 69 | def _export_keys_to(self, keys): 70 | return {key: getattr(self, '_%s' % key).tolist() for key in keys} 71 | 72 | 73 | class DefaultIO(BaseIO): 74 | """ 75 | A simple I/O implementation: can save and load a serializable instance. 76 | """ 77 | 78 | def __init__(self, serializable, filename, **params): 79 | super(DefaultIO, self).__init__(**params) 80 | self.serializable = serializable 81 | self.filename = filename 82 | 83 | def load(self): 84 | directory = self.load_dir 85 | if not directory is None: 86 | destination = os.path.join(directory, self.filename) 87 | if os.path.exists(destination): 88 | data = DefaultIO._load_dict(destination) 89 | if is_debug_logged(): 90 | debug('Loaded data: %s from %s' % (smart_str(data), destination)) 91 | self.serializable.import_from(data) 92 | return 93 | self.serializable.import_from({}) 94 | 95 | def save(self): 96 | directory = self.save_dir 97 | if not DefaultIO._prepare(directory): 98 | return 99 | destination = os.path.join(directory, self.filename) 100 | with open(destination, 'w') as file_: 101 | data = smart_str(self.serializable.export_to()) 102 | vlog('Exported data:', data) 103 | file_.write(data) 104 | debug('Data saved to:', destination) 105 | 106 | 107 | def _format_path(directory): 108 | if directory and '{' in directory and '}' in directory: 109 | now = datetime.datetime.now() 110 | return directory.format(date=now.strftime('%Y-%m-%d'), 111 | time=now.strftime('%H-%M-%S'), 112 | random_id=random_id()) 113 | return directory 114 | -------------------------------------------------------------------------------- /hyperengine/spec/sugar.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import math 7 | from scipy import stats 8 | 9 | from .nodes import * 10 | 11 | def wrap(node, transform): 12 | if transform is not None: 13 | return MergeNode(transform, node) 14 | return node 15 | 16 | def uniform(start=0.0, end=1.0, transform=None, name=None): 17 | node = UniformNode(start, end).with_name(name) 18 | return wrap(node, transform) 19 | 20 | def normal(mean=0.0, stdev=1.0, name=None): 21 | return NonUniformNode(ppf=stats.norm.ppf, loc=mean, scale=stdev).with_name(name) 22 | 23 | def choice(array, transform=None, name=None): 24 | if not [item for item in array if isinstance(item, BaseNode)]: 25 | node = ChoiceNode(*array).with_name(name) 26 | else: 27 | node = MergeChoiceNode(*array).with_name(name) 28 | return wrap(node, transform) 29 | 30 | def merge(nodes, function, name=None): 31 | if callable(nodes) and not callable(function): 32 | nodes, function = function, nodes 33 | if isinstance(nodes, BaseNode): 34 | nodes = [nodes] 35 | return MergeNode(function, *nodes).with_name(name) 36 | 37 | def random_bit(): 38 | return choice([0, 1]) 39 | 40 | def random_bool(): 41 | return choice([False, True]) 42 | 43 | def random_int(n): 44 | return choice(range(n)) 45 | 46 | def exp(node): return merge([node], math.exp) 47 | def expm1(node): return merge([node], math.expm1) 48 | def frexp(node): return merge([node], math.frexp) 49 | def ldexp(node, i): return merge([node], lambda x: math.ldexp(x, i)) 50 | 51 | def sqrt(node): return merge([node], math.sqrt) 52 | def pow(a, b): return a ** b 53 | 54 | def log(node, base=None): return merge([node], lambda x: math.log(x, base)) 55 | def log1p(node): return merge([node], math.log1p) 56 | def log10(node): return merge([node], math.log10) 57 | 58 | def sin(node): return merge([node], math.sin) 59 | def cos(node): return merge([node], math.cos) 60 | def tan(node): return merge([node], math.tan) 61 | 62 | def sinh(node): return merge([node], math.sinh) 63 | def cosh(node): return merge([node], math.cosh) 64 | def tanh(node): return merge([node], math.tanh) 65 | 66 | def asin(node): return merge([node], math.asin) 67 | def acos(node): return merge([node], math.acos) 68 | def atan(node): return merge([node], math.atan) 69 | def atan2(node): return merge([node], math.atan2) 70 | 71 | def asinh(node): return merge([node], math.asinh) 72 | def acosh(node): return merge([node], math.acosh) 73 | def atanh(node): return merge([node], math.atanh) 74 | 75 | def min_(*array): 76 | nodes = [item for item in array if isinstance(item, BaseNode)] 77 | if len(nodes) == 0: 78 | return min(*array) if len(array) > 1 else array[0] 79 | node = merge(nodes, min) if len(nodes) > 1 else nodes[0] 80 | 81 | rest = [item for item in array if not isinstance(item, BaseNode)] 82 | if rest: 83 | node = merge([node], lambda x: min(x, *rest)) 84 | return node 85 | 86 | def max_(*array): 87 | nodes = [item for item in array if isinstance(item, BaseNode)] 88 | if len(nodes) == 0: 89 | return max(*array) if len(array) > 1 else array[0] 90 | node = merge(nodes, max) if len(nodes) > 1 else nodes[0] 91 | 92 | rest = [item for item in array if not isinstance(item, BaseNode)] 93 | if rest: 94 | node = merge([node], lambda x: max(x, *rest)) 95 | return node 96 | 97 | def new(*args, **kwargs): 98 | from ..base import NamedDict 99 | if len(args) == 1 and len(kwargs) == 0: 100 | return NamedDict(args[0]) 101 | assert len(args) == 0, 'Failed to created a NamedDict with arguments: %s' % str(args) 102 | return NamedDict(kwargs) 103 | -------------------------------------------------------------------------------- /hyperengine/tests/named_dict_test.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | 7 | import six 8 | import unittest 9 | 10 | import numpy as np 11 | 12 | import hyperengine as hype 13 | from hyperengine.base import NamedDict 14 | 15 | 16 | class NamedDictTest(unittest.TestCase): 17 | def test_embedded_dict(self): 18 | spec = hype.spec.new({ 19 | 'foo': { 20 | 'bar': { 21 | 'baz': 999 22 | }, 23 | 'baz': [] 24 | } 25 | }) 26 | 27 | instance = self._instantiate(spec) 28 | self.assertEqual(repr(instance), "{'foo': {'bar': {'baz': 999}, 'baz': []}}") 29 | six.assertCountEqual(self, instance.foo.keys(), ['bar', 'baz']) 30 | 31 | self.assertTrue('foo' in instance) 32 | self.assertTrue('bar' in instance.foo) 33 | self.assertTrue('baz' in instance.foo.bar) 34 | 35 | self.assertEqual(type(instance), NamedDict) 36 | self.assertEqual(type(instance.foo), NamedDict) 37 | 38 | self.assertEqual(instance.foo.bar.baz, 999) 39 | self.assertEqual(instance.foo.baz, []) 40 | 41 | 42 | def test_embedded_list(self): 43 | spec = hype.spec.new( 44 | value = [[1], [[2]], (3, 4), {'foo': 5}], 45 | ) 46 | 47 | instance = self._instantiate(spec) 48 | self.assertEqual(type(instance), NamedDict) 49 | self.assertEqual(type(instance.value), list) 50 | self.assertEqual(type(instance.value[3]), NamedDict) 51 | self.assertEqual(repr(instance), "{'value': [[1], [[2]], [3, 4], {'foo': 5}]}") 52 | self.assertEqual(instance.value[3].foo, 5) 53 | 54 | 55 | def test_dict_inside_list(self): 56 | spec = hype.spec.new( 57 | foo = [ 58 | { 'bar': 0 }, 59 | { 'bar': 1 }, 60 | { 'bar': 2 }, 61 | ] 62 | ) 63 | 64 | instance = self._instantiate(spec) 65 | self.assertEqual(type(instance), NamedDict) 66 | self.assertEqual(type(instance.foo), list) 67 | self.assertEqual(type(instance.foo[0]), NamedDict) 68 | self.assertEqual(instance.foo[0].bar, 0) 69 | self.assertEqual(instance.foo[1].bar, 1) 70 | self.assertEqual(instance.foo[2].bar, 2) 71 | 72 | 73 | def test_real(self): 74 | hyper_params_spec = hype.spec.new( 75 | learning_rate=10 ** hype.spec.uniform(-2, -3), 76 | conv=hype.spec.new( 77 | filters=[hype.spec.choice([20, 32, 48]), hype.spec.choice([64, 96, 128])], 78 | residual=hype.spec.random_bit(), 79 | ), 80 | dropout=hype.spec.uniform(0.5, 0.9), 81 | ) 82 | 83 | instance = self._instantiate(hyper_params_spec) 84 | self.assertEqual(repr(instance), 85 | "{'conv': {'filters': [20, 64], 'residual': 0}, 'dropout': 0.500000, 'learning_rate': 0.001000}") 86 | six.assertCountEqual(self, instance.keys(), ['learning_rate', 'conv', 'dropout']) 87 | 88 | self.assertEqual(type(instance), NamedDict) 89 | self.assertEqual(type(instance.conv), NamedDict) 90 | 91 | self.assertEqual(instance.learning_rate, 0.001) 92 | self.assertEqual(instance['learning_rate'], 0.001) 93 | self.assertEqual(instance.get('learning_rate'), 0.001) 94 | 95 | self.assertEqual(instance.foo, None) 96 | self.assertEqual(instance.get('foo'), None) 97 | self.assertEqual(instance.get('foo', 'bar'), 'bar') 98 | 99 | six.assertCountEqual(self, instance.conv.keys(), ['filters', 'residual']) 100 | self.assertEqual(instance.conv.filters, [20, 64]) 101 | self.assertEqual(instance.conv.filters[0], 20) 102 | self.assertEqual(instance.conv.filters[1], 64) 103 | 104 | 105 | def _instantiate(self, spec): 106 | parsed = hype.spec.ParsedSpec(spec) 107 | points = np.zeros([parsed.size()]) 108 | instance = parsed.instantiate(points) 109 | return instance 110 | -------------------------------------------------------------------------------- /hyperengine/base/util.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import ast 7 | import collections 8 | import numbers 9 | import os 10 | import random 11 | import string 12 | import sys 13 | 14 | from six import iteritems 15 | from six.moves import urllib 16 | 17 | import numpy as np 18 | 19 | 20 | def smart_str(val): 21 | if type(val) in [float, np.float32, np.float64] and val: 22 | return '%.6f' % val if abs(val) > 1e-6 else '%e' % val 23 | if type(val) == dict: 24 | return '{%s}' % ', '.join(['%s: %s' % (repr(k), smart_str(val[k])) for k in sorted(val.keys())]) 25 | if type(val) in [list, tuple]: 26 | return '[%s]' % ', '.join(['%s' % smart_str(i) for i in val]) 27 | return repr(val) 28 | 29 | def str_to_dict(s): 30 | return ast.literal_eval(s) 31 | 32 | def zip_longest(list1, list2): 33 | len1 = len(list1) 34 | len2 = len(list2) 35 | for i in range(max(len1, len2)): 36 | yield (list1[i % len1], list2[i % len2]) 37 | 38 | def deep_update(dict_, upd): 39 | for key, value in iteritems(upd): 40 | if isinstance(value, collections.Mapping): 41 | recursive = deep_update(dict_.get(key, {}), value) 42 | dict_[key] = recursive 43 | else: 44 | dict_[key] = upd[key] 45 | return dict_ 46 | 47 | def mini_batch(total, size): 48 | return zip(range(0, total, size), 49 | range(size, total + size, size)) 50 | 51 | def random_id(size=6, chars=string.ascii_uppercase + string.digits): 52 | return ''.join(random.choice(chars) for _ in range(size)) 53 | 54 | def safe_concat(list_): 55 | list_ = [i for i in list_ if i is not None] 56 | if len(list_) == 0: 57 | return None 58 | if type(list_[0]) == np.ndarray: 59 | return np.concatenate(list_) 60 | return list_ 61 | 62 | def call(obj, *args): 63 | if callable(obj): 64 | return obj(*args) 65 | 66 | apply = getattr(obj, 'apply', None) 67 | if callable(apply): 68 | return apply(*args) 69 | 70 | def slice_dict(d, key_prefix): 71 | return {key[len(key_prefix):]: value for key, value in iteritems(d) if key.startswith(key_prefix)} 72 | 73 | def as_function(val, presets, default=None): 74 | if callable(val): 75 | return val 76 | 77 | preset = presets.get(val, default) 78 | if preset is not None: 79 | return preset 80 | 81 | raise ValueError('Value is not recognized: ', val) 82 | 83 | def as_numeric_function(val, presets, default=None): 84 | if isinstance(val, numbers.Number): 85 | def const(*_): 86 | return val 87 | return const 88 | 89 | return as_function(val, presets, default) 90 | 91 | 92 | def download_if_needed(url, path, filename=None): 93 | from .logging import info, debug 94 | 95 | if not os.path.exists(path): 96 | os.makedirs(path) 97 | filename = filename or os.path.basename(url) 98 | full_path = os.path.join(path, filename) 99 | if not os.path.exists(full_path): 100 | info('Downloading %s, please wait...' % filename) 101 | result_path, _ = urllib.request.urlretrieve(url, full_path, _report_hook) 102 | stat = os.stat(result_path) 103 | info('Successfully downloaded "%s" (%d Kb)' % (filename, stat.st_size / 1024)) 104 | return result_path 105 | else: 106 | debug('Already downloaded:', full_path) 107 | return full_path 108 | 109 | 110 | def _report_hook(block_num, block_size, total_size): 111 | read_so_far = block_num * block_size 112 | if total_size > 0: 113 | percent = read_so_far * 1e2 / total_size 114 | s = '\r%5.1f%% %*d / %d' % (percent, len(str(total_size)), read_so_far, total_size) 115 | sys.stdout.write(s) 116 | if read_so_far >= total_size: # near the end 117 | sys.stdout.write('\n') 118 | else: # total size is unknown 119 | sys.stdout.write('read %d\n' % (read_so_far,)) 120 | -------------------------------------------------------------------------------- /hyperengine/impl/tensorflow/tensorflow_runner.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | 7 | import tensorflow as tf 8 | 9 | from ...base import * 10 | from ...model.base_runner import BaseRunner 11 | from .tf_util import graph_vars, get_total_dim 12 | 13 | 14 | class TensorflowRunner(BaseRunner): 15 | def __init__(self, model=None, extra_feed={}, 16 | input='input', label='label', mode='mode', loss='loss', accuracy='accuracy', train='minimize'): 17 | super(TensorflowRunner, self).__init__() 18 | 19 | self._graph = model or tf.get_default_graph() 20 | assert isinstance(self._graph, tf.Graph), '"model" argument must be either tf.Graph instance or None' 21 | self._extra_feed = extra_feed 22 | assert isinstance(self._extra_feed, dict), '"extra_feed" must be a dictionary (tensor -> value)' 23 | 24 | self._x = self._find_tensor(input) 25 | self._y = self._find_tensor(label) 26 | self._mode = self._find_tensor(mode, mandatory=False) 27 | self._loss = self._find_tensor(loss, mandatory=False) 28 | self._accuracy = self._find_tensor(accuracy, mandatory=False) 29 | self._minimize = self._find_op(train) 30 | self._model_size = self._calc_model_size() 31 | 32 | def build_model(self): 33 | pass 34 | 35 | def init(self, **kwargs): 36 | self._session = kwargs['session'] 37 | init = self._find_op('initializer', tf.global_variables_initializer()) 38 | self._session.run(init) 39 | 40 | def run_batch(self, batch_x, batch_y): 41 | feed_dict = self._get_feed_dict(batch_x, batch_y, 'train') 42 | self._session.run(self._minimize, feed_dict=feed_dict) 43 | 44 | def evaluate(self, batch_x, batch_y): 45 | feed_dict = self._get_feed_dict(batch_x, batch_y, 'test') 46 | if self._loss is None and self._accuracy is None: 47 | return {} 48 | if self._accuracy is None: 49 | loss = self._session.run(self._loss, feed_dict=feed_dict) 50 | return {'loss': loss} 51 | if self._loss is None: 52 | accuracy = self._session.run(self._accuracy, feed_dict=feed_dict) 53 | return {'accuracy': accuracy} 54 | loss, accuracy = self._session.run([self._loss, self._accuracy], feed_dict=feed_dict) 55 | return {'loss': loss, 'accuracy': accuracy} 56 | 57 | def terminate(self): 58 | tf.reset_default_graph() 59 | 60 | def graph(self): 61 | return self._graph 62 | 63 | def model_size(self): 64 | return self._model_size 65 | 66 | def _get_feed_dict(self, batch_x, batch_y, mode): 67 | feed_dict = {self._x: batch_x, self._y: batch_y} 68 | if self._mode is not None: 69 | feed_dict[self._mode] = mode 70 | for k, v in self._extra_feed.items(): 71 | if isinstance(v, dict) and mode in v.keys(): 72 | v = v[mode] 73 | feed_dict[k] = v 74 | return feed_dict 75 | 76 | def _find_tensor(self, name, mandatory=True): 77 | try: 78 | return self._graph.get_tensor_by_name(name + ':0') 79 | except KeyError: 80 | if not mandatory: 81 | debug('Tensor not found in Tensorflow graph:', name) 82 | return None 83 | warn('Failed to infer a tensor "%s" in Tensorflow graph. ' 84 | 'Most likely, you should add "name=\'%s\'" in the placeholder/tensor definition' % (name, name)) 85 | raise 86 | 87 | def _find_op(self, name, default=None): 88 | try: 89 | return self._graph.get_operation_by_name(name) 90 | except KeyError: 91 | if default is not None: 92 | debug('Op not found in Tensorflow graph:', name) 93 | return default 94 | warn('Failed to infer an op "%s" in Tensorflow graph. ' 95 | 'Most likely, you should add "name=\'%s\'" in the op definition' % (name, name)) 96 | raise 97 | 98 | def _calc_model_size(self): 99 | return sum(get_total_dim(element) for element in graph_vars(self._graph)) 100 | -------------------------------------------------------------------------------- /hyperengine/examples/2_1_cnn_mnist.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import tensorflow as tf 7 | from tensorflow.examples.tutorials.mnist import input_data 8 | 9 | import hyperengine as hype 10 | 11 | ACTIVATIONS = {name: getattr(tf.nn, name) for name in ['relu', 'relu6', 'elu', 'sigmoid', 'tanh', 'leaky_relu']} 12 | 13 | def cnn_model(params): 14 | x = tf.placeholder(tf.float32, [None, 28, 28, 1], name='input') 15 | y = tf.placeholder(tf.float32, [None, 10], name='label') 16 | mode = tf.placeholder(tf.string, name='mode') 17 | training = tf.equal(mode, 'train') 18 | 19 | def conv_layer(input, hp): 20 | conv = tf.layers.conv2d(input, 21 | filters=hp.filter_num, 22 | kernel_size=hp.filter_size, 23 | padding='same', 24 | activation=ACTIVATIONS[hp.activation]) 25 | bn = tf.layers.batch_normalization(conv, training=training) if hp.batch_norm else conv 26 | dropped = tf.layers.dropout(bn, rate=hp.dropout, training=training) if hp.dropout else bn 27 | pool = tf.layers.max_pooling2d(dropped, pool_size=[2, 2], strides=[2, 2]) 28 | return pool 29 | 30 | def dense_layer(input, hp): 31 | flat = tf.reshape(input, [-1, input.shape[1] * input.shape[2] * input.shape[3]]) 32 | dense = tf.layers.dense(inputs=flat, units=hp.size, activation=tf.nn.relu) 33 | dropped = tf.layers.dropout(dense, rate=hp.dropout, training=training) 34 | return dropped 35 | 36 | layer = conv_layer(x, params.conv[0]) 37 | layer = conv_layer(layer, params.conv[1]) 38 | layer = dense_layer(layer, params.dense) 39 | logits = tf.layers.dense(inputs=layer, units=10) 40 | 41 | loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y), name='loss') 42 | optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate) 43 | train_op = optimizer.minimize(loss_op, name='minimize') 44 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), tf.float32), name='accuracy') 45 | 46 | return locals() # to avoid GC 47 | 48 | tf_data_sets = input_data.read_data_sets('temp-mnist/data', one_hot=True) 49 | convert = lambda data_set: hype.DataSet(data_set.images.reshape((-1, 28, 28, 1)), data_set.labels) 50 | data = hype.Data(train=convert(tf_data_sets.train), 51 | validation=convert(tf_data_sets.validation), 52 | test=convert(tf_data_sets.test)) 53 | 54 | def solver_generator(params): 55 | solver_params = { 56 | 'batch_size': 1000, 57 | 'eval_batch_size': 2500, 58 | 'epochs': 10, 59 | 'evaluate_test': True, 60 | 'eval_flexible': False, 61 | 'save_dir': 'temp-mnist/model-zoo/example-2-1-{date}-{random_id}', 62 | 'save_accuracy_limit': 0.9930, 63 | } 64 | cnn_model(params) 65 | solver = hype.TensorflowSolver(data=data, hyper_params=params, **solver_params) 66 | return solver 67 | 68 | 69 | hyper_params_spec = hype.spec.new( 70 | learning_rate = 10**hype.spec.uniform(-2, -3), 71 | conv = [ 72 | # Layer 1 73 | hype.spec.new( 74 | filter_num = hype.spec.choice([20, 32, 48]), 75 | filter_size = [hype.spec.choice([3, 5])] * 2, 76 | activation = hype.spec.choice(ACTIVATIONS.keys()), 77 | batch_norm = hype.spec.random_bool(), 78 | dropout = hype.spec.uniform(0.0, 0.3), 79 | ), 80 | # Layer 2 81 | hype.spec.new( 82 | filter_num = hype.spec.choice([64, 96, 128]), 83 | filter_size = [hype.spec.choice([3, 5])] * 2, 84 | activation = hype.spec.choice(ACTIVATIONS.keys()), 85 | batch_norm = hype.spec.random_bool(), 86 | dropout = hype.spec.uniform(0.0, 0.5), 87 | ), 88 | ], 89 | # Dense layer 90 | dense = hype.spec.new( 91 | size = hype.spec.choice([256, 512, 768, 1024]), 92 | dropout = hype.spec.uniform(0.0, 0.5), 93 | ), 94 | ) 95 | strategy_params = { 96 | 'io_load_dir': 'temp-mnist/example-2-1', 97 | 'io_save_dir': 'temp-mnist/example-2-1', 98 | } 99 | 100 | tuner = hype.HyperTuner(hyper_params_spec, solver_generator, **strategy_params) 101 | tuner.tune() 102 | -------------------------------------------------------------------------------- /hyperengine/model/data_set.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import numpy as np 7 | 8 | 9 | class DataProvider(object): 10 | pass 11 | 12 | 13 | class IterableDataProvider(DataProvider): 14 | def __init__(self): 15 | super(IterableDataProvider, self).__init__() 16 | self._size = 0 17 | self._step = 0 18 | self._epochs_completed = 0 19 | self._index_in_epoch = 0 20 | self._just_completed = False 21 | 22 | @property 23 | def size(self): 24 | """ 25 | Data size (number of rows) 26 | """ 27 | return self._size 28 | 29 | @property 30 | def step(self): 31 | """ 32 | The number of batches processed 33 | """ 34 | return self._step 35 | 36 | @property 37 | def index(self): 38 | """ 39 | Total index of input rows (over all epochs) 40 | """ 41 | return self._epochs_completed * self._size + self._index_in_epoch 42 | 43 | @property 44 | def index_in_epoch(self): 45 | """ 46 | The index of input rows in a current epoch 47 | """ 48 | return self._index_in_epoch 49 | 50 | @property 51 | def epochs_completed(self): 52 | """ 53 | A number of completed epochs 54 | """ 55 | return self._epochs_completed 56 | 57 | @property 58 | def just_completed(self): 59 | """ 60 | Whether the previous epoch was just completed 61 | """ 62 | return self._just_completed 63 | 64 | def reset_counters(self): 65 | """ 66 | Resets all counters. 67 | """ 68 | self._step = 0 69 | self._epochs_completed = 0 70 | self._index_in_epoch = 0 71 | self._just_completed = False 72 | 73 | def next_batch(self, batch_size): 74 | """ 75 | Returns the next `batch_size` examples from this data set. 76 | """ 77 | raise NotImplementedError 78 | 79 | def _inc_index(self): 80 | index = self._index_in_epoch + 1 81 | if index >= self._size: 82 | self._index_in_epoch = 0 83 | self._epochs_completed += 1 84 | self._just_completed = True 85 | else: 86 | self._index_in_epoch = index 87 | self._just_completed = False 88 | 89 | 90 | class DataSet(IterableDataProvider): 91 | """ 92 | A labeled data set. Both inputs and labels are stored as numpy arrays in memory. 93 | """ 94 | 95 | def __init__(self, x, y): 96 | super(DataSet, self).__init__() 97 | 98 | x = np.array(x) 99 | y = np.array(y) 100 | assert x.shape[0] == y.shape[0] 101 | 102 | self._size = x.shape[0] 103 | self._x = x 104 | self._y = y 105 | 106 | @property 107 | def x(self): 108 | return self._x 109 | 110 | @property 111 | def y(self): 112 | return self._y 113 | 114 | def next_batch(self, batch_size): 115 | if self._just_completed: 116 | permutation = np.arange(self._size) 117 | np.random.shuffle(permutation) 118 | self._x = self._x[permutation] 119 | self._y = self._y[permutation] 120 | 121 | self._step += 1 122 | start = self._index_in_epoch 123 | self._index_in_epoch += batch_size 124 | end = min(self._index_in_epoch, self._size) 125 | if self._index_in_epoch >= self._size: 126 | self._index_in_epoch = 0 127 | self._just_completed = end == self._size 128 | self._epochs_completed += int(self._just_completed) 129 | return self._x[start:end], self._y[start:end] 130 | 131 | 132 | def merge_data_sets(ds1, ds2): 133 | x = np.concatenate([ds1.x, ds2.x], axis=0) 134 | y = np.concatenate([ds1.y, ds2.y], axis=0) 135 | return DataSet(x, y) 136 | 137 | 138 | class Data(object): 139 | """ 140 | Holds a standard data division: training set, validation set and test set. 141 | """ 142 | 143 | def __init__(self, train, validation, test): 144 | assert train is not None, 'Training set must be not None' 145 | assert train is not validation, 'Validation set can not coincide with the training set' 146 | assert train is not test, 'Test set can not coincide with the training set' 147 | 148 | self.train = train 149 | self.validation = validation 150 | self.test = test 151 | 152 | def reset_counters(self): 153 | self.train.reset_counters() 154 | if self.validation: 155 | self.validation.reset_counters() 156 | if self.test: 157 | self.test.reset_counters() 158 | -------------------------------------------------------------------------------- /hyperengine/examples/2_2_cnn_cifar.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import tensorflow as tf 7 | 8 | from common import get_cifar10 9 | import hyperengine as hype 10 | 11 | ACTIVATIONS = {name: getattr(tf.nn, name) for name in ['relu', 'relu6', 'elu', 'sigmoid', 'tanh', 'leaky_relu']} 12 | DOWN_SAMPLES = {name: getattr(tf.layers, name) for name in ['max_pooling2d', 'average_pooling2d']} 13 | 14 | def cnn_model(params): 15 | x = tf.placeholder(tf.float32, [None, 32, 32, 3], name='input') 16 | y = tf.placeholder(tf.float32, [None, 10], name='label') 17 | mode = tf.placeholder(tf.string, name='mode') 18 | training = tf.equal(mode, 'train') 19 | 20 | def conv_layer(input, hp): 21 | conv = tf.layers.conv2d(input, 22 | filters=hp.filter_num, 23 | kernel_size=hp.filter_size, 24 | padding='same', 25 | activation=ACTIVATIONS[hp.activation]) 26 | bn = tf.layers.batch_normalization(conv, training=training) if hp.batch_norm else conv 27 | dropped = tf.layers.dropout(bn, rate=hp.dropout, training=training) if hp.dropout else bn 28 | pool = DOWN_SAMPLES[hp.down_sample](dropped, pool_size=[2, 2], strides=[2, 2]) 29 | return pool 30 | 31 | def dense_layer(input, hp): 32 | flat = tf.reshape(input, [-1, input.shape[1] * input.shape[2] * input.shape[3]]) 33 | dense = tf.layers.dense(inputs=flat, units=hp.size, activation=tf.nn.relu) 34 | dropped = tf.layers.dropout(dense, rate=hp.dropout, training=training) 35 | return dropped 36 | 37 | layer = conv_layer(x, params.conv[0]) 38 | layer = conv_layer(layer, params.conv[1]) 39 | layer = dense_layer(layer, params.dense) 40 | logits = tf.layers.dense(inputs=layer, units=10) 41 | 42 | loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y), name='loss') 43 | optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate) 44 | train_op = optimizer.minimize(loss_op, name='minimize') 45 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), tf.float32), name='accuracy') 46 | 47 | return locals() # to avoid GC 48 | 49 | (x_train, y_train), (x_val, y_val), (x_test, y_test) = get_cifar10(path='temp-cifar10/data', 50 | one_hot=True, 51 | validation_size=5000) 52 | data = hype.Data(train=hype.DataSet(x_train, y_train), 53 | validation=hype.DataSet(x_val, y_val), 54 | test=hype.DataSet(x_test, y_test)) 55 | 56 | def solver_generator(params): 57 | solver_params = { 58 | 'batch_size': 500, 59 | 'eval_batch_size': 500, 60 | 'epochs': 10, 61 | 'evaluate_test': True, 62 | 'eval_flexible': False, 63 | 'save_dir': 'temp-cifar10/model-zoo/example-2-2-{date}-{random_id}', 64 | 'save_accuracy_limit': 0.65, 65 | } 66 | cnn_model(params) 67 | solver = hype.TensorflowSolver(data=data, hyper_params=params, **solver_params) 68 | return solver 69 | 70 | 71 | hyper_params_spec = hype.spec.new( 72 | learning_rate = 10**hype.spec.uniform(-2, -3), 73 | conv = [ 74 | # Layer 1 75 | hype.spec.new( 76 | filter_num = hype.spec.choice([32, 48, 64]), 77 | filter_size = [hype.spec.choice([3, 5])] * 2, 78 | activation = hype.spec.choice(ACTIVATIONS.keys()), 79 | batch_norm = False, 80 | dropout = hype.spec.uniform(0.0, 0.2), 81 | down_sample = hype.spec.choice(DOWN_SAMPLES.keys()), 82 | ), 83 | # Layer 2 84 | hype.spec.new( 85 | filter_num = hype.spec.choice([96, 128]), 86 | filter_size = [hype.spec.choice([3, 5])] * 2, 87 | activation = hype.spec.choice(ACTIVATIONS.keys()), 88 | batch_norm = False, 89 | dropout = hype.spec.uniform(0.0, 0.4), 90 | down_sample = hype.spec.choice(DOWN_SAMPLES.keys()), 91 | ), 92 | ], 93 | # Dense layer 94 | dense = hype.spec.new( 95 | size = hype.spec.choice([256, 512, 768, 1024]), 96 | dropout = hype.spec.uniform(0.0, 0.5), 97 | ), 98 | ) 99 | strategy_params = { 100 | 'io_load_dir': 'temp-cifar10/example-2-2', 101 | 'io_save_dir': 'temp-cifar10/example-2-2', 102 | } 103 | 104 | tuner = hype.HyperTuner(hyper_params_spec, solver_generator, **strategy_params) 105 | tuner.tune() 106 | -------------------------------------------------------------------------------- /hyperengine/model/curve_predictor.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import numpy as np 7 | 8 | from ..base import * 9 | 10 | 11 | class BaseCurvePredictor(Serializable): 12 | def __init__(self, **params): 13 | self._x = np.array([]) 14 | self._y = np.array([]) 15 | self._burn_in = params.get('burn_in', 10) 16 | self._min_input_size = params.get('min_input_size', 3) 17 | 18 | self._curve_io = DefaultIO(self, filename='curve-data.xjson', **slice_dict(params, 'io_')) 19 | self._curve_io.load() 20 | 21 | @property 22 | def curves_number(self): 23 | return self._x.shape[0] 24 | 25 | @property 26 | def curve_length(self): 27 | if self.curves_number == 0: 28 | return 0 29 | return self._x.shape[1] 30 | 31 | def add_curve(self, curve, value): 32 | assert len(curve.shape) == 1 33 | 34 | curve = curve.reshape(1, -1) 35 | value = value.reshape(1) 36 | 37 | if self.curves_number == 0: 38 | self._x = curve 39 | self._y = value 40 | else: 41 | self._x = np.concatenate([self._x, curve], axis=0) 42 | self._y = np.concatenate([self._y, value], axis=0) 43 | 44 | info('Adding curve (value=%.4f). Current data shape: ' % value[0], self._x.shape) 45 | self._curve_io.save() 46 | 47 | def predict(self, curve): 48 | raise NotImplementedError() 49 | 50 | def stop_condition(self): 51 | raise NotImplementedError() 52 | 53 | def result_metric(self): 54 | raise NotImplementedError() 55 | 56 | def import_from(self, data): 57 | self._import_from_keys(data, keys=('x', 'y'), default_value=[]) 58 | 59 | def export_to(self): 60 | return self._export_keys_to(keys=('x', 'y')) 61 | 62 | 63 | class LinearCurvePredictor(BaseCurvePredictor): 64 | def __init__(self, **params): 65 | super(LinearCurvePredictor, self).__init__(**params) 66 | self._value_limit = params.get('value_limit') 67 | self._model = {} 68 | 69 | def add_curve(self, curve, value): 70 | super(LinearCurvePredictor, self).add_curve(curve, value) 71 | self._model = {} 72 | 73 | def predict(self, curve): 74 | size = curve.shape[0] 75 | if self.curves_number < self._burn_in or \ 76 | size < self._min_input_size or \ 77 | size >= max(self.curves_number, self.curve_length): 78 | return None 79 | 80 | w, error = self._build_model(size) 81 | value_prediction = self._eval(curve[:size], w) 82 | value_prediction = max(value_prediction, max(curve)) 83 | info('Prediction for the curve: %.4f (error=%.4f)' % (value_prediction, error)) 84 | return value_prediction, error 85 | 86 | def stop_condition(self): 87 | def condition(curve): 88 | curve = np.array(curve) 89 | interval = self.predict(curve) 90 | if interval: 91 | expected, error = interval 92 | upper_bound = expected + error 93 | limit = self._value_limit or np.max(self._y) 94 | if upper_bound < limit: 95 | info('Max expected value for the curve is %.4f. Stop now (curve size=%d/%d)' % 96 | (upper_bound, curve.shape[0], self.curve_length)) 97 | return True 98 | return False 99 | return condition 100 | 101 | def result_metric(self): 102 | def metric(curve): 103 | curve = np.array(curve) 104 | if curve.shape[0] < self.curve_length: 105 | expected, _ = self.predict(curve) 106 | info('Expected value for the curve is %.4f' % expected) 107 | return expected 108 | value = max(curve) 109 | self.add_curve(curve, value) 110 | return value 111 | return metric 112 | 113 | def _build_model(self, size): 114 | result = self._model.get(size) 115 | if result is None: 116 | w = self._compute_matrix(size) 117 | error = self._error(w, size) 118 | result = (w, error) 119 | self._model[size] = result 120 | return result 121 | 122 | def _compute_matrix(self, size): 123 | bias = np.ones(self.curves_number) 124 | x = np.column_stack([bias, self._x[:,:size]]) 125 | y = self._y 126 | return np.linalg.pinv(x.T.dot(x)).dot(x.T).dot(y) 127 | 128 | def _error(self, w, size): 129 | x = self._x[:,:size] 130 | y = self._y 131 | predictions = self._eval(x, w) 132 | return np.max(np.abs(predictions - y)) 133 | 134 | def _eval(self, x, w): 135 | if len(x.shape) > 1: 136 | bias = np.ones(x.shape[0]) 137 | x = np.column_stack([bias, x]) 138 | else: 139 | x = np.insert(x, 0, 1) 140 | return x.dot(w) 141 | -------------------------------------------------------------------------------- /hyperengine/spec/parsed_spec.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | 7 | import copy 8 | import numbers 9 | from six import iteritems 10 | 11 | import numpy as np 12 | 13 | from .nodes import BaseNode, AcceptsInputNode, JointNode 14 | 15 | 16 | class ParsedSpec(object): 17 | """ 18 | Responsible for parsing the spec and constructing a tree. 19 | When the spec is parsed, it can transform values from multi-dimensional [0,1] hypercube to domain values. 20 | """ 21 | def __init__(self, spec): 22 | self._spec = spec 23 | self._input_nodes = {} 24 | self._traverse_nodes(spec) 25 | 26 | def size(self): 27 | return len(self._input_nodes) 28 | 29 | def instantiate(self, points): 30 | assert len(points) == self.size() 31 | for node, index in iteritems(self._input_nodes): 32 | node.set_point(points[index]) 33 | 34 | spec_copy = copy.deepcopy(self._spec) 35 | spec_copy = self._traverse_and_replace(spec_copy) 36 | return spec_copy 37 | 38 | def get_names(self): 39 | return {index: node.name() for node, index in iteritems(self._input_nodes)} 40 | 41 | def _traverse_nodes(self, spec): 42 | self._visited = set() 43 | self._traverse_nodes_recursive(spec) 44 | self._visited = None 45 | 46 | def _visit(self, obj): 47 | if isinstance(obj, object) and hasattr(obj, '__dict__'): 48 | id_ = id(obj) 49 | if id_ in self._visited: 50 | return True 51 | self._visited.add(id_) 52 | return False 53 | 54 | def _traverse_nodes_recursive(self, spec, *path): 55 | if self._visit(spec): 56 | return 57 | 58 | if isinstance(spec, BaseNode): 59 | node = spec 60 | 61 | if isinstance(node, JointNode): 62 | for i, child in enumerate(node._children): 63 | self._traverse_nodes_recursive(child, *path) 64 | 65 | if isinstance(node, AcceptsInputNode) and not node in self._input_nodes: 66 | index = len(self._input_nodes) 67 | self._input_nodes[node] = index 68 | self._set_name(node, path) 69 | return 70 | 71 | if isinstance(spec, numbers.Number): 72 | return 73 | 74 | if isinstance(spec, dict): 75 | for key, value in iteritems(spec): 76 | self._traverse_nodes_recursive(value, key, *path) 77 | return 78 | 79 | if isinstance(spec, list) or isinstance(spec, tuple): 80 | for i, item in enumerate(spec): 81 | self._traverse_nodes_recursive(item, *path) 82 | return 83 | 84 | if isinstance(spec, object) and hasattr(spec, '__dict__'): 85 | for key, value in iteritems(spec.__dict__): 86 | if not (key.startswith('__') and key.endswith('__')): 87 | self._traverse_nodes_recursive(value, key, *path) 88 | return 89 | 90 | def _set_name(self, node, path): 91 | if node._name is None: 92 | name = '-'.join([str(i) for i in reversed(path)]) 93 | describe = node.describe() 94 | if describe: 95 | name = name + '-' + describe 96 | node._name = name 97 | 98 | def _traverse_and_replace(self, spec_copy): 99 | self._visited = set() 100 | spec_copy = self._traverse_and_replace_recursive(spec_copy) 101 | self._visited = None 102 | return spec_copy 103 | 104 | def _traverse_and_replace_recursive(self, spec_copy): 105 | if isinstance(spec_copy, BaseNode): 106 | return spec_copy.value() 107 | 108 | if isinstance(spec_copy, numbers.Number): 109 | return spec_copy 110 | 111 | if self._visit(spec_copy): 112 | return spec_copy 113 | 114 | if isinstance(spec_copy, dict): 115 | for key, value in iteritems(spec_copy): 116 | spec_copy[key] = self._traverse_and_replace_recursive(spec_copy[key]) 117 | return spec_copy 118 | 119 | if isinstance(spec_copy, list) or isinstance(spec_copy, tuple): 120 | replaced = [self._traverse_and_replace_recursive(item_copy) for item_copy in spec_copy] 121 | if isinstance(spec_copy, tuple): 122 | replaced = tuple(replaced) 123 | return replaced 124 | 125 | if isinstance(spec_copy, object) and hasattr(spec_copy, '__dict__'): 126 | for key, value in iteritems(spec_copy.__dict__): 127 | if not(key.startswith('__') and key.endswith('__')): 128 | setattr(spec_copy, key, self._traverse_and_replace_recursive(value)) 129 | return spec_copy 130 | 131 | return spec_copy 132 | 133 | 134 | def get_instance(spec): 135 | parsed = spec if isinstance(spec, ParsedSpec) else ParsedSpec(spec) 136 | points = np.random.uniform(0, 1, size=(parsed.size(),)) 137 | return parsed.instantiate(points) 138 | -------------------------------------------------------------------------------- /hyperengine/examples/4_1_word2vec_embedding.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | from collections import Counter, deque 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | 11 | from common import get_text8 12 | import hyperengine as hype 13 | 14 | 15 | class SentenceDataProvider(hype.IterableDataProvider): 16 | def __init__(self, **params): 17 | super(SentenceDataProvider, self).__init__() 18 | 19 | self._vocab_size = params.get('vocab_size', 50000) 20 | self._num_skips = params.get('num_skips', 2) 21 | self._skip_window = params.get('skip_window', 1) 22 | 23 | self._vocabulary = None 24 | self._dictionary = None 25 | self._data = None 26 | 27 | @property 28 | def vocabulary(self): 29 | return self._vocabulary 30 | 31 | @property 32 | def vocab_size(self): 33 | return self._vocab_size 34 | 35 | def build(self): 36 | hype.util.debug('Building the data provider') 37 | words = get_text8('temp-text8/data') 38 | self._vocabulary = [('UNK', None)] + Counter(words).most_common(self._vocab_size - 1) 39 | self._vocabulary = np.array([word for word, _ in self._vocabulary]) 40 | self._dictionary = {word: code for code, word in enumerate(self._vocabulary)} 41 | self._data = np.array([self._dictionary.get(word, 0) for word in words]) 42 | self._size = len(self._data) 43 | 44 | if hype.util.is_debug_logged(): 45 | hype.util.debug('Total words in text: %dM' % (len(words) / 1000000)) 46 | hype.util.debug('Text example: %s...' % ' '.join(words[:10])) 47 | hype.util.debug('Encoded text: %s...' % self._data[:10].tolist()) 48 | 49 | def next_batch(self, batch_size): 50 | self._step += 1 51 | return self._generate_batch(batch_size, self._num_skips, self._skip_window) 52 | 53 | def _generate_batch(self, batch_size, num_skips, skip_window): 54 | assert batch_size % num_skips == 0 55 | assert num_skips <= 2 * skip_window 56 | 57 | batch = np.ndarray(shape=(batch_size, ), dtype=np.int32) 58 | labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32) 59 | span = 2 * skip_window + 1 # [ skip_window target skip_window ] 60 | buffer = deque(maxlen=span) 61 | for _ in range(span): 62 | buffer.append(self._data[self._index_in_epoch]) 63 | self._inc_index() 64 | for i in range(batch_size // num_skips): 65 | target = skip_window # target label at the center of the buffer 66 | targets_to_avoid = [skip_window] 67 | for j in range(num_skips): 68 | while target in targets_to_avoid: 69 | target = np.random.randint(0, span) 70 | targets_to_avoid.append(target) 71 | batch[i * num_skips + j] = buffer[skip_window] 72 | labels[i * num_skips + j, 0] = buffer[target] 73 | buffer.append(self._data[self._index_in_epoch]) 74 | self._inc_index() 75 | return batch, labels 76 | 77 | 78 | def word2vec_model(params): 79 | # Input data. 80 | inputs = tf.placeholder(dtype=tf.int32, shape=[None], name='input') 81 | labels = tf.placeholder(dtype=tf.int32, shape=[None, 1], name='label') 82 | 83 | # Look up embeddings for inputs. 84 | embeddings = tf.Variable( 85 | tf.random_uniform([params.vocab_size, params.embedding_size], -1.0, 1.0) 86 | ) 87 | embed = tf.nn.embedding_lookup(embeddings, inputs) 88 | 89 | # Construct the variables for the NCE loss 90 | nce_weights = tf.Variable( 91 | tf.truncated_normal(shape=[params.vocab_size, params.embedding_size], 92 | stddev=1.0 / np.sqrt(params.embedding_size)) 93 | ) 94 | nce_biases = tf.Variable(tf.zeros([params.vocab_size])) 95 | 96 | # Compute the average NCE loss for the batch. 97 | # tf.nce_loss automatically draws a new sample of the negative labels each 98 | # time we evaluate the loss. 99 | loss = tf.reduce_mean( 100 | tf.nn.nce_loss(nce_weights, nce_biases, 101 | labels=labels, 102 | inputs=embed, 103 | num_sampled=params.negative_samples, 104 | num_classes=params.vocab_size), 105 | name='loss' 106 | ) 107 | optimizer = tf.train.AdamOptimizer(params.learning_rate) 108 | optimizer.minimize(loss, name='minimize') 109 | 110 | 111 | provider = SentenceDataProvider() 112 | provider.build() 113 | data = hype.Data(train=provider, validation=None, test=None) 114 | 115 | word2vec_model(params=hype.spec.new( 116 | vocab_size = provider.vocab_size, 117 | embedding_size = 128, 118 | negative_samples = 64, 119 | learning_rate = 0.01, 120 | )) 121 | 122 | solver_params = { 123 | 'batch_size': 1024, 124 | 'epochs': 5, 125 | 'eval_flexible': False, 126 | } 127 | solver = hype.TensorflowSolver(data=data, **solver_params) 128 | solver.train() 129 | -------------------------------------------------------------------------------- /hyperengine/examples/2_3_all_conv_net_cifar.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import tensorflow as tf 7 | 8 | from common import get_cifar10 9 | import hyperengine as hype 10 | 11 | ACTIVATIONS = {name: getattr(tf.nn, name) for name in ['relu', 'relu6', 'elu', 'sigmoid', 'tanh', 'leaky_relu']} 12 | DOWN_SAMPLES = {name: getattr(tf.layers, name) for name in ['max_pooling2d', 'average_pooling2d']} 13 | 14 | def cnn_model(params): 15 | x = tf.placeholder(tf.float32, [None, 32, 32, 3], name='input') 16 | y = tf.placeholder(tf.float32, [None, 10], name='label') 17 | mode = tf.placeholder(tf.string, name='mode') 18 | training = tf.equal(mode, 'train') 19 | 20 | def conv_layer(input, hp): 21 | layer = input 22 | for filter in hp.filters: 23 | layer = tf.layers.conv2d(layer, 24 | filters=filter[-1], 25 | kernel_size=filter[:-1], 26 | padding=hp.get('padding', 'same'), 27 | activation=ACTIVATIONS.get(hp.activation, None)) 28 | layer = tf.layers.batch_normalization(layer, training=training) if hp.batch_norm else layer 29 | layer = tf.layers.dropout(layer, rate=hp.dropout, training=training) if hp.dropout else layer 30 | layer = DOWN_SAMPLES[hp.down_sample](layer, pool_size=[2, 2], strides=[2, 2]) if hp.down_sample else layer 31 | return layer 32 | 33 | layer = x 34 | for conv_hp in params.conv: 35 | layer = conv_layer(layer, hp=conv_hp) 36 | logits = tf.squeeze(layer, axis=[1, 2]) 37 | 38 | loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y), name='loss') 39 | optimizer = tf.train.AdamOptimizer(learning_rate=params.optimizer.learning_rate, 40 | beta1=params.optimizer.beta1, 41 | beta2=params.optimizer.beta2, 42 | epsilon=params.optimizer.epsilon) 43 | train_op = optimizer.minimize(loss_op, name='minimize') 44 | accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1)), tf.float32), name='accuracy') 45 | 46 | return locals() # to avoid GC 47 | 48 | (x_train, y_train), (x_val, y_val), (x_test, y_test) = get_cifar10(path='temp-cifar10/data', 49 | one_hot=True, 50 | validation_size=5000) 51 | data = hype.Data(train=hype.DataSet(x_train, y_train), 52 | validation=hype.DataSet(x_val, y_val), 53 | test=hype.DataSet(x_test, y_test)) 54 | 55 | 56 | def solver_generator(params): 57 | solver_params = { 58 | 'batch_size': 500, 59 | 'eval_batch_size': 500, 60 | 'epochs': 10, 61 | 'evaluate_test': True, 62 | 'eval_flexible': False, 63 | 'save_dir': 'temp-cifar10/model-zoo/example-2-3-{date}-{random_id}', 64 | 'save_accuracy_limit': 0.70, 65 | } 66 | cnn_model(params) 67 | solver = hype.TensorflowSolver(data=data, hyper_params=params, **solver_params) 68 | return solver 69 | 70 | 71 | hyper_params_spec = hype.spec.new( 72 | optimizer = hype.spec.new( 73 | learning_rate = 10**hype.spec.uniform(-2, -4), 74 | beta1 = 0.9, 75 | beta2 = 0.999, 76 | epsilon = 1e-8, 77 | ), 78 | conv = [ 79 | # Layer 1: accepts 32x32 80 | hype.spec.new( 81 | filters = [[3, 3, hype.spec.choice([48, 64])]], 82 | activation = hype.spec.choice(ACTIVATIONS.keys()), 83 | batch_norm = False, 84 | dropout = None, 85 | down_sample = hype.spec.choice(DOWN_SAMPLES.keys()), 86 | ), 87 | # Layer 2: accepts 16x16 88 | hype.spec.new( 89 | filters = [[3, 3, hype.spec.choice([64, 96, 128])]], 90 | activation = hype.spec.choice(ACTIVATIONS.keys()), 91 | batch_norm = False, 92 | dropout = None, 93 | down_sample = hype.spec.choice(DOWN_SAMPLES.keys()), 94 | ), 95 | # Layer 3: accepts 8x8 96 | hype.spec.new( 97 | filters = [[1, 1, 64], 98 | [3, 3, hype.spec.choice([128, 192, 256])]], 99 | activation = hype.spec.choice(ACTIVATIONS.keys()), 100 | batch_norm = False, 101 | dropout = None, 102 | down_sample = hype.spec.choice(DOWN_SAMPLES.keys()), 103 | ), 104 | # Layer 4: accepts 4x4 105 | hype.spec.new( 106 | filters = [[1, 1, 96], 107 | [2, 2, hype.spec.choice([256, 384, 512])]], 108 | activation = hype.spec.choice(ACTIVATIONS.keys()), 109 | batch_norm = False, 110 | dropout = None, 111 | down_sample = hype.spec.choice(DOWN_SAMPLES.keys()), 112 | ), 113 | # Layer 5: accepts 2x2 114 | hype.spec.new( 115 | filters = [[1, 1, 128]], 116 | ), 117 | # Layer 6: accepts 2x2 118 | hype.spec.new( 119 | filters = [[2, 2, hype.spec.choice([512, 768])]], 120 | padding = 'valid', 121 | activation = hype.spec.choice(ACTIVATIONS.keys()), 122 | batch_norm = False, 123 | dropout = None, 124 | ), 125 | # Layer 7: accepts 1x1 126 | hype.spec.new( 127 | filters=[[1, 1, 10]], 128 | ), 129 | ], 130 | ) 131 | strategy_params = { 132 | 'io_load_dir': 'temp-cifar10/example-2-3', 133 | 'io_save_dir': 'temp-cifar10/example-2-3', 134 | } 135 | 136 | tuner = hype.HyperTuner(hyper_params_spec, solver_generator, **strategy_params) 137 | tuner.tune() 138 | -------------------------------------------------------------------------------- /hyperengine/examples/3_2_rnn_sms_spam_detector.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import numpy as np 7 | import tensorflow as tf 8 | from tensorflow.contrib.learn import preprocessing 9 | 10 | from common import get_sms_spam_data 11 | import hyperengine as hype 12 | 13 | 14 | inputs, labels = get_sms_spam_data('temp-sms-spam/data') 15 | hype.util.info('Data examples:') 16 | for text, target in zip(inputs[:15], labels[:15]): 17 | hype.util.info('%4s: %s' % (target, text[:100])) 18 | hype.util.info() 19 | 20 | 21 | def preprocess(params): 22 | # Data processing: encode text inputs into numeric vectors 23 | processor = preprocessing.VocabularyProcessor(max_document_length=params.max_sequence_length, 24 | min_frequency=params.min_frequency) 25 | encoded_inputs = list(processor.fit_transform(inputs)) 26 | vocab_size = len(processor.vocabulary_) 27 | 28 | # Set this to see verbose output: 29 | # hype.util.set_verbose() 30 | 31 | if hype.util.is_debug_logged(): 32 | hype.util.debug('Encoded text examples:') 33 | for i in range(3): 34 | hype.util.debug(' %s ->' % inputs[i]) 35 | hype.util.debug(' %s\n' % encoded_inputs[i].tolist()) 36 | 37 | encoded_inputs = np.array(encoded_inputs) 38 | encoded_labels = np.array([int(label == 'ham') for label in labels]) 39 | 40 | # Shuffle the data 41 | shuffled_ix = np.random.permutation(np.arange(len(encoded_labels))) 42 | x_shuffled = encoded_inputs[shuffled_ix] 43 | y_shuffled = encoded_labels[shuffled_ix] 44 | 45 | # Split into train/validation/test sets 46 | idx1 = int(len(y_shuffled) * 0.75) 47 | idx2 = int(len(y_shuffled) * 0.85) 48 | x_train, x_val, x_test = x_shuffled[:idx1], x_shuffled[idx1:idx2], x_shuffled[idx2:] 49 | y_train, y_val, y_test = y_shuffled[:idx1], y_shuffled[idx1:idx2], y_shuffled[idx2:] 50 | 51 | if hype.util.is_debug_logged(): 52 | hype.util.debug('Vocabulary size: %d' % vocab_size) 53 | hype.util.debug('Train/validation/test split: train=%d, val=%d, test=%d' % 54 | (len(y_train), len(y_val), len(y_test))) 55 | 56 | train = hype.DataSet(x_train, y_train) 57 | validation = hype.DataSet(x_val, y_val) 58 | test = hype.DataSet(x_test, y_test) 59 | data = hype.Data(train, validation, test) 60 | 61 | return data, vocab_size 62 | 63 | 64 | def rnn_model(params): 65 | data, vocab_size = preprocess(params) 66 | 67 | x = tf.placeholder(shape=[None, params.max_sequence_length], dtype=tf.int32, name='input') 68 | y = tf.placeholder(shape=[None], dtype=tf.int32, name='label') 69 | mode = tf.placeholder(tf.string, name='mode') 70 | training = tf.equal(mode, 'train') 71 | 72 | # Create embedding 73 | embedding_matrix = tf.Variable(tf.random_uniform([vocab_size, params.embedding_size], -1.0, 1.0)) 74 | embedding_output = tf.nn.embedding_lookup(embedding_matrix, x) 75 | 76 | if params.rnn_cell == 'basic_rnn': 77 | cell = tf.nn.rnn_cell.BasicRNNCell(num_units=params.rnn_hidden_size) 78 | elif params.rnn_cell == 'lstm': 79 | cell = tf.nn.rnn_cell.LSTMCell(num_units=params.rnn_hidden_size) 80 | elif params.rnn_cell == 'gru': 81 | cell = tf.nn.rnn_cell.GRUCell(num_units=params.rnn_hidden_size) 82 | else: 83 | raise ValueError('Unexpected hyper-parameter: %s' % params.rnn_cell) 84 | output, state = tf.nn.dynamic_rnn(cell, embedding_output, dtype=tf.float32) 85 | output = tf.cond(training, lambda: tf.nn.dropout(output, keep_prob=params.dropout_keep_prob), lambda: output) 86 | 87 | # Get output of RNN sequence 88 | # output = (?, max_sequence_length, rnn_hidden_size) 89 | # output_last = (?, rnn_hidden_size) 90 | output = tf.transpose(output, [1, 0, 2]) 91 | output_last = tf.gather(output, int(output.get_shape()[0]) - 1) 92 | 93 | # Final logits for binary classification: 94 | # logits = (?, 2) 95 | weight = tf.Variable(tf.truncated_normal([params.rnn_hidden_size, 2], stddev=0.1)) 96 | bias = tf.Variable(tf.constant(0.1, shape=[2])) 97 | logits = tf.matmul(output_last, weight) + bias 98 | 99 | # Loss function 100 | cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y) 101 | loss = tf.reduce_mean(cross_entropy, name='loss') 102 | tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), tf.cast(y, tf.int64)), tf.float32), name='accuracy') 103 | optimizer = tf.train.RMSPropOptimizer(params.learning_rate) 104 | optimizer.minimize(loss, name='minimize') 105 | 106 | return data 107 | 108 | 109 | def solver_generator(params): 110 | solver_params = { 111 | 'batch_size': 200, 112 | 'eval_batch_size': 200, 113 | 'epochs': 20, 114 | 'evaluate_test': True, 115 | 'eval_flexible': False, 116 | 'save_dir': 'temp-sms-spam/model-zoo/example-3-2-{date}-{random_id}', 117 | 'save_accuracy_limit': 0.97, 118 | } 119 | data = rnn_model(params) 120 | solver = hype.TensorflowSolver(data=data, hyper_params=params, **solver_params) 121 | return solver 122 | 123 | hyper_params_spec = hype.spec.new( 124 | max_sequence_length = hype.spec.choice(range(20, 50)), 125 | min_frequency = hype.spec.choice([1, 3, 5, 10]), 126 | embedding_size = hype.spec.choice([32, 64, 128]), 127 | rnn_cell = hype.spec.choice(['basic_rnn', 'lstm', 'gru']), 128 | rnn_hidden_size = hype.spec.choice([16, 32, 64]), 129 | dropout_keep_prob = hype.spec.uniform(0.5, 1.0), 130 | learning_rate = 10**hype.spec.uniform(-4, -3), 131 | ) 132 | 133 | strategy_params = { 134 | 'io_load_dir': 'temp-sms-spam/example-3-2', 135 | 'io_save_dir': 'temp-sms-spam/example-3-2', 136 | } 137 | tuner = hype.HyperTuner(hyper_params_spec, solver_generator, **strategy_params) 138 | tuner.tune() 139 | -------------------------------------------------------------------------------- /hyperengine/bayesian/utility.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import math 7 | import numpy as np 8 | from scipy import stats 9 | 10 | 11 | class BaseUtility(object): 12 | """ 13 | Utility (aka acquisition) is a function that evaluates a potential of the points in high-dimensional spaces. 14 | Utility can use prior information about the true values over certain points to model the target function, 15 | thus predict the points that are likely to be the maximum. 16 | """ 17 | 18 | def __init__(self, points, values, **params): 19 | super(BaseUtility, self).__init__() 20 | self.points = np.array(points) 21 | self.values = np.array(values) 22 | 23 | if len(self.points.shape) == 1: 24 | self.points = self.points.reshape(-1, 1) 25 | assert len(self.points.shape) == 2 26 | assert len(self.values.shape) == 1 27 | assert self.points.shape[0] == self.values.shape[0] 28 | 29 | self.dimension = self.points.shape[1] 30 | self.iteration = params.get('iteration', self.points.shape[0]) 31 | 32 | def compute_values(self, batch): 33 | """ 34 | Evaluates the utility function for a batch of points. Returns the numpy array. 35 | """ 36 | raise NotImplementedError() 37 | 38 | 39 | class BaseGaussianUtility(BaseUtility): 40 | """ 41 | Represents the utility function based on Gaussian Process models. 42 | See https://en.wikipedia.org/wiki/Gaussian_process 43 | """ 44 | 45 | def __init__(self, points, values, kernel, mu_prior=0, noise_sigma=0.0, **params): 46 | super(BaseGaussianUtility, self).__init__(points, values, **params) 47 | self.kernel = kernel 48 | 49 | mu_prior = np.array(mu_prior) 50 | if len(mu_prior.shape) == 0: 51 | mu_prior_values, mu_prior_star = mu_prior, mu_prior 52 | else: 53 | mu_prior_values, mu_prior_star = mu_prior[:-1], mu_prior[-1] 54 | 55 | kernel_matrix = self.kernel.compute(self.points) + np.eye(self.points.shape[0]) * noise_sigma**2 56 | self.k_inv = np.linalg.pinv(kernel_matrix) 57 | self.k_inv_f = np.dot(self.k_inv, (self.values - mu_prior_values)) 58 | self.mu_prior_star = mu_prior_star 59 | 60 | def mean_and_std(self, batch): 61 | assert len(batch.shape) == 2 62 | 63 | batch = np.array(batch) 64 | k_star = np.swapaxes(self.kernel.compute(self.points, batch), 0, 1) 65 | k_star_star = self.kernel.id(batch) 66 | 67 | mu_star = self.mu_prior_star + np.dot(k_star, self.k_inv_f) 68 | 69 | t_star = np.dot(self.k_inv, k_star.T) 70 | t_star = np.einsum('ij,ji->i', k_star, t_star) 71 | sigma_star = k_star_star - t_star 72 | 73 | return mu_star, sigma_star 74 | 75 | 76 | class ProbabilityOfImprovement(BaseGaussianUtility): 77 | """ 78 | Implements the PI method. 79 | See the following sources for more details: 80 | H. J. Kushner. A new method of locating the maximum of an arbitrary multipeak curve in the presence of noise. 81 | J. Basic Engineering, 86:97–106, 1964. 82 | """ 83 | 84 | def __init__(self, points, values, kernel, mu_prior=0, noise_sigma=0.0, **params): 85 | super(ProbabilityOfImprovement, self).__init__(points, values, kernel, mu_prior, noise_sigma, **params) 86 | self.epsilon = params.get('epsilon', 1e-8) 87 | self.max_value = np.max(self.values) 88 | 89 | def compute_values(self, batch): 90 | mu, sigma = self.mean_and_std(batch) 91 | z = (mu - self.max_value - self.epsilon) / sigma 92 | cdf = stats.norm.cdf(z) 93 | cdf[np.abs(sigma) < self.epsilon] = 0.0 94 | return cdf 95 | 96 | 97 | class ExpectedImprovement(BaseGaussianUtility): 98 | """ 99 | Implements the EI method. 100 | See the following sources for more details: 101 | J. Mockus, V. Tiesis, and A. Zilinskas. Toward Global Optimization, volume 2, 102 | chapter The Application of Bayesian Methods for Seeking the Extremum, pages 117–128. Elsevier, 1978. 103 | """ 104 | 105 | def __init__(self, points, values, kernel, mu_prior=0, noise_sigma=0.0, **params): 106 | super(ExpectedImprovement, self).__init__(points, values, kernel, mu_prior, noise_sigma, **params) 107 | self.epsilon = params.get('epsilon', 1e-8) 108 | self.max_value = np.max(self.values) 109 | 110 | def compute_values(self, batch): 111 | mu, sigma = self.mean_and_std(batch) 112 | z = (mu - self.max_value - self.epsilon) / sigma 113 | ei = (mu - self.max_value - self.epsilon) * stats.norm.cdf(z) + sigma * stats.norm.pdf(z) 114 | ei[np.abs(sigma) < self.epsilon] = 0.0 115 | return ei 116 | 117 | 118 | class UpperConfidenceBound(BaseGaussianUtility): 119 | """ 120 | Implements the UCB method. 121 | See the following sources for more details: 122 | Peter Auer, Using Confidence Bounds for Exploitation-Exploration Trade-offs, 123 | Journal of Machine Learning Research 3 (2002) 397-422, 2011. 124 | """ 125 | 126 | def __init__(self, points, values, kernel, mu_prior=0, noise_sigma=0.0, **params): 127 | super(UpperConfidenceBound, self).__init__(points, values, kernel, mu_prior, noise_sigma, **params) 128 | delta = params.get('delta', 0.5) 129 | self.beta = np.sqrt(2 * np.log(self.dimension * self.iteration**2 * math.pi**2 / (6 * delta))) 130 | 131 | def compute_values(self, batch): 132 | mu, sigma = self.mean_and_std(batch) 133 | return mu + self.beta * sigma 134 | 135 | 136 | class RandomPoint(BaseUtility): 137 | """ 138 | A naive random point method. All points are picked equally likely, thus utility method is constant everywhere. 139 | """ 140 | 141 | def __init__(self, points, values, **params): 142 | super(RandomPoint, self).__init__(points, values, **params) 143 | 144 | def compute_values(self, batch): 145 | return np.zeros(len(batch)) 146 | -------------------------------------------------------------------------------- /hyperengine/spec/nodes.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import operator 7 | 8 | 9 | class BaseNode(object): 10 | def __init__(self, name=None): 11 | super(BaseNode, self).__init__() 12 | self._domain_value = None 13 | self._name = name 14 | 15 | def value(self): 16 | return self._domain_value 17 | 18 | def name(self): 19 | return self._name 20 | 21 | def with_name(self, name): 22 | self._name = name 23 | return self 24 | 25 | def describe(self): 26 | return None 27 | 28 | def __add__(self, other): return _op2(self, other, operator.add) 29 | def __radd__(self, other): return _op2(self, other, operator.add, rev=True) 30 | 31 | def __sub__(self, other): return _op2(self, other, operator.sub) 32 | def __rsub__(self, other): return _op2(self, other, operator.sub, rev=True) 33 | 34 | def __mul__(self, other): return _op2(self, other, operator.mul) 35 | def __rmul__(self, other): return _op2(self, other, operator.mul, rev=True) 36 | 37 | def __div__(self, other): return _op2(self, other, operator.div) 38 | def __rdiv__(self, other): return _op2(self, other, operator.div, rev=True) 39 | 40 | def __mod__(self, other): return _op2(self, other, operator.mod) 41 | def __rmod__(self, other): return _op2(self, other, operator.mod, rev=True) 42 | 43 | def __floordiv__(self, other): return _op2(self, other, operator.floordiv) 44 | def __rfloordiv__(self, other): return _op2(self, other, operator.floordiv, rev=True) 45 | 46 | def __truediv__(self, other): return _op2(self, other, operator.truediv) 47 | def __rtruediv__(self, other): return _op2(self, other, operator.truediv, rev=True) 48 | 49 | def __pow__(self, other): return _op2(self, other, operator.pow) 50 | def __rpow__(self, other): return _op2(self, other, operator.pow, rev=True) 51 | 52 | def __and__(self, other): return _op2(self, other, operator.and_) 53 | def __rand__(self, other): return _op2(self, other, operator.and_, rev=True) 54 | 55 | def __or__(self, other): return _op2(self, other, operator.or_) 56 | def __ror__(self, other): return _op2(self, other, operator.or_, rev=True) 57 | 58 | def __xor__(self, other): return _op2(self, other, operator.__xor__) 59 | def __rxor__(self, other): return _op2(self, other, operator.__xor__, rev=True) 60 | 61 | def __lshift__(self, other): return _op2(self, other, operator.lshift) 62 | def __rlshift__(self, other): return _op2(self, other, operator.lshift, rev=True) 63 | 64 | def __rshift__(self, other): return _op2(self, other, operator.rshift) 65 | def __rrshift__(self, other): return _op2(self, other, operator.rshift, rev=True) 66 | 67 | def __neg__(self): return _op1(self, operator.neg) 68 | def __pos__(self): return _op1(self, operator.pos) 69 | def __abs__(self): return _op1(self, operator.abs) 70 | def __invert__(self): return _op1(self, operator.invert) 71 | 72 | 73 | def _op1(this, _operator): 74 | return MergeNode(_operator, this) 75 | 76 | def _op2(this, other, _operator, rev=False): 77 | if isinstance(other, BaseNode): 78 | if rev: 79 | return MergeNode(_operator, other, this) 80 | else: 81 | return MergeNode(_operator, this, other) 82 | 83 | if rev: 84 | return MergeNode(lambda x: _operator(other, x), this) 85 | else: 86 | return MergeNode(lambda x: _operator(x, other), this) 87 | 88 | 89 | class AcceptsInputNode(BaseNode): 90 | def __init__(self): 91 | super(AcceptsInputNode, self).__init__() 92 | self._point = None 93 | 94 | def set_point(self, point): 95 | self._point = point 96 | self._domain_value = self.to_domain_value(point) 97 | 98 | def to_domain_value(self, point): 99 | raise NotImplementedError() 100 | 101 | 102 | class UniformNode(AcceptsInputNode): 103 | def __init__(self, start=0.0, end=1.0): 104 | super(UniformNode, self).__init__() 105 | self._shift = min(start, end) 106 | self._scale = abs(end - start) 107 | self._describe = 'uniform(%f, %f)' % (start, end) 108 | 109 | def to_domain_value(self, point): 110 | return point * self._scale + self._shift 111 | 112 | def describe(self): 113 | return self._describe 114 | 115 | 116 | class NonUniformNode(AcceptsInputNode): 117 | def __init__(self, ppf, **args): 118 | super(NonUniformNode, self).__init__() 119 | self._ppf = ppf 120 | self._args = args 121 | 122 | def to_domain_value(self, point): 123 | return self._ppf(point, **self._args) 124 | 125 | def describe(self): 126 | return func_to_str(self._ppf) or 'complex' 127 | 128 | def func_to_str(func): 129 | try: 130 | if hasattr(func, 'im_self'): 131 | self = func.im_self 132 | if hasattr(self, '__class__'): 133 | self = self.__class__ 134 | return self.__name__ 135 | return func.__name__ 136 | except: 137 | return None 138 | 139 | 140 | class ChoiceNode(AcceptsInputNode): 141 | def __init__(self, *array): 142 | super(ChoiceNode, self).__init__() 143 | self._array = array 144 | 145 | def to_domain_value(self, point): 146 | index = int(point * len(self._array)) 147 | return self._array[min(index, len(self._array) - 1)] 148 | 149 | def describe(self): 150 | return 'choice(%d)' % len(self._array) 151 | 152 | 153 | class JointNode(BaseNode): 154 | def __init__(self, *children): 155 | super(JointNode, self).__init__() 156 | self._children = children 157 | 158 | 159 | class MergeNode(JointNode): 160 | def __init__(self, function, *children): 161 | super(MergeNode, self).__init__(*children) 162 | assert callable(function) 163 | self.function = function 164 | 165 | def value(self): 166 | if self._domain_value is None: 167 | self._domain_value = self.function(*[child.value() for child in self._children]) 168 | return self._domain_value 169 | 170 | 171 | class MergeChoiceNode(JointNode, AcceptsInputNode): 172 | def __init__(self, *children): 173 | super(MergeChoiceNode, self).__init__(*children) 174 | 175 | def value(self): 176 | if self._domain_value is None and self._point is not None: 177 | values = [child.value() if isinstance(child, BaseNode) else child for child in self._children] 178 | index = int(self._point * len(values)) 179 | self._domain_value = values[min(index, len(values) - 1)] 180 | return self._domain_value 181 | 182 | def to_domain_value(self, point): 183 | return None 184 | 185 | def describe(self): 186 | return 'choice(%d)' % len(self._children) 187 | -------------------------------------------------------------------------------- /hyperengine/examples/common.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import os 7 | import pickle 8 | import re 9 | import sys 10 | import tarfile 11 | import zipfile 12 | 13 | import numpy as np 14 | 15 | import hyperengine as hype 16 | 17 | 18 | ######################################################################################################################## 19 | # CIFAR-10 20 | ######################################################################################################################## 21 | 22 | 23 | def get_cifar10(path='temp-cifar10/data', 24 | url='http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz', 25 | one_hot=False, 26 | validation_size=None): 27 | tar_file = hype.util.download_if_needed(url, path) 28 | untar(tar_file) 29 | path = os.path.join(path, 'cifar-10-batches-py') 30 | 31 | x_train = [] 32 | y_train = [] 33 | for i in range(1, 6): 34 | batch_path = os.path.join(path, 'data_batch_%d' % i) 35 | data, labels = load_cifar10_batch(batch_path) 36 | if i == 1: 37 | x_train = data 38 | y_train = labels 39 | else: 40 | x_train = np.concatenate([x_train, data], axis=0) 41 | y_train = np.concatenate([y_train, labels], axis=0) 42 | 43 | batch_path = os.path.join(path, 'test_batch') 44 | x_test, y_test = load_cifar10_batch(batch_path) 45 | 46 | x_train = np.dstack((x_train[:, :1024], x_train[:, 1024:2048], 47 | x_train[:, 2048:])) / 255. 48 | x_train = np.reshape(x_train, [-1, 32, 32, 3]) 49 | x_test = np.dstack((x_test[:, :1024], x_test[:, 1024:2048], 50 | x_test[:, 2048:])) / 255. 51 | x_test = np.reshape(x_test, [-1, 32, 32, 3]) 52 | 53 | if one_hot: 54 | y_train = encode_one_hot(y_train, 10) 55 | y_test = encode_one_hot(y_test, 10) 56 | 57 | if validation_size is not None: 58 | x_val = x_train[:validation_size] 59 | y_val = y_train[:validation_size] 60 | x_train = x_train[validation_size:] 61 | y_train = y_train[validation_size:] 62 | return (x_train, y_train), (x_val, y_val), (x_test, y_test) 63 | 64 | return (x_train, y_train), (x_test, y_test) 65 | 66 | 67 | def load_cifar10_batch(filename): 68 | with open(filename, 'rb') as file_: 69 | if sys.version_info > (3, 0): 70 | d = pickle.load(file_, encoding='latin1') # Python3 71 | else: 72 | d = pickle.load(file_) # Python2 73 | return d['data'], d['labels'] 74 | 75 | 76 | def untar(filename): 77 | if filename.endswith('tar.gz'): 78 | tar = tarfile.open(filename) 79 | tar.extractall(path=os.path.dirname(filename)) 80 | tar.close() 81 | hype.util.debug('File "%s" extracted in current directory' % filename) 82 | else: 83 | hype.util.warn('Not a tar.gz file: "%s"' % filename) 84 | 85 | 86 | def encode_one_hot(y, num_classes=None): 87 | """ 88 | Convert class vector (integers from 0 to num_classes) to binary class matrix. 89 | Arguments: 90 | y: `array`. Class vector to convert. 91 | num_classes: `int`. Total number of classes. 92 | """ 93 | y = np.asarray(y, dtype='int32') 94 | if not num_classes: 95 | num_classes = np.max(y) + 1 96 | one_hot = np.zeros((len(y), num_classes)) 97 | one_hot[np.arange(len(y)), y] = 1.0 98 | return one_hot 99 | 100 | 101 | ######################################################################################################################## 102 | # Text8 (Wikipedia text corpus) 103 | ######################################################################################################################## 104 | 105 | 106 | # Mirror: 107 | # https://cs.fit.edu/~mmahoney/compression/text8.zip 108 | def get_text8(path='temp-text8/data', url='http://mattmahoney.net/dc/text8.zip'): 109 | zip_path = hype.util.download_if_needed(url, path) 110 | with zipfile.ZipFile(zip_path) as file_: 111 | data = file_.read(file_.namelist()[0]) 112 | return data.decode('ascii').split() 113 | 114 | 115 | ######################################################################################################################## 116 | # SMS spam collection 117 | ######################################################################################################################## 118 | 119 | 120 | # See https://archive.ics.uci.edu/ml/datasets/SMS+Spam+Collection 121 | def get_sms_spam_raw(path='temp-sms-spam/data', 122 | url='http://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip'): 123 | zip_path = hype.util.download_if_needed(url, path) 124 | with zipfile.ZipFile(zip_path) as file_: 125 | data = file_.read('SMSSpamCollection') 126 | text_data = data.decode(errors='ignore') 127 | text_data = text_data.encode('ascii', errors='ignore') 128 | text_data = text_data.decode().split('\n') 129 | return text_data 130 | 131 | def clean_text(text): 132 | text = re.sub(r'([^\s\w]|_|[0-9])+', '', text) 133 | text = ' '.join(text.split()) 134 | text = text.lower() 135 | return text 136 | 137 | def get_sms_spam_data(path='temp-sms-spam/data'): 138 | data = get_sms_spam_raw(path) 139 | data = [line.split('\t') for line in data if len(line) >= 1] 140 | [labels, inputs] = [list(line) for line in zip(*data)] 141 | inputs = [clean_text(line) for line in inputs] 142 | return inputs, labels 143 | 144 | 145 | ######################################################################################################################## 146 | # Wine quality set 147 | ######################################################################################################################## 148 | 149 | 150 | # See https://archive.ics.uci.edu/ml/datasets/Wine+Quality 151 | def get_wine_data(path='temp-wine/data', 152 | urls=('https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv', 153 | 'https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv'), 154 | val_ratio=0.8, 155 | test_ratio=0.9): 156 | full_data = None 157 | for url in urls: 158 | result_path = hype.util.download_if_needed(url, path) 159 | data = np.genfromtxt(result_path, delimiter=';', skip_header=True) 160 | if full_data is None: 161 | full_data = data 162 | else: 163 | full_data = np.vstack([full_data, data]) 164 | 165 | total_rows = full_data.shape[0] 166 | val_split = int(total_rows * val_ratio) if val_ratio is not None else None 167 | test_split = int(total_rows * test_ratio) if test_ratio is not None else None 168 | 169 | np.random.shuffle(full_data) 170 | x_train, y_train = full_data[:val_split,:-1], full_data[:val_split,-1] 171 | x_val, y_val = full_data[val_split:test_split,:-1], full_data[val_split:test_split,-1] 172 | x_test, y_test = full_data[test_split:,:-1], full_data[test_split:,-1] 173 | 174 | if val_ratio is None and test_ratio is None: 175 | return x_train, y_train 176 | 177 | if val_ratio is None: 178 | return x_train, y_train, x_test, y_test 179 | 180 | if test_ratio is None: 181 | return x_train, y_train, x_val, y_val 182 | 183 | return x_train, y_train, x_test, y_test, x_val, y_val 184 | -------------------------------------------------------------------------------- /hyperengine/model/base_solver.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | __author__ = 'maxim' 4 | 5 | import inspect 6 | import numpy as np 7 | 8 | from ..base import * 9 | from .data_set import IterableDataProvider 10 | 11 | reducers = { 12 | 'max': lambda curve: np.max(curve) if curve else 0, 13 | 'avg': lambda curve: np.mean(curve) if curve else 0, 14 | } 15 | 16 | class BaseSolver(object): 17 | """ 18 | Implements a training algorithm. 19 | """ 20 | 21 | def __init__(self, runner, data, hyper_params, augmentation=None, reducer='max', **params): 22 | data.reset_counters() 23 | self._train_set = data.train 24 | self._val_set = data.validation 25 | self._test_set = data.test 26 | self._augmentation = augmentation 27 | self._runner = runner 28 | self._hyper_params = hyper_params 29 | self._reducer = as_numeric_function(reducer, presets=reducers) 30 | self._max_val_accuracy = 0 31 | self._val_accuracy_curve = [] 32 | 33 | self._epochs = params.get('epochs', 1) 34 | self._dynamic_epochs = params.get('dynamic_epochs') 35 | self._stop_condition = params.get('stop_condition') 36 | self._batch_size = params.get('batch_size', 16) 37 | self._eval_batch_size = params.get('eval_batch_size', self._val_set.size if self._val_set else 0) 38 | self._eval_flexible = params.get('eval_flexible', True) 39 | self._eval_train_every = params.get('eval_train_every', 10) if not self._eval_flexible else 1e1000 40 | self._eval_validation_every = params.get('eval_validation_every', 100) if not self._eval_flexible else 1e1000 41 | self._eval_test = params.get('evaluate_test', False) 42 | 43 | def train(self): 44 | self._runner.build_model() 45 | info('Start training. Model size: %dk' % (self._runner.model_size() / 1000)) 46 | info('Hyper params: %s' % smart_str(self._hyper_params)) 47 | 48 | with self.create_session(): 49 | self._max_val_accuracy = self.init_session() 50 | while self._train_set.epochs_completed < self._epochs: 51 | batch_x, batch_y = self._train_set.next_batch(self._batch_size) 52 | batch_x = self.augment(batch_x) 53 | self._runner.run_batch(batch_x, batch_y) 54 | self._evaluate_train(batch_x, batch_y) 55 | 56 | eval_result = self._evaluate_validation() 57 | val_accuracy = eval_result.get('accuracy') if eval_result is not None else None 58 | if val_accuracy is not None: 59 | self._val_accuracy_curve.append(val_accuracy) 60 | self._update_epochs_dynamically() 61 | 62 | if val_accuracy > self._max_val_accuracy: 63 | self._max_val_accuracy = val_accuracy 64 | self.on_best_accuracy(val_accuracy, eval_result) 65 | 66 | if self._stop_condition and self._stop_condition(self._val_accuracy_curve): 67 | info('Solver stopped due to the stop condition') 68 | break 69 | 70 | self._evaluate_test() 71 | return self._reducer(self._val_accuracy_curve) 72 | 73 | def create_session(self): 74 | return None 75 | 76 | def init_session(self): 77 | return 0 78 | 79 | def augment(self, x): 80 | augmented = call(self._augmentation, x) 81 | return augmented if augmented is not None else x 82 | 83 | def terminate(self): 84 | pass 85 | 86 | def on_best_accuracy(self, accuracy, eval_result): 87 | pass 88 | 89 | def _update_epochs_dynamically(self): 90 | if self._dynamic_epochs is None: 91 | return 92 | 93 | curve = self._val_accuracy_curve 94 | max_acc = max(self._val_accuracy_curve) 95 | 96 | new_epochs = self._epochs 97 | args_spec = inspect.getargspec(self._dynamic_epochs) 98 | if args_spec.varargs is not None or args_spec.keywords is not None: 99 | new_epochs = self._dynamic_epochs(curve, max_acc) 100 | else: 101 | args = args_spec.args 102 | if len(args) == 1: 103 | arg = self._val_accuracy_curve if args[0] in ['curve', 'history'] else max_acc 104 | new_epochs = self._dynamic_epochs(arg) 105 | elif args == 2: 106 | new_epochs = self._dynamic_epochs(curve) 107 | else: 108 | warn('Invalid "dynamic_epochs" parameter: ' 109 | 'expected a function with either one or two arguments, but got %s' % args_spec) 110 | 111 | if self._epochs != new_epochs: 112 | self._epochs = new_epochs or self._epochs 113 | debug('Update epochs=%d' % new_epochs) 114 | 115 | def _evaluate_train(self, batch_x, batch_y): 116 | eval_this_step = self._train_set.step % self._eval_train_every == 0 117 | if eval_this_step and is_info_logged(): 118 | eval_ = self._runner.evaluate(batch_x, batch_y) 119 | self._log_iteration('train_accuracy', eval_.get('loss'), eval_.get('accuracy'), False) 120 | 121 | def _evaluate_validation(self): 122 | eval_this_step = self._train_set.step % self._eval_validation_every == 0 123 | epoch_just_completed = (self._train_set.just_completed and self._eval_flexible) 124 | 125 | if eval_this_step or epoch_just_completed: 126 | if self._val_set is None: 127 | warn('Validation set is not provided. Skip val evaluation') 128 | return 129 | 130 | eval_ = self._evaluate(self._val_set) 131 | self._log_iteration('validation_accuracy', eval_.get('loss'), eval_.get('accuracy'), True) 132 | return eval_ 133 | 134 | def _evaluate_test(self): 135 | if not self._eval_test: 136 | return 137 | 138 | if self._test_set is None: 139 | warn('Test set is not provided. Skip test evaluation') 140 | return 141 | 142 | eval_ = self._evaluate(self._test_set) 143 | test_accuracy = eval_.get('accuracy') 144 | if test_accuracy is None: 145 | warn('Test accuracy evaluation is not available') 146 | return 147 | 148 | info('Final test_accuracy=%.4f' % test_accuracy) 149 | return eval_ 150 | 151 | def _log_iteration(self, name, loss, accuracy, mark_best): 152 | message = 'Epoch %2d, iteration %7d' % (self._train_set.epochs_completed, self._train_set.index) 153 | if accuracy is not None: 154 | marker = ' *' if mark_best and (accuracy > self._max_val_accuracy) else '' 155 | if loss is None: 156 | info('%s: %s=%.4f%s' % (message, name, accuracy, marker)) 157 | else: 158 | info('%s: loss=%.6f, %s=%.4f%s' % (message, loss, name, accuracy, marker)) 159 | else: 160 | if loss is not None: 161 | info('%s: loss=%.6f' % (message, loss)) 162 | else: 163 | info('%s: -- no loss or accuracy defined --' % message) 164 | 165 | def _evaluate(self, data_set): 166 | assert isinstance(data_set, IterableDataProvider), 'Currently can evaluate only IterableDataProviders' 167 | 168 | data_set.reset_counters() 169 | result_acc = None 170 | result_loss = None 171 | while data_set.epochs_completed < 1: 172 | batch_x, batch_y = data_set.next_batch(self._eval_batch_size) 173 | eval_ = self._runner.evaluate(batch_x=batch_x, batch_y=batch_y) 174 | result_acc = (result_acc or 0) + eval_ ['accuracy'] * len(batch_x) if 'accuracy' in eval_ else None 175 | result_loss = (result_loss or 0) + eval_['loss'] * len(batch_x) if 'loss' in eval_ else None 176 | 177 | return { 178 | 'accuracy': result_acc / data_set.size if result_acc is not None else None, 179 | 'loss': result_loss / data_set.size if result_loss is not None else None, 180 | } 181 | -------------------------------------------------------------------------------- /hyperengine/bayesian/strategy.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import numpy as np 7 | 8 | from ..base import * 9 | 10 | from .kernel import RadialBasisFunction 11 | from .maximizer import MonteCarloUtilityMaximizer 12 | from .utility import ProbabilityOfImprovement, ExpectedImprovement, UpperConfidenceBound, RandomPoint 13 | 14 | 15 | mu_priors = { 16 | 'mean': lambda values: np.mean(values, axis=0), 17 | 'max': lambda values: np.max(values, axis=0), 18 | } 19 | 20 | kernels = { 21 | 'rbf': lambda params: RadialBasisFunction(**slice_dict(params, 'rbf_')), 22 | } 23 | 24 | utilities = { 25 | 'pi': lambda points, values, kernel, mu_prior, params: ProbabilityOfImprovement(points, values, kernel, mu_prior, 26 | **slice_dict(params, 'pi_')), 27 | 'ei': lambda points, values, kernel, mu_prior, params: ExpectedImprovement(points, values, kernel, mu_prior, 28 | **slice_dict(params, 'ei_')), 29 | 'ucb': lambda points, values, kernel, mu_prior, params: UpperConfidenceBound(points, values, kernel, mu_prior, 30 | **slice_dict(params, 'ucb_')), 31 | 'rand': lambda points, values, kernel, mu_prior, params: RandomPoint(points, values, **slice_dict(params, 'rand_')), 32 | } 33 | 34 | maximizers = { 35 | 'mc': lambda utility, sampler, params: MonteCarloUtilityMaximizer(utility, sampler, **slice_dict(params, 'mc_')), 36 | } 37 | 38 | 39 | class BaseStrategy(Serializable): 40 | """ 41 | A strategy represents an iterative algorithm of selecting points from a high-dimensional space: 42 | a sequential design strategy for global optimization of black-box functions that doesn't require derivatives. 43 | 44 | The caller should evaluate the target function for each point and report the value back to the strategy. 45 | The strategy is responsible to solve exploration / exploitation problem in order to iterate to the local maximum 46 | as fast as possible. 47 | 48 | The strategy can also save and restore the session to/from the file between different executions. 49 | """ 50 | 51 | def __init__(self, sampler, **params): 52 | self._sampler = sampler 53 | self._params = params 54 | 55 | self._points = np.array([]) 56 | self._values = np.array([]) 57 | 58 | self._strategy_io = DefaultIO(self, filename='strategy-session.xjson', **slice_dict(params, 'io_')) 59 | self._strategy_io.load() 60 | 61 | @property 62 | def iteration(self): 63 | """ 64 | Returns current iteration. 65 | """ 66 | return self._points.shape[0] 67 | 68 | @property 69 | def points(self): 70 | """ 71 | Returns all previously selected points 72 | """ 73 | return self._points 74 | 75 | @property 76 | def values(self): 77 | """ 78 | Returns all collected values. 79 | """ 80 | return self._values 81 | 82 | def next_proposal(self): 83 | """ 84 | Returns the next proposed point. 85 | """ 86 | raise NotImplementedError() 87 | 88 | def add_point(self, point, value): 89 | """ 90 | Adds the point along with its value to the strategy store. 91 | """ 92 | debug('Adding a point:', point) 93 | debug('Adding a value:', value) 94 | self._points = np.append(self._points, point).reshape([-1] + list(point.shape)) 95 | self._values = np.append(self._values, value) 96 | self._strategy_io.save() 97 | 98 | def import_from(self, data): 99 | self._import_from_keys(data, keys=('points', 'values'), default_value=[]) 100 | 101 | def export_to(self): 102 | return self._export_keys_to(keys=('points', 'values')) 103 | 104 | 105 | class BaseBayesianStrategy(BaseStrategy): 106 | """ 107 | Represents an abstract Bayesian optimization strategy. 108 | See https://en.wikipedia.org/wiki/Bayesian_optimization 109 | """ 110 | 111 | def __init__(self, sampler, **params): 112 | super(BaseBayesianStrategy, self).__init__(sampler, **params) 113 | self._kernel = None 114 | self._maximizer = None 115 | 116 | self._mu_prior_gen = as_numeric_function(params.get('mu_prior_gen', 'mean'), presets=mu_priors) 117 | self._kernel_gen = as_function(params.get('kernel_gen', 'rbf'), presets=kernels) 118 | self._maximizer_gen = as_function(params.get('maximizer_gen', 'mc'), presets=maximizers) 119 | 120 | self._burn_in = params.get('burn_in', 1) 121 | 122 | def _instantiate(self, method_gen): 123 | mu_prior = self._mu_prior_gen(self._values) 124 | debug('Select the prior: mu_prior=%.6f' % mu_prior) 125 | 126 | self._kernel = self._kernel_gen(self._params) 127 | self._method = method_gen(self._points, self._values, self._kernel, mu_prior, self._params) 128 | self._maximizer = self._maximizer_gen(self._method, self._sampler, self._params) 129 | info('Use utility method: %s' % self._method.__class__.__name__) 130 | 131 | return self._maximizer 132 | 133 | 134 | class BayesianStrategy(BaseBayesianStrategy): 135 | """ 136 | Represents a straight Bayesian strategy, based on a single utility method (not mixed). 137 | Utility method must implement `BaseUtility` interface. 138 | """ 139 | 140 | def __init__(self, sampler, **params): 141 | super(BayesianStrategy, self).__init__(sampler, **params) 142 | self._method = None 143 | self._method_gen = as_function(params.get('utility_gen', 'ucb'), presets=utilities) 144 | 145 | def next_proposal(self): 146 | if self.iteration < self._burn_in: 147 | return self._sampler.sample(size=1)[0] 148 | 149 | self._maximizer = self._instantiate(self._method_gen) 150 | return self._maximizer.compute_max_point() 151 | 152 | 153 | class BayesianPortfolioStrategy(BaseBayesianStrategy): 154 | """ 155 | Represents a Bayesian portfolio strategy, that combines several utility methods and 156 | picks a random method on each iteration. 157 | Utility methods must implement `BaseUtility` interface. 158 | """ 159 | 160 | def __init__(self, sampler, methods, **params): 161 | self._methods = [as_function(gen, presets=utilities) for gen in methods] 162 | self._probabilities = params.get('probabilities') 163 | self._scores = np.zeros(shape=len(methods)) 164 | self._index = None 165 | self._alpha = params.get('alpha', 0.9) 166 | 167 | super(BayesianPortfolioStrategy, self).__init__(sampler, **params) 168 | 169 | def next_proposal(self): 170 | if self.iteration < self._burn_in: 171 | return self._sampler.sample(size=1)[0] 172 | 173 | if self._probabilities is None: 174 | self._probabilities = softmax(self._scores) 175 | self._index = np.random.choice(range(len(self._scores)), p=self._probabilities) 176 | method_gen = self._methods[self._index] 177 | self._maximizer = self._instantiate(method_gen) 178 | return self._maximizer.compute_max_point() 179 | 180 | def add_point(self, point, value): 181 | if self.iteration >= self._burn_in and self._probabilities is None: 182 | score = value - np.mean(self._values) 183 | self._scores[self._index] = self._alpha * score + (1 - self._alpha) * self._scores[self._index] 184 | 185 | super(BayesianPortfolioStrategy, self).add_point(point, value) 186 | 187 | def import_from(self, data): 188 | self._import_from_keys(data, keys=('points', 'values'), default_value=[]) 189 | self._scores = np.array(data.get('scores', np.zeros(shape=len(self._methods)))) 190 | 191 | def export_to(self): 192 | return self._export_keys_to(keys=('points', 'values', 'scores')) 193 | 194 | 195 | def softmax(scores): 196 | scores -= np.mean(scores) 197 | exp = np.exp(scores) 198 | return exp / np.sum(exp) 199 | -------------------------------------------------------------------------------- /hyperengine/tests/curve_predictor_test.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import unittest 7 | 8 | import numpy as np 9 | 10 | from hyperengine.base import * 11 | from hyperengine import LinearCurvePredictor 12 | 13 | SYNTHETIC_TRAIN = [ 14 | ([40, 50, 55, 60, 63, 65, 68, 67, 69, 70, 72], 72), 15 | ([30, 34, 40, 42, 46, 46, 50, 52, 53, 55, 54], 55), 16 | ([32, 40, 45, 48, 54, 56, 57, 58, 60, 61, 61], 61), 17 | ([37, 45, 51, 54, 57, 60, 59, 63, 64, 67, 68], 68), 18 | ([44, 48, 53, 59, 63, 63, 64, 64, 68, 69, 70], 70), 19 | ([42, 49, 53, 57, 60, 63, 62, 65, 66, 68, 68], 68), 20 | ([41, 47, 55, 58, 61, 62, 65, 67, 69, 71, 71], 71), 21 | ([36, 43, 50, 52, 57, 61, 61, 62, 64, 66, 69], 69), 22 | ([20, 25, 30, 34, 38, 42, 46, 50, 53, 55, 54], 55), 23 | ([25, 27, 35, 40, 43, 47, 51, 52, 55, 56, 57], 57), 24 | ([29, 34, 40, 45, 50, 53, 56, 57, 57, 60, 59], 60), 25 | ([10, 16, 25, 30, 32, 35, 36, 35, 37, 37, 37], 37), 26 | ([41, 51, 52, 56, 61, 62, 65, 66, 68, 67, 69], 69), 27 | ([45, 48, 54, 57, 60, 62, 62, 65, 65, 66, 66], 66), 28 | ([22, 30, 33, 37, 39, 40, 45, 47, 48, 48, 51], 51), 29 | ([42, 51, 55, 56, 58, 61, 62, 64, 65, 66, 68], 68), 30 | ([34, 43, 45, 50, 52, 55, 61, 61, 60, 66, 64], 66), 31 | ([20, 24, 25, 30, 31, 33, 36, 37, 40, 42, 43], 43), 32 | ([44, 51, 55, 58, 60, 63, 65, 64, 66, 67, 67], 67), 33 | ([41, 49, 55, 58, 61, 61, 66, 67, 68, 68, 71], 71), 34 | ([34, 45, 48, 53, 55, 57, 61, 62, 63, 63, 64], 64), 35 | ] 36 | SYNTHETIC_TEST = [ 37 | ([43, 51, 55, 57, 62, 64, 67, 66, 68, 68, 71], 71), 38 | ([42, 50, 53, 59, 61, 63, 64, 66, 67, 68, 70], 70), 39 | ([39, 46, 51, 54, 59, 61, 62, 63, 65, 65, 66], 66), 40 | ([41, 51, 52, 56, 61, 62, 64, 66, 68, 68, 69], 69), 41 | ([44, 48, 54, 57, 60, 62, 65, 65, 66, 66, 68], 68), 42 | ([40, 51, 54, 58, 62, 66, 68, 69, 71, 70, 72], 72), 43 | ] 44 | 45 | 46 | class LinearCurvePredictorTest(unittest.TestCase): 47 | def test_synthetic(self): 48 | self.set = SYNTHETIC_TRAIN, SYNTHETIC_TEST 49 | self.check_prediction(size=3, max_diff=2.5, avg_diff=1.2) 50 | self.check_prediction(size=4, max_diff=2.0, avg_diff=1.2) 51 | self.check_prediction(size=5, max_diff=2.5, avg_diff=1.2) 52 | self.check_prediction(size=6, max_diff=2.5, avg_diff=1.2) 53 | self.check_prediction(size=7, max_diff=3.0, avg_diff=1.8) 54 | self.check_prediction(size=8, max_diff=1.5, avg_diff=1.0) 55 | self.check_prediction(size=9, max_diff=2.5, avg_diff=1.2) 56 | 57 | def test_cifar10_1(self): 58 | self.set = self.read_curve_data(train_size=20, test_size=10) 59 | self.check_prediction(size=3, max_diff=0.07, avg_diff=0.03, check_between=False) 60 | self.check_prediction(size=4, max_diff=0.05, avg_diff=0.03, check_between=False) 61 | self.check_prediction(size=5, max_diff=0.05, avg_diff=0.03, check_between=False) 62 | self.check_prediction(size=6, max_diff=0.05, avg_diff=0.03, check_between=False) 63 | self.check_prediction(size=7, max_diff=0.05, avg_diff=0.03, check_between=False) 64 | self.check_prediction(size=8, max_diff=0.05, avg_diff=0.03, check_between=False) 65 | self.check_prediction(size=9, max_diff=0.05, avg_diff=0.03, check_between=False) 66 | self.check_prediction(size=10, max_diff=0.05, avg_diff=0.03, check_between=False) 67 | self.check_prediction(size=11, max_diff=0.05, avg_diff=0.03, check_between=False) 68 | self.check_prediction(size=12, max_diff=0.05, avg_diff=0.03, check_between=False) 69 | self.check_prediction(size=13, max_diff=0.07, avg_diff=0.04, check_between=False) 70 | self.check_prediction(size=14, max_diff=0.07, avg_diff=0.04, check_between=False) 71 | self.check_prediction(size=15, max_diff=0.06, avg_diff=0.04, check_between=False) 72 | self.check_prediction(size=16, max_diff=0.08, avg_diff=0.04, check_between=False) 73 | self.check_prediction(size=17, max_diff=0.06, avg_diff=0.04, check_between=False) 74 | self.check_prediction(size=18, max_diff=0.06, avg_diff=0.04, check_between=False) 75 | self.check_prediction(size=19, max_diff=0.20, avg_diff=0.08, check_between=False) 76 | self.check_prediction(size=20, max_diff=0.07, avg_diff=0.03, check_between=False) 77 | 78 | def test_cifar10_2(self): 79 | self.set = self.read_curve_data(train_size=29, test_size=10) 80 | self.check_prediction(size=3, max_diff=0.04, avg_diff=0.02, check_between=False) 81 | self.check_prediction(size=4, max_diff=0.04, avg_diff=0.02, check_between=False) 82 | self.check_prediction(size=5, max_diff=0.04, avg_diff=0.02, check_between=False) 83 | self.check_prediction(size=6, max_diff=0.04, avg_diff=0.02, check_between=False) 84 | self.check_prediction(size=7, max_diff=0.04, avg_diff=0.02, check_between=False) 85 | self.check_prediction(size=8, max_diff=0.04, avg_diff=0.02, check_between=False) 86 | self.check_prediction(size=9, max_diff=0.04, avg_diff=0.02, check_between=False) 87 | self.check_prediction(size=10, max_diff=0.04, avg_diff=0.02, check_between=False) 88 | self.check_prediction(size=11, max_diff=0.04, avg_diff=0.02, check_between=False) 89 | self.check_prediction(size=12, max_diff=0.04, avg_diff=0.02, check_between=False) 90 | self.check_prediction(size=13, max_diff=0.04, avg_diff=0.02, check_between=False) 91 | self.check_prediction(size=14, max_diff=0.04, avg_diff=0.02, check_between=False) 92 | self.check_prediction(size=15, max_diff=0.04, avg_diff=0.02, check_between=False) 93 | self.check_prediction(size=16, max_diff=0.04, avg_diff=0.02, check_between=False) 94 | self.check_prediction(size=17, max_diff=0.04, avg_diff=0.02, check_between=False) 95 | self.check_prediction(size=18, max_diff=0.04, avg_diff=0.02, check_between=False) 96 | self.check_prediction(size=19, max_diff=0.04, avg_diff=0.02, check_between=False) 97 | self.check_prediction(size=20, max_diff=0.04, avg_diff=0.02, check_between=False) 98 | 99 | def test_cifar10_3(self): 100 | self.set = self.read_curve_data(train_size=50, test_size=10) 101 | self.check_prediction(size=3, max_diff=0.03, avg_diff=0.02, check_between=False) 102 | self.check_prediction(size=4, max_diff=0.04, avg_diff=0.02, check_between=False) 103 | self.check_prediction(size=5, max_diff=0.04, avg_diff=0.02, check_between=False) 104 | self.check_prediction(size=6, max_diff=0.04, avg_diff=0.02, check_between=False) 105 | self.check_prediction(size=7, max_diff=0.04, avg_diff=0.02, check_between=False) 106 | self.check_prediction(size=8, max_diff=0.03, avg_diff=0.02, check_between=False) 107 | self.check_prediction(size=9, max_diff=0.03, avg_diff=0.01, check_between=False) 108 | self.check_prediction(size=10, max_diff=0.03, avg_diff=0.01, check_between=False) 109 | self.check_prediction(size=11, max_diff=0.03, avg_diff=0.02, check_between=False) 110 | self.check_prediction(size=12, max_diff=0.03, avg_diff=0.02, check_between=False) 111 | self.check_prediction(size=13, max_diff=0.03, avg_diff=0.02, check_between=False) 112 | self.check_prediction(size=14, max_diff=0.02, avg_diff=0.01, check_between=False) 113 | self.check_prediction(size=15, max_diff=0.02, avg_diff=0.01, check_between=False) 114 | self.check_prediction(size=16, max_diff=0.02, avg_diff=0.01, check_between=False) 115 | self.check_prediction(size=17, max_diff=0.02, avg_diff=0.01, check_between=False) 116 | self.check_prediction(size=18, max_diff=0.02, avg_diff=0.01, check_between=False) 117 | self.check_prediction(size=19, max_diff=0.03, avg_diff=0.01, check_between=False) 118 | self.check_prediction(size=20, max_diff=0.02, avg_diff=0.01, check_between=False) 119 | 120 | def check_prediction(self, size, max_diff, avg_diff, check_between=True): 121 | train, test = self.set 122 | 123 | predictor = LinearCurvePredictor(burn_in=len(train), min_input_size=1) 124 | predictor._x = np.array([item[0] for item in train]) 125 | predictor._y = np.array([item[1] for item in train]) 126 | 127 | test_x = np.array([item[0][:size] for item in test]) 128 | test_y = np.array([item[1] for item in test]) 129 | 130 | total_error = 0 131 | for i in range(test_x.shape[0]): 132 | x = test_x[i] 133 | y = test_y[i] 134 | 135 | predicted, error = predictor.predict(x) 136 | diff = abs(predicted - y) 137 | 138 | if check_between: 139 | self.check_between(predicted-error, y, predicted+error) 140 | self.assertLess(diff, max_diff) 141 | total_error += diff 142 | 143 | avg_error = total_error / test_x.shape[0] 144 | self.assertLess(avg_error, avg_diff) 145 | 146 | def check_between(self, x, y, z): 147 | if x > z: x, z = z, x 148 | self.assertLess(x, y) 149 | self.assertLess(y, z) 150 | 151 | def read_curve_data(self, train_size, test_size): 152 | data = DefaultIO._load_dict('curve-test-data.xjson') 153 | self.assertTrue(data) 154 | 155 | x = data['x'][:train_size] 156 | y = data['y'][:train_size] 157 | test_x = data['x'][train_size:train_size+test_size] 158 | test_y = data['y'][train_size:train_size+test_size] 159 | 160 | return list(zip(x, y)), list(zip(test_x, test_y)) 161 | 162 | 163 | if __name__ == '__main__': 164 | unittest.main() 165 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | Hyper-parameters Tuning for Machine Learning 3 | ============================================ 4 | 5 | - `Overview <#overview>`__ 6 | - `About <#about>`__ 7 | - `Installation <#installation>`__ 8 | - `How to use <#how-to-use>`__ 9 | - `Features <#features>`__ 10 | - `Straight-forward specification <#straight-forward-specification>`__ 11 | - `Exploration-exploitation trade-off <#exploration-exploitation-trade-off>`__ 12 | - `Learning Curve Estimation <#learning-curve-estimation>`__ 13 | - `Bayesian Optimization <#bayesian-optimization>`__ 14 | 15 | -------- 16 | Overview 17 | -------- 18 | 19 | About 20 | ===== 21 | 22 | *HyperEngine* is a toolbox for `model selection and hyper-parameters tuning `__. 23 | It aims to provide most state-of-the-art techniques via intuitive API and with minimum dependencies. 24 | *HyperEngine* is **not a framework**, which means it doesn't enforce any structure or design to the main code, 25 | thus making integration local and non-intrusive. 26 | 27 | Installation 28 | ============ 29 | 30 | .. code-block:: shell 31 | 32 | pip install hyperengine 33 | 34 | Dependencies: 35 | 36 | - ``six``, ``numpy``, ``scipy`` 37 | - ``tensorflow`` (optional) 38 | - ``matplotlib`` (optional, only for development) 39 | 40 | Compatibility: 41 | 42 | .. image:: https://travis-ci.org/maxim5/hyper-engine.svg?branch=master 43 | :target: https://travis-ci.org/maxim5/hyper-engine 44 | 45 | - Python 2.7, 3.5, 3.6 46 | 47 | License: 48 | 49 | - `Apache 2.0 `__ 50 | 51 | *HyperEngine* is designed to be ML-platform agnostic, but currently provides only simple `TensorFlow `__ binding. 52 | 53 | How to use 54 | ========== 55 | 56 | Adapting your code to *HyperEngine* usually boils down to migrating hard-coded hyper-parameters to a dictionary (or an object) 57 | and giving names to particular tensors. 58 | 59 | **Before:** 60 | 61 | .. code-block:: python 62 | 63 | def my_model(): 64 | x = tf.placeholder(...) 65 | y = tf.placeholder(...) 66 | ... 67 | optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) 68 | ... 69 | 70 | **After:** 71 | 72 | .. code-block:: python 73 | 74 | def my_model(params): 75 | x = tf.placeholder(..., name='input') 76 | y = tf.placeholder(..., name='label') 77 | ... 78 | optimizer = tf.train.GradientDescentOptimizer(learning_rate=params['learning_rate']) 79 | ... 80 | 81 | # Now can run the model with any set of hyper-parameters 82 | 83 | 84 | The rest of the integration code is isolated and can be placed in the ``main`` script. 85 | See the examples of hyper-parameter tuning in `examples `__ package. 86 | 87 | -------- 88 | Features 89 | -------- 90 | 91 | Straight-forward specification 92 | ============================== 93 | 94 | The crucial part of hyper-parameter tuning is the definition of a *domain* 95 | over which the engine is going to optimize the model. Some variables are continuous (e.g., the learning rate), 96 | some variables are integer values in a certain range (e.g., the number of hidden units), some variables are categorical 97 | and represent architecture knobs (e.g., the choice of non-linearity). 98 | 99 | You can define all these variables and their ranges in ``numpy``-like fashion: 100 | 101 | .. code-block:: python 102 | 103 | hyper_params_spec = { 104 | 'optimizer': { 105 | 'learning_rate': 10**spec.uniform(-3, -1), # makes the continuous range [0.1, 0.001] 106 | 'epsilon': 1e-8, # constants work too 107 | }, 108 | 'conv': { 109 | 'filters': [[3, 3, spec.choice(range(32, 48))], # an integer between [32, 48] 110 | [3, 3, spec.choice(range(64, 96))], # an integer between [64, 96] 111 | [3, 3, spec.choice(range(128, 192))]], # an integer between [128, 192] 112 | 'activation': spec.choice(['relu','prelu','elu']), # a categorical range: 1 of 3 activations 113 | 'down_sample': { 114 | 'size': [2, 2], 115 | 'pooling': spec.choice(['max_pool', 'avg_pool']) # a categorical range: 1 of 2 pooling methods 116 | }, 117 | 'residual': spec.random_bool(), # either True or False 118 | 'dropout': spec.uniform(0.75, 1.0), # a uniform continuous range 119 | }, 120 | } 121 | 122 | Note that ``10**spec.uniform(-3, -1)`` is not the same *distribution* as ``spec.uniform(0.001, 0.1)`` 123 | (though they both define the same *range* of values). 124 | In the first case, the whole logarithmic spectrum ``(-3, -1)`` is equally probable, while in 125 | the second case, small values around ``0.001`` are much less likely than the values around the mean ``0.0495``. 126 | Specifying the following domain range for the learning rate - ``spec.uniform(0.001, 0.1)`` - will likely skew the results 127 | towards higher learning rates. This outlines the importance of random variable transformations and arithmetic operations. 128 | 129 | Exploration-exploitation trade-off 130 | ================================== 131 | 132 | Machine learning model selection is expensive. 133 | Each model evaluation requires full training from scratch and may take minutes to hours to days, 134 | depending on the problem complexity and available computational resources. 135 | *HyperEngine* provides the algorithm to explore the space of parameters efficiently, focus on the most promising areas, 136 | thus converge to the maximum as fast as possible. 137 | 138 | **Example 1**: the true function is 1-dimensional, ``f(x) = x * sin(x)`` (black curve) on [-10, 10] interval. 139 | Red dots represent each trial, red curve is the `Gaussian Process `__ mean, 140 | blue curve is the mean plus or minus one standard deviation. 141 | The optimizer randomly chose the negative mode as more promising. 142 | 143 | .. image:: /.images/figure_1.png 144 | :width: 80% 145 | :alt: 1D Bayesian Optimization 146 | :align: center 147 | 148 | **Example 2**: the 2-dimensional function ``f(x, y) = (x + y) / ((x - 1) ** 2 - sin(y) + 2)`` (black surface) on [0,9]x[0,9] square. 149 | Red dots represent each trial, the Gaussian Process mean and standard deviations are not shown for simplicity. 150 | Note that to achieve the maximum both variables must be picked accurately. 151 | 152 | .. image:: /.images/figure_2-1.png 153 | :width: 100% 154 | :alt: 2D Bayesian Optimization 155 | :align: center 156 | 157 | .. image:: /.images/figure_2-2.png 158 | :width: 100% 159 | :alt: 2D Bayesian Optimization 160 | :align: center 161 | 162 | The code for these and others examples is `here `__. 163 | 164 | Learning Curve Estimation 165 | ========================= 166 | 167 | *HyperEngine* can monitor the model performance during the training and stop early if it's learning too slowly. 168 | This is done via *learning curve prediction*. Note that this technique is compatible with Bayesian Optimization, since 169 | it estimates the model accuracy after full training - this value can be safely used to update Gaussian Process parameters. 170 | 171 | Example code: 172 | 173 | .. code-block:: python 174 | 175 | curve_params = { 176 | 'burn_in': 30, # burn-in period: 30 models 177 | 'min_input_size': 5, # start predicting after 5 epochs 178 | 'value_limit': 0.80, # stop if the estimate is less than 80% with high probability 179 | } 180 | curve_predictor = LinearCurvePredictor(**curve_params) 181 | 182 | Currently there is only one implementation of the predictor, ``LinearCurvePredictor``, 183 | which is very efficient, but requires relatively large burn-in period to predict model accuracy without flaws. 184 | 185 | Note that learning curves can be reused between different models and works quite well for the burn-in, 186 | so it's recommended to serialize and load curve data via ``io_save_dir`` and ``io_load_dir`` parameters. 187 | 188 | See also the following paper: 189 | `Speeding up Automatic Hyperparameter Optimization of Deep Neural Networks 190 | by Extrapolation of Learning Curves `__ 191 | 192 | --------------------- 193 | Bayesian Optimization 194 | --------------------- 195 | 196 | Implements the following `methods `__: 197 | 198 | - Probability of improvement (See H. J. Kushner. A new method of locating the maximum of an arbitrary multipeak curve in the presence of noise. J. Basic Engineering, 86:97–106, 1964.) 199 | - Expected Improvement (See J. Mockus, V. Tiesis, and A. Zilinskas. Toward Global Optimization, volume 2, chapter The Application of Bayesian Methods for Seeking the Extremum, pages 117–128. Elsevier, 1978) 200 | - `Upper Confidence Bound `__ 201 | - `Mixed / Portfolio strategy `__ 202 | - Naive random search. 203 | 204 | PI method prefers exploitation to exploration, UCB is the opposite. One of the best strategies we've seen is a mixed one: 205 | start with high probability of UCB and gradually decrease it, increasing PI probability. 206 | 207 | Default kernel function used is `RBF kernel `__, but it is extensible. 208 | -------------------------------------------------------------------------------- /hyperengine/tests/spec_test.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import six 7 | import unittest 8 | 9 | from hyperengine.spec import * 10 | 11 | 12 | class SpecTest(unittest.TestCase): 13 | def test_zero_nodes(self): 14 | def check_zero_nodes(spec): 15 | parsed = ParsedSpec(spec) 16 | self.assertEqual(parsed.size(), 0) 17 | self.assertEqual(spec, parsed.instantiate([])) 18 | 19 | check_zero_nodes(1) 20 | check_zero_nodes([]) 21 | check_zero_nodes([1, 2, 3]) 22 | check_zero_nodes((1, 2, 3)) 23 | check_zero_nodes({}) 24 | check_zero_nodes({'a': 0, 'b': 1}) 25 | check_zero_nodes({'a': [1, 2], 'b': {'key': (1, 2)}}) 26 | 27 | 28 | def test_uniform(self): 29 | spec = uniform() 30 | parsed = ParsedSpec(spec) 31 | self.assertEqual(parsed.size(), 1) 32 | self.assertEqual(0.0, parsed.instantiate([0.0])) 33 | self.assertEqual(0.5, parsed.instantiate([0.5])) 34 | self.assertEqual(1.0, parsed.instantiate([1.0])) 35 | 36 | 37 | def test_uniform_rev(self): 38 | spec = uniform(4, 0) 39 | parsed = ParsedSpec(spec) 40 | self.assertEqual(parsed.size(), 1) 41 | self.assertEqual(0.0, parsed.instantiate([0.0])) 42 | self.assertEqual(2.0, parsed.instantiate([0.5])) 43 | self.assertEqual(4.0, parsed.instantiate([1.0])) 44 | 45 | 46 | def test_uniform_negative(self): 47 | spec = uniform(-4, -2) 48 | parsed = ParsedSpec(spec) 49 | self.assertEqual(parsed.size(), 1) 50 | self.assertEqual(-4.0, parsed.instantiate([0.0])) 51 | self.assertEqual(-3.0, parsed.instantiate([0.5])) 52 | self.assertEqual(-2.0, parsed.instantiate([1.0])) 53 | 54 | 55 | def test_uniform_negative_rev(self): 56 | spec = uniform(-2, -4) 57 | parsed = ParsedSpec(spec) 58 | self.assertEqual(parsed.size(), 1) 59 | self.assertEqual(-4.0, parsed.instantiate([0.0])) 60 | self.assertEqual(-3.0, parsed.instantiate([0.5])) 61 | self.assertEqual(-2.0, parsed.instantiate([1.0])) 62 | 63 | 64 | def test_normal(self): 65 | spec = normal() 66 | parsed = ParsedSpec(spec) 67 | self.assertEqual(parsed.size(), 1) 68 | self.assertAlmostEqual(-1.0, parsed.instantiate([0.1587]), delta=0.001) 69 | self.assertAlmostEqual(-0.5, parsed.instantiate([0.3085]), delta=0.001) 70 | self.assertAlmostEqual( 0.0, parsed.instantiate([0.5000]), delta=0.001) 71 | self.assertAlmostEqual( 0.7, parsed.instantiate([0.7580]), delta=0.001) 72 | self.assertAlmostEqual( 0.9, parsed.instantiate([0.8159]), delta=0.001) 73 | 74 | 75 | def test_choice(self): 76 | spec = choice([10, 20, 30]) 77 | parsed = ParsedSpec(spec) 78 | self.assertEqual(parsed.size(), 1) 79 | self.assertEqual(10, parsed.instantiate([0.0])) 80 | self.assertEqual(20, parsed.instantiate([0.5])) 81 | self.assertEqual(30, parsed.instantiate([1.0])) 82 | 83 | 84 | def test_choice_str(self): 85 | spec = choice(['foo', 'bar']) 86 | parsed = ParsedSpec(spec) 87 | self.assertEqual(parsed.size(), 1) 88 | self.assertEqual('foo', parsed.instantiate([0.0])) 89 | self.assertEqual('bar', parsed.instantiate([1.0])) 90 | 91 | 92 | def test_merge(self): 93 | spec = merge([uniform(), uniform()], lambda x, y: x+y) 94 | parsed = ParsedSpec(spec) 95 | self.assertEqual(parsed.size(), 2) 96 | self.assertEqual(0.5, parsed.instantiate([0.0, 0.5])) 97 | self.assertEqual(1.5, parsed.instantiate([0.5, 1.0])) 98 | self.assertEqual(2.0, parsed.instantiate([1.0, 1.0])) 99 | 100 | 101 | def test_transform(self): 102 | spec = wrap(uniform(), lambda x: x*x) 103 | parsed = ParsedSpec(spec) 104 | self.assertEqual(parsed.size(), 1) 105 | self.assertEqual(0.0, parsed.instantiate([0.0])) 106 | self.assertEqual(4.0, parsed.instantiate([2.0])) 107 | 108 | 109 | def test_transform_merge(self): 110 | spec = wrap(merge([uniform(), uniform()], lambda x, y: x+y), lambda x: x*x) 111 | parsed = ParsedSpec(spec) 112 | self.assertEqual(parsed.size(), 2) 113 | self.assertEqual(1.0, parsed.instantiate([0.0, 1.0])) 114 | self.assertEqual(4.0, parsed.instantiate([1.0, 1.0])) 115 | 116 | 117 | def test_duplicate_nodes_1(self): 118 | node = uniform() 119 | spec = merge([node, node, node], lambda x, y, z: x+y+z) 120 | parsed = ParsedSpec(spec) 121 | self.assertEqual(parsed.size(), 1) 122 | self.assertEqual(3.0, parsed.instantiate([1.0])) 123 | self.assertEqual(9.0, parsed.instantiate([3.0])) 124 | 125 | 126 | def test_duplicate_nodes_2(self): 127 | node = uniform() 128 | spec = [[node, node]] 129 | parsed = ParsedSpec(spec) 130 | self.assertEqual(parsed.size(), 1) 131 | self.assertEqual([[1.0, 1.0]], parsed.instantiate([1.0])) 132 | 133 | 134 | def test_duplicate_nodes_3(self): 135 | spec = [uniform()] * 3 136 | parsed = ParsedSpec(spec) 137 | self.assertEqual(parsed.size(), 1) 138 | self.assertEqual([0.0, 0.0, 0.0], parsed.instantiate([0.0])) 139 | self.assertEqual([1.0, 1.0, 1.0], parsed.instantiate([1.0])) 140 | 141 | 142 | def test_merge_choice(self): 143 | spec = choice([uniform(0, 1), uniform(2, 3)]) 144 | parsed = ParsedSpec(spec) 145 | self.assertEqual(parsed.size(), 3) 146 | self.assertEqual(0.0, parsed.instantiate([0.0, 0.0, 0.0])) 147 | self.assertEqual(1.0, parsed.instantiate([1.0, 0.0, 0.0])) 148 | self.assertEqual(2.0, parsed.instantiate([0.0, 0.0, 0.9])) 149 | self.assertEqual(3.0, parsed.instantiate([0.0, 1.0, 0.9])) 150 | 151 | 152 | def test_if_condition(self): 153 | def if_cond(switch, size, num): 154 | if switch > 0.5: 155 | return [size, num, num] 156 | return [size, num] 157 | 158 | spec = merge([uniform(0, 1), uniform(1, 2), uniform(2, 3)], if_cond) 159 | parsed = ParsedSpec(spec) 160 | self.assertEqual(parsed.size(), 3) 161 | 162 | self.assertEqual([1, 2], parsed.instantiate([0, 0, 0])) 163 | self.assertEqual([2, 3], parsed.instantiate([0, 1, 1])) 164 | self.assertEqual([1, 2, 2], parsed.instantiate([1, 0, 0])) 165 | self.assertEqual([2, 3, 3], parsed.instantiate([1, 1, 1])) 166 | 167 | 168 | def test_object(self): 169 | class Dummy: pass 170 | dummy = Dummy 171 | dummy.value = uniform() 172 | dummy.foo = 'bar' 173 | dummy.ref = dummy 174 | 175 | spec = dummy 176 | parsed = ParsedSpec(spec) 177 | self.assertEqual(parsed.size(), 1) 178 | 179 | instance = parsed.instantiate([0]) 180 | self.assertEqual(0, instance.value) 181 | self.assertEqual('bar', instance.foo) 182 | self.assertEqual(instance, instance.ref) 183 | 184 | 185 | def test_dict(self): 186 | spec = {1: uniform(), 2: choice(['foo', 'bar']), 3: merge(lambda x: -x, uniform())} 187 | parsed = ParsedSpec(spec) 188 | self.assertEqual(parsed.size(), 3) 189 | self.assertEqual({1: 0.0, 2: 'foo', 3: 0.0}, parsed.instantiate([0, 0, 0])) 190 | self.assertEqual({1: 1.0, 2: 'bar', 3: -1.0}, parsed.instantiate([1, 1, 1])) 191 | 192 | 193 | def test_dict_deep_1(self): 194 | spec = {1: {'foo': uniform() } } 195 | parsed = ParsedSpec(spec) 196 | self.assertEqual(parsed.size(), 1) 197 | 198 | 199 | def test_dict_deep_2(self): 200 | spec = {'a': {'b': {'c': { 'd': uniform() } } } } 201 | parsed = ParsedSpec(spec) 202 | self.assertEqual(parsed.size(), 1) 203 | 204 | 205 | def test_math_operations_1(self): 206 | spec = uniform() + 1 207 | parsed = ParsedSpec(spec) 208 | self.assertEqual(parsed.size(), 1) 209 | self.assertEqual(2.0, parsed.instantiate([1.0])) 210 | 211 | 212 | def test_math_operations_2(self): 213 | spec = uniform() * (uniform() ** 2 + 1) / uniform() 214 | parsed = ParsedSpec(spec) 215 | self.assertEqual(parsed.size(), 3) 216 | self.assertEqual(2.0, parsed.instantiate([1.0, 1.0, 1.0])) 217 | self.assertEqual(1.0, parsed.instantiate([0.5, 1.0, 1.0])) 218 | self.assertEqual(1.0, parsed.instantiate([0.5, 0.0, 0.5])) 219 | 220 | 221 | def test_math_operations_3(self): 222 | spec = 2 / (1 + uniform()) * (3 - uniform() + 4 ** uniform()) 223 | parsed = ParsedSpec(spec) 224 | self.assertEqual(parsed.size(), 3) 225 | self.assertEqual(6.0, parsed.instantiate([1.0, 1.0, 1.0])) 226 | 227 | 228 | def test_math_operations_4(self): 229 | spec = choice(['foo', 'bar']) + '-' + choice(['abc', 'def']) 230 | parsed = ParsedSpec(spec) 231 | self.assertEqual(parsed.size(), 2) 232 | self.assertEqual('foo-abc', parsed.instantiate([0.0, 0.0])) 233 | self.assertEqual('bar-def', parsed.instantiate([1.0, 1.0])) 234 | 235 | 236 | def test_min_1(self): 237 | spec = min(uniform(), uniform(), 0.5) 238 | parsed = ParsedSpec(spec) 239 | self.assertEqual(parsed.size(), 2) 240 | self.assertEqual(0.5, parsed.instantiate([1.0, 0.7])) 241 | self.assertEqual(0.5, parsed.instantiate([1.0, 0.5])) 242 | self.assertEqual(0.0, parsed.instantiate([0.0, 0.5])) 243 | 244 | 245 | def test_min_2(self): 246 | spec = min(uniform(), 0.8, 0.5) 247 | parsed = ParsedSpec(spec) 248 | self.assertEqual(parsed.size(), 1) 249 | self.assertEqual(0.5, parsed.instantiate([1.0])) 250 | self.assertEqual(0.5, parsed.instantiate([0.5])) 251 | self.assertEqual(0.2, parsed.instantiate([0.2])) 252 | 253 | 254 | def test_min_3(self): 255 | spec = min(uniform(), uniform()) 256 | parsed = ParsedSpec(spec) 257 | self.assertEqual(parsed.size(), 2) 258 | self.assertEqual(0.5, parsed.instantiate([1.0, 0.5])) 259 | self.assertEqual(0.2, parsed.instantiate([0.2, 0.5])) 260 | 261 | 262 | def test_max_1(self): 263 | spec = max(0.5) 264 | parsed = ParsedSpec(spec) 265 | self.assertEqual(parsed.size(), 0) 266 | self.assertEqual(0.5, parsed.instantiate([])) 267 | 268 | 269 | def test_max_2(self): 270 | spec = max(0.5, 1.0) 271 | parsed = ParsedSpec(spec) 272 | self.assertEqual(parsed.size(), 0) 273 | self.assertEqual(1.0, parsed.instantiate([])) 274 | 275 | 276 | def test_max_3(self): 277 | spec = max(uniform()) 278 | parsed = ParsedSpec(spec) 279 | self.assertEqual(parsed.size(), 1) 280 | self.assertEqual(1.0, parsed.instantiate([1.0])) 281 | self.assertEqual(0.0, parsed.instantiate([0.0])) 282 | 283 | 284 | def test_name_1(self): 285 | aaa = uniform() 286 | bbb = choice(['foo']) 287 | ccc = uniform(-1, 1) 288 | ddd = uniform() 289 | spec = {'aaa': aaa, 'bbb': bbb, 'ccc': ccc **2, 'ddd': [ddd, ddd]} 290 | 291 | parsed = ParsedSpec(spec) 292 | self.assertEqual(parsed.size(), 4) 293 | self.assertTrue('aaa' in aaa.name()) 294 | self.assertTrue('uniform' in aaa.name()) 295 | self.assertTrue('bbb' in bbb.name()) 296 | self.assertTrue('choice' in bbb.name()) 297 | self.assertTrue('ccc' in ccc.name()) 298 | self.assertTrue('uniform' in ccc.name()) 299 | self.assertTrue('ddd' in ddd.name()) 300 | self.assertTrue('uniform' in ddd.name()) 301 | 302 | 303 | def test_name_2(self): 304 | norm_node = normal() 305 | choice_node = choice([uniform(), uniform(), uniform()]) 306 | spec = {'a': {'b': {'c': { 'd': norm_node, 0: choice_node } } } } 307 | 308 | # stats.norm.ppf is an instance method in python 2 309 | expected_normal_name = 'norm_gen' if six.PY2 else 'ppf' 310 | 311 | parsed = ParsedSpec(spec) 312 | self.assertEqual(parsed.size(), 5) 313 | self.assertTrue('a-b-c-d' in norm_node.name(), 'name=%s' % norm_node.name()) 314 | self.assertTrue(expected_normal_name in norm_node.name(), 'name=%s' % norm_node.name()) 315 | self.assertTrue('a-b-c-0' in choice_node.name(), 'name=%s' % choice_node.name()) 316 | self.assertTrue('choice' in choice_node.name(), 'name=%s' % choice_node.name()) 317 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /hyperengine/tests/strategy_test.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __author__ = 'maxim' 5 | 6 | import unittest 7 | 8 | from hyperengine.base import * 9 | from hyperengine import DefaultSampler, BayesianStrategy 10 | 11 | 12 | class BayesianStrategyTest(unittest.TestCase): 13 | # 1-D 14 | 15 | def test_1d_simple(self): 16 | self.run_opt(f=lambda x: np.abs(np.sin(x) / x), a=-10, b=10, start=5, global_max=1, steps=10, plot=False) 17 | self.run_opt(f=lambda x: x * x, a=-10, b=10, start=5, global_max=100, steps=10, plot=False) 18 | self.run_opt(f=lambda x: np.sin(np.log(np.abs(x))), a=-10, b=10, start=5, global_max=1, steps=10, plot=False) 19 | self.run_opt(f=lambda x: x / (np.sin(x) + 2), a=-8, b=8, start=3, global_max=4.8, steps=10, plot=False) 20 | 21 | def test_1d_periodic_max_1(self): 22 | self.run_opt(f=lambda x: x * np.sin(x), 23 | a=-10, b=10, start=3, 24 | global_max=7.9, delta=0.3, 25 | steps=30, 26 | plot=False) 27 | 28 | def test_1d_periodic_max_2(self): 29 | self.run_opt(f=lambda x: x * np.sin(x + 1) / 2, 30 | a=-12, b=16, start=3, 31 | global_max=6.55, delta=0.5, 32 | steps=30, 33 | plot=False) 34 | 35 | def test_1d_many_small_peaks(self): 36 | self.run_opt(f=lambda x: np.exp(np.sin(x * 5) * np.sqrt(x)), 37 | a=0, b=10, start=3, 38 | global_max=20.3, delta=1.0, 39 | steps=30, 40 | plot=False) 41 | 42 | # 2-D 43 | 44 | def test_2d_simple(self): 45 | self.run_opt(f=lambda x: x[0] + x[1], 46 | a=(0, 0), b=(10, 10), start=(5, 5), 47 | global_max=20, delta=1.0, 48 | steps=10, 49 | plot=False) 50 | 51 | def test_2d_peak_1(self): 52 | self.run_opt(f=(lambda x: (x[0] + x[1]) / ((x[0] - 1) ** 2 - np.sin(x[1]) + 2)), 53 | a=(0, 0), b=(9, 9), start=None, 54 | global_max=8.95, 55 | steps=20, 56 | plot=False) 57 | 58 | def test_2d_irregular_max_1(self): 59 | self.run_opt(f=(lambda x: (x[0] + x[1]) / (np.exp(-np.sin(x[0])))), 60 | a=(0, 0), b=(9, 9), start=None, 61 | global_max=46, 62 | steps=20, 63 | plot=False) 64 | 65 | def test_2d_irregular_max_2(self): 66 | self.run_opt(f=lambda x: np.sum(x * np.sin(x + 1) / 2, axis=0), 67 | a=(-10, -10), b=(15, 15), start=None, 68 | global_max=13.175, 69 | steps=40, 70 | plot=False) 71 | 72 | def test_2d_periodic_max_1(self): 73 | self.run_opt(f=lambda x: np.sin(x[0]) + np.cos(x[1]), 74 | a=(0, 0), b=(10, 10), start=(5, 5), 75 | global_max=2, 76 | steps=10, 77 | plot=False) 78 | 79 | def test_2d_periodic_max_2(self): 80 | self.run_opt(f=lambda x: np.sin(x[0]) * np.cos(x[1]), 81 | a=(0, 0), b=(10, 10), start=(5, 5), 82 | global_max=1, 83 | steps=30, 84 | plot=False) 85 | 86 | def test_2d_periodic_max_3(self): 87 | self.run_opt(f=lambda x: np.sin(x[0]) / (np.cos(x[1]) + 2), 88 | a=(0, 0), b=(10, 10), start=(5, 5), 89 | global_max=1, 90 | steps=40, 91 | plot=False) 92 | 93 | # 4-D 94 | 95 | def test_4d_simple_1(self): 96 | self.run_opt(f=lambda x: x[0] + 2 * x[1] - x[2] - 2 * x[3], 97 | a=(-10, -10, -10, -10), b=(10, 10, 10, 10), start=None, 98 | global_max=60, 99 | steps=50, 100 | plot=False) 101 | 102 | def test_4d_simple_2(self): 103 | self.run_opt(f=lambda x: x[0] + np.sin(x[1]) - x[2] + x[3], 104 | a=(-5, -5, -5, -5), b=(5, 5, 5, 5), start=None, 105 | global_max=16, delta=1.0, 106 | steps=20, 107 | plot=False) 108 | 109 | def test_4d_irregular_max(self): 110 | self.run_opt(f=lambda x: (np.sin(x[0] ** 2) + np.power(2, (x[1] - x[2]) / 5)) / (x[3] ** 2 + 1), 111 | a=(-5, -5, -5, -5), b=(5, 5, 5, 5), start=None, 112 | global_max=5, delta=1.0, 113 | steps=50, 114 | plot=False) 115 | 116 | # 10-D 117 | 118 | def test_10d_simple_1(self): 119 | self.run_opt(f=lambda x: x[1] + 5 * x[5] - x[7], 120 | a=[-1] * 10, b=[1] * 10, start=None, 121 | global_max=7, delta=0.5, 122 | steps=30, 123 | plot=False) 124 | 125 | def test_10d_simple_2(self): 126 | self.run_opt(f=lambda x: np.cos(x[0]) + np.sin(x[1]) + np.exp(-x[2]) - np.exp(x[3]), 127 | a=[-1] * 10, b=[1] * 10, start=None, 128 | global_max=4.2, delta=0.3, 129 | steps=30, 130 | plot=False) 131 | 132 | def test_10d_simple_3(self): 133 | self.run_opt(f=lambda x: (x[0] * x[1] - x[2] * x[3]) * x[4], 134 | a=[0] * 10, b=[1] * 10, start=None, 135 | global_max=1, delta=0.2, 136 | steps=30, 137 | plot=False) 138 | 139 | def test_10d_simple_4(self): 140 | self.run_opt(f=lambda x: x[0] * x[1] * x[2] * x[3] * x[4] * x[5], 141 | a=[0] * 10, b=[1] * 10, start=None, 142 | global_max=1, delta=0.5, 143 | steps=50, 144 | plot=False) 145 | 146 | def test_10d_simple_5(self): 147 | self.run_opt(f=lambda x: np.sum(x, axis=0), 148 | a=[0] * 10, b=[1] * 10, start=None, 149 | global_max=10, delta=1.5, 150 | steps=30, 151 | plot=False) 152 | 153 | def test_10d_irregular_max(self): 154 | self.run_opt(f=lambda x: (np.sin(x[0] ** 2) + np.power(2, (x[1] - x[2]))) / (x[3] ** 2 + 1), 155 | a=[0] * 10, b=[1] * 10, start=None, 156 | global_max=2.8, delta=0.2, 157 | steps=30, 158 | plot=False) 159 | 160 | # Realistic 161 | 162 | def test_realistic_4d(self): 163 | def f(x): 164 | init, size, reg, _ = x 165 | result = 10 * np.cos(size - 3) * np.cos(reg - size / 2) 166 | result = np.asarray(result) 167 | result[size > 6] = 10 - size[size > 6] 168 | result[size < 1] = size[size < 1] 169 | result[init > 4] = 7 - init[init > 4] 170 | return result 171 | 172 | self.run_opt(f=f, 173 | a=(0, 0, 0, 0), b=(10, 10, 10, 1), start=None, 174 | global_max=10, delta=0.5, 175 | steps=20) 176 | 177 | def test_realistic_10d(self): 178 | def f(x): 179 | init, num1, size1, activation1, dropout1, num2, size2, activation2, dropout2, fc_size = x 180 | result = np.sin(num1 * size1 + activation1) + np.cos(num2 * size2 + activation2) + fc_size 181 | result = np.asarray(result) 182 | result[size1 > 0.5] = 1 - size1[size1 > 0.5] 183 | result[size2 > 0.5] = 1 - size1[size2 > 0.5] 184 | result[dropout1 < 0.3] = dropout1[dropout1 < 0.3] 185 | result[dropout2 < 0.4] = dropout1[dropout2 < 0.4] 186 | result[init > 0.5] = np.exp(-init[init > 0.5]) 187 | return result 188 | 189 | self.run_opt(f=f, 190 | a=[0] * 10, b=[1] * 10, start=None, 191 | global_max=3, delta=0.3, 192 | steps=50) 193 | 194 | # Technical details 195 | 196 | def run_opt(self, f, a, b, start=None, global_max=None, delta=None, steps=10, plot=False): 197 | if plot: 198 | self._run(f, a, b, start, steps, batch_size=100000, stop_condition=None) 199 | self._plot(f, a, b) 200 | return 201 | 202 | errors = [] 203 | size_list = [1000, 10000, 50000, 100000] 204 | 205 | for batch_size in size_list: 206 | delta = delta or abs(global_max) / 10.0 207 | max_value = self._run(f, a, b, start, steps, batch_size, 208 | stop_condition=lambda x: abs(f(x) - global_max) <= delta) 209 | if abs(max_value - global_max) <= delta: 210 | return 211 | msg = 'Failure %6d: max=%.3f, expected=%.3f within delta=%.3f' % (batch_size, max_value, global_max, delta) 212 | errors.append(msg) 213 | 214 | log('\n '.join(errors)) 215 | self.fail('\n '.join(errors)) 216 | 217 | def _run(self, f, a, b, start, steps, batch_size, stop_condition): 218 | sampler = DefaultSampler() 219 | sampler.add(lambda: np.random.uniform(a, b)) 220 | self.strategy = BayesianStrategy(sampler, mc_batch_size=batch_size) 221 | 222 | if start is not None: 223 | self.strategy.add_point(np.asarray(start), f(start)) 224 | 225 | for i in range(steps): 226 | x = self.strategy.next_proposal() 227 | log('selected_point=%s -> true=%.6f' % (x, f(x))) 228 | self.strategy.add_point(x, f(x)) 229 | if stop_condition is not None and stop_condition(x): 230 | break 231 | 232 | i = np.argmax(self.strategy.values) 233 | log('Best found: %s -> %.6f' % (self.strategy.points[i], self.strategy.values[i])) 234 | return self.strategy.values[i] 235 | 236 | def _plot(self, f, a, b): 237 | artist = Artist(strategy=self.strategy) 238 | if type(a) in [tuple, list]: 239 | if len(a) == 2: 240 | artist.plot_2d(f, a, b) 241 | else: 242 | artist.scatter_plot_per_dimension() 243 | else: 244 | artist.plot_1d(f, a, b) 245 | 246 | def _eval_max(self, f, a, b): 247 | sampler = DefaultSampler() 248 | sampler.add(lambda: np.random.uniform(a, b)) 249 | batch = sampler.sample(1000000) 250 | batch = np.swapaxes(batch, 1, 0) 251 | return np.max(f(batch)) 252 | 253 | def _eval_at(self, val): 254 | log(self.strategy._method.compute_values(np.asarray([[val]]))) 255 | log(self.strategy._method.compute_values(self.strategy._method.points)) 256 | 257 | 258 | ######################################################################################################################## 259 | # Helper code 260 | ######################################################################################################################## 261 | 262 | 263 | import math 264 | 265 | import matplotlib.pyplot as plt 266 | from mpl_toolkits.mplot3d import Axes3D 267 | 268 | import numpy as np 269 | 270 | 271 | class Artist(object): 272 | def __init__(self, *args, **kwargs): 273 | super(Artist, self).__init__() 274 | if len(args) == 3: 275 | self.points, self.values, self.utility = args 276 | else: 277 | strategy = kwargs.get('strategy') 278 | if strategy is not None: 279 | self.points = strategy.points 280 | self.values = strategy.values 281 | self.utility = strategy._method 282 | self.names = kwargs.get('names', {}) 283 | 284 | def plot_1d(self, f, a, b, grid_size=1000): 285 | grid = np.linspace(a, b, num=grid_size).reshape((-1, 1)) 286 | mu, sigma = self.utility.mean_and_std(grid) 287 | 288 | plt.plot(grid, f(grid), color='black', linewidth=1.5, label='f') 289 | plt.plot(grid, mu, color='red', label='mu') 290 | plt.plot(grid, mu + sigma, color='blue', linewidth=0.4, label='mu+sigma') 291 | plt.plot(grid, mu - sigma, color='blue', linewidth=0.4) 292 | plt.plot(self.points, f(np.asarray(self.points)), 'o', color='red') 293 | plt.xlim([a - 0.5, b + 0.5]) 294 | # plt.legend() 295 | plt.show() 296 | 297 | def plot_2d(self, f, a, b, grid_size=200): 298 | grid_x = np.linspace(a[0], b[0], num=grid_size).reshape((-1, 1)) 299 | grid_y = np.linspace(a[1], b[1], num=grid_size).reshape((-1, 1)) 300 | x, y = np.meshgrid(grid_x, grid_y) 301 | 302 | merged = np.stack([x.flatten(), y.flatten()]) 303 | z = f(merged).reshape(x.shape) 304 | 305 | swap = np.swapaxes(merged, 0, 1) 306 | mu, sigma = self.utility.mean_and_std(swap) 307 | mu = mu.reshape(x.shape) 308 | sigma = sigma.reshape(x.shape) 309 | 310 | points = np.asarray(self.points) 311 | xs = points[:, 0] 312 | ys = points[:, 1] 313 | zs = f(np.swapaxes(points, 0, 1)) 314 | 315 | fig = plt.figure() 316 | ax = Axes3D(fig) 317 | ax.plot_surface(x, y, z, color='black', label='f', alpha=0.7, 318 | linewidth=0, antialiased=False) 319 | ax.plot_surface(x, y, mu, color='red', label='mu', alpha=0.5) 320 | ax.plot_surface(x, y, mu + sigma, color='blue', label='mu+sigma', alpha=0.3) 321 | ax.plot_surface(x, y, mu - sigma, color='blue', alpha=0.3) 322 | ax.scatter(xs, ys, zs, color='red', marker='o', s=100) 323 | # plt.legend() 324 | plt.show() 325 | 326 | def scatter_plot_per_dimension(self): 327 | points = np.array(self.points) 328 | values = np.array(self.values) 329 | n, d = points.shape 330 | rows = int(math.sqrt(d)) 331 | cols = (d + rows - 1) / rows 332 | 333 | _, axes = plt.subplots(rows, cols) 334 | axes = axes.reshape(-1) 335 | for j in range(d): 336 | axes[j].scatter(points[:, j], values, s=100, alpha=0.5) 337 | axes[j].set_title(self.names.get(j, str(j))) 338 | plt.show() 339 | 340 | def bar_plot_per_dimension(self): 341 | points = np.array(self.points) 342 | values = np.array(self.values) 343 | n, d = points.shape 344 | rows = int(math.sqrt(d)) 345 | cols = (d + rows - 1) / rows 346 | 347 | _, axes = plt.subplots(rows, cols) 348 | axes = axes.reshape(-1) 349 | for j in range(d): 350 | p = points[:, j] 351 | split = np.linspace(np.min(p), np.max(p), 10) 352 | bar_values = np.zeros((len(split),)) 353 | for k in range(len(split) - 1): 354 | interval = np.logical_and(split[k] < p, p < split[k+1]) 355 | if np.any(interval): 356 | bar_values[k] = np.mean(values[interval]) 357 | axes[j].bar(split, height=bar_values, width=split[1]-split[0]) 358 | plt.show() 359 | 360 | 361 | if __name__ == "__main__": 362 | unittest.main() 363 | -------------------------------------------------------------------------------- /hyperengine/tests/curve-test-data.xjson: -------------------------------------------------------------------------------- 1 | {'x': [[0.387800, 0.471400, 0.489200, 0.525000, 0.572200, 0.581800, 0.602800, 0.624600, 0.624200, 0.630800, 0.647200, 0.667800, 0.686200, 0.689400, 0.699000, 0.687000, 0.691600, 0.709200, 0.708000, 0.706400, 0.712600, 0.724200, 0.722600, 0.715200, 0.732200, 0.730400, 0.728600], [0.220800, 0.273800, 0.343600, 0.365800, 0.392800, 0.424000, 0.449200, 0.461800, 0.467600, 0.494800, 0.480600, 0.491800, 0.504200, 0.521400, 0.521800, 0.533000, 0.527400, 0.539600, 0.546400, 0.539200, 0.560400, 0.560200, 0.563600, 0.565800, 0.562600, 0.582000, 0.574400], [0.352800, 0.452600, 0.493200, 0.518800, 0.541800, 0.553400, 0.574800, 0.597800, 0.595800, 0.610600, 0.643600, 0.645600, 0.657200, 0.675800, 0.672400, 0.667000, 0.678800, 0.675400, 0.696200, 0.681400, 0.680600, 0.710600, 0.708200, 0.723400, 0.728400, 0.720200, 0.722800], [0.350000, 0.438800, 0.472600, 0.522600, 0.547000, 0.565800, 0.584000, 0.604200, 0.602600, 0.625400, 0.615200, 0.634800, 0.652800, 0.677600, 0.674200, 0.691800, 0.686600, 0.706200, 0.687200, 0.693600, 0.715400, 0.717800, 0.705200, 0.726600, 0.737400, 0.740800, 0.736400], [0.288600, 0.369400, 0.413000, 0.407000, 0.435400, 0.453400, 0.477800, 0.494400, 0.498600, 0.521400, 0.525200, 0.545800, 0.560000, 0.570000, 0.563400, 0.574600, 0.569000, 0.609600, 0.600000, 0.620800, 0.609600, 0.604600, 0.616000, 0.638000, 0.633000, 0.650200, 0.616800], [0.320800, 0.409400, 0.450200, 0.490600, 0.512800, 0.522600, 0.556200, 0.558200, 0.585000, 0.600600, 0.595200, 0.614000, 0.628000, 0.633400, 0.632000, 0.648400, 0.663800, 0.666000, 0.673200, 0.662400, 0.680400, 0.683600, 0.691200, 0.674400, 0.696600, 0.716400, 0.702400], [0.245400, 0.314800, 0.363600, 0.385000, 0.414600, 0.436200, 0.452000, 0.482200, 0.468000, 0.496400, 0.526000, 0.523400, 0.544200, 0.551600, 0.560000, 0.578800, 0.590800, 0.585000, 0.595200, 0.601000, 0.607800, 0.608200, 0.624600, 0.624600, 0.630800, 0.639400, 0.636800], [0.334200, 0.429600, 0.475600, 0.524800, 0.543400, 0.544600, 0.545200, 0.591600, 0.616400, 0.630200, 0.625400, 0.645400, 0.637400, 0.659800, 0.668400, 0.678800, 0.677200, 0.684800, 0.683800, 0.690000, 0.695800, 0.685800, 0.710800, 0.706600, 0.715200, 0.700400, 0.719200], [0.280800, 0.352000, 0.427400, 0.449200, 0.464400, 0.491400, 0.490400, 0.535800, 0.549200, 0.568400, 0.561600, 0.588800, 0.588400, 0.606000, 0.608000, 0.615400, 0.608000, 0.616000, 0.640200, 0.637000, 0.655400, 0.647800, 0.658600, 0.665200, 0.675200, 0.680400, 0.688800], [0.291800, 0.358200, 0.421800, 0.451000, 0.491000, 0.512800, 0.511800, 0.535600, 0.582200, 0.586200, 0.609800, 0.623600, 0.625600, 0.654400, 0.658600, 0.665400, 0.684000, 0.692600, 0.695600, 0.704000, 0.696800, 0.681400, 0.718400, 0.717400, 0.720600, 0.731800, 0.747000], [0.338000, 0.406200, 0.451000, 0.492400, 0.505600, 0.533600, 0.528000, 0.536800, 0.560400, 0.572800, 0.553000, 0.585400, 0.620400, 0.602400, 0.633000, 0.631800, 0.622200, 0.632200, 0.645400, 0.657800, 0.658200, 0.663600, 0.668800, 0.681800, 0.682000, 0.686200, 0.684000], [0.282000, 0.376800, 0.414000, 0.467400, 0.488600, 0.497200, 0.519400, 0.522400, 0.555600, 0.557400, 0.575000, 0.589400, 0.606000, 0.605800, 0.619800, 0.637400, 0.631800, 0.653000, 0.661200, 0.661400, 0.656800, 0.675200, 0.687000, 0.679800, 0.695400, 0.671600, 0.684800], [0.210000, 0.284800, 0.356600, 0.385600, 0.430800, 0.417800, 0.460400, 0.460600, 0.474200, 0.473800, 0.488400, 0.495600, 0.524200, 0.532000, 0.540400, 0.539000, 0.559800, 0.564200, 0.580400, 0.580400, 0.581800, 0.596000, 0.607400, 0.598000, 0.621200, 0.607400, 0.631200], [0.316400, 0.369200, 0.462200, 0.487400, 0.526000, 0.528600, 0.567200, 0.588200, 0.597400, 0.598400, 0.625200, 0.627200, 0.627400, 0.654800, 0.663200, 0.667600, 0.677400, 0.661000, 0.681400, 0.689200, 0.699800, 0.698400, 0.700000, 0.694800, 0.702200, 0.723600, 0.716800], [0.302600, 0.391600, 0.457400, 0.477800, 0.494400, 0.538600, 0.551000, 0.579200, 0.584800, 0.587000, 0.600400, 0.608800, 0.617800, 0.636400, 0.642800, 0.651600, 0.655800, 0.665600, 0.667000, 0.672200, 0.665400, 0.688400, 0.689000, 0.680000, 0.690200, 0.684800, 0.709000], [0.306000, 0.403400, 0.425200, 0.457200, 0.515600, 0.535800, 0.539600, 0.562600, 0.555200, 0.580000, 0.604000, 0.612600, 0.625600, 0.618800, 0.623000, 0.644800, 0.646000, 0.652200, 0.646000, 0.663400, 0.659600, 0.673000, 0.672600, 0.689600, 0.665800, 0.694400, 0.687800], [0.337800, 0.407400, 0.457600, 0.489000, 0.486200, 0.518600, 0.538800, 0.548600, 0.574800, 0.578200, 0.595600, 0.614200, 0.624600, 0.631200, 0.639600, 0.630000, 0.656600, 0.655800, 0.666600, 0.666600, 0.674400, 0.688400, 0.676800, 0.664200, 0.693600, 0.700200, 0.697000], [0.262400, 0.337200, 0.391600, 0.401200, 0.452000, 0.468400, 0.476000, 0.492600, 0.501600, 0.541200, 0.551600, 0.555000, 0.581600, 0.576000, 0.598000, 0.593200, 0.601800, 0.610600, 0.630200, 0.621200, 0.644600, 0.657000, 0.663000, 0.669400, 0.683400, 0.684800, 0.680400], [0.222000, 0.330800, 0.352400, 0.407000, 0.440600, 0.444800, 0.486200, 0.493000, 0.521000, 0.521600, 0.522200, 0.539000, 0.565400, 0.554400, 0.575000, 0.591800, 0.585200, 0.569400, 0.591600, 0.618200, 0.638600, 0.648200, 0.654600, 0.659800, 0.648000, 0.679800, 0.676400], [0.319400, 0.411600, 0.470000, 0.493600, 0.519800, 0.524800, 0.539000, 0.548200, 0.580200, 0.574200, 0.601200, 0.620800, 0.633000, 0.630600, 0.641400, 0.658000, 0.653000, 0.650600, 0.649600, 0.674000, 0.663800, 0.692200, 0.697000, 0.696400, 0.699000, 0.698200, 0.713600], [0.397600, 0.472400, 0.506800, 0.553400, 0.568000, 0.577600, 0.592000, 0.611200, 0.595400, 0.638800, 0.619800, 0.646400, 0.639800, 0.658200, 0.676400, 0.669800, 0.658000, 0.679800, 0.689000, 0.676200, 0.677000, 0.695400, 0.707000, 0.699400, 0.699600, 0.718400, 0.723600], [0.343000, 0.421000, 0.477200, 0.502200, 0.551400, 0.575600, 0.607400, 0.621200, 0.616400, 0.644800, 0.663400, 0.675400, 0.687400, 0.678000, 0.701000, 0.693600, 0.713800, 0.712400, 0.710600, 0.734200, 0.744600, 0.723400, 0.753800, 0.750200, 0.753400, 0.757400, 0.766000], [0.346400, 0.432000, 0.477800, 0.537400, 0.523800, 0.551800, 0.589600, 0.597600, 0.625600, 0.658200, 0.643400, 0.666200, 0.681800, 0.691400, 0.687800, 0.718000, 0.722800, 0.726800, 0.724600, 0.736200, 0.732800, 0.742000, 0.746400, 0.730800, 0.739400, 0.739800, 0.762200], [0.267000, 0.353200, 0.396000, 0.431200, 0.470600, 0.473400, 0.491600, 0.490400, 0.500000, 0.521800, 0.535800, 0.537000, 0.555600, 0.565600, 0.585200, 0.577400, 0.594600, 0.598000, 0.596400, 0.608000, 0.621400, 0.613600, 0.631800, 0.634400, 0.639400, 0.646200, 0.641600], [0.277600, 0.310600, 0.380000, 0.434200, 0.469200, 0.478800, 0.497000, 0.503200, 0.541400, 0.550400, 0.558800, 0.569600, 0.577600, 0.595800, 0.603800, 0.617600, 0.635200, 0.639000, 0.643200, 0.644200, 0.670400, 0.667400, 0.674600, 0.670800, 0.687800, 0.693000, 0.702600], [0.228000, 0.324400, 0.389600, 0.428200, 0.450400, 0.471800, 0.489000, 0.501200, 0.520000, 0.535000, 0.540200, 0.563400, 0.556000, 0.609000, 0.605600, 0.604200, 0.615800, 0.632600, 0.629000, 0.643600, 0.660000, 0.654800, 0.671200, 0.667200, 0.685200, 0.688600, 0.692200], [0.233800, 0.296800, 0.359400, 0.398800, 0.427000, 0.455800, 0.459000, 0.456600, 0.494000, 0.477000, 0.503000, 0.519200, 0.538400, 0.542800, 0.541200, 0.557000, 0.562000, 0.588400, 0.589800, 0.589200, 0.547000, 0.604400, 0.620600, 0.630000, 0.615200, 0.627800, 0.639200], [0.361400, 0.450400, 0.488600, 0.501400, 0.535800, 0.578600, 0.611800, 0.625800, 0.649400, 0.665600, 0.657000, 0.691000, 0.678600, 0.693600, 0.703200, 0.706200, 0.713000, 0.721800, 0.724800, 0.724400, 0.747600, 0.749400, 0.748800, 0.757800, 0.750600, 0.757800, 0.759600], [0.341000, 0.419000, 0.492600, 0.531200, 0.548000, 0.569200, 0.613600, 0.618800, 0.646400, 0.656600, 0.668600, 0.685200, 0.679000, 0.695200, 0.705000, 0.707200, 0.715400, 0.716400, 0.733200, 0.726000, 0.724600, 0.741000, 0.727800, 0.753200, 0.756200, 0.758800, 0.757800], [0.377800, 0.452200, 0.503400, 0.527400, 0.570400, 0.571800, 0.617000, 0.625800, 0.630000, 0.623000, 0.659200, 0.665600, 0.683800, 0.696800, 0.702600, 0.705400, 0.705000, 0.708400, 0.724000, 0.717400, 0.731000, 0.728400, 0.730000, 0.741000, 0.748400, 0.749400, 0.745600], [0.303000, 0.382600, 0.451000, 0.444000, 0.505000, 0.514000, 0.547200, 0.533400, 0.570800, 0.574000, 0.582400, 0.598200, 0.605200, 0.610000, 0.606200, 0.590800, 0.633400, 0.636000, 0.655800, 0.658000, 0.654000, 0.677200, 0.662800, 0.652600, 0.680400, 0.687600, 0.690000], [0.341000, 0.416200, 0.460200, 0.502200, 0.521200, 0.546800, 0.566000, 0.568800, 0.590400, 0.612400, 0.606400, 0.618800, 0.638400, 0.634600, 0.650400, 0.627200, 0.626000, 0.644600, 0.650000, 0.661800, 0.671200, 0.682800, 0.676200, 0.688000, 0.686000, 0.679200, 0.694200], [0.318800, 0.449800, 0.503200, 0.523400, 0.542800, 0.551000, 0.579400, 0.613600, 0.644600, 0.648200, 0.639800, 0.673800, 0.674200, 0.666000, 0.683800, 0.696000, 0.703200, 0.706200, 0.705800, 0.719400, 0.705400, 0.714400, 0.727200, 0.719400, 0.723600, 0.730000, 0.742200], [0.333400, 0.416400, 0.448600, 0.504400, 0.510200, 0.543200, 0.560600, 0.564600, 0.593800, 0.606000, 0.614800, 0.610000, 0.634800, 0.644200, 0.661200, 0.658000, 0.677800, 0.660200, 0.673000, 0.688600, 0.693600, 0.683800, 0.693000, 0.692200, 0.709400, 0.712000, 0.717400], [0.338800, 0.428200, 0.469200, 0.505400, 0.537000, 0.566000, 0.600800, 0.606200, 0.621400, 0.635600, 0.645000, 0.661600, 0.666400, 0.691600, 0.678600, 0.692200, 0.702200, 0.716800, 0.715600, 0.730200, 0.721600, 0.734200, 0.730000, 0.737400, 0.747600, 0.726200, 0.745000], [0.351400, 0.432600, 0.490200, 0.499200, 0.556400, 0.550600, 0.586800, 0.604200, 0.614400, 0.637200, 0.645000, 0.661200, 0.677800, 0.680800, 0.687200, 0.690000, 0.683000, 0.696000, 0.717800, 0.723600, 0.706400, 0.725600, 0.728400, 0.729200, 0.734800, 0.736200, 0.736800], [0.346200, 0.395000, 0.479000, 0.504400, 0.526200, 0.543800, 0.580600, 0.583600, 0.625800, 0.631600, 0.642800, 0.652400, 0.661800, 0.651200, 0.676000, 0.668800, 0.692200, 0.700800, 0.696800, 0.708400, 0.709400, 0.718200, 0.706800, 0.725800, 0.733200, 0.733600, 0.735000], [0.344800, 0.424200, 0.469800, 0.531400, 0.572400, 0.585600, 0.609000, 0.638000, 0.637400, 0.661800, 0.682200, 0.667400, 0.701600, 0.703200, 0.679400, 0.710800, 0.701000, 0.726200, 0.732800, 0.749200, 0.752000, 0.753400, 0.759400, 0.760800, 0.761400, 0.751000, 0.755600], [0.370400, 0.441400, 0.475400, 0.520400, 0.544200, 0.587200, 0.593800, 0.630800, 0.626000, 0.642400, 0.663000, 0.667000, 0.662800, 0.698600, 0.693800, 0.695600, 0.719000, 0.717800, 0.733200, 0.740200, 0.734200, 0.743600, 0.747800, 0.741200, 0.731600, 0.751800, 0.760600], [0.321000, 0.392400, 0.387400, 0.492000, 0.495600, 0.511400, 0.535400, 0.528600, 0.529400, 0.550800, 0.595400, 0.605200, 0.613000, 0.632400, 0.638800, 0.645000, 0.644400, 0.635600, 0.653000, 0.668000, 0.667200, 0.677400, 0.672600, 0.696800, 0.699600, 0.705400, 0.683200], [0.356600, 0.448200, 0.515800, 0.544800, 0.560400, 0.600800, 0.601400, 0.633200, 0.652800, 0.657800, 0.675800, 0.687000, 0.682400, 0.700200, 0.706800, 0.706400, 0.711000, 0.725200, 0.733600, 0.734000, 0.737000, 0.729800, 0.739000, 0.741000, 0.754600, 0.744200, 0.762000], [0.300600, 0.382600, 0.400800, 0.444800, 0.445400, 0.477800, 0.485400, 0.525400, 0.522800, 0.537400, 0.562600, 0.572000, 0.591600, 0.585000, 0.610400, 0.595000, 0.628200, 0.630600, 0.620000, 0.627800, 0.655400, 0.659200, 0.655200, 0.664200, 0.661600, 0.669600, 0.676400], [0.357200, 0.422600, 0.437200, 0.493400, 0.527800, 0.540800, 0.560400, 0.576600, 0.572200, 0.599000, 0.606400, 0.606600, 0.630000, 0.638200, 0.630600, 0.642200, 0.634200, 0.659800, 0.662000, 0.663800, 0.675400, 0.687000, 0.679800, 0.686800, 0.696000, 0.695400, 0.706600], [0.327400, 0.386600, 0.442600, 0.454200, 0.487600, 0.485600, 0.517600, 0.529200, 0.545800, 0.560600, 0.557200, 0.577800, 0.599000, 0.620000, 0.598000, 0.609200, 0.628600, 0.651600, 0.643000, 0.656800, 0.652000, 0.652000, 0.655200, 0.667800, 0.688400, 0.696400, 0.696800], [0.396200, 0.483800, 0.499200, 0.557800, 0.585200, 0.574400, 0.586000, 0.623800, 0.625000, 0.649400, 0.656200, 0.669200, 0.672000, 0.680000, 0.692600, 0.701400, 0.682800, 0.702600, 0.718000, 0.709400, 0.716800, 0.713000, 0.714000, 0.734600, 0.738200, 0.724800, 0.725400], [0.381800, 0.449400, 0.515800, 0.529200, 0.540800, 0.564400, 0.586600, 0.603400, 0.622800, 0.632600, 0.644400, 0.656400, 0.659400, 0.660400, 0.662400, 0.690000, 0.667200, 0.687600, 0.702800, 0.716000, 0.710200, 0.721600, 0.717200, 0.720000, 0.712600, 0.729800, 0.734400], [0.335800, 0.451200, 0.492400, 0.523000, 0.560200, 0.594400, 0.614000, 0.635200, 0.645400, 0.654000, 0.646000, 0.677200, 0.679000, 0.687600, 0.702000, 0.694000, 0.707200, 0.713000, 0.731600, 0.734000, 0.735800, 0.740000, 0.749800, 0.759200, 0.746200, 0.753800, 0.761600], [0.342000, 0.450200, 0.510000, 0.517600, 0.579800, 0.575800, 0.619600, 0.625400, 0.629800, 0.665600, 0.662000, 0.677200, 0.681200, 0.703800, 0.701600, 0.707200, 0.725600, 0.709400, 0.726400, 0.735200, 0.738400, 0.740600, 0.743000, 0.747000, 0.753600, 0.753200, 0.754600], [0.394000, 0.444000, 0.497800, 0.536400, 0.575200, 0.601200, 0.622400, 0.625600, 0.650600, 0.660000, 0.670200, 0.681600, 0.691800, 0.701000, 0.700600, 0.705600, 0.717200, 0.718800, 0.732200, 0.738600, 0.738400, 0.745600, 0.749800, 0.750200, 0.753800, 0.749400, 0.758000], [0.351400, 0.450200, 0.470200, 0.511000, 0.511400, 0.576200, 0.603400, 0.620200, 0.629200, 0.644800, 0.650400, 0.673800, 0.679800, 0.692800, 0.700800, 0.707000, 0.703800, 0.714800, 0.714800, 0.730200, 0.733000, 0.729000, 0.728000, 0.742200, 0.736200, 0.758600, 0.752400], [0.346000, 0.421600, 0.491200, 0.536200, 0.575400, 0.600400, 0.612200, 0.634600, 0.657400, 0.667400, 0.678200, 0.691800, 0.704400, 0.714400, 0.708400, 0.735200, 0.731800, 0.747800, 0.755200, 0.752400, 0.757200, 0.757800, 0.755600, 0.758400, 0.757800, 0.765000, 0.761000], [0.347400, 0.454600, 0.505600, 0.538400, 0.581800, 0.591000, 0.619200, 0.632400, 0.636200, 0.659400, 0.657200, 0.669400, 0.685200, 0.703400, 0.703000, 0.703400, 0.722800, 0.723400, 0.733000, 0.732200, 0.742000, 0.748000, 0.745600, 0.750400, 0.763000, 0.762000, 0.755000], [0.314400, 0.447000, 0.479600, 0.514800, 0.538200, 0.560000, 0.591200, 0.595000, 0.631400, 0.625200, 0.655600, 0.654600, 0.662000, 0.685000, 0.683000, 0.692000, 0.709400, 0.697800, 0.703000, 0.720000, 0.736000, 0.717000, 0.731400, 0.724600, 0.727400, 0.739400, 0.724400], [0.377200, 0.469000, 0.528800, 0.524600, 0.575000, 0.612000, 0.626400, 0.656200, 0.665000, 0.688800, 0.694400, 0.670600, 0.717600, 0.716200, 0.720400, 0.725000, 0.726000, 0.738800, 0.729600, 0.739000, 0.752800, 0.767600, 0.754200, 0.764600, 0.761000, 0.765600, 0.776400], [0.387200, 0.463800, 0.507000, 0.562800, 0.582400, 0.586800, 0.604000, 0.631800, 0.659400, 0.669400, 0.673000, 0.687600, 0.690600, 0.684200, 0.687600, 0.706200, 0.698800, 0.717200, 0.725400, 0.737800, 0.729200, 0.739400, 0.734400, 0.744600, 0.741400, 0.737600, 0.741600], [0.380400, 0.459600, 0.528400, 0.566000, 0.553600, 0.625000, 0.641800, 0.646400, 0.663800, 0.689400, 0.691800, 0.705800, 0.700000, 0.700200, 0.716200, 0.703000, 0.736000, 0.731800, 0.738400, 0.729400, 0.744200, 0.743800, 0.746000, 0.734800, 0.748400, 0.759400, 0.759200], [0.365200, 0.440600, 0.474600, 0.525600, 0.557800, 0.568000, 0.588400, 0.607600, 0.622800, 0.636200, 0.656600, 0.654600, 0.674200, 0.675200, 0.697800, 0.702400, 0.703000, 0.712200, 0.707800, 0.714000, 0.725800, 0.731400, 0.737600, 0.737600, 0.737800, 0.739400, 0.756000], [0.367600, 0.462600, 0.494800, 0.500000, 0.570400, 0.593000, 0.616600, 0.607400, 0.641400, 0.673200, 0.668000, 0.669600, 0.708000, 0.705800, 0.714800, 0.723400, 0.717600, 0.724600, 0.741400, 0.739200, 0.738200, 0.739800, 0.747800, 0.753000, 0.761600, 0.755000, 0.764200], [0.357800, 0.432400, 0.483800, 0.511600, 0.541800, 0.561400, 0.604200, 0.626800, 0.632800, 0.656000, 0.667200, 0.674600, 0.703400, 0.686800, 0.705600, 0.708600, 0.700800, 0.720400, 0.721000, 0.747400, 0.729200, 0.752600, 0.754400, 0.738000, 0.764800, 0.765000, 0.761800], [0.372000, 0.432600, 0.519400, 0.535000, 0.546000, 0.585600, 0.593200, 0.628200, 0.632600, 0.659600, 0.678800, 0.645200, 0.676000, 0.688400, 0.692800, 0.719200, 0.715600, 0.711600, 0.710200, 0.723000, 0.719800, 0.737400, 0.738400, 0.729800, 0.739200, 0.732200, 0.746400], [0.345800, 0.466000, 0.516200, 0.526000, 0.560600, 0.592600, 0.617400, 0.632600, 0.657400, 0.659000, 0.678800, 0.687800, 0.686600, 0.701400, 0.668600, 0.700600, 0.707000, 0.704600, 0.723600, 0.723400, 0.735600, 0.735600, 0.731600, 0.752400, 0.735200, 0.762600, 0.764000], [0.359000, 0.460400, 0.484000, 0.536800, 0.566000, 0.602200, 0.587800, 0.641600, 0.632000, 0.668000, 0.676000, 0.675200, 0.674200, 0.674800, 0.707000, 0.693600, 0.693600, 0.719600, 0.715800, 0.726400, 0.735000, 0.740600, 0.748400, 0.743200, 0.740200, 0.746600, 0.746400], [0.337400, 0.435600, 0.474200, 0.548800, 0.563000, 0.590400, 0.590000, 0.611400, 0.645600, 0.661600, 0.674200, 0.664000, 0.676200, 0.699000, 0.717200, 0.705600, 0.721000, 0.720800, 0.734400, 0.731600, 0.736400, 0.740600, 0.750600, 0.750800, 0.762000, 0.757200, 0.755200], [0.379600, 0.463200, 0.501400, 0.538200, 0.566800, 0.589600, 0.614800, 0.621000, 0.629600, 0.667000, 0.680400, 0.691200, 0.693600, 0.696400, 0.684200, 0.702400, 0.720400, 0.722200, 0.731400, 0.735800, 0.729800, 0.742400, 0.731600, 0.751600, 0.759200, 0.740400, 0.733200], [0.365200, 0.444400, 0.509200, 0.541000, 0.584000, 0.594600, 0.616400, 0.610000, 0.647800, 0.661600, 0.661600, 0.659600, 0.696000, 0.702800, 0.718600, 0.723200, 0.721800, 0.724400, 0.724800, 0.731000, 0.736200, 0.739000, 0.750800, 0.734200, 0.744800, 0.749400, 0.750400], [0.370000, 0.436800, 0.513200, 0.532600, 0.562600, 0.603800, 0.620800, 0.639000, 0.657200, 0.658200, 0.685400, 0.691600, 0.697200, 0.724800, 0.714000, 0.730400, 0.740600, 0.746200, 0.749400, 0.763200, 0.757800, 0.758800, 0.770600, 0.767200, 0.775600, 0.773400, 0.771600], [0.397200, 0.468600, 0.516400, 0.535200, 0.540200, 0.591600, 0.605200, 0.629800, 0.620000, 0.651400, 0.651800, 0.637000, 0.684200, 0.685400, 0.696400, 0.673000, 0.709800, 0.714800, 0.717600, 0.728200, 0.719200, 0.740400, 0.745800, 0.745400, 0.739200, 0.743400, 0.745800], [0.325400, 0.437800, 0.477400, 0.490800, 0.511800, 0.564000, 0.577200, 0.600400, 0.609200, 0.627400, 0.644400, 0.652400, 0.647600, 0.671600, 0.659000, 0.688800, 0.699200, 0.720600, 0.706600, 0.728600, 0.723400, 0.723200, 0.714200, 0.736000, 0.740400, 0.736200, 0.745000], [0.336400, 0.414200, 0.483600, 0.491200, 0.546400, 0.571000, 0.601600, 0.597200, 0.619200, 0.618400, 0.637600, 0.662600, 0.679000, 0.687600, 0.699800, 0.697400, 0.714400, 0.719600, 0.724200, 0.722600, 0.730200, 0.732000, 0.736600, 0.737400, 0.750600, 0.758200, 0.757000], [0.352600, 0.391200, 0.473200, 0.506000, 0.552800, 0.582800, 0.602200, 0.621200, 0.647400, 0.657000, 0.667800, 0.679800, 0.660000, 0.689000, 0.711000, 0.712600, 0.718600, 0.700000, 0.725800, 0.739800, 0.737400, 0.742200, 0.740800, 0.751200, 0.744000, 0.759400, 0.761400], [0.333000, 0.415400, 0.474200, 0.507000, 0.541000, 0.551600, 0.569600, 0.594200, 0.617800, 0.633800, 0.645800, 0.674600, 0.672800, 0.705000, 0.697000, 0.711800, 0.722600, 0.718600, 0.748800, 0.719000, 0.737400, 0.746200, 0.743000, 0.757800, 0.762600, 0.752000, 0.766200]], 'y': [0.732200, 0.582000, 0.728400, 0.740800, 0.650200, 0.716400, 0.639400, 0.719200, 0.688800, 0.747000, 0.686200, 0.695400, 0.631200, 0.723600, 0.709000, 0.694400, 0.700200, 0.684800, 0.679800, 0.713600, 0.723600, 0.766000, 0.762200, 0.646200, 0.702600, 0.692200, 0.639200, 0.759600, 0.758800, 0.749400, 0.690000, 0.694200, 0.742200, 0.717400, 0.747600, 0.736800, 0.735000, 0.761400, 0.760600, 0.705400, 0.762000, 0.676400, 0.706600, 0.696800, 0.738200, 0.734400, 0.761600, 0.754600, 0.758000, 0.758600, 0.765000, 0.763000, 0.739400, 0.776400, 0.744600, 0.759400, 0.756000, 0.764200, 0.765000, 0.746400, 0.764000, 0.748400, 0.762000, 0.759200, 0.750800, 0.775600, 0.745800, 0.745000, 0.758200, 0.761400, 0.766200]} --------------------------------------------------------------------------------