├── Mariana ├── __init__.py ├── tests │ ├── __init__.py │ ├── theano_device_check1.py │ ├── datasetmaps_tests.py │ └── decorators_tests.py ├── compatibility │ ├── __init__.py │ └── lasagne.py ├── HTML_Templates │ ├── __init__.py │ ├── vulcan │ │ ├── __init__.py │ │ ├── weblibs │ │ │ ├── mariana_logo.png │ │ │ ├── vulcan.css │ │ │ └── vulcan.js │ │ ├── vulcan.html │ │ └── vulcan.py │ └── template.py ├── doc │ ├── source │ │ ├── candies.rst │ │ ├── wrappers.rst │ │ ├── convolution.rst │ │ ├── layers.rst │ │ ├── costs.rst │ │ ├── activation.rst │ │ ├── regularisation.rst │ │ ├── decorators.rst │ │ ├── scenari.rst │ │ ├── network.rst │ │ ├── installation.rst │ │ ├── gpu.rst │ │ ├── intro.rst │ │ ├── training.rst │ │ ├── tips.rst │ │ ├── index.rst │ │ └── conf.py │ ├── Makefile │ └── make.bat ├── settings.py ├── reshaping.py ├── useful.py ├── candies.py ├── regularizations.py ├── activations.py ├── costs.py ├── recurrence.py ├── custom_types.py ├── convolution.py ├── training │ ├── datasetmaps.py │ └── trainers.py ├── initializations.py ├── abstraction.py ├── decorators.py ├── scenari.py └── sampling.py ├── MANIFEST.in ├── .travis.yml ├── .gitignore ├── DESCRIPTION.rst ├── setup.py ├── CHANGELOG.rst ├── README.rst └── LICENSE /Mariana/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Mariana/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Mariana/compatibility/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Mariana/HTML_Templates/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Mariana/HTML_Templates/vulcan/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.rst 2 | include LICENCE -------------------------------------------------------------------------------- /Mariana/doc/source/candies.rst: -------------------------------------------------------------------------------- 1 | Candies 2 | =============== 3 | 4 | Just some niceties 5 | 6 | .. automodule:: Mariana.candies 7 | :members: 8 | 9 | -------------------------------------------------------------------------------- /Mariana/HTML_Templates/vulcan/weblibs/mariana_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tariqdaouda/Mariana/HEAD/Mariana/HTML_Templates/vulcan/weblibs/mariana_logo.png -------------------------------------------------------------------------------- /Mariana/doc/source/wrappers.rst: -------------------------------------------------------------------------------- 1 | Wrappers 2 | ================== 3 | 4 | These things wrap Theano stuff. 5 | 6 | .. automodule:: Mariana.wrappers 7 | :members: -------------------------------------------------------------------------------- /Mariana/doc/source/convolution.rst: -------------------------------------------------------------------------------- 1 | Convolution 2 | =============== 3 | 4 | Here's what Mariana offers in terms of convlutions. 5 | 6 | .. automodule:: Mariana.convolution 7 | :members: -------------------------------------------------------------------------------- /Mariana/doc/source/layers.rst: -------------------------------------------------------------------------------- 1 | Layers 2 | =============== 3 | 4 | These are the layer types currently provided by Mariana, feel free to add your own. 5 | 6 | .. automodule:: Mariana.layers 7 | :members: -------------------------------------------------------------------------------- /Mariana/doc/source/costs.rst: -------------------------------------------------------------------------------- 1 | Costs 2 | ========== 3 | 4 | Costs are passed to output layers as arguments and are used to update the parameters of the network. 5 | 6 | .. automodule:: Mariana.costs 7 | :members: 8 | 9 | -------------------------------------------------------------------------------- /Mariana/doc/source/activation.rst: -------------------------------------------------------------------------------- 1 | Activations 2 | =============== 3 | 4 | Here's a list of all the activation functions Mariana provides. Feel free to add your own. 5 | 6 | .. automodule:: Mariana.activations 7 | :members: 8 | 9 | -------------------------------------------------------------------------------- /Mariana/HTML_Templates/template.py: -------------------------------------------------------------------------------- 1 | class HTMLTemplate_ABC(object): 2 | """The class that all templates must follow""" 3 | def __init__(self): 4 | super(HTMLTemplate_ABC, self).__init__() 5 | 6 | def render(self, filename) : 7 | """write the thing to disk""" 8 | raise NotImplementedError("Should be implemented in child: %s" % self.name) 9 | -------------------------------------------------------------------------------- /Mariana/doc/source/regularisation.rst: -------------------------------------------------------------------------------- 1 | Regularizations 2 | =============== 3 | 4 | Regularizations are added to costs and are there to prevent over-fitting. As costs, regularizations can be applied to both hidden and output layers, but unlike costs, hidden layers do not inherit regularizations for output layers. 5 | 6 | .. automodule:: Mariana.regularizations 7 | :members: 8 | 9 | -------------------------------------------------------------------------------- /Mariana/doc/source/decorators.rst: -------------------------------------------------------------------------------- 1 | Decorators 2 | ========== 3 | 4 | Mariana layers can take decorators as arguments. Decorators modify the layer's behaviour and can be used for example, to mask parts of the output to the next layers (for dropout or denoising auto-encoders), and to specify custom weight initialisations or sparsity features. 5 | 6 | .. automodule:: Mariana.decorators 7 | :members: -------------------------------------------------------------------------------- /Mariana/doc/source/scenari.rst: -------------------------------------------------------------------------------- 1 | Scenarii 2 | =============== 3 | 4 | Learning scenarii are used to manage the hyper-parameters and update the parameters of layers. 5 | Scenarii can be applied to both hidden and output layers, and if a hidden layer does not 6 | a scenario of it's own, it will inherit one from an output layer. 7 | 8 | .. automodule:: Mariana.scenari 9 | :members: 10 | 11 | -------------------------------------------------------------------------------- /Mariana/HTML_Templates/vulcan/weblibs/vulcan.css: -------------------------------------------------------------------------------- 1 | body { 2 | font: 10pt sans; 3 | } 4 | 5 | a{ 6 | color: #1983a5; 7 | } 8 | .vulcan-button-primary{ 9 | background-color: #1983a5; 10 | opacity: 0.8; 11 | } 12 | .vulcan-button-primary:hover{ 13 | background-color: #1983a5; 14 | opacity: 1; 15 | } 16 | 17 | .vulcan-button-secondary{ 18 | background-color: #f68920; 19 | opacity: 0.8; 20 | } 21 | 22 | .vulcan-button-secondary:hover{ 23 | background-color: #f68920; 24 | opacity: 1; 25 | } -------------------------------------------------------------------------------- /Mariana/settings.py: -------------------------------------------------------------------------------- 1 | #Personal tribute to nomad soul 2 | OMICRON_SIGNATURE = ">|\/| /-\ |-> | /-\ |\| /-\>" 3 | 4 | VERBOSE = False 5 | SAVE_MESSAGE_LOG = True 6 | SAVE_MESSAGE_LOG_FILE = "Mariana_logs.txt" 7 | 8 | #The seed used for generating random stuff 9 | import time 10 | RANDOM_SEED = int(time.time()) - 1523432180 11 | 12 | TYPE_INPUT_LAYER = "input" 13 | TYPE_OUTPUT_LAYER = "output" 14 | TYPE_HIDDEN_LAYER = "hidden" 15 | 16 | import theano 17 | DEVICE_IS_GPU = (theano.config.device.find("gpu") > -1) 18 | INTX="int32" 19 | FLOATX=theano.config.floatX 20 | -------------------------------------------------------------------------------- /Mariana/doc/source/network.rst: -------------------------------------------------------------------------------- 1 | Networks 2 | =============== 3 | 4 | Mariana Networks are graphs of connected layers and are in charge of calling the behind the scenes Theano functions. 5 | Networks can also be saved into files, reloaded, and exported to DOT format so they can be visualized in a DOT visualizer such as graphviz. 6 | Networks are not supposed to be manually instanciated, but are automatically created when layers are connected. 7 | You can forget about all the graph manipulation functions and simply use the shorthand **">"** to connect layer together. 8 | 9 | .. automodule:: Mariana.network 10 | :members: 11 | 12 | -------------------------------------------------------------------------------- /Mariana/doc/source/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============= 3 | 4 | First, make sure you have the latest version of Theano_. 5 | 6 | Mariana can be cloned from github_, it has two branches *master* for the stable version and *dev* for the latest developements. 7 | If you are planning to do some GPU learning or some theano fine tuning you should also check out the documentation of Theano_. 8 | 9 | .. _Theano: http://www.deeplearning.net/software/theano/ 10 | .. _github: https://github.com/tariqdaouda/Mariana/ 11 | 12 | **Clone it from git!**:: 13 | 14 | git clone https://github.com/tariqdaouda/Mariana.git 15 | cd Mariana 16 | python setup.py develop 17 | 18 | **Upgrade**:: 19 | 20 | git pull 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | 3 | notifications: 4 | email: false 5 | 6 | language: python 7 | 8 | python: 9 | - "2.7" 10 | 11 | addons: 12 | apt: 13 | packages: 14 | - liblapack-dev 15 | - gfortran 16 | 17 | before_install: 18 | - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh 19 | - bash miniconda.sh -b -p $HOME/miniconda 20 | - export PATH="$HOME/miniconda/bin:$PATH" 21 | - conda update --yes conda 22 | 23 | install: 24 | - conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy scipy 25 | - pip install -r https://raw.githubusercontent.com/Lasagne/Lasagne/v0.1/requirements.txt 26 | - pip install git+https://www.github.com/Lasagne/Lasagne 27 | - pip install coverage 28 | - python setup.py install 29 | 30 | script: coverage run -m unittest discover Mariana/tests/ 31 | 32 | after_success: bash <(curl -s https://codecov.io/bash) 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | 56 | # my stuff 57 | test/ 58 | trash/ 59 | Mariana/test/ 60 | Mariana/trash 61 | -------------------------------------------------------------------------------- /Mariana/tests/theano_device_check1.py: -------------------------------------------------------------------------------- 1 | #Checks the device used, coming from: http://deeplearning.net/software/theano/tutorial/using_gpu.html 2 | #more details on the link 3 | #Launch on GPU: THEANO_FLAGS=mode=FAST_RUN,device=cuda,floatX=float32 python theano_device_check1.py 4 | 5 | from theano import function, config, shared, tensor 6 | import numpy 7 | import time 8 | 9 | vlen = 10 * 30 * 768 # 10 x #cores x # threads per core 10 | iters = 1000 11 | 12 | rng = numpy.random.RandomState(22) 13 | x = shared(numpy.asarray(rng.rand(vlen), config.floatX)) 14 | f = function([], tensor.exp(x)) 15 | print(f.maker.fgraph.toposort()) 16 | t0 = time.time() 17 | for i in range(iters): 18 | r = f() 19 | t1 = time.time() 20 | print("Looping %d times took %f seconds" % (iters, t1 - t0)) 21 | print("Result is %s" % (r,)) 22 | if numpy.any([isinstance(x.op, tensor.Elemwise) and 23 | ('Gpu' not in type(x.op).__name__) 24 | for x in f.maker.fgraph.toposort()]): 25 | print('Used the cpu') 26 | else: 27 | print('Used the gpu') -------------------------------------------------------------------------------- /DESCRIPTION.rst: -------------------------------------------------------------------------------- 1 | Mariana: If you can draw it, you can make it. 2 | ============================================= 3 | 4 | Named after the deepest place on earth (Mariana trench), Mariana is a Python Machine Learning Framework built on top of Theano, that focuses on ease of use. The full documentation is available here_. 5 | 6 | .. _here: http://bioinfo.iric.ca/~daoudat/Mariana/ 7 | 8 | Creating Neural Networks with Mariana 9 | ===================================== 10 | 11 | .. code:: python 12 | 13 | import Mariana.activations as MA 14 | import Mariana.decorators as MD 15 | import Mariana.layers as ML 16 | import Mariana.costs as MC 17 | import Mariana.regularizations as MR 18 | import Mariana.scenari as MS 19 | 20 | **The instant MLP with dropout, L1 regularization and ReLUs** 21 | 22 | .. code:: python 23 | 24 | ls = MS.GradientDescent(lr = 0.01) 25 | cost = MC.NegativeLogLikelihood() 26 | 27 | i = ML.Input(28*28, name = "inputLayer") 28 | h = ML.Hidden(300, activation = MA.reLU, decorators = [MD.BinomialDropout(0.2)], regularizations = [ MR.L1(0.0001) ]) 29 | o = ML.SoftmaxClassifier(9, learningScenario = ls, costObject = cost, regularizations = [ MR.L1(0.0001) ]) 30 | 31 | MLP = i > h > o 32 | -------------------------------------------------------------------------------- /Mariana/reshaping.py: -------------------------------------------------------------------------------- 1 | import Mariana.initializations as MI 2 | 3 | import Mariana.compatibility.lasagne as MLASAGNE 4 | import lasagne.layers as LasagneLayers 5 | 6 | __all__ = ["Reshape", "Flatten", "Dimshuffle", "Padding", "Slice"] 7 | 8 | class Reshape(MLASAGNE.LasagneLayer): 9 | """ 10 | reshape the output of a layer to a new shape. 11 | newShape can be a tuple or a TensorVariable. 12 | Each elements of the the tuple can be: 13 | * a positive int to denote the size of the dimension 14 | * a list of a single element, ex: [i], to use the size of of the ith input element 15 | * -1 To infer the size of the dimension based on the sizes of all other dimension. There cannot be more than one -1. 16 | """ 17 | def __init__( 18 | self, 19 | newShape, 20 | name=None, 21 | **kwargs 22 | ): 23 | super(Reshape, self).__init__( 24 | LasagneLayers.ReshapeLayer, 25 | lasagneHyperParameters={ 26 | "shape": newShape, 27 | }, 28 | initializations=[], 29 | name=name, 30 | lasagneKwargs={}, 31 | **kwargs 32 | ) 33 | # self.shape = newShape -------------------------------------------------------------------------------- /Mariana/useful.py: -------------------------------------------------------------------------------- 1 | import theano 2 | import numpy 3 | import theano.tensor as tt 4 | 5 | import Mariana.settings as MSET 6 | 7 | def iCast_theano(thing) : 8 | """intelligently cast ints and floats into the corresct datatypes (for theano variables)""" 9 | if str(thing.dtype).find("int") > -1 : 10 | return tt.cast(thing, MSET.INTX) 11 | else : 12 | return tt.cast(thing, theano.config.floatX) 13 | 14 | def iCast_numpy(thing) : 15 | """intelligently cast ints and floats into the corresct datatypes (for numpy variables)""" 16 | 17 | if str(thing.dtype).find("int") > -1 : 18 | return numpy.asarray(thing, dtype=MSET.INTX) 19 | else : 20 | return numpy.asarray(thing, dtype=theano.config.floatX) 21 | 22 | def sparsify(numpy_array, coef) : 23 | """return a sparse version of *numpy_array*, *coef* is the sparcity coefficient should be within [0; 1], 24 | where 1 means a matrix of zeros and 0 returns numpy_array as is""" 25 | assert coef >= 0. and coef <= 1. 26 | if coef == 0 : 27 | return numpy_array 28 | elif coef == 1 : 29 | return numpy.zeros(numpy_array.shape) 30 | 31 | mask = numpy.random.uniform(numpy_array.shape) 32 | v = numpy_array * ( mask > coef) 33 | return v -------------------------------------------------------------------------------- /Mariana/doc/source/gpu.rst: -------------------------------------------------------------------------------- 1 | Running on GPU 2 | ================ 3 | 4 | Mariana is fully capable of running on both GPU and CPU and running Mariana on GPU is the same as running Theano on GPU. 5 | This is a quick tutorial on how to get you started, if you need more information you can have a look at the documention of Theano_ . 6 | 7 | .. _Theano: http://deeplearning.net/software/theano/tutorial/using_gpu.html 8 | 9 | To make sure your machine is GPU enabled, go to the examples folder and run:: 10 | 11 | THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python theano_device_check1.py 12 | 13 | You can also run it on CPU mode to compare the performances:: 14 | 15 | THEANO_FLAGS=mode=FAST_RUN,device=cpu,floatX=float32 python theano_device_check1.py 16 | 17 | To run Mariana on GPU mode you can use:: 18 | 19 | THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python .py 20 | 21 | It is also a good practice to make sure your data is in float32 (as GPUs cannot handle more). Because Mariana has your best interest at heart, 22 | she will tell you on wich device she is running the functions (unless you set *settings.VERBOSE* to *False*). She also will warn you if you specify that 23 | you want to use the GPU but have some float64s that slow down the performances. -------------------------------------------------------------------------------- /Mariana/tests/datasetmaps_tests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import Mariana.layers as ML 4 | import Mariana.layers as ML 5 | import Mariana.decorators as dec 6 | import Mariana.costs as MC 7 | import Mariana.regularizations as MR 8 | import Mariana.scenari as MS 9 | import Mariana.activations as MA 10 | import Mariana.training.datasetmaps as MD 11 | 12 | import theano.tensor as tt 13 | import numpy 14 | 15 | class DastasetMapsTests(unittest.TestCase): 16 | 17 | def setUp(self) : 18 | pass 19 | 20 | def tearDown(self) : 21 | pass 22 | 23 | def test_classSets(self) : 24 | def sample(cls) : 25 | o = cls.getAll("onehot") 26 | n = cls.getAll("classNumber") 27 | p = cls.getAll("input") 28 | 29 | return o, n, p 30 | 31 | l1 = numpy.arange(100) 32 | l2 = numpy.arange(10) + 10 33 | 34 | cls = MD.ClassSets( sets = [ ("l1", l1), ("l2", l2) ], sampleSize = len(l1) ) 35 | 36 | o, n, p = sample(cls) 37 | for i in xrange(len(o)) : 38 | if n[i] == 0. : 39 | self.assertEquals(o[i][1], 0.) 40 | self.assertEquals(o[i][0], 1.) 41 | else : 42 | self.assertEquals(o[i][0], 0.) 43 | self.assertEquals(o[i][1], 1.) 44 | 45 | nbTrials = 10000 46 | nb2 = 0. 47 | for i in xrange(nbTrials) : 48 | o, n, p = sample(cls) 49 | for j in xrange(len(p)) : 50 | if p[j] > 10 : 51 | nb2 += 1 52 | 53 | f = nb2/float(len(p)*nbTrials) 54 | r = abs(f-0.5) 55 | self.assertTrue(r < 2) 56 | 57 | if __name__ == '__main__' : 58 | import Mariana.settings as MSET 59 | MSET.VERBOSE = False 60 | unittest.main() 61 | -------------------------------------------------------------------------------- /Mariana/HTML_Templates/vulcan/vulcan.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {TITLE} 5 | {LIBS} 6 | 7 | 8 | 9 |
10 |
11 |
12 |

13 | 14 | Documentation 15 | 16 |

17 |

{TITLE}

18 |

19 |
20 | {MODEL_NOTES} 21 |
22 | 23 | 24 | 25 |
26 |
27 |
28 | 29 |
30 |
31 |
32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /Mariana/candies.py: -------------------------------------------------------------------------------- 1 | import sys, time 2 | import Mariana.settings as MSET 3 | 4 | MESSAGE_LOG_FILE = None 5 | 6 | def friendly(msgSubject, msg, warning=False, flush = True) : 7 | """Prints a friendly message""" 8 | 9 | global MESSAGE_LOG_FILE 10 | 11 | m = " " + msg.replace("\n", '\n ') 12 | 13 | if warning : 14 | subject = "WARNING: " + msgSubject 15 | else : 16 | subject = msgSubject 17 | 18 | s = """\n%s:\n%s\n%s\n\n Cheers :),\n\n Mariana\n""" %(subject, "-"*(len(subject) + 1), m) 19 | if MSET.VERBOSE : 20 | print s 21 | if flush : 22 | sys.stdout.flush() 23 | 24 | if MSET.SAVE_MESSAGE_LOG : 25 | if not MESSAGE_LOG_FILE : 26 | MESSAGE_LOG_FILE = open(MSET.SAVE_MESSAGE_LOG_FILE, "w") 27 | MESSAGE_LOG_FILE.write("\ntimestamp:%s, human time:%s\n%s" % (time.time(), time.ctime(), s)) 28 | if flush : 29 | MESSAGE_LOG_FILE.flush() 30 | 31 | def fatal(msgSubject, msg, toRaise = ValueError, flush = True) : 32 | """Death is upon us""" 33 | 34 | global MESSAGE_LOG_FILE 35 | 36 | m = " " + msg.replace("\n", '\n ') 37 | 38 | subject = msgSubject 39 | 40 | s = """\n%s:\n%s\n%s\n\n %s\nSorry,\n\n Mariana\n""" %(subject, "-"*(len(subject) + 1), m, toRaise.message) 41 | 42 | if MSET.SAVE_MESSAGE_LOG : 43 | if not MESSAGE_LOG_FILE : 44 | MESSAGE_LOG_FILE = open(MSET.SAVE_MESSAGE_LOG_FILE, "w") 45 | MESSAGE_LOG_FILE.write("\ntimestamp:%s, human time:%s\n%s" % (time.time(), time.ctime(), s)) 46 | if flush : 47 | MESSAGE_LOG_FILE.flush() 48 | 49 | raise toRaise 50 | 51 | def warning(msg) : 52 | print "\n=========\nWARNING: %s\n=========\n" % msg -------------------------------------------------------------------------------- /Mariana/regularizations.py: -------------------------------------------------------------------------------- 1 | import Mariana.abstraction as MABS 2 | 3 | __all__ = ["SingleLayerRegularizer_ABC", "L1", "L2", "ActivationL1"] 4 | 5 | class SingleLayerRegularizer_ABC(MABS.UntrainableAbstraction_ABC, MABS.Apply_ABC) : 6 | """An abstract regularization to be applied to a layer.""" 7 | 8 | def __init__(self, streams=["train"], **kwargs): 9 | self.streams = streams 10 | super(SingleLayerRegularizer_ABC, self).__init__(**kwargs) 11 | 12 | def apply(self, layer, variable, stream) : 13 | """Apply to a layer and update networks's log""" 14 | 15 | if stream in self.streams : 16 | variable += self.run(layer, stream) 17 | 18 | return variable 19 | 20 | def run(self, layer, stream) : 21 | """Returns the expression to be added to the cost""" 22 | raise NotImplemented("Must be implemented in child") 23 | 24 | class L1(SingleLayerRegularizer_ABC) : 25 | """ 26 | Will add this to the cost. Weights will tend towards 0 27 | resulting in sparser weight matrices. 28 | .. math:: 29 | 30 | factor * abs(Weights) 31 | """ 32 | def __init__(self, factor) : 33 | super(L1, self).__init__() 34 | self.setHP("factor", factor) 35 | 36 | def run(self, layer, stream) : 37 | return self.getHP("factor") * ( abs(layer.getP("W")()).sum() ) 38 | 39 | class L2(SingleLayerRegularizer_ABC) : 40 | """ 41 | Will add this to the cost. Causes the weights to stay small 42 | .. math:: 43 | 44 | factor * (Weights)^2 45 | """ 46 | def __init__(self, factor) : 47 | super(L2, self).__init__() 48 | self.setHP("factor", factor) 49 | 50 | def run(self, layer, stream) : 51 | return self.getHP("factor") * ( (layer.getP("W")() ** 2).sum() ) 52 | 53 | class ActivationL1(SingleLayerRegularizer_ABC) : 54 | """ 55 | L1 on the activations. Neurone activations will tend towards 56 | 0, resulting into sparser representations. 57 | 58 | Will add this to the cost 59 | .. math:: 60 | 61 | factor * abs(activations) 62 | """ 63 | def __init__(self, factor, stream) : 64 | super(ActivationL1, self).__init__() 65 | SingleLayerRegularizer_ABC.__init__(self) 66 | self.setHP("factor", factor) 67 | 68 | def run(self, layer) : 69 | return self.getHP("factor") * ( abs(layer.outputs[streams]).sum() ) -------------------------------------------------------------------------------- /Mariana/activations.py: -------------------------------------------------------------------------------- 1 | import theano.tensor as tt 2 | import Mariana.abstraction as MABS 3 | 4 | __all__ = ["Activation_ABC", "Pass", "Sigmoid", "Tanh", "ReLU", "Softmax"] 5 | 6 | class Activation_ABC(MABS.TrainableAbstraction_ABC, MABS.Apply_ABC): 7 | """All activations must inherit from this class""" 8 | 9 | def apply(self, layer, x) : 10 | """Apply to a layer and update networks's log""" 11 | for s in x.streams : 12 | x[s] = self.run(x[s]) 13 | 14 | def run(self, x) : 15 | """the actual activation run that will be applied to the neurones.""" 16 | raise NotImplemented("Must be implemented in child") 17 | 18 | class Pass(Activation_ABC): 19 | """ 20 | simply returns x 21 | """ 22 | 23 | def run(self, x): 24 | return x 25 | Linear = Pass 26 | 27 | class Sigmoid(Activation_ABC): 28 | """ 29 | .. math:: 30 | 31 | 1/ (1/ + exp(-x))""" 32 | def run(self, x): 33 | return tt.nnet.sigmoid(x) 34 | 35 | class Swish(Activation_ABC): 36 | """ 37 | .. math:: 38 | 39 | x * (1/ (1/ + exp(-x)) )""" 40 | def run(self, x): 41 | return x*tt.nnet.sigmoid(x) 42 | 43 | class Softplus(Activation_ABC): 44 | """ 45 | .. math:: 46 | 47 | ln(1 + exp(x))""" 48 | def run(self, x): 49 | return tt.nnet.softplus(x) 50 | 51 | class Sin(Activation_ABC): 52 | """ 53 | .. math:: 54 | 55 | x * (1/ (1/ + exp(-x)) )""" 56 | def run(self, x): 57 | return tt.Sin(x) 58 | 59 | class Tanh(Activation_ABC): 60 | """ 61 | .. math:: 62 | 63 | tanh(x)""" 64 | def run(self, x): 65 | return tt.tanh(x) 66 | 67 | class ReLU(Activation_ABC): 68 | """ 69 | .. math:: 70 | 71 | if pre_act < 0 return leakiness; else return pre_act""" 72 | def __init__(self, leakiness=0): 73 | super(ReLU, self).__init__() 74 | self.setHP("leakiness", leakiness) 75 | 76 | def run(self, x): 77 | return tt.nnet.relu(x, alpha=self.getHP("leakiness")) 78 | 79 | class Softmax(Activation_ABC): 80 | """Softmax to get a probabilistic output 81 | 82 | .. math:: 83 | 84 | scale * exp(x_i/T)/ sum_k( exp(x_k/T) ) 85 | """ 86 | def __init__(self, scale = 1, temperature = 1): 87 | super(Softmax, self).__init__() 88 | self.setHP("temperature", temperature) 89 | self.setHP("scale", scale) 90 | 91 | def run(self, x): 92 | return self.getHP("scale") * tt.nnet.softmax(x/self.getHP("temperature")) 93 | -------------------------------------------------------------------------------- /Mariana/tests/decorators_tests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import Mariana.layers as ML 4 | import Mariana.initializations as MI 5 | import Mariana.decorators as MD 6 | import Mariana.costs as MC 7 | import Mariana.regularizations as MR 8 | import Mariana.scenari as MS 9 | import Mariana.activations as MA 10 | 11 | import theano.tensor as tt 12 | import numpy 13 | 14 | class DecoratorTests(unittest.TestCase): 15 | 16 | def setUp(self) : 17 | pass 18 | 19 | def tearDown(self) : 20 | pass 21 | 22 | # @unittest.skip("skipping") 23 | def test_batch_norm(self) : 24 | import theano, numpy 25 | 26 | def batchnorm(W, b, data) : 27 | return numpy.asarray( W * ( (data-numpy.mean(data)) / numpy.std(data) ) + b, dtype= theano.config.floatX) 28 | 29 | data = numpy.random.randn(1, 100).astype(theano.config.floatX) 30 | batch = MD.BatchNormalization(testMu=0, testSigma=1) 31 | inp = ML.Input(100, name='inp', decorators=[batch]) 32 | 33 | model = inp.network 34 | model.init() 35 | 36 | m1 = numpy.mean( model["inp"].propagate["train"]({"inp.inputs": data})["inp.propagate.train"] ) 37 | m2 = numpy.mean( batchnorm(batch.getP("gamma").getValue(), batch.getP("beta").getValue(), data) ) 38 | 39 | epsilon = 1e-6 40 | self.assertTrue ( (m1 - m2) < epsilon ) 41 | 42 | # @unittest.skip("skipping") 43 | def test_mask(self) : 44 | import theano, numpy 45 | 46 | inp = ML.Input(100, 'inp', decorators=[MD.Mask(mask = numpy.zeros(100))]) 47 | model = inp.network 48 | model.init() 49 | 50 | data = numpy.random.randn(1, 100).astype(theano.config.floatX) 51 | out = model["inp"].propagate["train"]({"inp.inputs": data})["inp.propagate.train"] 52 | 53 | self.assertEqual(sum(out[0]), 0) 54 | 55 | # @unittest.skip("skipping") 56 | def test_dropout(self) : 57 | import theano, numpy 58 | 59 | inp = ML.Input(100, 'inp', decorators=[MD.BinomialDropout(dropoutRatio = 0.8)]) 60 | model = inp.network 61 | model.init() 62 | 63 | data = numpy.random.randn(1, 100).astype(theano.config.floatX) +1 64 | out = model["inp"].propagate["train"]({"inp.inputs": data})["inp.propagate.train"] 65 | 66 | self.assertTrue(sum(out[0] != 0) < sum(data[0] != 0)) 67 | 68 | if __name__ == '__main__' : 69 | import Mariana.settings as MSET 70 | MSET.VERBOSE = False 71 | unittest.main() 72 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | from codecs import open 3 | from os import path 4 | 5 | here = path.abspath(path.dirname(__file__)) 6 | 7 | # Get the long description from the relevant file 8 | with open(path.join(here, 'DESCRIPTION.rst'), encoding='utf-8') as f: 9 | long_description = f.read() 10 | 11 | setup( 12 | name='Mariana', 13 | 14 | version='2.0.0rc1', 15 | 16 | description="The Cutest Deep Learning Framework", 17 | long_description=long_description, 18 | 19 | url='https://github.com/tariqdaouda/mariana', 20 | 21 | author='Tariq Daouda', 22 | author_email='tariq.daouda@umontreal.ca', 23 | 24 | license='ApacheV2.0', 25 | 26 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 27 | classifiers=[ 28 | # How mature is this project? Common values are 29 | # 3 - Alpha 30 | # 4 - Beta 31 | # 5 - Production/Stable 32 | 'Development Status :: 5 - Production/Stable', 33 | 34 | 'Intended Audience :: Science/Research', 35 | 'Intended Audience :: Developers', 36 | 'Topic :: Scientific/Engineering :: Bio-Informatics', 37 | 'Topic :: Scientific/Engineering :: Machine-learning', 38 | 'Topic :: Scientific/Engineering :: Deep-learning', 39 | 'Topic :: Software Development :: Libraries', 40 | 41 | 'License :: OSI Approved :: Apache Software License', 42 | 43 | 'Programming Language :: Python :: 2.7', 44 | ], 45 | 46 | keywords='Machine Learning deeplearning neural networks', 47 | 48 | packages=find_packages(exclude=['trash']), 49 | 50 | # List run-time dependencies here. These will be installed by pip when your 51 | # project is installed. For an analysis of "install_requires" vs pip's 52 | # requirements files see: 53 | # https://packaging.python.org/en/latest/technical.html#install-requires-vs-requirements-files 54 | install_requires=["theano", "pyGeno", "simplejson", "numpy", "Lasagne"], 55 | 56 | # If there are data files included in your packages that need to be 57 | # installed, specify them here. If using Python 2.6 or less, then these 58 | # have to be included in MANIFEST.in as well. 59 | #~ package_data={ 60 | #~ 'sample': ['package_data.dat'], 61 | #~ }, 62 | 63 | # Although 'package_data' is the preferred approach, in some case you may 64 | # need to place data files outside of your packages. 65 | # see http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files 66 | # In this case, 'data_file' will be installed into '/my_data' 67 | #~ data_files=[('my_data', ['data/data_file'])], 68 | 69 | # To provide executable scripts, use entry points in preference to the 70 | # "scripts" keyword. Entry points provide cross-platform support and allow 71 | # pip to create the appropriate form of executable for the target platform. 72 | entry_points={ 73 | 'console_scripts': [ 74 | 'sample=sample:main', 75 | ], 76 | }, 77 | ) 78 | -------------------------------------------------------------------------------- /Mariana/costs.py: -------------------------------------------------------------------------------- 1 | import theano 2 | import theano.tensor as tt 3 | import Mariana.abstraction as MABS 4 | 5 | __all__ = ["Cost_ABC", "Null", "NegativeLogLikelihood", "MeanSquaredError", "CrossEntropy", "CategoricalCrossEntropy", "BinaryCrossEntropy"] 6 | 7 | class Cost_ABC(MABS.UntrainableAbstraction_ABC, MABS.Apply_ABC) : 8 | """This is the interface a Cost must expose. In order for the trainer/recorder to know which attributes are hyper-parameters, 9 | this class must also include a list attribute **self.hyperParameters** containing the names of all attributes that must be considered 10 | as hyper-parameters.""" 11 | 12 | def __init__(self, reverse=False, streams=["test", "train"], **kwargs) : 13 | """use reverse = True, to have the opposite of cost""" 14 | super(Cost_ABC, self).__init__(streams=streams, **kwargs) 15 | self.setHP("reverse", reverse) 16 | 17 | def logApply(self, layer, **kwargs) : 18 | message = "Applying '%s' on layer '%s'" % (self.name, self.getHP('parameter'), layer.name) 19 | if self.getHP("reverse") : 20 | message += " (reverse)" 21 | self.logEvent(message) 22 | 23 | def apply(self, layer, targets, outputs, stream) : 24 | """Apply to a layer and update networks's log""" 25 | 26 | if self.getHP("reverse") : 27 | return -self.run(targets, outputs, stream) 28 | else : 29 | return self.run(targets, outputs, stream) 30 | 31 | def run(self, targets, outputs, stream) : 32 | """The cost function. Must be implemented in child""" 33 | raise NotImplemented("Must be implemented in child") 34 | 35 | class Null(Cost_ABC) : 36 | """No cost at all""" 37 | def run(self, targets, outputs, stream) : 38 | return tt.sum(outputs*0 + targets*0) 39 | 40 | class NegativeLogLikelihood(Cost_ABC) : 41 | """For a probalistic output, works great with a softmax output layer""" 42 | def run(self, targets, outputs, stream) : 43 | cost = -tt.mean(tt.log(outputs)[tt.arange(targets.shape[0]), targets]) 44 | return cost 45 | 46 | class MeanSquaredError(Cost_ABC) : 47 | """The all time classic""" 48 | def run(self, targets, outputs, stream) : 49 | cost = tt.mean((outputs - targets) ** 2) 50 | return cost 51 | 52 | class AbsoluteAverage(Cost_ABC) : 53 | """Average absolute value""" 54 | def run(self, targets, outputs, stream) : 55 | cost = tt.mean((outputs - targets)) 56 | return cost 57 | 58 | class CategoricalCrossEntropy(Cost_ABC) : 59 | """Returns the average number of bits needed to identify an event.""" 60 | def run(self, targets, outputs, stream) : 61 | cost = tt.mean( tt.nnet.categorical_crossentropy(outputs, targets) ) 62 | return cost 63 | 64 | CrossEntropy = CategoricalCrossEntropy 65 | # class CrossEntropy(CategoricalCrossEntropy) : 66 | # """Short hand for CategoricalCrossEntropy""" 67 | # pass 68 | 69 | class BinaryCrossEntropy(Cost_ABC) : 70 | """Use this one for binary data""" 71 | def run(self, targets, outputs, stream) : 72 | cost = tt.mean( tt.nnet.binary_crossentropy(outputs, targets) ) 73 | return cost -------------------------------------------------------------------------------- /Mariana/doc/source/intro.rst: -------------------------------------------------------------------------------- 1 | Examples 2 | ============= 3 | 4 | Full 5 | ----- 6 | 7 | Please have a look at the examples_ folder on github_. You will find the following files: 8 | * mnist_mlp_: Implementation and training of an MLP (neural network with one hidden layer) that makes use of the full setup with dataset mappers, trainer, stop criteria and decorators. 9 | * vanilla_mnist_perceptron_mlp_: Implementations of both a Perceptron (no hidden layers) and ans MLP (one hidden layer) but trained without dataset mappers and trainers. 10 | * ae_: a very basic auto encoder that learns to encode 8 bits into 3. 11 | 12 | 13 | .. _examples: https://github.com/tariqdaouda/Mariana/tree/master/examples 14 | .. _github: https://github.com/tariqdaouda/Mariana 15 | .. _mnist_mlp: https://github.com/tariqdaouda/Mariana/tree/master/examples/mnist_mlp.py 16 | .. _vanilla_mnist_perceptron_mlp: https://github.com/tariqdaouda/Mariana/tree/master/examples/vanilla_mnist_perceptron_mlp.py 17 | .. _ae: https://github.com/tariqdaouda/Mariana/tree/master/examples/autoencoder.py 18 | 19 | Snippets 20 | ------------- 21 | 22 | These are the basics, importations first: 23 | 24 | .. code:: python 25 | 26 | import Mariana.activations as MA 27 | import Mariana.decorators as MD 28 | import Mariana.layers as ML 29 | import Mariana.costs as MC 30 | import Mariana.regularizations as MR 31 | import Mariana.scenari as MS 32 | 33 | **This is an MLP in Mariana, with dropout, L1 regularization and ReLUs** 34 | 35 | .. code:: python 36 | 37 | ls = MS.GradientDescent(lr = 0.01) 38 | cost = MC.NegativeLogLikelihood() 39 | 40 | i = ML.Input(28*28, name = "inputLayer") 41 | h = ML.Hidden(300, activation = MA.ReLU(), decorators = [MD.BinomialDropout(0.2)], regularizations = [ MR.L1(0.0001) ]) 42 | o = ML.SoftmaxClassifier(9, learningScenario = ls, costObject = cost, regularizations = [ MR.L1(0.0001) ]) 43 | 44 | MLP = i > h > o 45 | 46 | **This is an autoencoder with tied weights** 47 | 48 | .. code:: python 49 | 50 | ls = MS.GradientDescent(lr = 0.001) 51 | cost = MC.MeanSquaredError() 52 | 53 | i = ML.Input(10, name = "inputLayer") 54 | h = ML.Hidden(2, activation = MA.Tanh(), decorators = [ MD.GlorotTanhInit() ]) 55 | o = ML.Regression(10, activation = MA.Tanh(), costObject = cost, learningScenario = ls) 56 | 57 | ae = i > h > o 58 | 59 | #tied weights, we need to force the initialisation of the weight first 60 | ae.init() 61 | o.W = h.W.T 62 | 63 | Training, Testing and Propagating without a trainer: 64 | 65 | .. code:: python 66 | 67 | #train the model for output 'o' function will update parameters and return the current cost 68 | print MLP.train(o, inputLayer = train_set[0][i : i +miniBatchSize], targets = train_set[1][i : i +miniBatchSize] ) 69 | 70 | #the same as train but does not updated the parameters 71 | print MLP.test(o, inputLayer = test_set[0][i : i +miniBatchSize], targets = test_set[1][i : i +miniBatchSize] ) 72 | 73 | #the propagate will return the output for the output layer 'o' 74 | print MLP.propagate(o, inputLayer = test_set[0][i : i +miniBatchSize]) 75 | 76 | That's it for the snippets. There's also a trainer that make things even easier: `Training, Testing and Driving Nets in Mariana`_ 77 | 78 | .. _`Training, Testing and Driving Nets in Mariana`: training.html#trainers 79 | -------------------------------------------------------------------------------- /Mariana/doc/source/training.rst: -------------------------------------------------------------------------------- 1 | 2 | Training, Testing and Driving Nets in Mariana 3 | ============================================= 4 | All the concepts introduced here are demonstrasted in the `Examples`_. section. 5 | 6 | .. _Examples: 7 | 8 | Vanilla Driving 9 | --------------- 10 | 11 | Models in Mariana can be driven output layer by output layer using the model functions *train*, *test* and *propagate* 12 | 13 | .. code:: python 14 | 15 | import Mariana.activations as MA 16 | import Mariana.decorators as MD 17 | import Mariana.layers as ML 18 | import Mariana.costs as MC 19 | import Mariana.regularizations as MR 20 | import Mariana.scenari as MS 21 | 22 | ls = MS.GradientDescent(lr = 0.01) 23 | cost = MC.NegativeLogLikelihood() 24 | 25 | i = ML.Input(28*28, name = "inputLayer") 26 | h = ML.Hidden(300, activation = MA.ReLU(), decorators = [MD.BinomialDropout(0.2)], regularizations = [ MR.L1(0.0001) ]) 27 | o = ML.SoftmaxClassifier(9, learningScenario = ls, costObject = cost, regularizations = [ MR.L1(0.0001) ]) 28 | 29 | MLP = i > h > o 30 | 31 | ...load you dataset etc... 32 | 33 | #train the model for output 'o' function will update parameters and return the current cost 34 | print MLP.train(o, inputLayer = train_set[0][i : i +miniBatchSize], targets = train_set[1][i : i +miniBatchSize] ) 35 | 36 | #the same as train but does not updated the parameters 37 | print MLP.test(o, inputLayer = test_set[0][i : i +miniBatchSize], targets = test_set[1][i : i +miniBatchSize] ) 38 | 39 | #the propagate will return the output for the output layer 'o' 40 | print MLP.propagate(o, inputLayer = test_set[0][i : i +miniBatchSize]) 41 | 42 | Trainers 43 | -------- 44 | 45 | A trainer takes care of the whole training process. If the process dies unexpectedly during training it will 46 | automatically save the last version of the model as well as logs explaining what happened. The trainer can also take as argument a list of stopCriterias, and be paired with a recorder whose job is to record the training evolution. Trainers must has a .store dictionary exposed that represents the current state of the process. The store is used by stop criteria, recorders and learning scenarii. 47 | 48 | .. automodule:: Mariana.training.trainers 49 | :members: 50 | 51 | Dataset maps 52 | ------------ 53 | 54 | Mariana is dataset format agnostic and uses **DatasetMaps** to associate input and output layers 55 | with the data they must receive (cf. mnist_mlp_ for a full example). Here's a short example of how it works:: 56 | 57 | i = ML.Input(...) 58 | o = ML.Output(...) 59 | 60 | trainSet = RandomSeries(images = train_set[0], classes = train_set[1]) 61 | 62 | DatasetMapper = dm 63 | dm.map(i, trainSet.images) 64 | dm.map(o, trainSet.classes) 65 | 66 | .. _mnist_mlp: https://github.com/tariqdaouda/Mariana/tree/master/Mariana/examples/mnist_mlp.py 67 | 68 | .. automodule:: Mariana.training.datasetmaps 69 | :members: 70 | 71 | Stop criteria 72 | -------------- 73 | 74 | Stop criteria observe the trainer's store and tell it when it should stop by raising an EndOfTraining exception. 75 | 76 | .. automodule:: Mariana.training.stopcriteria 77 | :members: 78 | 79 | Recorders 80 | --------- 81 | 82 | Recorders are objects meant to be plugged into trainers to record the advancement of the training. They take trainer as argument, look into it's **.store** attribute and do something smart and useful with it. As with everything else in Mariana feel free to write your own recorders. 83 | 84 | .. automodule:: Mariana.training.recorders 85 | :members: 86 | -------------------------------------------------------------------------------- /Mariana/doc/source/tips.rst: -------------------------------------------------------------------------------- 1 | Tips and FAQs 2 | =============== 3 | 4 | 5 | What can you do with Mariana 6 | ---------------------------- 7 | 8 | Any type of deep, shallow, feed-forward, back-prop trained Neural Network should work. Convolutional Nets and Recurrent Nets are not supported yet but they will be. 9 | 10 | 11 | A word about the **'>'** 12 | ------------------------- 13 | 14 | When communicating about neural networks people often draw sets of connected layers. That's the idea behind Mariana: layers are first defined, then connected using the **'>'** operator. 15 | 16 | Can it run on GPU? 17 | ------------------ 18 | 19 | At the heart of Mariana are Theano functions, so the answer is yes. The guys behind Theano really did an awesome 20 | job of optimization, so it should be pretty fast, wether you're running on CPU or GPU. 21 | 22 | 23 | Less Verbosity 24 | --------------- 25 | 26 | .. code:: python 27 | 28 | import Mariana.settings as MSET 29 | 30 | MSET.VERBOSE = False 31 | 32 | .. Modifiying hyper-parameters during training 33 | .. -------------------------------------------- 34 | 35 | .. Learning scenarii have an **update(self, trainer)** function that is called by the trainer at each epoch. Trainers have a **.store** dictionary attribute that stores values relative to the current epoch (for example the current epoch number is contained in **trainer.store["runInfos"]["epoch"]**). The role of this function is to modify the attribute of the learning scenario according to the values in the store. 36 | .. You may need to create your own learning scenario, for that, simply write a class that inherits from an existing learning scenario or from the provided base class. 37 | 38 | Launching runs on the GPU 39 | -------------------------- 40 | For more information please have a look at the documentation of Theano. The easiest way, once Theano has been correctly set up is to run : 41 | 42 | .. code:: shell 43 | 44 | THEANO_FLAGS="device='gpu'" python my_model.py 45 | 46 | 47 | Getting the outputs of intermediate layers 48 | ------------------------------------------- 49 | 50 | By initialising a layer with the argument:: 51 | 52 | saveOutputs=True 53 | 54 | You tell Mariana to keep the last outputs of that layer stored, you can then access them using the layer's "getLastOutputs()"" function. 55 | 56 | Changing the seed random parameters generation 57 | ---------------------------------------------- 58 | 59 | .. code:: python 60 | 61 | import Mariana.settings as MSET 62 | 63 | MSET.RANDOM_SEED = 5826 64 | 65 | Saving and resuming training 66 | ----------------------------- 67 | 68 | Models can be saved using the **save()** function: 69 | 70 | .. code:: python 71 | 72 | mlp.save("myMLP") 73 | 74 | Loading is a simple unpickling: 75 | 76 | .. code:: python 77 | 78 | import cPickle 79 | 80 | mlp = cPickle.load(open("myMLP.mariana.pkl")) 81 | mlp.train(...) 82 | 83 | Cloning layers and re-using layers 84 | ----------------------------------- 85 | 86 | Mariana allows you to clone layers so you can train a model, extract one of it's layers, and use it for another model. 87 | 88 | .. code:: python 89 | 90 | h2 = h.clone() 91 | 92 | You can also transform an output layer into a hidden layer, that you can include afterwards in an other model. 93 | 94 | .. code:: python 95 | 96 | h3 = o.toHidden() 97 | 98 | And a hidden layer to an output layer using: 99 | 100 | .. code:: python 101 | 102 | o = h.toOutput(ML.Regression, costObject = cost, learningScenario = ls) 103 | 104 | Visualizing networks 105 | --------------------- 106 | 107 | Networks can be exported to graphs in the DOT format: 108 | 109 | .. code:: python 110 | 111 | #to simply print it 112 | print mlp.toDOT() 113 | 114 | #to save it 115 | mlp.saveDOT("myMLP.dot") 116 | 117 | You can then visualize the graph with any DOT visualizer such a graphviz. -------------------------------------------------------------------------------- /Mariana/doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. Mariana documentation master file, created by 2 | sphinx-quickstart on Thu Jun 11 13:02:52 2015. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Mariana: Deep Neural Networks should be Easy to Write 7 | ===================================================== 8 | .. image:: https://img.shields.io/badge/python-2.7-blue.svg 9 | 10 | Mariana's goal is to create a powerful language through which complex deep neural networks can be meaningfully expressed and easily manipulated. It's here to empower researchers, teachers and students, while greatly facilitating AI knowledge transfer into other domains 11 | 12 | Mariana is a Python Machine Learning Framework built on top of Theano_. She Mariana lives on github_. 13 | 14 | 15 | .. _Theano: http://www.deeplearning.net/software/theano/ 16 | .. _github: https://github.com/tariqdaouda/Mariana/ 17 | 18 | More Documentation: 19 | ------------------- 20 | 21 | * More examples and presentation material available here_. 22 | * YouTube presentation (english_). 23 | * YouTube presentation (french_). 24 | 25 | .. _here: https://github.com/tariqdaouda/Mariana_talks 26 | .. _english: https://youtu.be/dGS_Qny1E9E 27 | .. _french: https://youtu.be/TzRYF1lPP84?t=8m15s 28 | 29 | ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn 30 | 31 | Why is it cool? 32 | ---------------- 33 | 34 | **If you can draw it, you can write it.** 35 | 36 | Mariana provides an interface so simple and intuitive that writing models becomes a breeze. 37 | Networks are graphs of connected layers and that allows for the craziest deepest architectures 38 | you can think of, as well as for a super light and clean interface. The paradigm is simple 39 | create layers and connect them using **'>'**. Plugging per layer regularizations, costs and things 40 | such as dropout and custom initilisations is also super easy. 41 | 42 | Here's a snippet for an instant MLP with dropout, ReLU units and L1 regularization: 43 | 44 | .. code:: python 45 | 46 | import Mariana.activations as MA 47 | import Mariana.decorators as MD 48 | import Mariana.layers as ML 49 | import Mariana.costs as MC 50 | import Mariana.regularizations as MR 51 | import Mariana.scenari as MS 52 | 53 | ls = MS.GradientDescent(lr = 0.01) 54 | cost = MC.NegativeLogLikelihood() 55 | 56 | i = ML.Input(28*28, name = "inputLayer") 57 | h = ML.Hidden(300, activation = MA.ReLU(), decorators = [MD.BinomialDropout(0.2)], regularizations = [ MR.L1(0.0001) ]) 58 | o = ML.SoftmaxClassifier(9, learningScenario = ls, costObject = cost, regularizations = [ MR.L1(0.0001) ]) 59 | 60 | MLP = i > h > o 61 | 62 | Here are some full fledged examples_. 63 | 64 | Mariana also supports trainers_ that encapsulate the whole training to make things even easier. 65 | 66 | So in short: 67 | 68 | * no YAML 69 | * completely modular and extendable 70 | * use the trainer to encapsulate your training in a safe environement 71 | * write your models super fast 72 | * save your models and resume training 73 | * export your models into DOT format to obtain clean and easy to communicate graphs 74 | * free your imagination and experiment 75 | * no requirements concerning the format of the datasets 76 | 77 | 78 | .. _trainers: training.html#trainers 79 | .. _examples: intro.html 80 | 81 | 82 | Extendable 83 | ---------- 84 | 85 | Mariana is an extendable framework that provides abstract classes allowing you to define new types of layers, learning scenarios, costs, stop criteria, recorders and trainers. Feel free to taylor it to your needs. 86 | 87 | Contents: 88 | 89 | .. toctree:: 90 | :maxdepth: 2 91 | 92 | installation 93 | intro 94 | training 95 | gpu 96 | layers 97 | initializations 98 | convolution 99 | network 100 | costs 101 | scenari 102 | activation 103 | decorators 104 | regularisation 105 | wrappers 106 | candies 107 | tips 108 | 109 | Indices and tables 110 | ================== 111 | 112 | * :ref:`genindex` 113 | * :ref:`modindex` 114 | * :ref:`search` 115 | 116 | -------------------------------------------------------------------------------- /Mariana/recurrence.py: -------------------------------------------------------------------------------- 1 | import Mariana.initializations as MI 2 | 3 | import Mariana.compatibility.lasagne as MLASAGNE 4 | import lasagne.layers as LasagneLayers 5 | 6 | __all__ = ["RecurrentDense", "LSTM", "GatedRecurrentUnit", "GRU"] 7 | 8 | class RecurrentDense(MLASAGNE.LasagneLayer): 9 | """The classical recurrent layer with dense input to hidden and hidden to hidden connections. 10 | For a full explanation of the arguments please checkout lasagne's doc on lasagne.layers.RecurrentLayer. 11 | """ 12 | def __init__( 13 | self, 14 | size, 15 | name, 16 | initializations=[MI.Uniform('W_in_to_hid'), MI.Uniform('W_hid_to_hid'), MI.SingleValue('b', 0)], 17 | backwards=False, 18 | learnInit=False, 19 | gradientSteps=-1, 20 | gradClipping=0, 21 | unrollScan=False, 22 | # precomputeInput=False, 23 | onlyReturnFinal=False, 24 | **kwargs 25 | ): 26 | super(RecurrentDense, self).__init__( 27 | LasagneLayers.RecurrentLayer, 28 | lasagneHyperParameters={ 29 | "num_units": size, 30 | "backwards": backwards, 31 | "learn_init": learnInit, 32 | "gradient_steps": gradientSteps, 33 | "grad_clipping": gradClipping, 34 | "unroll_scan": unrollScan, 35 | "precompute_input": False, 36 | "mask_input": None, 37 | "only_return_final": onlyReturnFinal, 38 | }, 39 | initializations=initializations, 40 | name=name, 41 | lasagneKwargs={}, 42 | **kwargs 43 | ) 44 | 45 | self.addHyperParameters( 46 | { 47 | # "maxSequenceLength": maxSequenceLength, 48 | "backwards": backwards, 49 | "learnInit": learnInit, 50 | "gradientSteps": gradientSteps, 51 | "gradClipping": gradClipping, 52 | "unrollScan": unrollScan, 53 | # "precomputeInput": precomputeInput, 54 | "onlyReturnFinal": onlyReturnFinal 55 | } 56 | ) 57 | 58 | def getShape_abs(self) : 59 | if not self.inLayer : 60 | return None 61 | return self.lasagneLayer[self.streams[0]].get_output_shape_for([self.inLayer.getShape_abs()]) 62 | 63 | class GateConfig(object): 64 | """Legacy from lasagne. Holds the configuration for a gate but in Mariana way.""" 65 | # def __init__(self, 66 | # W_in=MI.Normal('W_in', 0.1, 0), 67 | # W_hid=MI.Normal('W_hid', 0.1, 0), 68 | # W_in_initialization=MI.Normal('W_cell', 0.1, 0), 69 | # W_in_initialization=MI.SingleValue('b', 0), 70 | # activation=MI.Sigmoid() 71 | # ): 72 | 73 | def __init__(self, 74 | initializations=[MI.Normal('W_in', 0.1, 0), MI.Normal('W_hid', 0.1, 0), MI.Normal('W_cell', 0.1, 0), MI.SingleValue('b', 0)], 75 | activation=MI.Sigmoid() 76 | ): 77 | 78 | super(GateConfig, self).__init__() 79 | self.initializations = initializations 80 | self.activation = activation 81 | 82 | 83 | class LSTM(LasagneLayers.LSTMLayer) : 84 | def __init__( 85 | self, 86 | size, 87 | name, 88 | inGateConfig=GateConfig(), 89 | forgateGateConfig=GateConfig(), 90 | cellGateConfig=GateConfig(), 91 | outgateGateConfig=GateConfig(W_cell=None, activation=MA.Tanh()), 92 | initializations=[MI.Uniform('W_in_to_hid'), MI.Uniform('W_hid_to_hid'), MI.SingleValue('b', 0)], 93 | backwards=False, 94 | learnInit=False, 95 | gradientSteps=-1, 96 | gradClipping=0, 97 | unrollScan=False, 98 | # precomputeInput=False, 99 | onlyReturnFinal=False, 100 | **kwargs 101 | ): 102 | 103 | # class lasagne.layers.LSTMLayer(incoming, 104 | # # num_units, 105 | # ingate=lasagne.layers.Gate(), 106 | # forgetgate=lasagne.layers.Gate(), 107 | # cell=lasagne.layers.Gate( W_cell=None, nonlinearity=lasagne.nonlinearities.tanh), 108 | # outgate=lasagne.layers.Gate(), 109 | # # nonlinearity=lasagne.nonlinearities.tanh, 110 | # cell_init=lasagne.init.Constant(0.), 111 | # hid_init=lasagne.init.Constant(0.), 112 | # peepholes=True, 113 | # # backwards=False, 114 | # # learn_init=False, 115 | # # gradient_steps=-1, 116 | # # grad_clipping=0, 117 | # # unroll_scan=False, 118 | # # precompute_input=True, 119 | # # mask_input=None, 120 | # # only_return_final=False, 121 | # **kwargs 122 | # ) -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | CHANGELOG 2 | ========= 3 | 4 | 1.0.3rc: 5 | -------- 6 | 7 | * bug fix: decorators now can intelligently cast ints and floats 8 | * bug fix: fixed misplaced decorators 9 | * Brand new awesome learning encapsulation paradygm, that is way more flexible and powerfull. Arbitrary infos such as weights means, maxes etc... can now be logged and saved on any type of device. 10 | * Layers can have multiple types (INPUT, HIDDEN, OUTPUT) 11 | * Embedding layares can now be hidden layers as well as inputs 12 | * Null cost redefined as a function of outputs and targets 13 | * GeometricalEarlyStopping can now work descending (default) or ascending 14 | * Better abstraction of saving criteria 15 | * Minor refactoring of GGPlot2 recorder 16 | * Added SavePeriod to periodically save the model 17 | * Embedding has now a paramater that allows the masking of inputs by using the label 0 18 | * Scale in softmax 19 | * Mandatory setCreationArgument() is gone for good 20 | * New saving method allows for Layers to be passed as constructor arguments 21 | * Parameter initializations/updates now go through layer functions initParameter and updateParameter 22 | * bug fix: Loading a saved model will no longer trigger parameter reinitialisations 23 | * bug fix: Precition accuracy used to be the same as the classification accuracy 24 | * Added HeWeights and ScaledVarianceWeights initializations 25 | * bug fix: GeometricEarlyStopping's patience is now truly reset when a better score is achieved 26 | * refact: Output layers no longer need to have weights/bias 27 | * feature: Outputlayer can now track the parameters of all layers in a network. Not only those that belong to branches that directly lead to them. 28 | * feature: Layers now perform basic sanity checks. 29 | * added test for Autoencode layer 30 | * bug fix: _setShape() now gives a last chance to set the layers shape before initialization. Fixes a bug where weight matrices could be wrongly initiliazed when the input was a Composite layer 31 | 32 | 1.0.2rc: 33 | -------- 34 | 35 | * Fixed multiple inputs and added test 36 | * Minor doc updates and cleaning 37 | * printLog() of network works even in the model does not compile, and shows the exception message at the end 38 | 39 | 1.0.1rc: 40 | -------- 41 | * Theano functions can now have several outputs. Model function no longer return an array, but an ordered dict where each key conrrespond to a given output 42 | * Theano function wrapper will now need more arguments, such as the names given to each output 43 | * Added accuracy functions such as: testAndAccuracy, and trainAndAccuracy that return both the score and the accuracy 44 | * Updated trainer/recorder/stopCriteria to support function multiple outputs. They now have more parameters 45 | * trainer now lets you define which function to use for train, test and validation 46 | * Added SavingRules (children of SavingRule_ABC) to decide when the model should be saved by the recorder. SavingRules are passed through the argument whenToSave 47 | * Created SaveMin and SaveMax SavingRules 48 | * EndOfTraining exceptions are now handeled independently from other exceptionin trainer. 49 | 50 | 1.0.0rc: 51 | -------- 52 | 53 | * The begining of a new era for Mariana. 54 | * There is as new abstraction type: initalialization (initializations.py). 55 | * Added batch normalization layer. 56 | * New Layer_ABC functions: getParameter, getParameterDict, getParameterNames, getParameterShape. The last one must be definded for initializations to work. 57 | * GlorotTanhInit is now an initialization. 58 | * Most abstractions now have a common interface. 59 | * More consistent and sane layer implementation. 60 | * All layers now have: activation, regularizations, initializations, learningScenario, decorators and name. 61 | * Layer types have been moved to Network. 62 | * Classifier_ABC is no more. 63 | * New abstract class WeightBias_ABC. 64 | * Networks now have a log, that can be pretty printed using printLog(). 65 | * saveOutputs argument is no more 66 | * All layers now have propagate() model function that returns their outputs. 67 | * Output layers can now also serve as hidden layers. 68 | * ToHidden() and toOutput() are no more. 69 | * SoftmaxClassifier() now has an accuracy function. 70 | * AutoEncoder layer now takes a layer name as argument. 71 | * Functions to save parameters of a network in npy or HDF5 formats. 72 | * Save() is now based on clone() and can now handle many layers and still uses pickle (Yeah I said that I am going to do something using HDF5 and JSON, but it is not worth the trouble). 73 | * CloneBare() is no more. 74 | * Clone() can now clone any layer based on the constructor arguments but you need to call the introspective self._setCreationArguments() at the end of the constructor. 75 | * Network.load() to load models saved by save(). 76 | * Embedding for Conv nets. 77 | * Added example for hierarchical softmax. 78 | * Many other things and little adjustements that make the code more beautiful. 79 | -------------------------------------------------------------------------------- /Mariana/compatibility/lasagne.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | import Mariana.layers as ML 3 | import Mariana.custom_types as MTYPES 4 | import lasagne 5 | 6 | __all__=["LasagneLayer", "LasagneStreamedLayer", "IAmAnnoyed"] 7 | 8 | class IAmAnnoyed(Exception) : 9 | """What Mariana raises when you annoy her""" 10 | def __init__(self, msg) : 11 | self.message = msg 12 | 13 | def __str__(self) : 14 | return self.message 15 | 16 | def __repr__(self): 17 | return self.message 18 | 19 | class LasagneStreamedLayer(object): 20 | """Wraps lasagne layers to give them a stream interface""" 21 | def __init__(self, incomingShape, streams, lasagneLayerCls, hyperParameters, initParameters, lasagneKwargs={}): 22 | super(LasagneStreamedLayer, self).__init__() 23 | self.streams = streams 24 | self.lasagneLayer = OrderedDict() 25 | 26 | kwargs = {} 27 | kwargs.update(hyperParameters) 28 | kwargs.update(initParameters) 29 | kwargs.update(lasagneKwargs) 30 | 31 | self.parameters = {} 32 | for f in streams : 33 | if len(self.parameters) == 0 : 34 | self.lasagneLayer[f] = lasagneLayerCls(incoming = incomingShape, **kwargs) 35 | for k in initParameters : 36 | self.parameters[k] = getattr(self.lasagneLayer[f], k) 37 | kwargs.update(self.parameters) 38 | else : 39 | self.lasagneLayer[f] = lasagneLayerCls(incoming = incomingShape, **kwargs) 40 | 41 | def __getitem__(self, k) : 42 | return self.lasagneLayer[k] 43 | 44 | def __setitem__(self, k, v) : 45 | self.lasagneLayer[k] = v 46 | 47 | class LasagneLayer(ML.Layer_ABC) : 48 | """This very special class allows you to incorporate a Lasagne layer seemlessly inside a Mariana network. 49 | An incorporated lasagne is just like a regular layer, with streams and all the other Mariana niceties. 50 | initializations must be specified with Mariana initializers, and please don't pass it an 'incoming', 'nonlinearity' argument. 51 | It is Mariana's job to do the shape inference and activate the layers, and she can get pretty upset if you try to tell her how to do her job. 52 | If you need to specifiy a specific value for some paramters, use the HardSet() initializer. 53 | 54 | Here's an examples:: 55 | 56 | from lasagne.layers.dense import DenseLayer 57 | 58 | hidden = LasagneLayer( 59 | DenseLayer, 60 | initializations=[MI.GlorotNormal('W'), MI.SingleValue('b', 0)], 61 | lasagneHyperParameters={"num_units": 10}, 62 | activation = MA.Tanh(), 63 | learningScenari = [MS.GradientDescent(lr = 0.1, momentum=0)], 64 | name = "HiddenLayer2" 65 | ) 66 | 67 | """ 68 | 69 | def __init__(self, lasagneLayerCls, initializations, lasagneHyperParameters={}, lasagneKwargs={}, **kwargs) : 70 | import inspect 71 | 72 | super(LasagneLayer, self).__init__(initializations=initializations, **kwargs) 73 | 74 | self.lasagneLayerCls = lasagneLayerCls 75 | 76 | self.lasagneHyperParameters = lasagneHyperParameters 77 | 78 | if "nonlinearity" in self.lasagneHyperParameters : 79 | raise IAmAnnoyed("There's an 'nonlinearity' argument in the hyperParameters. Use activation = <...>. Just like you would do for any other layer.") 80 | 81 | if "incoming" in self.lasagneHyperParameters : 82 | raise IAmAnnoyed("There's an 'incoming' argument in the hyperParameters. Don't tell me how to do my job!") 83 | 84 | self.addHyperParameters(self.lasagneHyperParameters) 85 | if "nonlinearity" in inspect.getargspec(lasagneLayerCls.__init__)[0] : 86 | self.lasagneHyperParameters["nonlinearity"] = None 87 | self.lasagneKwargs = lasagneKwargs 88 | 89 | self.lasagneLayer = None 90 | # self.inLayer = None 91 | self.lasagneParameters = {} 92 | 93 | for init in self.abstractions["initializations"] : 94 | self.setP(init.getHP("parameter"), MTYPES.Parameter("%s.%s" % (self.name, init.getHP("parameter")))) 95 | init.setup(self) 96 | self.lasagneParameters[init.getHP("parameter")] = init.run 97 | 98 | def setShape_abs(self) : 99 | inLayer = self.network.getInConnections(self)[0] 100 | if not self.lasagneLayer and inLayer.getShape_abs() is not None: 101 | self.lasagneLayer = LasagneStreamedLayer(incomingShape=inLayer.getShape_abs(), streams=self.streams, lasagneLayerCls=self.lasagneLayerCls, hyperParameters=self.lasagneHyperParameters, initParameters=self.lasagneParameters) 102 | 103 | def femaleConnect(self, layer) : 104 | # self.inLayer = layer 105 | self.setShape_abs() 106 | 107 | def getShape_abs(self) : 108 | try : 109 | inLayer = self.network.getInConnections(self)[0] 110 | except IndexError : 111 | return None 112 | return self.lasagneLayer[self.streams[0]].get_output_shape_for(inLayer.getShape_abs()) 113 | 114 | def _initParameters(self) : 115 | # self._setShape(self.inLayer) 116 | self.lasagneParameters = None 117 | for k, v in self.lasagneLayer.parameters.iteritems() : 118 | self.parameters[k].setValue(v, forceCast=False) 119 | 120 | def getParameterShape_abs(self, k) : 121 | v = getattr(self.lasagneLayer[self.streams[0]], k) 122 | return v.get_value().shape 123 | 124 | def setOutputs_abs(self) : 125 | inLayer = self.network.getInConnections(self)[0] 126 | for f in self.outputs.streams : 127 | self.outputs[f] = self.lasagneLayer[f].get_output_for(inLayer.outputs[f]) 128 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | What will happen now that Theano is no longer developed? 2 | ========================================================= 3 | 4 | **Mariana works!** I still use it almost everyday. 5 | 6 | I am still taking care of the maintenance and may still add some minor features. For the future, the most straightforward path would be a complete port to Tensorflow or PyTorch. Let me know if you'd like to help! 7 | 8 | T . 9 | 10 | 11 | .. image:: https://github.com/tariqdaouda/Mariana/blob/master/MarianaLogo.png 12 | logo by `Sawssan Kaddoura`_. 13 | 14 | .. _Sawssan Kaddoura: http://sawssankaddoura.com 15 | 16 | Mariana V1 is here_ 17 | 18 | .. _here: https://github.com/tariqdaouda/Mariana/tree/master 19 | 20 | MARIANA: The Cutest Deep Learning Framework 21 | ============================================= 22 | .. image:: https://img.shields.io/badge/python-2.7-blue.svg 23 | 24 | **MARIANA V2.** 25 | 26 | Mariana is meant to be a **efficient language** through which complex deep neural networks can be easily expressed and easily manipulated. It's simple enough for beginners and doesn't get much complicated. Intuitive, user-friendly and yet flexible enough for research. It's here to empower **researchers**, **teachers** and **students** alike, while greatly facilitating **AI knowledge transfer** into other domains. 27 | 28 | Mariana is also compatible with google's GPUs and colab. For a running basic example you can check here on colab_. 29 | 30 | .. _colab: https://colab.research.google.com/github/tariqdaouda/Mariana/blob/V2-dev/Mariana_basics_example.ipynb 31 | 32 | .. code:: python 33 | 34 | import Mariana.layers as ML 35 | import Mariana.scenari as MS 36 | import Mariana.costs as MC 37 | import Mariana.activations as MA 38 | import Mariana.regularizations as MR 39 | 40 | import Mariana.settings as MSET 41 | 42 | ls = MS.GradientDescent(lr = 0.01, momentum=0.9) 43 | cost = MC.NegativeLogLikelihood() 44 | 45 | inp = ML.Input(28*28, name = "InputLayer") 46 | h1 = ML.Hidden(300, activation = MA.ReLU(), name = "Hidden1", regularizations = [ MR.L1(0.0001) ]) 47 | h2 = ML.Hidden(300, activation = MA.ReLU(), name = "Hidden2", regularizations = [ MR.L1(0.0001) ]) 48 | o = ML.SoftmaxClassifier(10, learningScenari = [ls], cost = cost, name = "Probabilities") 49 | 50 | #Connecting layers 51 | inp > h1 > h2 52 | concat = ML.C([inp, h2]) 53 | 54 | MLP_skip = concat > o 55 | MLP_skip.init() 56 | 57 | #Visualizing 58 | MLP_skip.saveHTML("mySkipMLP") 59 | 60 | #training: 61 | for i in xrange(1000) : 62 | MLP_skip["Probabilities"].train({"InputLayer.inputs": train_set[0], "Probabilities.targets": train_set[1]}) 63 | 64 | #testing 65 | print MLP_skip["Probabilities"].test({"InputLayer.inputs": test_set[0], "Probabilities.targets": test_set[1]}) 66 | 67 | V2's most exciting stuff 68 | ========================= 69 | 70 | V2 is almost a complete rewite of V1. It is much better. 71 | 72 | What's done 73 | ----------- 74 | 75 | * **New built-in visualization**: Interactive visuallization that shows architecture along with parameters, hyper-parameters as well as user defined notes. A great tool for collaboration. 76 | * **Function mixins (my favorite)**: Mariana functions can now be added together! The result is a function that performs the actions of all its components at once. Let's say we have two output layers and want a function that optimises on losses for both outputs. Creating it is as simple as: f = out1.train + out2.train, and then calling f. Mariana will derive f from both functions adding the costs, calculating gradients and updates seemlessly in the background 77 | * **Easy access to gradients and updates**: Just call .getGradients(), .getUpdates() on any function to get a view of either gradients of updates for all parameters. 78 | * **User friendly error messages**: Functions will tell you what arguments they expect. 79 | * **Very very clean code with Streams!**: You probably heard of batchnorm... and how it has a different behaviour in training and in testing. Well that simple fact can be the cause of some very messy DL code. With streams all this is over. Streams are parallel universes of execution for functions. You can define your own streams and have as many as you want. For batchnorm it mean that depending on the stream you call your function in (test or train), the behaviour will be different, even though you only changed one word. 80 | * **Chainable optimization rules**: As in the previous version, layers inherit their learning scenari from outputs, but have the possibility to redifine them. This is still true, but rules can now be chained. Here's how to define a layer with fixed bias: l = Dense( learningScenari=[GradientDescent(lr = 0.1), Fixed('b')]) 81 | * **Just in time function compilation**: All functions (including mixins) are only compiled if needed. 82 | * **Lasagne compatible**: Every lasagne layer can be seemlessly imported and used into Mariana 83 | * **Convolutions, Deconvolutions (Transpose convolution), all sorts of convolutions...** 84 | * **Much easier to extend**: The (almost) complete rewrite made for a much more cleaner code that is much more easy to extend. It is now much simpler to create your own layers, decorators, etc... Function that you need to implement end with *_abs* and Mariana has whole new bunch of custom type that support streams. 85 | * **New merge layer**: Need a layer that is a linear combination of other layers? The new MergeLayer is perfect for that newLayer = M(layer1 + (layers3 * layer3) + 4 ) 86 | * **New concatenation layer**: newLayer = C([Layer1, layer2]) 87 | * **Unlimited number of inputs per layer**: Each layer used to be limited to one. Now it is infinit 88 | * **Abstractions are now divided into trainable (layers, decorators, activations) and untrainable (scenari, costs, initializations)**: All trainable abstractions can hold parameters and have untrainable abstractions applied to them. PReLU will finally join ReLU as an activation! 89 | * Fancy ways to go downhill: **Adam, Adagrad**, ... 90 | 91 | What's almost done 92 | ------------------- 93 | 94 | * Inclusion of popular recurrences (LSTM, recurent layers, ...) 95 | 96 | What's next 97 | ----------- 98 | 99 | * Complete refactorisation of training encapsulation. Training encapsulation was the least popular aspect of Mariana so far. I will completely rewrite it to give it the same level of intuitiveness as the rest of the framework. The next iterration will be a huge improvement. 100 | * Arbitrary recurrences in the graph 101 | -------------------------------------------------------------------------------- /Mariana/custom_types.py: -------------------------------------------------------------------------------- 1 | import theano 2 | import Mariana.useful as MUSE 3 | 4 | class Variable(object): 5 | """docstring for Variable""" 6 | def __init__(self, variableType=None, streams=["train", "test"], **theano_kwargs): 7 | super(Variable, self).__init__() 8 | self.streams = streams 9 | self.variables = {} 10 | self.variableType = variableType 11 | self.ties = set() 12 | 13 | if variableType : 14 | self.set(variableType, **theano_kwargs) 15 | else : 16 | self.dtype = None 17 | for f in self.streams : 18 | self.variables[f] = None 19 | 20 | self.tied = False 21 | 22 | def isSet(self) : 23 | for f in self.streams : 24 | if not self.variables[f] : return False 25 | 26 | return True 27 | 28 | def isTied(self, stream) : 29 | return stream in self.ties 30 | 31 | def tie(self, var, stream = None) : 32 | if stream is None : 33 | if set(self.streams) != set(var.streams) : 34 | raise ValueError( "%s does not have the same streams. Self: %s, var: %s" % (var, self.streams, var.streams) ) 35 | for f in self.streams : 36 | self.variables[f] = var.variables[f] 37 | self.ties.add(f) 38 | else : 39 | self.variables[stream] = var[stream] 40 | self.ties.add(stream) 41 | 42 | def set(self, variableType, *theano_args, **theano_kwargs) : 43 | self.variableType = variableType 44 | for f in self.streams : 45 | self.variables[f] = variableType(*theano_args, **theano_kwargs) 46 | if f in self.ties : 47 | self.ties.remove(f) 48 | self.dtype = self.variables[f].dtype 49 | 50 | def getValue(self, stream) : 51 | v = self[stream] 52 | if v is None : 53 | raise ValueError("Variable has an empty value for stream: %s" % stream) 54 | 55 | return self[stream].get_value() 56 | 57 | def setValue(self, stream, value) : 58 | if stream not in self.streams : 59 | raise KeyError("There is no stream by the name of: '%s'" % stream) 60 | 61 | self[stream].set_value(value) 62 | if stream in self.ties : 63 | self.ties.remove(stream) 64 | 65 | def __getitem__(self, stream) : 66 | # print self.variables[stream] 67 | 68 | try : 69 | return self.variables[stream] 70 | except KeyError : 71 | raise KeyError("There is no stream by the name of: '%s'" % stream) 72 | 73 | def __setitem__(self, stream, newVal) : 74 | try : 75 | self.variables[stream] = newVal 76 | except KeyError : 77 | raise KeyError("There is no stream by the name of: '%s'" % stream) 78 | if stream in self.ties : 79 | self.ties.remove(stream) 80 | 81 | def __contains__(self, stream) : 82 | """check if the stream is supported""" 83 | return stream in self.streams 84 | 85 | def __repr__(self) : 86 | return "< Mariana %s, streams: %s>" % (self.__class__.__name__, self.streams) 87 | 88 | class Inputs(Variable): 89 | """docstring for Input""" 90 | 91 | class Targets(Variable): 92 | """docstring for Input""" 93 | 94 | class Parameter(object): 95 | """docstring for Parameter""" 96 | def __init__(self, name): 97 | super(Parameter, self).__init__() 98 | self.name = name 99 | self.theano_var = None 100 | self.tiedParams = {} 101 | self.master = None 102 | 103 | def _tie(self, master) : 104 | self.master = master 105 | 106 | def isTied(self) : 107 | return self.master is not None 108 | # try : 109 | # print self 110 | # return self.master is not None 111 | # except AttributeError : 112 | # return False 113 | 114 | def isShared(self) : 115 | return isinstance(self.theano_var, theano.compile.sharedvalue.SharedVariable) 116 | 117 | def isSet(self) : 118 | return self.theano_var is not None or self.isTied() 119 | 120 | def tie(self, otherParam, transpose=False) : 121 | if otherParam not in self.tiedParams : 122 | self.tiedParams[otherParam] = transpose 123 | otherParam.tie(self, transpose) 124 | 125 | def __call__(self) : 126 | return self.getVar() 127 | 128 | def getVar(self) : 129 | if self.isTied() : 130 | if self.tiedParams[self.master] : 131 | return self.master().T 132 | else : 133 | return self.master() 134 | else : 135 | return self.theano_var 136 | 137 | def hasValue(self) : 138 | return self.theano_var is not None 139 | 140 | def setValue(self, value, forceCast = True) : 141 | if isinstance(value, theano.Variable) : 142 | if forceCast : 143 | v = MUSE.iCast_theano(value) 144 | else : 145 | v = value 146 | else : 147 | if forceCast : 148 | v = theano.shared(value = MUSE.iCast_numpy(value), name = self.name) 149 | else : 150 | v = theano.shared(value = value, name = self.name) 151 | 152 | self.theano_var = v 153 | for p in self.tiedParams : 154 | p._tie(self) 155 | 156 | def updateValue(self, value, forceCast=False) : 157 | if forceCast : 158 | v = theano.shared(value = MUSE.iCast_numpy(value), name = self.name) 159 | else : 160 | v = value 161 | 162 | if v.shape != self.getShape() : 163 | print("Warning update has a different shape: %s -> %s" %(self.shape, v.shape)) 164 | self.theano_var.set_value(v) 165 | 166 | def getValue(self) : 167 | if self.theano_var is None : 168 | return None 169 | return self.theano_var.get_value() 170 | 171 | def getShape(self) : 172 | if self.theano_var is None : 173 | return None 174 | return self.getValue().shape 175 | 176 | def __repr__(self) : 177 | return "< Mariana Parameter: %s, %s. Tied to: %s>" % (self.name, self.getShape(), self.master) 178 | 179 | class Losses(object): 180 | """Contains the loss for every stream""" 181 | def __init__(self, layer, cost, targets, outputs): 182 | super(Losses, self).__init__() 183 | self.streams=targets.streams 184 | 185 | self.layer = layer 186 | self.cost = cost 187 | self.targets = targets 188 | self.outputs = outputs 189 | 190 | self.store = {} 191 | for k in self.streams : 192 | self.store[k] = self.cost.apply(self.layer, self.targets[k], self.outputs[k], stream = k) 193 | 194 | def __getitem__(self, k) : 195 | return self.store[k] 196 | 197 | def __setitem__(self, k, v) : 198 | self.store[k] = v 199 | 200 | def __contains__(self, stream) : 201 | """check if the stream is supported""" 202 | return stream in self.streams 203 | 204 | def __repr__(self) : 205 | return "< Mariana %s, streams: %s>" % (self.__class__.__name__, self.streams) 206 | 207 | -------------------------------------------------------------------------------- /Mariana/convolution.py: -------------------------------------------------------------------------------- 1 | import Mariana.initializations as MI 2 | 3 | import Mariana.compatibility.lasagne as MLASAGNE 4 | import lasagne.layers.conv as LasagneCONV 5 | 6 | __all__ = ["Convolution1D", "Convolution2D", "Convolution3D", "TransposeConvolution2D", "Deconv2D", "TransposeConvolution3D", "Deconv3D", "DilatedConv2DLayer"] 7 | 8 | class Convolution1D(MLASAGNE.LasagneLayer): 9 | """This layer wraps lasagnes's Conv1DLayer layer and performs a 1D convolution over each channel. 10 | For a full explanation of the arguments please checkout lasagne's doc""" 11 | def __init__( 12 | self, 13 | numFilters, 14 | filterSize, 15 | name, 16 | stride=1, 17 | pad=0, 18 | untieBiases=False, 19 | flipFilters=True, 20 | initializations=[MI.GlorotNormal('W'), MI.SingleValue('b', 0)], 21 | **kwargs 22 | ): 23 | super(Convolution1D, self).__init__( 24 | LasagneCONV.Conv1DLayer, 25 | initializations=initializations, 26 | lasagneHyperParameters={ 27 | "num_filters": numFilters, 28 | "filter_size": filterSize, 29 | "stride": stride, 30 | "pad": pad, 31 | "untie_biases": untieBiases, 32 | "flip_filters": flipFilters 33 | }, 34 | lasagneKwargs={}, 35 | **kwargs 36 | ) 37 | 38 | class Convolution2D(MLASAGNE.LasagneLayer): 39 | """This layer wraps lasagnes's Conv2DLayer layer and performs a 2D convolution over each channel. 40 | For a full explanation of the arguments please checkout lasagne's doc""" 41 | def __init__( 42 | self, 43 | numFilters, 44 | filterHeight, 45 | filterWidth, 46 | name, 47 | stride=(1, 1), 48 | pad=0, 49 | untieBiases=False, 50 | initializations=[MI.GlorotNormal('W'), MI.SingleValue('b', 0)], 51 | flipFilters=True, 52 | **kwargs 53 | ): 54 | super(Convolution2D, self).__init__( 55 | LasagneCONV.Conv2DLayer, 56 | initializations=initializations, 57 | lasagneHyperParameters={ 58 | "num_filters": numFilters, 59 | "filter_size": (filterHeight, filterWidth), 60 | "stride": stride, 61 | "pad": pad, 62 | "untie_biases": untieBiases, 63 | "flip_filters": flipFilters 64 | }, 65 | lasagneKwargs={}, 66 | name=name, 67 | **kwargs 68 | ) 69 | 70 | class Convolution3D(MLASAGNE.LasagneLayer): 71 | """This layer wraps lasagnes's Conv3DLayer layer and performs a 3D convolution over each channel. 72 | For a full explanation of the arguments please checkout lasagne's doc""" 73 | def __init__( 74 | self, 75 | numFilters, 76 | filterHeight, 77 | filterWidth, 78 | filterDepth, 79 | name, 80 | stride=(1, 1, 1), 81 | pad=0 , 82 | untieBiases=False, 83 | initializations=[MI.GlorotNormal('W'), MI.SingleValue('b', 0)], 84 | flipFilters=True, 85 | **kwargs 86 | ): 87 | super(Convolution3D, self).__init__( 88 | LasagneCONV.Conv3DLayer, 89 | initializations=initializations, 90 | lasagneHyperParameters={ 91 | "numFilters": numFilters, 92 | "filter_size": (filterHeight, filterWidth, filterDepth), 93 | "stride": stride, 94 | "pad": pad, 95 | "untie_biases": untieBiases, 96 | "flip_filters": flipFilters 97 | }, 98 | lasagneKwargs={}, 99 | name=name, 100 | **kwargs 101 | ) 102 | 103 | class TransposeConvolution2D(MLASAGNE.LasagneLayer): 104 | """This layer wraps lasagnes's TransposedConv2DLayer layer and performs a 2D transpose convolution (deconvolution) over each channel. 105 | For a full explanation of the arguments please checkout lasagne's doc""" 106 | def __init__( 107 | self, 108 | numFilters, 109 | filterHeight, 110 | filterWidth, 111 | name, 112 | stride=(1, 1), 113 | crop=0, 114 | untieBiases=False, 115 | initializations=[MI.GlorotNormal('W'), MI.SingleValue('b', 0)], 116 | flipFilters=True, 117 | **kwargs 118 | ): 119 | super(TransposeConvolution2D, self).__init__( 120 | LasagneCONV.TransposedConv2DLayer, 121 | initializations=initializations, 122 | lasagneHyperParameters={ 123 | "num_filters": numFilters, 124 | "filter_size": (filterHeight, filterWidth), 125 | "stride": stride, 126 | "crop": crop, 127 | "untie_biases": untieBiases, 128 | "flip_filters": flipFilters 129 | }, 130 | lasagneKwargs={}, 131 | name=name, 132 | **kwargs 133 | ) 134 | Deconv2D = TransposeConvolution2D 135 | 136 | class TransposeConvolution3D(MLASAGNE.LasagneLayer): 137 | """This layer wraps lasagnes's TransposedConv3DLayer layer and performs a 3D transpose convolution (deconvolution) over each channel. 138 | For a full explanation of the arguments please checkout lasagne's doc""" 139 | def __init__( 140 | self, 141 | numFilters, 142 | filterHeight, 143 | filterWidth, 144 | filterDepth, 145 | name, 146 | stride=(1, 1, 1), 147 | crop=0 , 148 | untieBiases=False, 149 | initializations=[MI.GlorotNormal('W'), MI.SingleValue('b', 0)], 150 | flipFilters=True, 151 | **kwargs 152 | ): 153 | super(Convolution3D, self).__init__( 154 | LasagneCONV.TransposedConv3DLayer, 155 | initializations=initializations, 156 | lasagneHyperParameters={ 157 | "num_filters": numFilters, 158 | "filter_size": (filterHeight, filterWidth, filterDepth), 159 | "stride": stride, 160 | "crop": crop, 161 | "untie_biases": untieBiases, 162 | "flip_filters": flipFilters 163 | }, 164 | lasagneKwargs={}, 165 | name=name, 166 | **kwargs 167 | ) 168 | Deconv3D = TransposeConvolution3D 169 | 170 | class DilatedConvolution2D(MLASAGNE.LasagneLayer): 171 | """This layer wraps lasagnes's DilatedConv2DLayer layer and performs a 2D dilated convolution (deconvolution) over each channel. 172 | For a full explanation of the arguments please checkout lasagne's doc""" 173 | def __init__( 174 | self, 175 | numFilters, 176 | filterHeight, 177 | filterWidth, 178 | name, 179 | dilation=(1, 1), 180 | stride=(1, 1), 181 | pad=0, 182 | untieBiases=False, 183 | initializations=[MI.GlorotNormal('W'), MI.SingleValue('b', 0)], 184 | flipFilters=True, 185 | **kwargs 186 | ): 187 | super(TransposeConvolution2D, self).__init__( 188 | LasagneCONV.DilatedConv2DLayer, 189 | initializations=initializations, 190 | lasagneHyperParameters={ 191 | "num_filters": numFilters, 192 | "filter_size": (filterHeight, filterWidth), 193 | "stride": stride, 194 | "pad": pad, 195 | "dilation": dilation, 196 | "untie_biases": untieBiases, 197 | "flip_filters": flipFilters 198 | }, 199 | lasagneKwargs={}, 200 | name=name, 201 | **kwargs 202 | ) -------------------------------------------------------------------------------- /Mariana/doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | dirhtml: 60 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 63 | 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | pickle: 70 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 71 | @echo 72 | @echo "Build finished; now you can process the pickle files." 73 | 74 | json: 75 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 76 | @echo 77 | @echo "Build finished; now you can process the JSON files." 78 | 79 | htmlhelp: 80 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 81 | @echo 82 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 83 | ".hhp project file in $(BUILDDIR)/htmlhelp." 84 | 85 | qthelp: 86 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 87 | @echo 88 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 89 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 90 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Mariana.qhcp" 91 | @echo "To view the help file:" 92 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Mariana.qhc" 93 | 94 | applehelp: 95 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 96 | @echo 97 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 98 | @echo "N.B. You won't be able to view it unless you put it in" \ 99 | "~/Library/Documentation/Help or install it in your application" \ 100 | "bundle." 101 | 102 | devhelp: 103 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 104 | @echo 105 | @echo "Build finished." 106 | @echo "To view the help file:" 107 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Mariana" 108 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Mariana" 109 | @echo "# devhelp" 110 | 111 | epub: 112 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 113 | @echo 114 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 115 | 116 | latex: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo 119 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 120 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 121 | "(use \`make latexpdf' here to do that automatically)." 122 | 123 | latexpdf: 124 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 125 | @echo "Running LaTeX files through pdflatex..." 126 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 127 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 128 | 129 | latexpdfja: 130 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 131 | @echo "Running LaTeX files through platex and dvipdfmx..." 132 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 133 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 134 | 135 | text: 136 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 137 | @echo 138 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 139 | 140 | man: 141 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 142 | @echo 143 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 144 | 145 | texinfo: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo 148 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 149 | @echo "Run \`make' in that directory to run these through makeinfo" \ 150 | "(use \`make info' here to do that automatically)." 151 | 152 | info: 153 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 154 | @echo "Running Texinfo files through makeinfo..." 155 | make -C $(BUILDDIR)/texinfo info 156 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 157 | 158 | gettext: 159 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 160 | @echo 161 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 162 | 163 | changes: 164 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 165 | @echo 166 | @echo "The overview file is in $(BUILDDIR)/changes." 167 | 168 | linkcheck: 169 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 170 | @echo 171 | @echo "Link check complete; look for any errors in the above output " \ 172 | "or in $(BUILDDIR)/linkcheck/output.txt." 173 | 174 | doctest: 175 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 176 | @echo "Testing of doctests in the sources finished, look at the " \ 177 | "results in $(BUILDDIR)/doctest/output.txt." 178 | 179 | coverage: 180 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 181 | @echo "Testing of coverage in the sources finished, look at the " \ 182 | "results in $(BUILDDIR)/coverage/python.txt." 183 | 184 | xml: 185 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 186 | @echo 187 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 188 | 189 | pseudoxml: 190 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 191 | @echo 192 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 193 | -------------------------------------------------------------------------------- /Mariana/doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 10 | set I18NSPHINXOPTS=%SPHINXOPTS% source 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | echo. coverage to run coverage check of the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | 51 | REM Check if sphinx-build is available and fallback to Python version if any 52 | %SPHINXBUILD% 2> nul 53 | if errorlevel 9009 goto sphinx_python 54 | goto sphinx_ok 55 | 56 | :sphinx_python 57 | 58 | set SPHINXBUILD=python -m sphinx.__init__ 59 | %SPHINXBUILD% 2> nul 60 | if errorlevel 9009 ( 61 | echo. 62 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 63 | echo.installed, then set the SPHINXBUILD environment variable to point 64 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 65 | echo.may add the Sphinx directory to PATH. 66 | echo. 67 | echo.If you don't have Sphinx installed, grab it from 68 | echo.http://sphinx-doc.org/ 69 | exit /b 1 70 | ) 71 | 72 | :sphinx_ok 73 | 74 | 75 | if "%1" == "html" ( 76 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 80 | goto end 81 | ) 82 | 83 | if "%1" == "dirhtml" ( 84 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 88 | goto end 89 | ) 90 | 91 | if "%1" == "singlehtml" ( 92 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 93 | if errorlevel 1 exit /b 1 94 | echo. 95 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 96 | goto end 97 | ) 98 | 99 | if "%1" == "pickle" ( 100 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can process the pickle files. 104 | goto end 105 | ) 106 | 107 | if "%1" == "json" ( 108 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 109 | if errorlevel 1 exit /b 1 110 | echo. 111 | echo.Build finished; now you can process the JSON files. 112 | goto end 113 | ) 114 | 115 | if "%1" == "htmlhelp" ( 116 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 117 | if errorlevel 1 exit /b 1 118 | echo. 119 | echo.Build finished; now you can run HTML Help Workshop with the ^ 120 | .hhp project file in %BUILDDIR%/htmlhelp. 121 | goto end 122 | ) 123 | 124 | if "%1" == "qthelp" ( 125 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 129 | .qhcp project file in %BUILDDIR%/qthelp, like this: 130 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Mariana.qhcp 131 | echo.To view the help file: 132 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Mariana.ghc 133 | goto end 134 | ) 135 | 136 | if "%1" == "devhelp" ( 137 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. 141 | goto end 142 | ) 143 | 144 | if "%1" == "epub" ( 145 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 149 | goto end 150 | ) 151 | 152 | if "%1" == "latex" ( 153 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 157 | goto end 158 | ) 159 | 160 | if "%1" == "latexpdf" ( 161 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 162 | cd %BUILDDIR%/latex 163 | make all-pdf 164 | cd %~dp0 165 | echo. 166 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdfja" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf-ja 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "text" ( 181 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 182 | if errorlevel 1 exit /b 1 183 | echo. 184 | echo.Build finished. The text files are in %BUILDDIR%/text. 185 | goto end 186 | ) 187 | 188 | if "%1" == "man" ( 189 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 190 | if errorlevel 1 exit /b 1 191 | echo. 192 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 193 | goto end 194 | ) 195 | 196 | if "%1" == "texinfo" ( 197 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 198 | if errorlevel 1 exit /b 1 199 | echo. 200 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 201 | goto end 202 | ) 203 | 204 | if "%1" == "gettext" ( 205 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 206 | if errorlevel 1 exit /b 1 207 | echo. 208 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 209 | goto end 210 | ) 211 | 212 | if "%1" == "changes" ( 213 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 214 | if errorlevel 1 exit /b 1 215 | echo. 216 | echo.The overview file is in %BUILDDIR%/changes. 217 | goto end 218 | ) 219 | 220 | if "%1" == "linkcheck" ( 221 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 222 | if errorlevel 1 exit /b 1 223 | echo. 224 | echo.Link check complete; look for any errors in the above output ^ 225 | or in %BUILDDIR%/linkcheck/output.txt. 226 | goto end 227 | ) 228 | 229 | if "%1" == "doctest" ( 230 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Testing of doctests in the sources finished, look at the ^ 234 | results in %BUILDDIR%/doctest/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "coverage" ( 239 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of coverage in the sources finished, look at the ^ 243 | results in %BUILDDIR%/coverage/python.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "xml" ( 248 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 252 | goto end 253 | ) 254 | 255 | if "%1" == "pseudoxml" ( 256 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 257 | if errorlevel 1 exit /b 1 258 | echo. 259 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 260 | goto end 261 | ) 262 | 263 | :end 264 | -------------------------------------------------------------------------------- /Mariana/training/datasetmaps.py: -------------------------------------------------------------------------------- 1 | import Mariana.layers as ML 2 | 3 | 4 | # pkl = {train: {3:, 5:}, test...} 5 | # dataset = MD.ClassDataset(pkl) 6 | # d = DataFeeder(fct) 7 | # d.streamTranslate("10%", "train") 8 | # d.feed("input.inputs", data.images) 9 | # d.sample(100) 10 | # d.all() 11 | 12 | class DatasetMapper(object): 13 | """a DatasetMapper maps Input and Output layer to the data they must receive. 14 | It's much less complicated than it sounds cf. doc for map(), you can ignore the others. 15 | The DatasetMapper is intended to be used with a Trainer 16 | 17 | :param miniBatchSize: Size of miniBatches, use None for the complete dataset 18 | :param rerollFreq:How many epochs before internally shuffling the subsets). Freq=10 means 1 reroll each 10 epochs. Use a None to prevent rerolls 19 | 20 | """ 21 | 22 | def __init__(self, runFunctionName, miniBatchSize=None, rerollFreq=1): 23 | self.datasets = set() 24 | self.maps = {} 25 | self.layersByName = {} 26 | self.inputLayers = [] 27 | self.outputLayers = [] 28 | self.minLength = float('inf') 29 | self.minFullLength = float('inf') 30 | 31 | self.miniBatchSize = miniBatchSize 32 | if miniBatchSize is None : 33 | self.rerollFreq = None 34 | else : 35 | self.rerollFreq = rerollFreq 36 | 37 | self.epochNumber = 0 38 | self.batchNumber = 0 39 | self.runFunctionName = runFunctionName 40 | self.runFunction = None 41 | 42 | def mapInput(self, layer, setHandle) : 43 | """ 44 | Maps an input or an output layer to Dataset's subset:: 45 | 46 | i = InputLayer(...) 47 | 48 | trainSet = RandomSeries(images = train_set[0], classes = train_set[1]) 49 | 50 | DatasetMapper = dm 51 | dm.map(i, trainSet.images) 52 | """ 53 | import Mariana.settings as MSET 54 | 55 | if MSET.TYPE_INPUT_LAYER not in layer.types : 56 | raise ValueError("%s is not an input layer (type: %s)" % (layer.name, layer.types)) 57 | 58 | self.inputLayers.append(layer) 59 | self.maps[layer] = ( (layer.name, setHandle), ) 60 | self.layersByName[layer.name] = layer 61 | 62 | self.datasets.add(setHandle.dataset) 63 | if len(setHandle.dataset) < self.minLength : 64 | self.minLength = len(setHandle.dataset) 65 | if setHandle.dataset.getFullLength() < self.minFullLength : 66 | self.minFullLength = setHandle.dataset.getFullLength() 67 | 68 | if self.runFunction is None : 69 | self.runFunction = getattr(layer.network, self.runFunctionName) 70 | 71 | def mapOutput(self, layer, setHandle, inputName = 'targets') : 72 | """ 73 | Maps an input or an output layer to Dataset's subset:: 74 | 75 | o = OutputLayer(...) 76 | 77 | trainSet = RandomSeries(images = train_set[0], classes = train_set[1]) 78 | 79 | DatasetMapper = dm 80 | dm.map(o, trainSet.classes, "targets") 81 | 82 | inputName: train and test functions often need addictional inputs such as the targets. With this 83 | parameter you can associate one of these additional inputs (as defined in a theano function) to 84 | a dataset. This argument is optional, the default value is 'targets' which should work for all 85 | out of the box Mariana stuff. 86 | """ 87 | import Mariana.settings as MSET 88 | 89 | if MSET.TYPE_OUTPUT_LAYER not in layer.types : 90 | raise ValueError("%s is not an output layer (type: %s)" % (layer.name, layer.types)) 91 | 92 | self.outputLayers.append(layer) 93 | self.layersByName[layer.name] = layer 94 | 95 | k = (inputName, setHandle) 96 | 97 | try : 98 | self.maps[layer].append(k) 99 | except KeyError : 100 | self.maps[layer] = [ k ] 101 | 102 | self.datasets.add(setHandle.dataset) 103 | if len(setHandle.dataset) < self.minLength : 104 | self.minLength = len(setHandle.dataset) 105 | if setHandle.dataset.getFullLength() < self.minFullLength : 106 | self.minFullLength = setHandle.dataset.getFullLength() 107 | 108 | if self.runFunction is None : 109 | self.runFunction = getattr(layer.network, self.runFunctionName) 110 | 111 | def reroll(self, force=False) : 112 | """internally shuffle subsets periodically according to reroll frequency. This is called once by the trainer at each epoch""" 113 | if force or ( self.rerollFreq is not None and self.epochNumber%self.rerollFreq == 0 ): 114 | for d in self.datasets : 115 | d.reroll() 116 | 117 | def next(self, layerList=None, strict=False) : 118 | """returns the next batch of data 119 | 120 | :param list layerList: The list of layers for which to return data. If None, will return for all layers. 121 | :param bool strict: Will raise a KeyError exception if a layer from the list has no associated map, default behaviour is to ignore it. 122 | 123 | """ 124 | if self.miniBatchSize is None : 125 | if self.batchNumber > 0 : 126 | self.batchNumber = 0 127 | self.epochNumber += 1 128 | self.reroll() 129 | raise StopIteration("That was the last batch") 130 | 131 | batch = self.getAll(layerList, strict) 132 | else : 133 | try : 134 | batch = self.getBatch(self.batchNumber*self.miniBatchSize, self.miniBatchSize, layerList, strict) 135 | except IndexError : 136 | self.batchNumber = 0 137 | self.epochNumber += 1 138 | self.reroll() 139 | raise StopIteration("That was the last batch") 140 | 141 | self.batchNumber += 1 142 | return batch 143 | 144 | def getBatch(self, i, size, layerList=None, strict=False) : 145 | """ 146 | Returns a dictionary:: 147 | 148 | layer.name => elements 149 | 150 | :param list layerList: The list of layers for which to return data. If None, will return for all layers. 151 | :param bool strict: Will raise a KeyError exception if a layer from the list has no associated map, default behaviour is to ignore it. 152 | """ 153 | if i >= self.minLength : 154 | raise IndexError("index i '%s', out of range '%s'" % (i, size)) 155 | 156 | res = {} 157 | if layerList is None : 158 | layers = self.maps.keys() 159 | else : 160 | layers = layerList 161 | 162 | for l in layers : 163 | try : 164 | layer = self.layersByName[l] 165 | except KeyError : 166 | layer = l 167 | try : 168 | for name, handle in self.maps[layer] : 169 | res[name] = handle.dataset.get(handle.subset, i, size) 170 | except KeyError: 171 | if strict : 172 | raise KeyError("There's no layer: %s" % layer) 173 | return res 174 | 175 | def getAll(self, layerList=None, strict=False) : 176 | """Same as getBatch() but returns the totality of the subset for each layer 177 | 178 | :param list layerList: The list of layers for which to return data. If None, will return for all layers. 179 | :param bool strict: Will raise a KeyError exception if a layer from the list has no associated map, default behaviour is to ignore it. 180 | """ 181 | 182 | if layerList is None : 183 | layers = self.maps.keys() 184 | else : 185 | layers = layerList 186 | 187 | res = {} 188 | for l in layers : 189 | try : 190 | layer = self.layersByName[l] 191 | except KeyError : 192 | layer = l 193 | try : 194 | for name, handle in self.maps[layer] : 195 | res[name] = handle.dataset.getAll(handle.subset) 196 | except KeyError: 197 | if strict : 198 | raise KeyError("There's no layer: %s" % layer) 199 | return res 200 | 201 | def getMinFullLength(self) : 202 | return self.minFullLength 203 | 204 | def __len__(self) : 205 | return self.minLength 206 | 207 | def __iter__(self): 208 | return self 209 | 210 | -------------------------------------------------------------------------------- /Mariana/HTML_Templates/vulcan/vulcan.py: -------------------------------------------------------------------------------- 1 | import inspect, sys, os, shutil 2 | import Mariana.HTML_Templates.template as MTMP 3 | 4 | 5 | class Vulcan(MTMP.HTMLTemplate_ABC): 6 | """A theme""" 7 | TEMPLATE=""" 8 | 9 | 10 | 11 | {TITLE} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 |
23 |
24 |
25 |

26 | 27 | Documentation 28 | 29 |

30 |

{TITLE}

31 |

32 |
33 | {MODEL_NOTES} 34 |
35 | 36 | 37 | 38 |
39 |
40 |
41 | 42 |
43 |
44 |
45 | 46 | 47 | 48 | 49 | 50 | 51 | """ 52 | 53 | def __init__(self): 54 | super(Vulcan, self).__init__() 55 | self.dirname = os.path.dirname(inspect.getfile(sys.modules[__name__])) 56 | 57 | # f = open(os.path.join(self.dirname, "vulcan.html")) 58 | # self.html = f.read() 59 | # f.close() 60 | 61 | self.weblibsDir = os.path.join(self.dirname, "weblibs") 62 | # self.libs = """ 63 | 64 | # """ 65 | 66 | # self.libs_remote = """ 67 | # 68 | # 69 | # 70 | # 71 | # 72 | # 73 | # 74 | # """ 75 | 76 | def formatNotes(self, notes) : 77 | tmp = """ 78 |
79 | {trs} 80 |
81 | """ 82 | 83 | tmpTrs =""" 84 |
85 |

{title}

86 |

{text}

87 |
88 | """ 89 | 90 | trs = [] 91 | for k, v in notes.iteritems() : 92 | trs.append(tmpTrs.format(title = k, text = v)) 93 | 94 | return tmp.format(trs = "\n".join(trs)) 95 | 96 | def render(self, filename, networkJson, save) : 97 | import time 98 | import json 99 | 100 | title = os.path.basename(filename) 101 | noSpaceTitle = title.replace(" ", "-") 102 | if save : 103 | libsFolder = "%s_weblibs" % noSpaceTitle 104 | currFolder = os.path.dirname(filename) 105 | else : 106 | libsFolder = "https://cdn.rawgit.com/tariqdaouda/Mariana/V2-dev/Mariana/HTML_Templates/vulcan/weblibs/" 107 | 108 | layers = [] 109 | for l in networkJson["layers"] : 110 | absCheck = set() 111 | dct = {"name": l, "shape": networkJson["layers"][l]['shape'], "level": networkJson["layers"][l]['level'], "abstractions": {"size": 1, "layer":[{"name": "Class", "value": networkJson["layers"][l]['class']}]} } 112 | for cat in ["parameters", "hyperParameters", "notes"] : 113 | dct[cat] = {"size": 0} 114 | dct[cat]["layer"] = [] 115 | for pName, pVal in networkJson["layers"][l][cat].iteritems() : 116 | if cat == "notes" : 117 | pKey = pName 118 | else : 119 | pKey = "%s.%s" % (l, pName) 120 | 121 | dct[cat]["layer"].append({"name": pKey, "value": pVal}) 122 | dct[cat]["size"] += 1 123 | 124 | for absCat, abstractions in networkJson["layers"][l]["abstractions"].iteritems() : 125 | dct[cat][absCat] = [] 126 | if absCat not in dct["abstractions"] : 127 | dct["abstractions"][absCat] = [] 128 | 129 | for absName, absVal in abstractions.iteritems() : 130 | if absName not in absCheck : 131 | absCheck.add(absName) 132 | dct["abstractions"]["size"] += 1 133 | dct["abstractions"][absCat].append({"name": absName, "value": ""}) 134 | 135 | try : 136 | for pName, pVal in absVal[cat].iteritems() : 137 | if cat == "notes" : 138 | pKey = pName 139 | else : 140 | pKey = "{absName}.{pName}".format(absName = absName, pName = pName) 141 | 142 | dct[cat][absCat].append({"name": pKey, "value": pVal}) 143 | dct[cat]["size"] += 1 144 | except KeyError : 145 | pass 146 | layers.append([l, dct]) 147 | 148 | # if save : 149 | # libs = self.libs_local 150 | # else : 151 | # libs = self.libs_remote 152 | 153 | html = self.TEMPLATE.format( 154 | TITLE=title, 155 | # LIBS=libs, 156 | LIBS_FOLDER=libsFolder, 157 | DOCUMENTATION_URL="http://bioinfo.iric.ca/~daoudat/Mariana/", 158 | GITHUB_URL="https://github.com/tariqdaouda/Mariana", 159 | MODEL_NOTES=self.formatNotes(networkJson["notes"]), 160 | MACHINE_TIME=time.time(), 161 | USER_TIME=time.ctime().replace("_", " "), 162 | LAYERS_JSON=json.dumps(layers), 163 | EDGES_JSON=json.dumps(networkJson["edges"]) 164 | ) 165 | 166 | if save : 167 | if os.path.isdir(self.weblibsDir) : 168 | dstFolder = os.path.join(currFolder, libsFolder) 169 | if os.path.isdir(dstFolder) : 170 | shutil.rmtree(dstFolder) 171 | shutil.copytree(self.weblibsDir, dstFolder) 172 | 173 | f = open(os.path.join(currFolder, "%s.html" % noSpaceTitle), "w") 174 | f.write(html) 175 | f.close() 176 | 177 | return html -------------------------------------------------------------------------------- /Mariana/initializations.py: -------------------------------------------------------------------------------- 1 | import Mariana.settings as MSET 2 | import Mariana.useful as MUSE 3 | import Mariana.abstraction as MABS 4 | import Mariana.activations as MA 5 | 6 | import lasagne.init as LI 7 | 8 | import numpy 9 | import theano 10 | import theano.tensor as tt 11 | 12 | __all__= [ 13 | "Initialization_ABC", 14 | "Identity", 15 | "HardSet", 16 | "SingleValue", 17 | "Normal", 18 | "Uniform", 19 | "FanInFanOut_ABC", 20 | "GlorotNormal", 21 | "GlorotUniform", 22 | "HeNormal", 23 | "HeUniform", 24 | ] 25 | 26 | class Initialization_ABC(MABS.UntrainableAbstraction_ABC, MABS.Apply_ABC) : 27 | """This class defines the interface that an Initialization must offer. 28 | 29 | :param string parameter: the name of the parameter to be initialized 30 | :param float parameter: how sparse should the result be. 0 => no sparsity, 1 => a matrix of zeros 31 | """ 32 | 33 | def __init__(self, parameter, sparsity=0., **kwargs): 34 | super(Initialization_ABC, self).__init__(**kwargs) 35 | 36 | self.addHyperParameters({ 37 | "parameter": parameter, 38 | "sparsity": sparsity 39 | }) 40 | 41 | # def _apply(self, abstraction, **kwargs) : 42 | # if not abstraction.hasP(self.getHP("parameter")) : 43 | # raise ValueError ("'%s' does not have a parameter '%s'" % (abstraction, self.getHP("parameter") ) ) 44 | 45 | # if not abstraction.getP(self.getHP("parameter")).isTied() : 46 | # super(Initialization_ABC, self)._apply(abstraction, **kwargs) 47 | 48 | def logApply(self, layer, **kwargs) : 49 | message = "Applying '%s' on parameter: '%s' of layer '%s'" % (self.name, self.getHP('parameter'), layer.name) 50 | self.logEvent(message) 51 | 52 | def apply(self, abstraction, **kwargs) : 53 | 54 | retShape = abstraction._getParameterShape_abs(self.getHP("parameter")) 55 | # print self 56 | v = MUSE.iCast_numpy(self.run(retShape)) 57 | if (v.shape != retShape) : 58 | raise ValueError("Initialization has a wrong shape: %s, parameter shape is: %s " % (v.shape, retShape)) 59 | 60 | v = MUSE.sparsify(v, self.getHP("sparsity")) 61 | v = numpy.asarray(v, dtype = theano.config.floatX) 62 | # print self, v.dtype 63 | abstraction.setP(self.getHP("parameter"), v) 64 | 65 | def run(self, shape) : 66 | """The function that all Initialization_ABCs must implement""" 67 | raise NotImplemented("This one should be implemented in child") 68 | 69 | class Null(Initialization_ABC) : 70 | """Return None, mainly for some cases of lasagne compatibility""" 71 | def run(self, shape) : 72 | return None 73 | 74 | class Identity(Initialization_ABC) : 75 | """Identity matrix. Its your job to make sure that the parameter is a square matrix""" 76 | def run(self, shape) : 77 | sv = None 78 | for s in shape : 79 | if not sv : 80 | sv = s 81 | elif sv != s : 82 | raise ValueError("Shape must be square, got: %s" % shape) 83 | 84 | v = numpy.identity(shape, dtype = theano.config.floatX) 85 | return v 86 | 87 | class HardSet(Initialization_ABC) : 88 | """Sets the parameter to value. It's your job to make sure that the shape is correct""" 89 | def __init__(self, parameter, value, **kwargs) : 90 | super(HardSet, self).__init__(**kwargs) 91 | self.value = numpy.asarray(value, dtype=theano.config.floatX) 92 | 93 | def run(self, shape) : 94 | return self.value 95 | 96 | class SingleValue(Initialization_ABC) : 97 | """Initialize to a given value""" 98 | def __init__(self, parameter, value, **kwargs) : 99 | super(SingleValue, self).__init__(parameter, **kwargs) 100 | self.setHP("value", value) 101 | 102 | def run(self, shape) : 103 | return numpy.ones(shape) * self.getHP("value") 104 | 105 | class Normal(Initialization_ABC): 106 | """ 107 | Initializes using a random normal distribution. 108 | **Small** uses my personal initialization than I find works very well in most cases with a uniform distribution, simply divides by the sum of the weights. 109 | """ 110 | def __init__(self, parameter, mean=0, std=1, small=False, **kwargs): 111 | super(Normal, self).__init__(parameter, **kwargs) 112 | self.addHyperParameters({ 113 | "std": std, 114 | "mean": mean, 115 | "small": small 116 | }) 117 | 118 | def run(self, shape) : 119 | v = numpy.random.normal(self.getHP("mean"), self.getHP("std"), size=shape) 120 | if self.getHP("small") : 121 | return v / numpy.sum(v) 122 | return v 123 | 124 | class Uniform(Initialization_ABC): 125 | """ 126 | Initializes using a uniform distribution 127 | **Small** uses my personal initialization than I find can work very well, simply divides by the sum of the weights. 128 | """ 129 | def __init__(self, parameter, low=0, high=1, small=False, **kwargs): 130 | super(Uniform, self).__init__(parameter, **kwargs) 131 | self.setHP("low", int(low)) 132 | self.setHP("high", int(high)) 133 | self.setHP("small", small) 134 | 135 | def run(self, shape) : 136 | v = numpy.random.uniform(low=self.getHP("low"), high=self.getHP("high"), size=shape) 137 | if self.getHP("small") : 138 | return v / sum(v) 139 | return v 140 | 141 | class FanInFanOut_ABC(Initialization_ABC) : 142 | """ 143 | Abtract class for fan_in/_out inits (Glorot and He) 144 | Over the time people have introduced 145 | ways to make it work with other various activation functions by modifying a gain factor. 146 | You can force the gain using the *forceGain* argument, otherwise Mariana will choose 147 | one for you depending on the abstraction's activation. 148 | 149 | * ReLU: sqrt(2) 150 | 151 | * LeakyReLU: sqrt(2/(1+alpha**2)) where alpha is the leakiness 152 | 153 | * Everything else : 1.0 154 | 155 | This is an abtract class: see *GlorotNormal*, *GlorotUniform* 156 | """ 157 | def __init__(self, parameter, forceGain=None, **kwargs) : 158 | super(FanInFanOut_ABC, self).__init__(parameter, **kwargs) 159 | self.setHP("forceGain", forceGain) 160 | self.gain = None 161 | 162 | @classmethod 163 | def _getGain(cls, activation) : 164 | """returns the gain with respesct to an activation function""" 165 | if activation.__class__ is MA.ReLU : 166 | if activation.leakiness == 0 : 167 | return numpy.sqrt(2) 168 | else : 169 | return numpy.sqrt(2/(1+activation.leakiness**2)) 170 | return 1.0 171 | 172 | def setup(self, abstraction) : 173 | self.gain = self._getGain(abstraction.abstractions["activation"]) 174 | 175 | def apply(self, abstraction) : 176 | import Mariana.activations as MA 177 | 178 | forceGain = self.getHP("forceGain") 179 | if forceGain : 180 | self.gain = forceGain 181 | else : 182 | self.gain = self._getGain(abstraction.abstractions["activation"]) 183 | 184 | return super(FanInFanOut_ABC, self).apply(abstraction) 185 | 186 | class XNormal(FanInFanOut_ABC) : 187 | """ 188 | Initialization strategy introduced by Glorot et al. 2010 on a Normal distribution. 189 | Uses lasagne as backend. 190 | """ 191 | def run(self, shape) : 192 | return LI.GlorotNormal(gain = self.gain).sample(shape) 193 | 194 | XavierNormal = XNormal 195 | GlorotNormal = XNormal 196 | 197 | class XUniform(FanInFanOut_ABC) : 198 | """ 199 | Initialization strategy introduced by Glorot et al. 2010 on a Uniform distribution. 200 | If you use tanh() as activation try this one first. 201 | Uses lasagne as backend. 202 | """ 203 | def run(self, shape) : 204 | return LI.GlorotUniform(gain = self.gain).sample(shape) 205 | 206 | XavierUniform = XUniform 207 | GlorotUniform = XUniform 208 | 209 | class HeNormal(FanInFanOut_ABC) : 210 | """ 211 | Initialization proposed by He et al. for ReLU in *Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification*, 2015. 212 | 213 | On a Normal distribution, Uses lasagne as backend. 214 | """ 215 | def run(self, shape) : 216 | return LI.HeNormal(gain = self.gain).sample(shape) 217 | 218 | class HeUniform(FanInFanOut_ABC) : 219 | """ 220 | Initialization proposed by He et al. for ReLU in *Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification*, 2015. 221 | 222 | On a Uniform distribution, Uses lasagne as backend. 223 | """ 224 | def run(self, shape) : 225 | return LI.HeUniform(gain = self.gain).sample(shape) -------------------------------------------------------------------------------- /Mariana/abstraction.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | import Mariana.custom_types as MTYPES 3 | 4 | __all__ = ["Logger_ABC", "Abstraction_ABC", "UntrainableAbstraction_ABC", "TrainableAbstraction_ABC", "Apply_ABC"] 5 | 6 | class Logger_ABC(object): 7 | """Interface for objects that log events""" 8 | def __init__(self, **kwargs): 9 | super(Logger_ABC, self).__init__() 10 | self.log = [] 11 | self.notes = OrderedDict() 12 | 13 | def logEvent(self, message, **moreStuff) : 14 | """log an event""" 15 | import time 16 | 17 | entry = { 18 | "date": time.ctime(), 19 | "timestamp": time.time(), 20 | "message": message, 21 | } 22 | entry.update(moreStuff) 23 | self.log.append(entry) 24 | 25 | def getLog(self) : 26 | """return the log""" 27 | return self.log 28 | 29 | def addNote(self, title, text) : 30 | """add a note""" 31 | self.notes[title] = text 32 | 33 | def printLog(self) : 34 | """JSON pretty printed log""" 35 | import json 36 | print json.dumps(self.getLog(), indent=4, sort_keys=True) 37 | 38 | class Abstraction_ABC(Logger_ABC): 39 | """ 40 | This class represents a layer modifier. This class must includes a list attribute **self.hyperParameters** containing the names of all attributes that must be considered 41 | as hyper-parameters. 42 | """ 43 | def __init__(self, streams=["train", "test"], **kwargs): 44 | super(Abstraction_ABC, self).__init__() 45 | 46 | self.streams = streams 47 | self.hyperParameters = OrderedDict() 48 | 49 | self._mustInit=True 50 | 51 | def isTrainable(self) : 52 | raise NotImplementedError("Must be implemented in child") 53 | 54 | def getParameters(self) : 55 | raise NotImplementedError("Must be implemented in child") 56 | 57 | def addHyperParameters(self, dct) : 58 | """adds to the list of hyper params, dct must be a dict""" 59 | self.hyperParameters.update(dct) 60 | 61 | def setHyperparameter(k, v) : 62 | """sets a single hyper parameter""" 63 | self.setHP(k, v) 64 | 65 | def getHyperparameter(k, v) : 66 | """get a single hyper parameter""" 67 | self.getHP(k, v) 68 | 69 | def setHP(self, k, v) : 70 | """setHyperparameter() alias""" 71 | self.hyperParameters[k] = v 72 | 73 | def getHP(self, k) : 74 | """getHyperparameter() alias""" 75 | return self.hyperParameters[k] 76 | 77 | def getHyperParameters(k, v) : 78 | """return all hyper parameter""" 79 | return self.hyperParameters 80 | 81 | def toDictionary(self) : 82 | """A dct representation of the object""" 83 | res = { 84 | "name": str(self.name), 85 | "hyperParameters": OrderedDict(self.hyperParameters), 86 | "notes": OrderedDict(self.notes), 87 | "class": self.__class__.__name__ 88 | } 89 | 90 | return res 91 | 92 | def __repr__(self) : 93 | return "< %s, hps: %s >" % (self.__class__.__name__, dict(self.hyperParameters)) 94 | 95 | class UntrainableAbstraction_ABC(Abstraction_ABC): 96 | """docstring for UntrainableAbstraction_ABC""" 97 | 98 | def isTrainable(self) : 99 | return False 100 | 101 | def getParameters(self) : 102 | return {} 103 | 104 | def __hash__(self) : 105 | return hash(self.__class__.__name__ + str(self.hyperParameters.items())) 106 | 107 | def __eq__(self, a) : 108 | return self.__class__ is a.__class__ and self.hyperParameters == a.hyperParameters 109 | 110 | class TrainableAbstraction_ABC(Abstraction_ABC): 111 | """docstring for TrainableAbstraction_ABC""" 112 | def __init__(self, initializations=[], learningScenari=[], regularizations=[], **kwargs): 113 | super(TrainableAbstraction_ABC, self).__init__(**kwargs) 114 | 115 | self.abstractions={ 116 | "initializations": initializations, 117 | "learningScenari": learningScenari, 118 | "regularizations": regularizations, 119 | } 120 | 121 | self.parameters = OrderedDict() 122 | # self._mustInit = True 123 | 124 | def getAbstractions(self) : 125 | res = [] 126 | for absts in self.abstractions.itervalues() : 127 | for ab in absts : 128 | res.append(ab) 129 | return res 130 | 131 | def isTrainable(self) : 132 | return True 133 | 134 | def setParameter(k, v) : 135 | """Brutally set the value of a parameter. No checks applied""" 136 | self.setP(k, v) 137 | 138 | def setParameters(self, dct) : 139 | """to set a bunch of them with a dict""" 140 | for k, v in dct.iteritems() : 141 | self.setP(k, v) 142 | 143 | def getParameter(self, k) : 144 | """get a single parameter""" 145 | self.getP(k) 146 | 147 | def setP(self, param, value) : 148 | """setParameter() alias""" 149 | if isinstance(value, MTYPES.Parameter) : 150 | self.parameters[param] = value 151 | else : 152 | self.parameters[param].setValue(value) 153 | 154 | def getP(self, k) : 155 | """getParameter() alias""" 156 | return self.parameters[k] 157 | 158 | def getParameters(self) : 159 | """return all parameter""" 160 | return self.parameters 161 | 162 | def hasParameter(self, p) : 163 | """returns wether I have parameter called p""" 164 | return p in self.parameters 165 | 166 | def hasP(self, p) : 167 | """hasParameter() alias""" 168 | return self.hasParameter(p) 169 | 170 | def _getParameterShape_abs(self, param) : 171 | if param not in self.parameters : 172 | raise ValueError("Unknown parameter: %s for %s" % (param, self)) 173 | return self.getParameterShape_abs(param) 174 | 175 | def getParameterShape_abs(self, param) : 176 | """Should return the shape of the parameter. This has to be implemented in order for the initializations to work (and maybe some other stuff as well)""" 177 | raise NotImplementedError("Should be implemented in child") 178 | 179 | def _parametersSanityCheck(self) : 180 | "perform basic parameter checks on layers, automatically called on initialization" 181 | for k, v in self.getParameters().iteritems() : 182 | if not v.isSet() : 183 | raise ValueError("Parameter '%s' of '%s' has not been initialized" % (k, self.name) ) 184 | 185 | def _initParameters(self, forceReset=False) : 186 | """creates the parameters if necessary""" 187 | selfParams = set() 188 | for k, v in self.parameters.iteritems() : 189 | if not v.isSet() : 190 | selfParams.add(k) 191 | 192 | initParams = set() 193 | if self._mustInit or forceReset : 194 | for init in self.abstractions["initializations"] : 195 | if not self.getP(init.getHP("parameter")).isSet() : 196 | init._apply(self) 197 | initParams.add(init.getHP("parameter")) 198 | # print len(selfParams), len(initParams) 199 | # print self, selfParams, initParams 200 | if len(selfParams) != len(initParams) : 201 | raise ValueError("Parameters: %s of %s, where not supplied initializations" % ((selfParams - initParams), self) ) 202 | 203 | self._mustInit=False 204 | 205 | def toDictionary(self) : 206 | """A dct representation of the object""" 207 | 208 | res = super(TrainableAbstraction_ABC, self).toDictionary() 209 | ps = OrderedDict() 210 | for k, v in self.parameters.iteritems() : 211 | ps[k] = {"shape": self.getParameterShape_abs(k)} 212 | 213 | res["parameters"] = ps 214 | 215 | return res 216 | 217 | def __repr__(self) : 218 | return "< %s, hps: %s, ps: %s >" % (self.__class__.__name__, self.hyperParameters, self.parameters) 219 | 220 | class Apply_ABC(object): 221 | """Interface for abstractions that are applyied to other abstractions (all but layers)""" 222 | 223 | def __init__(self, **kwargs): 224 | super(Apply_ABC, self).__init__() 225 | self.name = self.__class__.__name__ 226 | self._mustInit=True 227 | # self.parent = None 228 | 229 | # def setParent(self, layer) : 230 | # message = "'%s' has been claimed by '%s'" % (self.name, layer.name) 231 | # self.logEvent(message) 232 | # self.parent = layer 233 | 234 | # def unsetParent(self) : 235 | # message = "'%s' is once again unclaimed" % (self.name) 236 | # self.logEvent(message) 237 | # self.parent = None 238 | 239 | def logApply(self, layer, **kwargs) : 240 | message = "Applying : '%s' on layer '%s'" % (self.name, layer.name) 241 | self.logEvent(message) 242 | 243 | def setup(self, abstraction) : 244 | """setups the untrainable abstraction for a given trainable abstraction. Called just before apply(). By default does nothing, but allows some degree of fine tuning if necesssary""" 245 | pass 246 | 247 | def _apply(self, layer, **kwargs) : 248 | """Logs and calls self.apply()""" 249 | self.setup(layer) 250 | self.logApply(layer, **kwargs) 251 | self.apply(layer, **kwargs) 252 | 253 | def apply(self, layer, **kwargs) : 254 | """Apply to a layer, basically logs stuff and then calls run""" 255 | raise NotImplementedError("Must be implemented in child") 256 | 257 | def run(self, **kwargs) : 258 | """the actual worker function that does whaters the abstraction is supposed to do""" 259 | raise NotImplementedError("Must be implemented in child") 260 | -------------------------------------------------------------------------------- /Mariana/decorators.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import theano 3 | import theano.tensor as tt 4 | import Mariana.settings as MSET 5 | import Mariana.abstraction as MABS 6 | import Mariana.initializations as MI 7 | import Mariana.useful as MUSE 8 | import Mariana.custom_types as MTYPES 9 | 10 | __all__= ["Decorator_ABC", "BatchNormalization", "Center", "Normalize", "Mask", "RandomMask", "BinomialDropout", "Clip", "AdditiveGaussianNoise", "MultiplicativeGaussianNoise"] 11 | 12 | class Decorator_ABC(MABS.TrainableAbstraction_ABC, MABS.Apply_ABC) : 13 | """A decorator is a modifier that is applied on a layer's output. They are always the last the abstraction to be applied.""" 14 | 15 | def __init__(self, streams, **kwargs): 16 | super(Decorator_ABC, self).__init__(**kwargs) 17 | self.streams = set(self.streams) 18 | self.setHP("streams", streams) 19 | 20 | def apply(self, layer, stream) : 21 | """Apply to a layer and update networks's log""" 22 | if stream in self.streams : 23 | return self.run(layer, stream=stream) 24 | 25 | def run(self, layer, stream) : 26 | """The function that all decorator_ABCs must implement""" 27 | raise NotImplemented("This one should be implemented in child") 28 | 29 | class Mask(Decorator_ABC): 30 | """Applies a fixed mask to the outputs of the layer. This layers does this: 31 | 32 | .. math:: 33 | 34 | outputs = outputs * mask 35 | 36 | If you want to remove some parts of the output use 0s, if you want to keep them as they are use 1s. 37 | Anything else will lower or increase the values by the given factors. 38 | 39 | :param array/list mask: It should have the same dimensions as the layer's outputs. 40 | 41 | """ 42 | def __init__(self, mask, streams=["train"], **kwargs): 43 | super(Mask, self).__init__(streams, **kwargs) 44 | self.mask = tt.cast(mask, theano.config.floatX) 45 | self.setHP("mask", mask) 46 | 47 | def run(self, layer, stream) : 48 | layer.outputs[stream] = layer.outputs[stream] * self.mask 49 | 50 | class RandomMask(Decorator_ABC): 51 | """ 52 | This decorator takes a list of masks and will randomly apply them to the outputs of the layer it runs. 53 | Could be used as a fast approximation for dropout. 54 | """ 55 | def __init__(self, masks, streams=["train"], **kwargs): 56 | super(RandomMask, self).__init__(streams, **kwargs) 57 | self.masks = tt.cast(mask, theano.config.floatX) 58 | self.setHP("masks", masks) 59 | 60 | def run(self, layer, stream) : 61 | rnd = tt.shared_randomstreams.RandomStreams(seed=MSET.RANDOM_SEED) 62 | maskId = rnd.random_integers(low=0, high=self.nbMasks-1, ndim=1) 63 | mask = self.masks[maskId] 64 | 65 | layer.outputs[stream] = layer.outputs[stream] * mask 66 | 67 | class BinomialDropout(Decorator_ABC): 68 | """Stochastically mask some parts of the output of the layer. Use it to make things such as denoising autoencoders and dropout layers""" 69 | def __init__(self, dropoutRatio, streams=["train"], **kwargs): 70 | super(BinomialDropout, self).__init__(streams, **kwargs) 71 | assert (dropoutRatio >= 0 and dropoutRatio <= 1) 72 | # self.dropoutRatio = dropoutRatio 73 | # self.seed = MSET.RANDOM_SEED 74 | self.setHP("dropoutRatio", dropoutRatio) 75 | 76 | def run(self, layer, stream) : 77 | if self.getHP("dropoutRatio") > 0 : 78 | rnd = tt.shared_randomstreams.RandomStreams(seed=MSET.RANDOM_SEED) 79 | mask = rnd.binomial(n = 1, p = (1-self.getHP("dropoutRatio")), size = layer.outputs[stream].shape, dtype=theano.config.floatX) 80 | # cast to stay in GPU float limit 81 | mask = MUSE.iCast_theano(mask) 82 | layer.outputs[stream] = layer.outputs[stream] * mask 83 | 84 | class Center(Decorator_ABC) : 85 | """Centers the outputs by substracting the mean""" 86 | def __init__(self, streams=["train"], **kwargs): 87 | super(Center, self).__init__(streams, **kwargs) 88 | 89 | def run(self, layer, stream) : 90 | layer.outputs[stream] = layer.outputs[stream]-tt.mean(layer.outputs[stream]) 91 | 92 | class Normalize(Decorator_ABC) : 93 | """ 94 | Normalizes the outputs by substracting the mean and dividing by the standard deviation 95 | 96 | :param float epsilon: Actually it is not the std that is used but the approximation: sqrt(Variance + epsilon). Use this parameter to set the epsilon value 97 | """ 98 | 99 | def __init__(self, epsilon=1e-6, streams=["train"]) : 100 | super(Normalize, self).__init__(streams, **kwargs) 101 | self.setHP("epsilon", epsilon) 102 | 103 | def run(self, layer, stream) : 104 | std = tt.sqrt( tt.var(layer.outputs[stream]) + self.epsilon ) 105 | layer.outputs[stream] = ( layer.outputs[stream]-tt.mean(layer.output) / std ) 106 | 107 | # class BatchNormalization(Decorator_ABC): 108 | # """Applies Batch Normalization to the outputs of the layer. 109 | # Implementation according to Sergey Ioffe and Christian Szegedy (http://arxiv.org/abs/1502.03167) 110 | 111 | # .. math:: 112 | 113 | # \\gamma * \\frac{x - \\mu}{\\sqrt{\\sigma^2 + \\epsilon}} + \\beta 114 | 115 | # Where \\gamma and \\beta are learned and std stands for the standard deviation. The mean and the std are computed accross the whole minibatch. 116 | 117 | # :param float epsilon: Actually it is not the std that is used but the approximation: sqrt(Variance + epsilon). Use this parameter to set the epsilon value 118 | # """ 119 | 120 | # def __init__(self, testMu, testSigma, initializations=[MI.SingleValue('gamma', 1), MI.SingleValue('beta', 0)], epsilon=1e-6, streams=["train", "test"], **kwargs) : 121 | # super(BatchNormalization, self).__init__(initializations=initializations, streams=streams, **kwargs) 122 | # self.setHP("testMu", testMu) 123 | # self.setHP("testSigma", testSigma) 124 | # self.setHP("epsilon", epsilon) 125 | 126 | # self.addParameters({ 127 | # "gamma": MTYPES.Parameter("gamma"), 128 | # "beta": MTYPES.Parameter("beta") 129 | # }) 130 | 131 | # def getParameterShape_abs(self, param, **kwargs) : 132 | # return self.parent.getShape_abs() 133 | 134 | # def run(self, layer, stream) : 135 | # if stream == "train" : 136 | # mu = tt.mean(layer.outputs[stream]) 137 | # sigma = tt.sqrt( tt.var(layer.outputs[stream]) + self.getHP("epsilon") ) 138 | # elif stream == "test" : 139 | # mu = self.getHP("testMu") 140 | # sigma = self.getHP("testSigma") 141 | 142 | # layer.outputs[stream] = self.getP("gamma")() * ( (layer.outputs[stream] - mu) / sigma ) + self.getP("beta")() 143 | 144 | class Clip(Decorator_ABC): 145 | """Clips the neurone activations, preventing them to go beyond the specified range""" 146 | 147 | def __init__(self, lower, upper, streams=["train"], **kwargs) : 148 | super(Clip, self).__init__(streams, **kwargs) 149 | assert lower < upper 150 | self.setHP("lower", lower) 151 | self.setHP("upper", upper) 152 | 153 | def run(self, layer, stream) : 154 | layer.outputs[stream] = layer.outputs[stream].clip(self.lower, self.upper) 155 | 156 | # class ForceUnitRange(Decorator_ABC): 157 | # """Force the output to be between [0, 1]""" 158 | 159 | # def __init__(self, streams=["train"], **kwargs): 160 | # super(ForceUnitRange, self).__init__(streams, **kwargs) 161 | 162 | # def run(self, layer, stream) : 163 | # layer.outputs[stream] = (layer.outputs[stream] - min(layer.outputs[stream])) 164 | # layer.outputs[stream] /= max(layer.outputs[stream]) 165 | 166 | class AddGaussianNoise(Decorator_ABC): 167 | """Add gaussian noise to the output of the layer""" 168 | 169 | def __init__(self, std, avg=0, streams=["train"], **kwargs): 170 | assert std > 0 171 | super(AddGaussianNoise, self).__init__(streams, **kwargs) 172 | self.setHP("std", std) 173 | self.setHP("avg", avg) 174 | 175 | def run(self, layer, stream) : 176 | rnd = tt.shared_randomstreams.RandomStreams(seed=MSET.RANDOM_SEED) 177 | randomVals = rnd.normal(size = layer.getIntrinsicShape(), avg=self.getHP("avg"), std=self.getHP("std") ) 178 | layer.outputs[stream] = layer.outputs[stream] + randomVals 179 | 180 | class MultGaussianNoise(Decorator_ABC): 181 | """Multiply gaussian noise to the output of the layer""" 182 | 183 | def __init__(self, std, avg=0, streams=["train"], **kwargs): 184 | assert std > 0 185 | super(MultGaussianNoise, self).__init__(streams, **kwargs) 186 | self.setHP("std", std) 187 | self.setHP("avg", avg) 188 | 189 | def run(self, layer, stream) : 190 | rnd = tt.shared_randomstreams.RandomStreams(seed=MSET.RANDOM_SEED) 191 | randomVals = rnd.normal(size = layer.getIntrinsicShape(), avg=self.getHP("avg"), std=self.getHP("std") ) 192 | layer.outputs[stream] = layer.outputs[stream] * randomVals 193 | 194 | class Scale(Decorator_ABC): 195 | """Multiplies the output by scale""" 196 | 197 | def __init__(self, scale, streams=["train", "test"], **kwargs): 198 | super(Scale, self).__init__(streams, **kwargs) 199 | self.setHP("scale", scale) 200 | 201 | def run(self, layer, stream) : 202 | layer.outputs[stream] = layer.outputs[stream] * self.getHP("scale") 203 | 204 | class Shift(Decorator_ABC): 205 | """Shifts (addiction) the output by scale""" 206 | 207 | def __init__(self, shift, streams=["train", "test"], **kwargs): 208 | super(Shift, self).__init__(streams, **kwargs) 209 | self.setHP("shift", shift) 210 | 211 | def run(self, layer, stream) : 212 | layer.outputs[stream] = layer.outputs[stream] + self.getHP("shift") -------------------------------------------------------------------------------- /Mariana/training/trainers.py: -------------------------------------------------------------------------------- 1 | import cPickle, time, sys, os, traceback, types, json, signal 2 | import numpy 3 | from collections import OrderedDict 4 | import theano.tensor as tt 5 | 6 | import Mariana.network as MNET 7 | import Mariana.training.future.recorders as MREC 8 | import Mariana.training.future.stopcriteria as MSTOP 9 | import Mariana.candies as MCAN 10 | import Mariana.settings as MSET 11 | 12 | __all__ = ["Trainer_ABC", "DefaultTrainer"] 13 | 14 | class OutputTrainingSchedule_ABC(object) : 15 | 16 | def __init__(self, name) : 17 | self.name = self.__class__.__name__ 18 | 19 | def getOutput(self, aMap) : 20 | """return an ouput to be trained on""" 21 | NotImplemented("Must be implemented in child") 22 | 23 | # class Synchronous(OutputTrainingSchedule_ABC) : 24 | 25 | # def __init__(self, outputLayers=None) : 26 | # self.outputLayers = outputLayers 27 | 28 | # def getOutput(self, aMap) : 29 | # layerList = aMap.inputLayers 30 | # if self.outputLayers : 31 | # outputs = self.outputLayers 32 | # else : 33 | # outputs = aMap.outputLayers 34 | 35 | # scores = {} 36 | # for batchData in aMap : 37 | # for output in outputs : 38 | # layerList.append(output) 39 | # res = aMap.runFunction(output, **batchData) 40 | # scores[output.name] = {} 41 | # for k, v in res.iteritems() : 42 | # try : 43 | # scores[output.name][k].append(v) 44 | # except KeyError: 45 | # scores[output.name][k] = [v] 46 | # layerList.pop(-1) 47 | 48 | # for output in aMap.outputLayers : 49 | # for k, v in scores[output.name].iteritems() : 50 | # scores[output.name][k] = numpy.mean(v) 51 | 52 | # return scores 53 | 54 | class Trainer_ABC(object) : 55 | """This is the general interface of trainer""" 56 | 57 | def __init__(self) : 58 | """ 59 | The store is initialised to:: 60 | 61 | self.store = { 62 | "runInfos" : { 63 | "epoch" : 0, 64 | "runtime" : 0 65 | }, 66 | "scores" : {} 67 | } 68 | 69 | """ 70 | 71 | self.store = { 72 | "runInfos" : { 73 | "epoch" : 0, 74 | "runtime" : 0 75 | }, 76 | "scores" : {}, 77 | "currentSet": None, 78 | "minibatchOutput": None 79 | } 80 | 81 | def start(self, runName, saveIfMurdered=True, **kwargs) : 82 | """Starts the training and encapsulates it into a safe environement. 83 | If the training stops because of an Exception or SIGTEM, the trainer 84 | will save logs, the store, and the last version of the model. 85 | """ 86 | 87 | import simplejson, signal, cPickle 88 | 89 | def _handler_sig_term(sig, frame) : 90 | _dieGracefully("SIGTERM", None) 91 | sys.exit(sig) 92 | 93 | def _dieGracefully(exType, tb = None) : 94 | if type(exType) is types.StringType : 95 | exName = exType 96 | else : 97 | exName = exType.__name__ 98 | 99 | death_time = time.ctime().replace(' ', '_') 100 | filename = "dx-xb_" + runName + "_death_by_" + exName + "_" + death_time 101 | sys.stderr.write("\n===\nDying gracefully from %s, and saving myself to:\n...%s\n===\n" % (exName, filename)) 102 | self.model.save(filename) 103 | f = open(filename + ".traceback.log", 'w') 104 | f.write("Mariana training Interruption\n=============================\n") 105 | f.write("\nDetails\n-------\n") 106 | f.write("Name: %s\n" % runName) 107 | f.write("pid: %s\n" % os.getpid()) 108 | f.write("Killed by: %s\n" % str(exType)) 109 | f.write("Time of death: %s\n" % death_time) 110 | f.write("Model saved to: %s\n" % filename) 111 | 112 | if tb is not None : 113 | f.write("\nTraceback\n---------\n") 114 | f.write(str(traceback.extract_tb(tb)).replace("), (", "),\n(").replace("[(","[\n(").replace(")]",")\n]")) 115 | f.flush() 116 | f.close() 117 | f = open(filename + ".store.pkl", "wb") 118 | cPickle.dump(self.store, f) 119 | f.close() 120 | 121 | signal.signal(signal.SIGTERM, _handler_sig_term) 122 | if MSET.VERBOSE : 123 | print "\n" + "Training starts." 124 | MCAN.friendly("Process id", "The pid of this run is: %d" % os.getpid()) 125 | 126 | try : 127 | return self.run(runName, **kwargs) 128 | except MSTOP.EndOfTraining as e : 129 | print e.message 130 | death_time = time.ctime().replace(' ', '_') 131 | filename = "finished_" + runName + "_" + death_time 132 | f = open(filename + ".stopreason.txt", 'w') 133 | f.write("Name: %s\n" % runName) 134 | f.write("pid: %s\n" % os.getpid()) 135 | f.write("Time of death: %s\n" % death_time) 136 | f.write("Epoch of death: %s\n" % self.store["runInfos"]["epoch"]) 137 | f.write("Stopped by: %s\n" % e.stopCriterion.name) 138 | f.write("Reason: %s\n" % e.message) 139 | 140 | f.flush() 141 | f.close() 142 | model.save(filename) 143 | f = open(filename + ".store.pkl", "wb") 144 | cPickle.dump(self.store, f) 145 | f.close() 146 | 147 | except KeyboardInterrupt : 148 | if not saveIfMurdered : 149 | raise 150 | exType, ex, tb = sys.exc_info() 151 | _dieGracefully(exType, tb) 152 | raise 153 | except : 154 | if not saveIfMurdered : 155 | raise 156 | exType, ex, tb = sys.exc_info() 157 | _dieGracefully(exType, tb) 158 | raise 159 | 160 | def run(self, **kwargs) : 161 | """Abtract function must be implemented in child. This function should implement the whole training process""" 162 | raise NotImplemented("Must be implemented in child") 163 | 164 | class DefaultTrainer(Trainer_ABC) : 165 | """The default trainer should serve for most purposes""" 166 | 167 | def __init__(self, 168 | setMaps, 169 | model, 170 | ouputSchedule=Synchronous(), 171 | # stopCriteria=[], 172 | onStart=[], 173 | onEpochStart=[], 174 | onSetStart=[], 175 | onMiniBatchStart=[], 176 | onMiniBatchEnd=[], 177 | onSetEnd=[], 178 | onEpochEnd=[], 179 | onEnd=[], 180 | ) : 181 | """ 182 | :param DatasetsetMaps trainsetMaps: Layer mappings for the training set 183 | :param DatasetsetMaps testtrainsetMaps: Layer mappings for the testing set 184 | :param DatasetsetMaps validationsetMaps: Layer mappings for the validation set, if you do not wich to set one, pass None as argument 185 | :param int trainMiniBatchSize: The size of a training minibatch, use DefaultTrainer.ALL_SET for the whole set 186 | :param list stopCriteria: List of StopCriterion objects 187 | :param int testMiniBatchSize: The size of a testing minibatch, use DefaultTrainer.ALL_SET for the whole set 188 | :param int validationMiniBatchSize: The size of a validationMiniBatchSize minibatch 189 | :param bool saveIfMurdered: Die gracefully in case of Exception or SIGTERM and save the current state of the model and logs 190 | :param string trainFunctionName: The name of the function to use for training 191 | :param string testFunctionName: The name of the function to use for testing 192 | :param string validationFunctionName: The name of the function to use for testing in validation 193 | """ 194 | 195 | Trainer_ABC.__init__(self) 196 | 197 | assert isinstance(setMaps, dict) 198 | self.setMaps = setMaps 199 | 200 | # self.stopCriteria = stopCriteria 201 | self.onStart = onStart 202 | self.onEnd = onEnd 203 | self.onSetStart = onSetStart 204 | self.onSetEnd = onSetEnd 205 | self.onMiniBatchStart = onMiniBatchStart 206 | self.onMiniBatchEnd = onMiniBatchEnd 207 | self.onEpochStart = onEpochStart 208 | self.onEpochEnd = onEpochEnd 209 | 210 | self.model = model 211 | self.ouputSchedule = ouputSchedule 212 | self.startTime = None 213 | 214 | def start(self, runName) : 215 | """starts the training, cf. run() for the a description of the arguments""" 216 | self.startTime = time.time() 217 | self._runHooks(self.onStart) 218 | 219 | Trainer_ABC.start( self, runName) 220 | 221 | self._runHooks(self.onEnd) 222 | 223 | def _runHooks(self, hooks) : 224 | for hook in hooks : 225 | hook.commit(self) 226 | 227 | def run(self, name) : 228 | """ 229 | :param str runName: The name of this run 230 | :param Recorder recorder: A recorder object 231 | :param int trainingOrder: 232 | * DefaultTrainer.SEQUENTIAL_TRAINING: Each output will be trained indipendetly on it's own epoch 233 | * DefaultTrainer.SIMULTANEOUS_TRAINING: All outputs are trained within the same epoch with the same inputs 234 | * Both are in O(m*n), where m is the number of mini batches and n the number of outputs 235 | * DefaultTrainer.RANDOM_PICK_TRAINING: Will pick one of the outputs at random for each example 236 | 237 | :param bool reset: Should the trainer be reset before starting the run 238 | :param dict moreHyperParameters: If provided, the fields in this dictionary will be included into the log .csv file 239 | """ 240 | while True : 241 | self._runHooks(self.onEpochStart) 242 | 243 | for mapName, aMap in self.setMaps.iteritems() : 244 | self.store["currentMap"] = mapName 245 | self._runHooks(self.onSetStart) 246 | 247 | tmpStore = {} 248 | for batchData in aMap : 249 | self._runHooks(self.onMiniBatchStart) 250 | for output in self.ouputSchedule.getOutputs(self) : 251 | res = aMap.runFunction(output, **batchData) 252 | scores[output.name] = {} 253 | for k, v in res.iteritems() : 254 | try : 255 | tmpStore[k].append(v) 256 | except KeyError : 257 | tmpStore[k] = [v] 258 | self.store["minibatchOutput"] = res 259 | self._runHooks(self.onMiniBatchEnd) 260 | 261 | for output in aMap.outputLayers : 262 | for k, v in scores[output.name].iteritems() : 263 | scores[output.name][k] = numpy.mean(v) 264 | 265 | self._runHooks(self.onSetEnd) 266 | 267 | # self.store["scores"][mapName] = self.ouputSchedule.run(aMap) 268 | 269 | 270 | self.store["runInfos"]["epoch"] += 1 271 | self.store["runInfos"]["runtime"] = (time.time() - self.startTime) 272 | 273 | self._runHooks(self.onEpochEnd) 274 | 275 | # for crit in self.stopCriteria : 276 | # if crit.stop(self) : 277 | # raise MSTOP.EndOfTraining(crit) 278 | 279 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /Mariana/doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Mariana documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Jun 11 13:02:52 2015. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | import shlex 18 | 19 | # If extensions (or modules to document with autodoc) are in another directory, 20 | # add these directories to sys.path here. If the directory is relative to the 21 | # documentation root, use os.path.abspath to make it absolute, like shown here. 22 | #sys.path.insert(0, os.path.abspath('.')) 23 | 24 | # -- General configuration ------------------------------------------------ 25 | 26 | # If your documentation needs a minimal Sphinx version, state it here. 27 | #needs_sphinx = '1.0' 28 | 29 | # Add any Sphinx extension module names here, as strings. They can be 30 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 31 | # ones. 32 | extensions = [ 33 | 'sphinx.ext.autodoc', 34 | 'sphinx.ext.doctest', 35 | 'sphinx.ext.intersphinx', 36 | 'sphinx.ext.todo', 37 | 'sphinx.ext.coverage', 38 | 'sphinx.ext.mathjax', 39 | 'sphinx.ext.ifconfig', 40 | 'sphinx.ext.viewcode', 41 | ] 42 | 43 | # Add any paths that contain templates here, relative to this directory. 44 | templates_path = ['_templates'] 45 | 46 | # The suffix(es) of source filenames. 47 | # You can specify multiple suffix as a list of string: 48 | # source_suffix = ['.rst', '.md'] 49 | source_suffix = '.rst' 50 | 51 | # The encoding of source files. 52 | #source_encoding = 'utf-8-sig' 53 | 54 | # The master toctree document. 55 | master_doc = 'index' 56 | 57 | # General information about the project. 58 | project = u'Mariana' 59 | copyright = u'Tariq Daouda' 60 | author = u'Tariq Daouda' 61 | 62 | # The version info for the project you're documenting, acts as replacement for 63 | # |version| and |release|, also used in various other places throughout the 64 | # built documents. 65 | # 66 | # The short X.Y version. 67 | version = '1.0.x' 68 | # The full version, including alpha/beta/rc tags. 69 | release = '1.0.xrc' 70 | 71 | # The language for content autogenerated by Sphinx. Refer to documentation 72 | # for a list of supported languages. 73 | # 74 | # This is also used if you do content translation via gettext catalogs. 75 | # Usually you set "language" from the command line for these cases. 76 | language = None 77 | 78 | # There are two options for replacing |today|: either, you set today to some 79 | # non-false value, then it is used: 80 | #today = '' 81 | # Else, today_fmt is used as the format for a strftime call. 82 | #today_fmt = '%B %d, %Y' 83 | 84 | # List of patterns, relative to source directory, that match files and 85 | # directories to ignore when looking for source files. 86 | exclude_patterns = [] 87 | 88 | # The reST default role (used for this markup: `text`) to use for all 89 | # documents. 90 | #default_role = None 91 | 92 | # If true, '()' will be appended to :func: etc. cross-reference text. 93 | #add_function_parentheses = True 94 | 95 | # If true, the current module name will be prepended to all description 96 | # unit titles (such as .. function::). 97 | #add_module_names = True 98 | 99 | # If true, sectionauthor and moduleauthor directives will be shown in the 100 | # output. They are ignored by default. 101 | #show_authors = False 102 | 103 | # The name of the Pygments (syntax highlighting) style to use. 104 | pygments_style = 'sphinx' 105 | 106 | # A list of ignored prefixes for module index sorting. 107 | #modindex_common_prefix = [] 108 | 109 | # If true, keep warnings as "system message" paragraphs in the built documents. 110 | #keep_warnings = False 111 | 112 | # If true, `todo` and `todoList` produce output, else they produce nothing. 113 | todo_include_todos = True 114 | 115 | 116 | # -- Options for HTML output ---------------------------------------------- 117 | 118 | # The theme to use for HTML and HTML Help pages. See the documentation for 119 | # a list of builtin themes. 120 | # html_theme = "nature" 121 | html_theme = "sphinx_rtd_theme" 122 | 123 | # Theme options are theme-specific and customize the look and feel of a theme 124 | # further. For a list of options available for each theme, see the 125 | # documentation. 126 | #html_theme_options = {} 127 | 128 | # Add any paths that contain custom themes here, relative to this directory. 129 | #html_theme_path = [] 130 | 131 | # The name for this set of Sphinx documents. If None, it defaults to 132 | # " v documentation". 133 | #html_title = None 134 | 135 | # A shorter title for the navigation bar. Default is the same as html_title. 136 | #html_short_title = None 137 | 138 | # The name of an image file (relative to this directory) to place at the top 139 | # of the sidebar. 140 | html_logo = "MarianaLogo.png" 141 | 142 | # The name of an image file (within the static path) to use as favicon of the 143 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 144 | # pixels large. 145 | #html_favicon = None 146 | 147 | # Add any paths that contain custom static files (such as style sheets) here, 148 | # relative to this directory. They are copied after the builtin static files, 149 | # so a file named "default.css" will overwrite the builtin "default.css". 150 | html_static_path = ['_static'] 151 | 152 | # Add any extra paths that contain custom files (such as robots.txt or 153 | # .htaccess) here, relative to this directory. These files are copied 154 | # directly to the root of the documentation. 155 | #html_extra_path = [] 156 | 157 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 158 | # using the given strftime format. 159 | #html_last_updated_fmt = '%b %d, %Y' 160 | 161 | # If true, SmartyPants will be used to convert quotes and dashes to 162 | # typographically correct entities. 163 | #html_use_smartypants = True 164 | 165 | # Custom sidebar templates, maps document names to template names. 166 | #html_sidebars = {} 167 | 168 | # Additional templates that should be rendered to pages, maps page names to 169 | # template names. 170 | #html_additional_pages = {} 171 | 172 | # If false, no module index is generated. 173 | #html_domain_indices = True 174 | 175 | # If false, no index is generated. 176 | #html_use_index = True 177 | 178 | # If true, the index is split into individual pages for each letter. 179 | #html_split_index = False 180 | 181 | # If true, links to the reST sources are added to the pages. 182 | #html_show_sourcelink = True 183 | 184 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 185 | #html_show_sphinx = True 186 | 187 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 188 | #html_show_copyright = True 189 | 190 | # If true, an OpenSearch description file will be output, and all pages will 191 | # contain a tag referring to it. The value of this option must be the 192 | # base URL from which the finished HTML is served. 193 | #html_use_opensearch = '' 194 | 195 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 196 | #html_file_suffix = None 197 | 198 | # Language to be used for generating the HTML full-text search index. 199 | # Sphinx supports the following languages: 200 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 201 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' 202 | #html_search_language = 'en' 203 | 204 | # A dictionary with options for the search language support, empty by default. 205 | # Now only 'ja' uses this config value 206 | #html_search_options = {'type': 'default'} 207 | 208 | # The name of a javascript file (relative to the configuration directory) that 209 | # implements a search results scorer. If empty, the default will be used. 210 | #html_search_scorer = 'scorer.js' 211 | 212 | # Output file base name for HTML help builder. 213 | htmlhelp_basename = 'Marianadoc' 214 | 215 | # -- Options for LaTeX output --------------------------------------------- 216 | 217 | latex_elements = { 218 | # The paper size ('letterpaper' or 'a4paper'). 219 | #'papersize': 'letterpaper', 220 | 221 | # The font size ('10pt', '11pt' or '12pt'). 222 | #'pointsize': '10pt', 223 | 224 | # Additional stuff for the LaTeX preamble. 225 | #'preamble': '', 226 | 227 | # Latex figure (float) alignment 228 | #'figure_align': 'htbp', 229 | } 230 | 231 | # Grouping the document tree into LaTeX files. List of tuples 232 | # (source start file, target name, title, 233 | # author, documentclass [howto, manual, or own class]). 234 | latex_documents = [ 235 | (master_doc, 'Mariana.tex', u'Mariana Documentation', 236 | u'Tariq Daouda', 'manual'), 237 | ] 238 | 239 | # The name of an image file (relative to this directory) to place at the top of 240 | # the title page. 241 | #latex_logo = None 242 | 243 | # For "manual" documents, if this is true, then toplevel headings are parts, 244 | # not chapters. 245 | #latex_use_parts = False 246 | 247 | # If true, show page references after internal links. 248 | #latex_show_pagerefs = False 249 | 250 | # If true, show URL addresses after external links. 251 | #latex_show_urls = False 252 | 253 | # Documents to append as an appendix to all manuals. 254 | #latex_appendices = [] 255 | 256 | # If false, no module index is generated. 257 | #latex_domain_indices = True 258 | 259 | 260 | # -- Options for manual page output --------------------------------------- 261 | 262 | # One entry per manual page. List of tuples 263 | # (source start file, name, description, authors, manual section). 264 | man_pages = [ 265 | (master_doc, 'mariana', u'Mariana Documentation', 266 | [author], 1) 267 | ] 268 | 269 | # If true, show URL addresses after external links. 270 | #man_show_urls = False 271 | 272 | 273 | # -- Options for Texinfo output ------------------------------------------- 274 | 275 | # Grouping the document tree into Texinfo files. List of tuples 276 | # (source start file, target name, title, author, 277 | # dir menu entry, description, category) 278 | texinfo_documents = [ 279 | (master_doc, 'Mariana', u'Mariana Documentation', 280 | author, 'Mariana', 'One line description of project.', 281 | 'Miscellaneous'), 282 | ] 283 | 284 | # Documents to append as an appendix to all manuals. 285 | #texinfo_appendices = [] 286 | 287 | # If false, no module index is generated. 288 | #texinfo_domain_indices = True 289 | 290 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 291 | #texinfo_show_urls = 'footnote' 292 | 293 | # If true, do not generate a @detailmenu in the "Top" node's menu. 294 | #texinfo_no_detailmenu = False 295 | 296 | 297 | # -- Options for Epub output ---------------------------------------------- 298 | 299 | # Bibliographic Dublin Core info. 300 | epub_title = project 301 | epub_author = author 302 | epub_publisher = author 303 | epub_copyright = copyright 304 | 305 | # The basename for the epub file. It defaults to the project name. 306 | #epub_basename = project 307 | 308 | # The HTML theme for the epub output. Since the default themes are not optimized 309 | # for small screen space, using the same theme for HTML and epub output is 310 | # usually not wise. This defaults to 'epub', a theme designed to save visual 311 | # space. 312 | #epub_theme = 'epub' 313 | 314 | # The language of the text. It defaults to the language option 315 | # or 'en' if the language is not set. 316 | #epub_language = '' 317 | 318 | # The scheme of the identifier. Typical schemes are ISBN or URL. 319 | #epub_scheme = '' 320 | 321 | # The unique identifier of the text. This can be a ISBN number 322 | # or the project homepage. 323 | #epub_identifier = '' 324 | 325 | # A unique identification for the text. 326 | #epub_uid = '' 327 | 328 | # A tuple containing the cover image and cover page html template filenames. 329 | #epub_cover = () 330 | 331 | # A sequence of (type, uri, title) tuples for the guide element of content.opf. 332 | #epub_guide = () 333 | 334 | # HTML files that should be inserted before the pages created by sphinx. 335 | # The format is a list of tuples containing the path and title. 336 | #epub_pre_files = [] 337 | 338 | # HTML files shat should be inserted after the pages created by sphinx. 339 | # The format is a list of tuples containing the path and title. 340 | #epub_post_files = [] 341 | 342 | # A list of files that should not be packed into the epub file. 343 | epub_exclude_files = ['search.html'] 344 | 345 | # The depth of the table of contents in toc.ncx. 346 | #epub_tocdepth = 3 347 | 348 | # Allow duplicate toc entries. 349 | #epub_tocdup = True 350 | 351 | # Choose between 'default' and 'includehidden'. 352 | #epub_tocscope = 'default' 353 | 354 | # Fix unsupported image types using the Pillow. 355 | #epub_fix_images = False 356 | 357 | # Scale large images. 358 | #epub_max_image_width = 0 359 | 360 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 361 | #epub_show_urls = 'inline' 362 | 363 | # If false, no index is generated. 364 | #epub_use_index = True 365 | 366 | 367 | # Example configuration for intersphinx: refer to the Python standard library. 368 | intersphinx_mapping = {'https://docs.python.org/': None} 369 | 370 | #Document both the class and the __init__() 371 | autoclass_content = 'both' 372 | -------------------------------------------------------------------------------- /Mariana/HTML_Templates/vulcan/weblibs/vulcan.js: -------------------------------------------------------------------------------- 1 | Vue.component('focus-switcher', { 2 | template: ` 3 |
4 | 5 |
6 | {{nbElements}} 7 |
8 | `, 9 | props: { 10 | nbElements: { 11 | type: Number, 12 | default: function(){return 0} 13 | }, 14 | active: { 15 | type: Boolean, 16 | default: function(){return false} 17 | }, 18 | focusName: { 19 | type: String, 20 | required: true 21 | }, 22 | iconName: { 23 | type: String, 24 | required: true 25 | }, 26 | node: { 27 | required: true 28 | } 29 | }, 30 | data: function() { 31 | return { 32 | icon: "icon: " + this.iconName, 33 | classObject: { 34 | "uk-button": true, 35 | "uk-button-small": true, 36 | "uk-button-default": true, 37 | }, 38 | selected: false 39 | } 40 | }, 41 | mounted: function(){ 42 | if (this.active) { 43 | this.changeFocus() 44 | } 45 | }, 46 | methods: { 47 | unSelect: function(){ 48 | this.selected = false 49 | this.classObject["uk-button-default"] = true 50 | this.classObject["uk-button-primary"] = false 51 | this.classObject["vulcan-button-primary"] = false 52 | }, 53 | select: function(){ 54 | this.selected = true 55 | this.classObject["uk-button-default"] = false 56 | this.classObject["uk-button-primary"] = true 57 | this.classObject["vulcan-button-primary"] = true 58 | }, 59 | changeFocus: function(){ 60 | if (this.nbElements > 0) { 61 | this.$emit('focusChanged', this, this.node, this.focusName) 62 | this.select() 63 | } 64 | } 65 | } 66 | }); 67 | Vue.component('graph-view', { 68 | template: ` 69 |
70 |
71 |
72 | 75 | 78 | 81 | 84 |
85 | 88 |
89 |
90 | 93 | 96 |
97 |
98 | 99 |
100 |
    101 |
  • 102 | 105 | 108 |
    109 |
    110 | 111 | 112 | 113 | 114 |
    115 |
    116 |
      117 |
    • 118 | 119 | {{category}} 120 | 121 |
      122 | 123 | {{kv.name}}: 124 | none 125 | {{kv.value}} 126 |
      127 |       128 |
      • 129 |
    130 |
    • 131 |
132 |
133 |
`, 134 | 135 | props: ["nodesp", "edgesp", "physicsp"], 136 | qtemplate: `
`, 137 | data : function(){ 138 | createNodes = function(layers, color, highlightColor, hoverColor){ 139 | categories = ["layer", "initializations", "activation", "regularizations", "decorators", "learningScenari"] 140 | superCats = ["parameters", "hyperParameters", "notes", "abstractions"] 141 | nodes = [] 142 | namesToI = {} 143 | maxLvl = 0 144 | console.log(layers) 145 | for (var i = 0; i < layers.length; i++) { 146 | availableCats = {} 147 | for (var k = 0; k < superCats.length; k++) { 148 | availableCats[superCats[k]] = [] 149 | for (var j = 0; j < categories.length; j++) { 150 | // console.log(layers[i][1]) 151 | // console.log(layers[i][1][superCats[k]]) 152 | console.log(layers[i][1][superCats[k]]) 153 | 154 | if (layers[i][1][superCats[k]][categories[j]].length > 0) { 155 | availableCats[superCats[k]].push(categories[j]) 156 | }; 157 | }; 158 | }; 159 | node = { id: i, label: layers[i][0], shape: "box", font: {color: "white"}, 160 | color: { 161 | background: color, 162 | border: "black", 163 | highlight:{ 164 | background:highlightColor, 165 | border:"black" 166 | }, 167 | hover:{ 168 | background:hoverColor, 169 | border:"black" 170 | } 171 | }, 172 | level: layers[i][1].level, 173 | shadow:true, 174 | Mariana: { 175 | open:false, 176 | focus: undefined, 177 | focusSwitchers: new Set(), 178 | data: layers[i][1], 179 | availableCats: availableCats 180 | }, 181 | } 182 | 183 | if (layers[i][1].level > maxLvl) { 184 | maxLvl = layers[i][1].level 185 | }; 186 | 187 | namesToI[layers[i][0]] = i 188 | console.log(node) 189 | nodes.push(node) 190 | } 191 | return {nodes: nodes, maxLvl: maxLvl, namesToI} 192 | } 193 | 194 | createEdges = function(edges, namesToI, color) { 195 | for (var i = edges.length - 1; i >= 0; i--) { 196 | edges[i].to = namesToI[edges[i].to] 197 | edges[i].from = namesToI[edges[i].from] 198 | edges[i].id = i 199 | edges[i].shadow = true 200 | edges[i].arrows = 'to' 201 | edges[i].color = color 202 | } 203 | return edges 204 | } 205 | 206 | nodeColor = 'rgba(25, 131, 165, 0.5)' 207 | nodeHighlightColor = 'rgba(25, 131, 165, 1)' 208 | nodeHoverColor = 'rgba(25, 131, 165, 1)' 209 | 210 | edgeColor = "#f68920" 211 | nodesObj = JSON.parse(this.nodesp) 212 | res = createNodes(nodesObj, nodeColor, nodeHighlightColor, nodeHoverColor) 213 | nodes = res.nodes 214 | nodesNamesToI = res.namesToI 215 | maxLvl = res.maxLvl 216 | nodes[0].Mariana.open = true 217 | 218 | edgesObj = JSON.parse(this.edgesp) 219 | edges = createEdges(edgesObj, nodesNamesToI, edgeColor) 220 | ret = { 221 | nodes: nodes, 222 | nodeColor: nodeColor, 223 | nodesNamesToI: nodesNamesToI, 224 | edges: edges, 225 | edgeColor: edgeColor, 226 | network: null, 227 | maxLvl: maxLvl, 228 | height: (maxLvl +1) * 150, 229 | direction: "UD", 230 | showShapes: true, 231 | physics: this.physicsp === "true", 232 | drawed:false 233 | } 234 | return ret 235 | }, 236 | mounted: function(){ 237 | this.draw() 238 | }, 239 | methods: { 240 | increaseHeight: function(){ 241 | this.height = this.height + 150 242 | this.draw() 243 | }, 244 | decreaseHeight: function(){ 245 | this.height = this.height - 150 246 | this.draw() 247 | }, 248 | toggleNode: function(node){ 249 | if (!node.Mariana.open) { 250 | node.Mariana.open=true 251 | this.network.selectNodes([node.id]) 252 | }else{ 253 | node.Mariana.open=false 254 | } 255 | }, 256 | changeFocus: function(switcher, node, focus){ 257 | node.Mariana.focus = focus; 258 | node.Mariana.focusSwitchers.add(switcher) 259 | 260 | switchers = Array.from(node.Mariana.focusSwitchers) 261 | for (var i = switchers.length - 1; i >= 0; i--) { 262 | if (switcher != switchers[i]) { 263 | switchers[i].unSelect() 264 | } 265 | } 266 | if (!node.Mariana.open) { 267 | if (this.drawed) { 268 | this.toggleNode(node) 269 | } 270 | } 271 | }, 272 | toggleShapes: function(){ 273 | this.showShapes = !this.showShapes 274 | this.draw() 275 | }, 276 | changeDirection: function(dir){ 277 | this.direction = dir 278 | this.draw() 279 | }, 280 | destroy: function(){ 281 | if (this.network !== null) { 282 | this.network.destroy(); 283 | } 284 | this.network = null; 285 | }, 286 | draw: function() { 287 | this.destroy(); 288 | for (var i = this.nodes.length - 1; i >= 0; i--) { 289 | if (this.showShapes) { 290 | this.nodes[i].label = this.nodes[i].Mariana.data.name + "\n" + "("+ this.nodes[i].Mariana.data.shape + ")" 291 | }else{ 292 | this.nodes[i].label = this.nodes[i].Mariana.data.name 293 | }; 294 | }; 295 | 296 | var data = { 297 | nodes: this.nodes, 298 | edges: this.edges 299 | }; 300 | var options = { 301 | edges: { 302 | smooth: { 303 | type: 'cubicBezier', 304 | forceDirection: (this.direction == "UD" || this.direction == "DU") ? 'vertical' : 'horizontal', 305 | roundness: 0.4 306 | } 307 | }, 308 | layout: { 309 | hierarchical: { 310 | direction: this.direction 311 | } 312 | }, 313 | physics:{ 314 | enabled: this.physics 315 | } 316 | }; 317 | var container = this.$refs.network; 318 | this.network = new vis.Network(container, data, options); 319 | 320 | var self = this 321 | this.network.on('select', function (params) { 322 | for (var i = self.nodes.length - 1; i >= 0; i--) { 323 | self.nodes[i].Mariana.open = false 324 | }; 325 | 326 | for (var i = params.edges.length - 1; i >= 0; i--) { 327 | var ed = self.edges[params.edges[i]] 328 | self.nodes[ed.to].Mariana.open = true 329 | self.nodes[ed.from].Mariana.open = true 330 | }; 331 | 332 | for (var i = params.nodes.length - 1; i >= 0; i--) { 333 | self.nodes[params.nodes[0]].Mariana.open = true 334 | }; 335 | }); 336 | this.drawed = true 337 | } 338 | 339 | } 340 | }) 341 | 342 | new Vue({ 343 | el: '#all' 344 | }) -------------------------------------------------------------------------------- /Mariana/scenari.py: -------------------------------------------------------------------------------- 1 | import theano, numpy 2 | import theano.tensor as tt 3 | from collections import OrderedDict 4 | import lasagne.updates as LUP 5 | 6 | import Mariana.abstraction as MABS 7 | 8 | 9 | __all__ = ["LearningScenario_ABC", "ParameterGradUpdates", "OptimizerFreeResults", "OptimizerResult", "Fixed", "GradientDescent"] 10 | 11 | class IncompatibleLearningScenarios(Exception) : 12 | def __init__(self, msg) : 13 | self.message = msg 14 | 15 | def __str__(self) : 16 | return self.message 17 | 18 | def __repr__(self): 19 | return self.message 20 | 21 | class ConflictResolve(object): 22 | """In Mariana scenari can be chained. The children of that class defines what to do in case of conflict""" 23 | def __init__(self, warning=False): 24 | super(ConflictResolve, self).__init__() 25 | self.warning=warning 26 | 27 | def apply(self, previous, current) : 28 | if self.warning : 29 | print("Resolving conflict between scenari using %s" % self.__class__.__name__) 30 | 31 | return self.resolve(previous, current) 32 | 33 | def resolve(self, previous, current) : 34 | raise NotImplemented("Should be implemented in child") 35 | 36 | class Overwrite(ConflictResolve): 37 | """Overwrite the previous value""" 38 | def resolve(self, previous, current) : 39 | return current 40 | 41 | class Ignore(ConflictResolve): 42 | """Ignore the update and keep the previous value""" 43 | def resolve(self, previous, current) : 44 | return previous 45 | 46 | class Die(ConflictResolve): 47 | """No conflic resolve, crashes everything""" 48 | def resolve(self, previous, current) : 49 | if previous.gradient is not None or previous.update is not None: 50 | raise IncompatibleLearningScenarios("Learning scenario is incompatible with previous ones (%s)" % previous) 51 | 52 | class ParameterGradUpdates(object): 53 | """docstring for ParameterGradUpdates""" 54 | def __init__(self, parameter, name, gradient, update): 55 | super(ParameterGradUpdates, self).__init__() 56 | self.name = name 57 | self.parameter = parameter 58 | self.update = update 59 | self.gradient = gradient 60 | 61 | class OptimizerResult(ParameterGradUpdates): 62 | """use this a return object for an optimizer""" 63 | def __init__(self, parameter, name, gradient, update): 64 | super(OptimizerResult, self).__init__(parameter, name, gradient, update) 65 | self.coParameters = [] 66 | 67 | def addCoParameter(self, parameter, name, gradient, update) : 68 | param = ParameterGradUpdates(parameter, name, gradient, update) 69 | self.coParameters.append(param) 70 | 71 | def __repr__(self) : 72 | return "< optimizer result for: %s (id: %s)>" % (self.parameter, id(self)) 73 | 74 | class LearningScenario_ABC(MABS.UntrainableAbstraction_ABC, MABS.Apply_ABC): 75 | """ 76 | This is the interface all optimizations rules must expose. 77 | """ 78 | def __init__(self, applyTo=None, inheritable=True, conflictResolve=Die(), **kwargs) : 79 | super(LearningScenario_ABC, self).__init__(**kwargs) 80 | if applyTo : 81 | self.applyTo = set(applyTo) 82 | self.setHP("applyTo", applyTo) 83 | else : 84 | self.applyTo = applyTo 85 | 86 | self.inheritable = inheritable 87 | self.conflictResolve = conflictResolve 88 | # self.memory = {} 89 | 90 | def isInheritable(self) : 91 | return self.inheritable 92 | 93 | def apply(self, abstraction, parameterName, loss, previous=None) : 94 | """Apply to a abstraction and update networks's log""" 95 | 96 | if self.applyTo is not None and parameterName not in self.applyTo : 97 | return None 98 | 99 | try: 100 | parameter = abstraction.getP(parameterName) 101 | except : 102 | raise KeyError("%s has no parameter %s"%(abstraction, parameterName)) 103 | 104 | # if (parameter in self.memory) and not self.force : 105 | # return self.memory[parameter] 106 | 107 | v = self.run(parameter=parameter, parameterName=parameterName, loss=loss, abstraction=abstraction, previous=previous) 108 | if previous : 109 | try : 110 | return self.conflictResolve.apply(previous, v) 111 | except IncompatibleLearningScenarios : 112 | raise IncompatibleLearningScenarios("Learning scenario: '%s' is incompatible with previous updates (abstraction: '%s', previous: '%s')" % (self.__class__.__name__, abstraction.name, previous)) 113 | 114 | # for cp in v.coParameters : 115 | # abstraction.registerCoParameter(cp.name, cp) 116 | 117 | return v 118 | 119 | def run(self, parameter, parameterName, loss, abstraction, previous) : 120 | """return the updates for the parameters of abstraction. Must be implemented in child""" 121 | raise NotImplemented("Must be implemented in child") 122 | 123 | class Independent(LearningScenario_ABC): 124 | "Indicates that the abstraction does not inherit optimization rules form the outputs. Must be placed at the first positon of the list." 125 | def __init__(self): 126 | super(Independent, self).__init__(inheritable=False) 127 | 128 | def run(*args, **kwargs) : 129 | return None 130 | 131 | class Fixed(LearningScenario_ABC): 132 | "No learning, the abstraction parameters stay fixed" 133 | def __init__(self, applyTo=None, inheritable=False, conflictResolve=Overwrite(), **kwargs): 134 | super(Fixed, self).__init__(applyTo, inheritable, conflictResolve, **kwargs) 135 | 136 | def run(self, parameter, **kwargs) : 137 | ret = OptimizerResult(parameter, None, None, None) 138 | return ret 139 | 140 | class GradientDescent(LearningScenario_ABC): 141 | "The GradientDescent scenario has a fixed learning rate." 142 | def __init__(self, lr, momentum=0, reverse=False, conflictResolve=Die(), **kwargs): 143 | """ 144 | use reverse = True for gradient ascent. 145 | """ 146 | super(GradientDescent, self).__init__(conflictResolve=conflictResolve, **kwargs) 147 | 148 | self.addHyperParameters({ 149 | "lr": lr, 150 | "momentum": momentum, 151 | "reverse": reverse 152 | }) 153 | 154 | def run(self, parameter, parameterName, loss, **kwargs) : 155 | pVar = parameter.getVar() 156 | gparam = tt.grad(loss, pVar) 157 | if self.getHP("momentum") == 0 : 158 | if not self.getHP("reverse") : 159 | param_update = parameter.getVar() - self.getHP("lr") * gparam 160 | else : 161 | param_update = parameter.getVar() + self.getHP("lr") * gparam 162 | 163 | ret = OptimizerResult(parameter.getVar(), parameterName, gparam, param_update) 164 | else : 165 | momentum_param = theano.shared(parameter.getValue()*0., broadcastable=parameter.getVar().broadcastable, name="momentum.%s" % (parameterName)) 166 | momentum_update = self.getHP("momentum") * momentum_param + (1-self.getHP("momentum"))*gparam 167 | if not self.getHP("reverse") : 168 | param_update = parameter.getVar() - self.getHP("lr") * momentum_param 169 | else : 170 | param_update = parameter.getVar() + self.getHP("lr") * momentum_param 171 | 172 | ret = OptimizerResult(parameter.getVar(), parameterName, gparam, param_update) 173 | ret.addCoParameter(momentum_param, "momentum", None, momentum_update) 174 | 175 | return ret 176 | 177 | SGD = GradientDescent 178 | 179 | #class GradientClipping(LearningScenario_ABC): 180 | # "Clips previous update to a minimum and maximum value." 181 | # def __init__(self, minimum, maximum, conflictResolve=Overwrite(), **kwargs): 182 | # """ 183 | # """ 184 | # super(GradientClipping, self).__init__(conflictResolve=conflictResolve, **kwargs) 185 | # self.addHyperParameters({ 186 | # "minimum": minimum, 187 | # "maximum": maximum, 188 | # }) 189 | 190 | # def run(self, parameter, parameterName, loss, **kwargs) : 191 | # previous = kwargs["previous"] 192 | # previous.gradient = tt.clip(previous.gradient, self.getHP("minimum"), self.getHP("maximum")) 193 | 194 | # return previous 195 | 196 | class Adam(LearningScenario_ABC): 197 | "The Adam. Uses lasagne as backend" 198 | def __init__(self, lr=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8, conflictResolve=Die(), **kwargs): 199 | """ 200 | use reverse = True for gradient ascent. 201 | """ 202 | super(Adam, self).__init__(**kwargs) 203 | 204 | self.addHyperParameters({ 205 | "lr": lr, 206 | "beta1": beta1, 207 | "beta2": beta2, 208 | "epsilon": epsilon, 209 | }) 210 | 211 | def run(self, parameter, parameterName, loss, **kwargs) : 212 | pVar = parameter.getVar() 213 | gparam = tt.grad(loss, pVar) 214 | updates = LUP.adam( [ gparam ], [pVar], learning_rate=self.getHP("lr"), beta1=self.getHP("beta1"), beta2=self.getHP("beta2"), epsilon=self.getHP("epsilon")) 215 | 216 | ret = OptimizerResult(pVar, parameterName, gparam, updates[pVar]) 217 | i = 0 218 | for param, update in updates.items() : 219 | if param is not pVar : 220 | name = "%s_adam_%s" % (parameterName, i) 221 | ret.addCoParameter(param, name, None, update) 222 | i += 1 223 | 224 | return ret 225 | 226 | class Adamax(LearningScenario_ABC): 227 | "The Adamax. Uses lasagne as backend" 228 | def __init__(self, lr=0.002, beta1=0.9, beta2=0.999, epsilon=1e-8, conflictResolve=Die(), **kwargs): 229 | super(Adamax, self).__init__(conflictResolve=conflictResolve, **kwargs) 230 | 231 | self.addHyperParameters({ 232 | "lr": lr, 233 | "beta1": beta1, 234 | "beta2": beta2, 235 | "epsilon": epsilon, 236 | }) 237 | 238 | def run(self, parameter, parameterName, loss, **kwargs) : 239 | pVar = parameter.getVar() 240 | gparam = tt.grad(loss, pVar) 241 | updates = LUP.adamax( [ gparam ], [pVar], learning_rate=self.getHP("lr"), beta1=self.getHP("beta1"), beta2=self.getHP("beta2"), epsilon=self.getHP("epsilon")) 242 | 243 | ret = OptimizerResult(pVar, parameterName, gparam, updates[pVar]) 244 | i = 0 245 | for param, update in updates.items() : 246 | if param is not pVar : 247 | name = "%s_adamax_%s" % (parameterName, i) 248 | ret.addCoParameter(param, name, None, update) 249 | i += 1 250 | 251 | return ret 252 | 253 | class Adadelta(LearningScenario_ABC): 254 | "The Adadelta. Uses lasagne as backend" 255 | def __init__(self, lr=1.0, rho=0.9, epsilon=1e-6, conflictResolve=Die(), **kwargs): 256 | super(Adadelta, self).__init__(conflictResolve=conflictResolve, **kwargs) 257 | 258 | self.addHyperParameters({ 259 | "lr": lr, 260 | "rho": rho, 261 | "epsilon": epsilon, 262 | }) 263 | 264 | def run(self, parameter, parameterName, loss, **kwargs) : 265 | pVar = parameter.getVar() 266 | gparam = tt.grad(loss, pVar) 267 | updates = LUP.adadelta( [ gparam ], [pVar], learning_rate=self.getHP("lr"), rho=self.getHP("rho"), epsilon=self.getHP("epsilon")) 268 | 269 | ret = OptimizerResult(pVar, parameterName, gparam, updates[pVar]) 270 | i = 0 271 | for param, update in updates.items() : 272 | if param is not pVar : 273 | name = "%s_adadelta_%s" % (parameterName, i) 274 | ret.addCoParameter(param, name, None, update) 275 | i += 1 276 | 277 | return ret 278 | 279 | class Adagrad(LearningScenario_ABC): 280 | "The Adagrad. Uses lasagne as backend" 281 | def __init__(self, lr=1.0, epsilon=1e-6, conflictResolve=Die(), **kwargs): 282 | super(Adagrad, self).__init__(conflictResolve=conflictResolve, **kwargs) 283 | 284 | self.addHyperParameters({ 285 | "lr": lr, 286 | "epsilon": epsilon, 287 | }) 288 | 289 | def run(self, parameter, parameterName, loss, **kwargs) : 290 | pVar = parameter.getVar() 291 | gparam = tt.grad(loss, pVar) 292 | updates = LUP.adagrad( [ gparam ], [pVar], learning_rate=self.getHP("lr"), epsilon=self.getHP("epsilon")) 293 | 294 | ret = OptimizerResult(pVar, parameterName, gparam, updates[pVar]) 295 | i = 0 296 | for param, update in updates.items() : 297 | if param is not pVar : 298 | name = "%s_adagrad_%s" % (parameterName, i) 299 | ret.addCoParameter(param, name, None, update) 300 | i += 1 301 | 302 | return ret 303 | 304 | class RMSProp(LearningScenario_ABC): 305 | "The RMSProp. Uses lasagne as backend" 306 | def __init__(self, lr=1.0, rho=0.9, epsilon=1e-6, conflictResolve=Die(), **kwargs): 307 | super(RMSProp, self).__init__(conflictResolve=conflictResolve, **kwargs) 308 | 309 | self.addHyperParameters({ 310 | "lr": lr, 311 | "rho": rho, 312 | "epsilon": epsilon, 313 | }) 314 | 315 | def run(self, parameter, parameterName, loss, **kwargs) : 316 | pVar = parameter.getVar() 317 | gparam = tt.grad(loss, pVar) 318 | updates = LUP.rmsprop( [ gparam ], [pVar], learning_rate=self.getHP("lr"), rho=self.getHP("rho"), epsilon=self.getHP("epsilon")) 319 | 320 | ret = OptimizerResult(pVar, parameterName, gparam, updates[pVar]) 321 | i = 0 322 | for param, update in updates.items() : 323 | if param is not pVar : 324 | name = "%s_rmsprop_%s" % (parameterName, i) 325 | ret.addCoParameter(param, name, None, update) 326 | i += 1 327 | 328 | return ret 329 | -------------------------------------------------------------------------------- /Mariana/sampling.py: -------------------------------------------------------------------------------- 1 | import Mariana.initializations as MI 2 | 3 | import Mariana.compatibility.lasagne as MLASAGNE 4 | import lasagne.layers.pool as LasagnePOOL 5 | 6 | __all__ = ["MaxPooling1D", "MaxPooling2D", "MaxPooling3D", "AveragePooling1D", "AveragePooling2D", "AveragePooling3D", "RepeatedUpscaling1D", "DilatedUpscaling1D", "RepeatedUpscaling2D", "DilatedUpscaling2D", "RepeatedUpscaling3D", "DilatedUpscaling3D", "WinnerTakesAll", "MaxSpatialPyramidPooling", "AverageSpatialPyramidPooling"] 7 | 8 | class MaxPooling1D(MLASAGNE.LasagneLayer): 9 | """This layer wraps lasagnes's MaxPool1DLayer layer and performs a 1D max pooling over each channel. 10 | For a full explanation of the arguments please checkout lasagne's doc""" 11 | def __init__( 12 | self, 13 | poolSize, 14 | name, 15 | stride=None, 16 | pad=0, 17 | **kwargs 18 | ): 19 | 20 | super(MaxPooling1D, self).__init__( 21 | LasagnePOOL.MaxPool1DLayer, 22 | initializations=[], 23 | lasagneHyperParameters={ 24 | "pool_size": poolSize, 25 | "stride": stride, 26 | "pad": pad, 27 | "ignore_border": True, 28 | }, 29 | name=name, 30 | 31 | ) 32 | 33 | class MaxPooling2D(MLASAGNE.LasagneLayer): 34 | """This layer wraps lasagnes's MaxPool2DLayer layer and performs a 2D max pooling over each channel. 35 | For a full explanation of the arguments please checkout lasagne's doc""" 36 | def __init__( 37 | self, 38 | poolHeight, 39 | poolWidth, 40 | name, 41 | stride=None, 42 | pad=(0, 0), 43 | **kwargs 44 | ): 45 | super(MaxPooling2D, self).__init__( 46 | LasagnePOOL.MaxPool2DLayer, 47 | initializations=[], 48 | lasagneHyperParameters={ 49 | "pool_size": (poolHeight, poolWidth), 50 | "stride": stride, 51 | "pad": pad, 52 | "ignore_border": True, 53 | }, 54 | name=name, 55 | 56 | ) 57 | 58 | class MaxPooling3D(MLASAGNE.LasagneLayer): 59 | """This layer wraps lasagnes's MaxPool3DLayer layer and performs a 3D max pooling over each channel. 60 | For a full explanation of the arguments please checkout lasagne's doc""" 61 | def __init__( 62 | self, 63 | poolHeight, 64 | poolWidth, 65 | poolDepth, 66 | name, 67 | stride=None, 68 | pad=(0, 0, 0), 69 | **kwargs 70 | ): 71 | super(MaxPooling3D, self).__init__( 72 | LasagnePOOL.MaxPool3DLayer, 73 | initializations=[], 74 | lasagneHyperParameters={ 75 | "pool_size": (poolHeight, poolWidth, poolDepth), 76 | "stride": stride, 77 | "pad": pad, 78 | "ignore_border": True, 79 | }, 80 | name=name, 81 | 82 | ) 83 | 84 | class AveragePooling1D(MLASAGNE.LasagneLayer): 85 | """This layer wraps lasagnes's Pool1DLayer layer and performs a 1D average pooling over each channel. 86 | For a full explanation of the arguments please checkout lasagne's doc""" 87 | def __init__( 88 | self, 89 | poolSize, 90 | name, 91 | stride=None, 92 | pad=0, 93 | includePadding=False, 94 | **kwargs 95 | ): 96 | if includePadding : 97 | mode = "average_inc_pad" 98 | else: 99 | mode = "average_exc_pad" 100 | 101 | super(AveragePooling1D, self).__init__( 102 | LasagnePOOL.Pool1DLayer, 103 | initializations=[], 104 | lasagneHyperParameters={ 105 | "pool_size": poolSize, 106 | "stride": stride, 107 | "pad": pad, 108 | "ignore_border": True, 109 | "mode": mode 110 | }, 111 | name=name, 112 | 113 | ) 114 | 115 | class AveragePooling2D(MLASAGNE.LasagneLayer): 116 | """This layer wraps lasagnes's Pool2DLayer layer and performs a 2D average pooling over each channel. 117 | For a full explanation of the arguments please checkout lasagne's doc""" 118 | def __init__( 119 | self, 120 | poolHeight, 121 | poolWidth, 122 | name, 123 | stride=None, 124 | pad=(0, 0), 125 | includePadding=False, 126 | **kwargs 127 | ): 128 | if includePadding : 129 | mode = "average_inc_pad" 130 | else: 131 | mode = "average_exc_pad" 132 | 133 | super(AveragePooling2D, self).__init__( 134 | LasagnePOOL.Pool2DLayer, 135 | initializations=[], 136 | lasagneHyperParameters={ 137 | "pool_size": (poolHeight, poolWidth), 138 | "stride": stride, 139 | "pad": pad, 140 | "ignore_border": True, 141 | "mode": mode 142 | }, 143 | name=name, 144 | ) 145 | 146 | class AveragePooling3D(MLASAGNE.LasagneLayer): 147 | """This layer wraps lasagnes's Pool3DLayer layer and performs a 3D average pooling over each channel. 148 | For a full explanation of the arguments please checkout lasagne's doc""" 149 | def __init__( 150 | self, 151 | poolHeight, 152 | poolWidth, 153 | poolDepth, 154 | name, 155 | stride=None, 156 | pad=(0, 0, 0), 157 | includePadding=False, 158 | **kwargs 159 | ): 160 | if includePadding : 161 | mode = "average_inc_pad" 162 | else: 163 | mode = "average_exc_pad" 164 | 165 | super(AveragePooling3D, self).__init__( 166 | LasagnePOOL.Pool3DLayer, 167 | initializations=[], 168 | lasagneHyperParameters={ 169 | "pool_size": (poolHeight, poolWidth, poolDepth), 170 | "stride": stride, 171 | "pad": pad, 172 | "ignore_border": True, 173 | "mode": mode 174 | }, 175 | name=name, 176 | ) 177 | 178 | class RepeatedUpscaling1D(MLASAGNE.LasagneLayer): 179 | """This layer wraps lasagnes's Upscale1DLayer layer and performs a 1D repeated upscaling over each channel. 180 | For a full explanation of the arguments please checkout lasagne's doc""" 181 | def __init__( 182 | self, 183 | scaleFactor, 184 | name, 185 | **kwargs 186 | ): 187 | 188 | super(RepeatedUpscaling1D, self).__init__( 189 | LasagnePOOL.Upscale1DLayer, 190 | initializations=[], 191 | lasagneHyperParameters={ 192 | "scale_factor": scaleFactor, 193 | "mode": "repeat" 194 | }, 195 | name=name, 196 | 197 | ) 198 | 199 | class DilatedUpscaling1D(MLASAGNE.LasagneLayer): 200 | """This layer wraps lasagnes's Upscale1DLayer layer and performs a 1D dilated upscaling over each channel. 201 | For a full explanation of the arguments please checkout lasagne's doc""" 202 | def __init__( 203 | self, 204 | scaleFactor, 205 | name, 206 | **kwargs 207 | ): 208 | 209 | super(DilatedUpscaling1D, self).__init__( 210 | LasagnePOOL.Upscale1DLayer, 211 | initializations=[], 212 | lasagneHyperParameters={ 213 | "scale_factor": scaleFactor, 214 | "mode": "dilate" 215 | }, 216 | name=name, 217 | 218 | ) 219 | 220 | class RepeatedUpscaling2D(MLASAGNE.LasagneLayer): 221 | """This layer wraps lasagnes's Upscale2DLayer layer and performs a 2D repeated upscaling over each channel. 222 | For a full explanation of the arguments please checkout lasagne's doc""" 223 | def __init__( 224 | self, 225 | heightScaleFactor, 226 | widthScaleFactor, 227 | name, 228 | **kwargs 229 | ): 230 | 231 | super(RepeatedUpscaling2D, self).__init__( 232 | LasagnePOOL.Upscale2DLayer, 233 | initializations=[], 234 | lasagneHyperParameters={ 235 | "scale_factor": (heightScaleFactor, widthScaleFactor), 236 | "mode": "repeat" 237 | }, 238 | name=name, 239 | 240 | ) 241 | 242 | class DilatedUpscaling2D(MLASAGNE.LasagneLayer): 243 | """This layer wraps lasagnes's Upscale2DLayer layer and performs a 2D dilated upscaling over each channel. 244 | For a full explanation of the arguments please checkout lasagne's doc""" 245 | def __init__( 246 | self, 247 | heightScaleFactor, 248 | widthScaleFactor, 249 | name, 250 | **kwargs 251 | ): 252 | 253 | super(DilatedUpscaling2D, self).__init__( 254 | LasagnePOOL.Upscale2DLayer, 255 | initializations=[], 256 | lasagneHyperParameters={ 257 | "scale_factor": (heightScaleFactor, widthScaleFactor), 258 | "mode": "dilate" 259 | }, 260 | name=name, 261 | 262 | ) 263 | 264 | class RepeatedUpscaling3D(MLASAGNE.LasagneLayer): 265 | """This layer wraps lasagnes's Upscale3DLayer layer and performs a 3D repeated upscaling over each channel. 266 | For a full explanation of the arguments please checkout lasagne's doc""" 267 | def __init__( 268 | self, 269 | heightScaleFactor, 270 | widthScaleFactor, 271 | depthScaleFactor, 272 | name, 273 | **kwargs 274 | ): 275 | 276 | super(RepeatedUpscaling3D, self).__init__( 277 | LasagnePOOL.Upscale3DLayer, 278 | initializations=[], 279 | lasagneHyperParameters={ 280 | "scale_factor": (heightScaleFactor, widthScaleFactor, depthScaleFactor), 281 | "mode": "repeat" 282 | }, 283 | name=name, 284 | 285 | ) 286 | 287 | class DilatedUpscaling3D(MLASAGNE.LasagneLayer): 288 | """This layer wraps lasagnes's Upscale3DLayer layer and performs a 3D dilated upscaling over each channel. 289 | For a full explanation of the arguments please checkout lasagne's doc""" 290 | def __init__( 291 | self, 292 | heightScaleFactor, 293 | widthScaleFactor, 294 | depthScaleFactor, 295 | name, 296 | **kwargs 297 | ): 298 | super(DilatedUpscaling3D, self).__init__( 299 | LasagnePOOL.Upscale3DLayer, 300 | initializations=[], 301 | lasagneHyperParameters={ 302 | "scale_factor": (heightScaleFactor, widthScaleFactor, depthScaleFactor), 303 | "mode": "dilate" 304 | }, 305 | name=name, 306 | 307 | ) 308 | 309 | class WinnerTakesAll(MLASAGNE.LasagneLayer): 310 | """This layer wraps lasagnes's FeatureWTALayer layer and performs a Winner takes all pooling over each channel. 311 | For a full explanation of the arguments please checkout lasagne's doc""" 312 | def __init__( 313 | self, 314 | poolSize, 315 | depthScaleFactor, 316 | name, 317 | axis=1, 318 | **kwargs 319 | ): 320 | 321 | super(WinnerTakesAll, self).__init__( 322 | LasagnePOOL.FeatureWTALayer, 323 | initializations=[], 324 | lasagneHyperParameters={ 325 | "pool_size": poolSize, 326 | "axis": axis 327 | }, 328 | name=name, 329 | 330 | ) 331 | 332 | class MaxSpatialPyramidPooling(MLASAGNE.LasagneLayer): 333 | """This layer wraps lasagnes's SpatialPyramidPooling layer and performs a pyramid max pooling over each channel. 334 | This variant of max pooling can be applied on inputs of arbitrary lengths. 335 | For a full explanation of the arguments please checkout lasagne's doc""" 336 | def __init__( 337 | self, 338 | name, 339 | poolDims=[4, 2, 1], 340 | implementation="fast", 341 | **kwargs 342 | ): 343 | 344 | super(MaxSpatialPyramidPooling, self).__init__( 345 | LasagnePOOL.SpatialPyramidPooling, 346 | initializations=[], 347 | lasagneHyperParameters={ 348 | "pool_size": poolSize, 349 | "implementation": implementation, 350 | "mode": 'max' 351 | }, 352 | name=name, 353 | 354 | ) 355 | 356 | 357 | class AverageSpatialPyramidPooling(MLASAGNE.LasagneLayer): 358 | """This layer wraps lasagnes's SpatialPyramidPooling layer and performs a pyramid average pooling over each channel. 359 | This variant of average pooling can be applied on inputs of arbitrary lengths. 360 | For a full explanation of the arguments please checkout lasagne's doc""" 361 | def __init__( 362 | self, 363 | name, 364 | poolDims=[4, 2, 1], 365 | implementation="fast", 366 | includePadding=False, 367 | **kwargs 368 | ): 369 | if includePadding : 370 | mode = "average_inc_pad" 371 | else: 372 | mode = "average_exc_pad" 373 | 374 | super(AverageSpatialPyramidPooling, self).__init__( 375 | LasagnePOOL.SpatialPyramidPooling, 376 | initializations=[], 377 | lasagneHyperParameters={ 378 | "pool_size": poolSize, 379 | "implementation": implementation, 380 | "mode": mode 381 | }, 382 | name=name, 383 | 384 | ) 385 | 386 | --------------------------------------------------------------------------------