├── salt ├── roots │ ├── salt │ │ ├── top.sls │ │ ├── etc │ │ │ ├── theanorc │ │ │ └── matplotlibrc │ │ ├── bin │ │ │ ├── ipynb.upstart │ │ │ ├── sanity_check.py │ │ │ ├── run_tests │ │ │ └── distribute_setup.py │ │ └── charmer │ │ │ └── init.sls │ └── pillar │ │ ├── top.sls │ │ └── python34.sls └── minion ├── .gitignore ├── TODO ├── notebooks ├── Demo │ ├── cm.csv │ └── graph.html ├── Hello World.ipynb └── Snake Charmer QA.ipynb ├── Vagrantfile ├── FAQ.md ├── CUSTOMIZING.md └── README.md /salt/roots/salt/top.sls: -------------------------------------------------------------------------------- 1 | base: 2 | '*': 3 | - charmer 4 | 5 | -------------------------------------------------------------------------------- /salt/roots/salt/etc/theanorc: -------------------------------------------------------------------------------- 1 | [blas] 2 | ldflags = -lblas -lgfortran 3 | 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | *.log 3 | log 4 | data 5 | notebooks/last_commit.txt 6 | 7 | -------------------------------------------------------------------------------- /salt/roots/pillar/top.sls: -------------------------------------------------------------------------------- 1 | base: 2 | 'charmed34': 3 | - python34 4 | 5 | 6 | -------------------------------------------------------------------------------- /salt/minion: -------------------------------------------------------------------------------- 1 | file_client: local 2 | 3 | log_file: /srv/log/minion 4 | 5 | providers: 6 | service: upstart 7 | -------------------------------------------------------------------------------- /salt/roots/salt/bin/ipynb.upstart: -------------------------------------------------------------------------------- 1 | description "Run IPython Notebook server" 2 | 3 | start on filesystem or runlevel [2345] 4 | stop on runlevel [!2345] 5 | 6 | # if you want it to automatically restart if it crashes, leave the next line in 7 | respawn 8 | 9 | limit nofile 100000 100000 10 | 11 | {% set pyver = pillar['pyver'] %} 12 | {% set pyver_ints = pyver|replace('.', '') %} 13 | 14 | script 15 | cd /home/vagrant/notebooks 16 | su -c "python{{ pyver }} -m IPython notebook --matplotlib inline --ip='*' --port 88{{ pyver_ints }}" vagrant 17 | end script 18 | 19 | -------------------------------------------------------------------------------- /salt/roots/salt/bin/sanity_check.py: -------------------------------------------------------------------------------- 1 | #! {{ python }} 2 | 3 | import urllib.request 4 | from sys import argv 5 | 6 | {% set pkgs = pillar['pip_pkgs'] %} 7 | {# FIXME #} 8 | {% for pkg in pkgs %} 9 | 10 | {% if pkg['import'] is defined %} 11 | {% set name = pkg['import'] %} 12 | {% else %} 13 | {% set name = pkg['name']|lower %} 14 | {% endif %} 15 | 16 | print("Checking {{ name }} is intact...") 17 | import {{ name }} 18 | try: 19 | print("Imported {{ name }} %s OK" % {{ name }}.__version__) 20 | except AttributeError: 21 | print("Imported {{ name }} OK") 22 | 23 | {% endfor %} 24 | 25 | req = urllib.request.Request('{{ nb_url }}') 26 | response = urllib.request.urlopen(req) 27 | 28 | print("Your VM is up and running: {{ nb_url }}") 29 | 30 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | Add PyToolz, xlrd, dill 2 | 3 | 4 | Try Pyrasite, and Ian Ozsvald's memory profiler 5 | 6 | 7 | Add tests for lifelines 8 | 9 | 10 | Add link to this doc http://nbviewer.ipython.org/github/ptwobrussell/Mining-the-Social-Web-2nd-Edition/blob/master/ipynb/_Appendix%20C%20-%20Python%20%26%20IPython%20Notebook%20Tips.ipynb 11 | 12 | 13 | Standardize output file naming format -- name sympy and pytables tests the same way we do theano 14 | 15 | 16 | Check whether we need to use os.walk for sympy or pytables tests 17 | 18 | 19 | Install optional backends for matplotlib (latex etc.) 20 | 21 | 22 | .matplotlibrc to use Agg by default 23 | 24 | 25 | Set up ramdisk for theano cache during Salt provisioning: 26 | 27 | mount -t ramfs -o size=200m ramfs /home/vagrant/theano_ram_cache 28 | 29 | Then get Theano to use this during testing: 30 | 31 | THEANO_FLAGS="base_compiledir=/home/vagrant/theano_ram_cache" $PYTHON `which theano-nose` --theano 32 | 33 | * Should we cache Theano artifacts on the host? 34 | 35 | 36 | Need to add documentation on clearing Theano cache 37 | 38 | 39 | Re-enable Theano typed_list tests when this feature is released (https://github.com/andrewclegg/snake-charmer/issues/17) 40 | 41 | 42 | Build ATLAS from source, for speed 43 | 44 | 45 | OpenCV looks like a big project: 46 | 47 | https://help.ubuntu.com/community/OpenCV 48 | 49 | http://opencvlover.blogspot.co.uk/2014/01/installing-opencv-on-ubuntu.html 50 | 51 | http://stackoverflow.com/questions/20953273/install-opencv-for-python-3-3/21212023#21212023 52 | 53 | 54 | -------------------------------------------------------------------------------- /notebooks/Demo/cm.csv: -------------------------------------------------------------------------------- 1 | source,target,value 2 | Russian,French,0.03278688524590164 3 | Russian,Spanish,0.02564102564102564 4 | Russian,Ukrainian,0.22727272727272727 5 | French,German,0.14035087719298245 6 | French,Italian,0.018867924528301886 7 | French,Spanish,0.15384615384615385 8 | French,Dutch,0.1891891891891892 9 | French,Polish,0.08823529411764706 10 | Swedish,French,0.03278688524590164 11 | Swedish,Italian,0.03773584905660377 12 | Swedish,Spanish,0.02564102564102564 13 | Swedish,Dutch,0.08108108108108109 14 | German,French,0.09836065573770492 15 | German,Swedish,0.01639344262295082 16 | German,Italian,0.07547169811320754 17 | German,Spanish,0.10256410256410256 18 | German,Dutch,0.21621621621621623 19 | German,Polish,0.029411764705882353 20 | Italian,French,0.03278688524590164 21 | Italian,Swedish,0.01639344262295082 22 | Italian,German,0.15789473684210525 23 | Italian,Spanish,0.05128205128205128 24 | Italian,Dutch,0.08108108108108109 25 | Italian,Portuguese,0.1 26 | Spanish,French,0.08196721311475409 27 | Spanish,German,0.05263157894736842 28 | Spanish,Dutch,0.10810810810810811 29 | Spanish,Portuguese,0.2 30 | Dutch,French,0.08196721311475409 31 | Dutch,German,0.15789473684210525 32 | Dutch,Spanish,0.05128205128205128 33 | Dutch,Polish,0.08823529411764706 34 | Polish,French,0.06557377049180328 35 | Polish,German,0.17543859649122806 36 | Polish,Italian,0.018867924528301886 37 | Polish,Dutch,0.08108108108108109 38 | Polish,Japanese,0.2 39 | Ukrainian,Russian,0.06451612903225806 40 | Chinese,Dutch,0.10810810810810811 41 | Chinese,Japanese,0.4 42 | Portuguese,German,0.017543859649122806 43 | Portuguese,Spanish,0.07692307692307693 44 | Arabic,Chinese,0.09090909090909091 45 | Arabic,Farsi,0.4 46 | Farsi,Chinese,0.09090909090909091 47 | Japanese,Dutch,0.05405405405405406 48 | Hebrew,Dutch,0.05405405405405406 49 | Catalan,Spanish,0.02564102564102564 50 | Catalan,Dutch,0.02702702702702703 51 | Hungarian,Dutch,0.05405405405405406 52 | Bulgarian,Russian,0.016129032258064516 53 | Bulgarian,Ukrainian,0.09090909090909091 54 | Finnish,French,0.03278688524590164 55 | Finnish,Dutch,0.02702702702702703 56 | Turkish,German,0.03508771929824561 57 | Czech,Spanish,0.02564102564102564 58 | Norwegian,Dutch,0.02702702702702703 59 | Esperanto,French,0.01639344262295082 60 | Esperanto,Dutch,0.02702702702702703 61 | Indonesian,German,0.017543859649122806 62 | -------------------------------------------------------------------------------- /notebooks/Hello World.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:b385549ffbada96541e76ca5768491391e29f674ef8a86908276e3c39308fa79" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "markdown", 13 | "metadata": {}, 14 | "source": [ 15 | "# Welcome to Snake Charmer" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "## Run this for basic system information" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "collapsed": false, 28 | "input": [ 29 | "import sys\n", 30 | "%load_ext rmagic\n", 31 | "%load_ext octavemagic\n", 32 | "print('Python version %s\\n' % (sys.version))\n", 33 | "!cat /proc/version; echo; cat /etc/*-release; echo; free -m; echo\n", 34 | "!echo -n \"Last Github \"; cat ~/notebooks/last_commit.txt; echo\n", 35 | "%R -o rVer rVer = R.version$version.string\n", 36 | "print(rVer[0])\n", 37 | "%octave disp(['Octave v' (v = ver())(1).Version]);" 38 | ], 39 | "language": "python", 40 | "metadata": {}, 41 | "outputs": [] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "## Run this for a list of installed Python packages" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "collapsed": false, 53 | "input": [ 54 | "# Same as \"pip list\" for current interpreter's pip. Std. library excluded.\n", 55 | "from pip.util import get_installed_distributions\n", 56 | "print('\\n'.join(str(x)\n", 57 | " for x in sorted(get_installed_distributions(),\n", 58 | " key=lambda dist: dist.project_name.lower())))" 59 | ], 60 | "language": "python", 61 | "metadata": {}, 62 | "outputs": [] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "## Run this for a list of installed Ubuntu packages" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "collapsed": false, 74 | "input": [ 75 | "!dpkg -l" 76 | ], 77 | "language": "python", 78 | "metadata": {}, 79 | "outputs": [] 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "## Run this for a graph" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "collapsed": false, 91 | "input": [ 92 | "import numpy as np\n", 93 | "import prettyplotlib as ppl\n", 94 | "np.random.seed(12)\n", 95 | "fig, ax = ppl.plt.subplots(1)\n", 96 | "ppl.hist(ax, np.random.randn(1000))" 97 | ], 98 | "language": "python", 99 | "metadata": {}, 100 | "outputs": [] 101 | } 102 | ], 103 | "metadata": {} 104 | } 105 | ] 106 | } -------------------------------------------------------------------------------- /salt/roots/salt/bin/run_tests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PYTHON=$1 4 | DIR=$2 5 | 6 | rm -rf $DIR 7 | mkdir $DIR 8 | cd $DIR # child processes inherit CWD by default 9 | 10 | 11 | # PyTables, SymPy and PyMC are currently slowest -- and Theano too if we take 12 | # it off FAST_COMPILE. 13 | 14 | 15 | # Put nicely-behaved well-distributed tests that use all cores evenly first -- 16 | # just coincidence that both of these need root 17 | 18 | 19 | # Numba 20 | 21 | sudo $PYTHON -c "import numba; numba.testing.multitest()" > numba.out 2>&1 22 | 23 | # IPython.parallel 24 | 25 | sudo $PYTHON `which iptest3` --all show parallel > ipython.parallel.out 2>&1 26 | 27 | 28 | # Then just bring the others in slowly, with progressively lower priority 29 | 30 | 31 | # PyTables 32 | 33 | nice -n -14 $PYTHON /usr/local/lib/$PYTHON/dist-packages/tables/tests/test_all.py --heavy > pytables.out 2>&1 & 34 | 35 | 36 | # SymPy 37 | 38 | sleep 30 39 | 40 | nice -n -13 nosetests --exe sympy > sympy.out 2>&1 & 41 | 42 | 43 | # PyMC 44 | 45 | sleep 30 46 | 47 | nice -n -12 $PYTHON -c "import pymc; pymc.test('full')" > pymc.out 2>&1 & 48 | 49 | 50 | # Theano 51 | 52 | sleep 30 53 | 54 | export OLDFLAGS=$THEANO_FLAGS 55 | 56 | export THEANO_FLAGS='mode=FAST_COMPILE' # Although maybe it's a better test with full optimizations included? 57 | 58 | nice -n -11 $PYTHON `which theano-nose` --batch=100 --theano > theano.out 2>&1 & 59 | 60 | export THEANO_FLAGS=$OLDFLAGS 61 | 62 | 63 | # Matpotlib 64 | 65 | sleep 30 66 | 67 | nice -n -10 $PYTHON -c "import matplotlib; matplotlib.test()" > matplotlib.out 2>&1 & 68 | 69 | 70 | # NumPy 71 | 72 | sleep 30 73 | 74 | nice -n -9 $PYTHON -c "import numpy; numpy.test('full')" > numpy.out 2>&1 & 75 | 76 | 77 | # SciPy 78 | 79 | sleep 30 80 | 81 | nice -n -8 $PYTHON -c "import scipy; scipy.test('full')" > scipy.out 2>&1 & 82 | 83 | 84 | # IPython 85 | 86 | nice -n -7 $PYTHON `which iptest3` --all show html kernel.inprocess autoreload terminal core kernel lib qt nbformat testing nbconvert config extensions utils > ipython.OTHERS.out 2>&1 & 87 | 88 | 89 | # Pandas 90 | 91 | sleep 30 92 | 93 | nice -n -6 $PYTHON /usr/local/bin/nosetests pandas > pandas.out 2>&1 & 94 | 95 | 96 | # scikit-learn 97 | 98 | sleep 30 99 | 100 | nice -n -5 $PYTHON /usr/local/bin/nosetests sklearn --exe > scikit-learn.out 2>&1 & 101 | 102 | 103 | # Joblib 104 | 105 | sleep 30 106 | 107 | nice -n -4 $PYTHON /usr/local/bin/nosetests joblib > joblib.out 2>&1 & 108 | 109 | 110 | # Pillow 111 | 112 | sleep 30 113 | 114 | (ln -s /srv/cache/src/Pillow/Images . && nice -n -3 $PYTHON /srv/cache/src/Pillow/selftest.py --installed) > pillow.out 2>&1 & 115 | 116 | 117 | # NLTK 118 | 119 | sleep 30 120 | 121 | nice -n -2 $PYTHON /usr/local/lib/$PYTHON/dist-packages/nltk/test/runtests.py > nltk.out 2>&1 & 122 | 123 | 124 | # gensim 125 | 126 | sleep 30 127 | 128 | (cd /srv/cache/src/gensim && nice -n -1 $PYTHON setup.py test) > gensim.out 2>&1 & 129 | 130 | 131 | # Cython 132 | 133 | sleep 30 134 | 135 | (cd /srv/cache/src/Cython/ && nice -n 0 $PYTHON runtests.py -vv) > cython.out 2>&1 & 136 | 137 | 138 | # Bottleneck 139 | 140 | sleep 30 141 | 142 | nice -n 1 $PYTHON -c "import bottleneck as bn; bn.test()" > bottleneck.out 2>&1 & 143 | 144 | 145 | # Statsmodels 146 | 147 | sleep 30 148 | 149 | nice -n 2 $PYTHON -c "import statsmodels as sm; sm.test()" > statsmodels.out 2>&1 & 150 | 151 | 152 | # Numexpr 153 | 154 | sleep 30 155 | 156 | nice -n 3 $PYTHON /usr/local/bin/nosetests numexpr > numexpr.out 2>&1 & 157 | 158 | 159 | # Patsy 160 | 161 | sleep 30 162 | 163 | nice -n 4 $PYTHON /usr/local/bin/nosetests patsy > patsy.out 2>&1 & 164 | 165 | 166 | wait 167 | 168 | -------------------------------------------------------------------------------- /notebooks/Demo/graph.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 41 | 42 | 189 | 190 | 191 | 192 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | require 'vagrant/util/which' 5 | 6 | require 'yaml' 7 | 8 | def get_env(key, default) 9 | if ENV.has_key? key 10 | return YAML.load(ENV[key]) 11 | else 12 | return default 13 | end 14 | end 15 | 16 | def mkdir(*dirs) 17 | dirs.each do |dir| 18 | Dir.mkdir dir unless File.exists? dir 19 | end 20 | end 21 | 22 | def install_plugins(vagrant_exe, *plugins) 23 | needs_restart = false 24 | 25 | plugins.each do |plugin| 26 | unless Vagrant.has_plugin? plugin 27 | # Install vbguest via another vagrant process; wait for completion 28 | cmd1 = [vagrant_exe, "plugin", "install", plugin] 29 | system *cmd1 30 | needs_restart = true 31 | end 32 | end 33 | 34 | if needs_restart 35 | # Restart vagrant with same args as current invocation 36 | cmd2 = ARGV 37 | cmd2.unshift vagrant_exe 38 | exec *cmd2 39 | end 40 | end 41 | 42 | begin 43 | Vagrant.require_version ">= 1.5.2" 44 | rescue NoMethodError 45 | raise 'Snake Charmer is only supported on Vagrant >= 1.5.2, please upgrade. Thanks.' 46 | end 47 | 48 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 49 | VAGRANTFILE_API_VERSION = "2" 50 | 51 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 52 | 53 | vagrant_exe = Vagrant::Util::Which.which("vagrant") 54 | unless vagrant_exe 55 | raise 'vagrant not found in path' 56 | end 57 | git_exe = Vagrant::Util::Which.which("git") 58 | 59 | install_plugins vagrant_exe, "vagrant-vbguest" 60 | 61 | # Get working directories from environment, using defaults if not supplied, 62 | # and create them if necessary 63 | notebook_dir = get_env("CHARMER_NOTEBOOK_DIR", "notebooks") 64 | data_dir = get_env("CHARMER_DATA_DIR", "data") 65 | salt_root = get_env("CHARMER_SALT_ROOT", "salt/roots/salt") 66 | pillar_root = get_env("CHARMER_PILLAR_ROOT", "salt/roots/pillar") 67 | cache_dir = get_env("CHARMER_CACHE_DIR", ".cache") 68 | log_dir = get_env("CHARMER_LOG_ROOT", "log") 69 | mkdir notebook_dir, data_dir, cache_dir, cache_dir + "/apt", cache_dir + "/apt/partial", log_dir 70 | 71 | # Workaround for https://www.virtualbox.org/ticket/12879 72 | # (missing symlink in guest additions package) 73 | 74 | config.vbguest.installer = Class.new(VagrantVbguest::Installers::Ubuntu) do 75 | 76 | # Aftr running standard install process, put the missing 77 | # symlink in place if it's not there already 78 | def install(opts=nil, &block) 79 | super 80 | fix_link = "test -e /usr/lib/VBoxGuestAdditions || ln -s /usr/lib/x86_64-linux-gnu/VBoxGuestAdditions /usr/lib/VBoxGuestAdditions" 81 | communicate.sudo(fix_link, opts, &block) 82 | end 83 | 84 | end 85 | 86 | config.vm.box = "hashicorp/precise64" 87 | 88 | config.ssh.forward_agent = true 89 | config.ssh.forward_x11 = true 90 | 91 | # Disable default synced folder -- it will lead to messiness... 92 | config.vm.synced_folder ".", "/vagrant", disabled: true 93 | 94 | # These locations are shared between all VMs 95 | config.vm.synced_folder notebook_dir, "/home/vagrant/notebooks" 96 | config.vm.synced_folder data_dir, "/home/vagrant/data" 97 | config.vm.synced_folder salt_root, "/srv/salt" 98 | config.vm.synced_folder pillar_root, "/srv/pillar" 99 | config.vm.synced_folder cache_dir, "/srv/cache" 100 | 101 | config.vm.define "charmed34" do |charmed34| 102 | 103 | # These locations are unique to each VM 104 | my_log = log_dir + "/charmed34" 105 | mkdir my_log 106 | charmed34.vm.synced_folder my_log, "/srv/log" 107 | 108 | # Bound ports -- IPython Notebook and SSH 109 | charmed34.vm.network "forwarded_port", guest: 8834, host: 8834, host_ip: "127.0.0.1" 110 | charmed34.vm.network "forwarded_port", guest: 22, host: 2234, host_ip: "127.0.0.1" 111 | 112 | # Unbound ports 9034, 9134 ... 9934 for user use -- forwarded automatically 113 | for i in 90..99 114 | port = (i * 100) + 34 115 | charmed34.vm.network "forwarded_port", guest: port, host: port 116 | end 117 | 118 | # Wipe salt minion log ready for fresh run 119 | charmed34.vm.provision "shell", 120 | inline: "truncate -s 0 /srv/log/minion" 121 | 122 | # Install and configure packages via salt 123 | charmed34.vm.provision :salt do |salt| 124 | salt.install_type = "stable" 125 | 126 | salt.minion_config = "salt/minion" 127 | salt.run_highstate = true 128 | salt.pillar({ 129 | "run_tests" => get_env("CHARMER_TEST", false), # not currently used 130 | "slimline" => get_env("CHARMER_SLIM", false) 131 | }) 132 | end 133 | 134 | # Stop salt-minion service to save resources, and disable it; 135 | # then run post-install sanity check 136 | charmed34.vm.provision "shell", 137 | inline: "service salt-minion stop; echo manual > /etc/init/salt-minion.override; /root/bin/sanity_check.py" 138 | 139 | # VM settings 140 | charmed34.vm.provider "virtualbox" do |v| 141 | v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] 142 | v.name = "charmed34" 143 | v.memory = get_env("CHARMER_RAM", 2048) 144 | v.cpus = get_env("CHARMER_CPUS", 1) 145 | end 146 | 147 | charmed34.vm.hostname = "charmed34" 148 | 149 | end 150 | 151 | # Write top of git log into data so we can show it in Hello World notebook 152 | msg = "commit not found" 153 | if git_exe 154 | last_commit = `"#{git_exe}" log --no-merges -n 1` 155 | if $?.success? 156 | msg = last_commit 157 | end 158 | end 159 | File.open("notebooks/last_commit.txt", "w") do |f| 160 | f.write(last_commit) 161 | end 162 | 163 | end 164 | 165 | # TODO 166 | # 167 | # Loop through set of python versions and configure 168 | # a VM for each one, something like this: 169 | # http://maci0.wordpress.com/2013/11/09/dynamic-multi-machine-vagrantfile/ 170 | 171 | -------------------------------------------------------------------------------- /salt/roots/pillar/python34.sls: -------------------------------------------------------------------------------- 1 | pyver: '3.4' 2 | 3 | # Deb packages (with optional versions) to install via apt -- order is not important 4 | 5 | apt_pkgs: 6 | - freetds-dev: 0.91-1 7 | - g++-4.6: 4.6.3-1ubuntu5 8 | - gfortran: 4:4.6.3-1ubuntu5 9 | - git: 1:1.7.9.5-1 10 | - htop: 1.0.1-1 11 | - libatlas-base-dev: 3.8.4-3build1 12 | - libatlas-dev: 3.8.4-3build1 13 | - libatlas3gf-base: 3.8.4-3build1 14 | - libboost-program-options-dev: 1.48.0.2 15 | - libfreetype6-dev: 2.4.8-1ubuntu2.1 16 | - libhdf5-serial-dev: 1.8.4-patch1-3ubuntu2 17 | - libjpeg8-dev: 8c-2ubuntu7 18 | - liblapack-dev: 3.3.1-1 19 | - liblapack3gf: 3.3.1-1 20 | - liblcms1-dev: 1.19.dfsg-1ubuntu3 21 | - liblzo2-dev: 2.06-1ubuntu0.1 22 | - libpng12-dev: 1.2.46-3ubuntu4 23 | - libpq-dev: 9.1.14-0ubuntu0.12.04 24 | - libtiff4-dev: 3.9.5-2ubuntu1.6 25 | - libwebp-dev: 0.1.3-2.1ubuntu1 26 | - libxml2-dev: 2.7.8.dfsg-5.1ubuntu4 27 | - libxslt1-dev: 1.1.26-8ubuntu1.3 28 | - libzmq-dev: 2.1.11-1ubuntu1 29 | - llvm-3.3-dev: 1:3.3-5ubuntu4~precise1 30 | - octave3.2: 3.2.4-12 31 | - pandoc: 1.9.1.1-1 32 | - python-software-properties: 0.82.7.7 33 | - r-base-dev: 2.14.1-1 34 | - r-recommended: 2.14.1-1 35 | - tcl8.5-dev: 8.5.11-1ubuntu1 36 | - tk8.5-dev: 8.5.11-1 37 | - zlib1g-dev: 1:1.2.3.4.dfsg-3ubuntu4 38 | 39 | # Additional debs for (optional) OpenCV -- work in progress! 40 | 41 | opencv_deps: 42 | - libopencv-dev 43 | - checkinstall 44 | - cmake 45 | - pkg-config 46 | - yasm 47 | - libjpeg-dev 48 | - libjasper-dev 49 | - libavcodec-dev 50 | - libavformat-dev 51 | - libswscale-dev 52 | - libdc1394-22-dev 53 | - libxine-dev 54 | - libgstreamer0.10-dev 55 | - libgstreamer-plugins-base0.10-dev 56 | - libv4l-dev 57 | - python-dev 58 | - python-numpy 59 | - libtbb-dev 60 | - libqt4-dev 61 | - libgtk2.0-dev 62 | - libfaac-dev 63 | - libmp3lame-dev 64 | - libopencore-amrnb-dev 65 | - libopencore-amrwb-dev 66 | - libtheora-dev 67 | - libvorbis-dev 68 | - libxvidcore-dev 69 | - x264 70 | - v4l-utils 71 | - ffmpeg 72 | 73 | # Python packages to install via pip -- a list of dicts, so order is maintained 74 | 75 | pip_pkgs: 76 | - name: openpyxl 77 | ver: ==1.8.5 78 | - name: numpy 79 | ver: ==1.8.2 80 | - name: scipy 81 | ver: ==0.14.0 82 | - name: Theano 83 | git: https://github.com/Theano/Theano.git 84 | rev: ba81e61d6545d5a82fbf9c85b9e6db2fd3c12ea3 85 | - name: numpydoc 86 | git: https://github.com/numpy/numpydoc.git 87 | rev: 56a1b39e8b16f2292f6b2b54ce26d259003fadad 88 | - name: Bottleneck 89 | ver: ==0.8.0 90 | - name: matplotlib 91 | git: https://github.com/matplotlib/matplotlib.git 92 | rev: 4b1bd6301d69f856deca9c614af563f5fb4d1e90 93 | - name: patsy 94 | ver: ==0.2.1 95 | - name: numexpr 96 | ver: ==2.4 97 | - name: Cython 98 | git: https://github.com/cython/cython.git 99 | rev: c646da0b6527bf2ad18e706eb4273c712ac356a8 100 | - name: tables 101 | ver: ==3.1.1 102 | - name: beautifulsoup4 103 | ver: ==4.2.1 104 | import: bs4 105 | - name: lxml 106 | ver: ==3.3.5 107 | - name: deap 108 | ver: ==1.0.1 109 | - name: fastcluster 110 | ver: ==1.1.13 111 | - name: sympy 112 | git: https://github.com/sympy/sympy.git 113 | rev: 254d99c1c2bfbd4f634f61cd5a444c8a3e79450b 114 | - name: pandas 115 | ver: ==0.14 116 | - name: lifelines 117 | ver: ==0.4.0.0 118 | - name: nose 119 | ver: ==1.3.3 120 | - name: mock 121 | ver: ==1.0.1 122 | - name: statsmodels 123 | git: https://github.com/statsmodels/statsmodels.git 124 | rev: 74b4dc920456a82e11d4ebd865bb698769b0b767 125 | - name: pymc 126 | git: https://github.com/pymc-devs/pymc 127 | rev: 950c18e0fc6ac5f7dd9c1a4c79100aa7b6d95014 128 | - name: psycopg2 129 | ver: ==2.5.2 130 | - name: pymssql 131 | ver: ==2.1.0 132 | - name: brewer2mpl 133 | ver: ==1.4 134 | - name: prettyplotlib 135 | git: https://github.com/olgabot/prettyplotlib.git 136 | rev: fc04d6e4f5edf1d402f51abeacc0dfd5132f60be 137 | - name: seaborn 138 | ver: ==0.3.1 139 | - name: ipython[all] 140 | import: IPython 141 | ver: ==2.0.0 142 | - name: scikit-learn 143 | ver: ==0.15.1 144 | import: sklearn 145 | - name: runipy 146 | ver: ==0.0.8 147 | - name: Pillow 148 | import: PIL 149 | git: https://github.com/python-imaging/Pillow.git 150 | rev: 84a701a82b33896a4d6997743c2131ab0a40c588 151 | - name: joblib 152 | git: https://github.com/joblib/joblib.git 153 | rev: f76129c888394a6b32aab8e6436c74c0b2ee842c 154 | - name: pexpect 155 | ver: ==3.2 156 | - name: rpy2 157 | ver: ==2.3.10 158 | - name: oct2py 159 | ver: ==1.3.0 160 | - name: pyyaml 161 | import: yaml 162 | ver: ==3.11 163 | - name: nltk 164 | git: https://github.com/nltk/nltk.git 165 | rev: 24e257a6fd29df503e9ffe628f96604012c8a8e1 166 | - name: Pyro4 167 | import: Pyro4 168 | ver: ==4.25 169 | - name: gensim 170 | git: https://github.com/piskvorky/gensim.git 171 | rev: d1c6d58b9acfec97e6c5d677c652293ae2119276 172 | export: true 173 | setup: python3.4 setup.py develop 174 | - name: llvmpy 175 | git: https://github.com/llvmpy/llvmpy.git 176 | rev: 34900d27481bd3457a4fac03569eb152b0fa8678 177 | setup: (LLVM_CONFIG_PATH=/usr/bin/llvm-config-3.3 python3.4 setup.py install) 178 | - name: numba 179 | git: https://github.com/numba/numba.git 180 | rev: 4b9a36e7b344823fa2d2bb85221a14ff9e0abb03 181 | export: true 182 | setup: pip3.4 install --install-option=build_ext --install-option="--inplace" -e . 183 | - name: emcee 184 | ver: ==2.1.0 185 | - name: pystan 186 | git: https://github.com/stan-dev/pystan.git 187 | rev: 15ce22cfc4714ccb0f3b1f256d4704d81ac6ff81 188 | - name: mpld3 189 | ver: ==0.2 190 | - name: wabbit_wappa 191 | git: https://github.com/snake-charmer-devs/wabbit_wappa.git 192 | rev: a9f7a4d34d44ff6f6000850558d833efe07bbb95 193 | - name: requests 194 | ver: ==2.3.0 195 | 196 | # Misc stuff not installed through usual channels 197 | 198 | vw_rev: 0c8da21c979951a36373ca0cb601d4c5bd056038 199 | 200 | -------------------------------------------------------------------------------- /salt/roots/salt/charmer/init.sls: -------------------------------------------------------------------------------- 1 | # DO NOT SWITCH STATE AUTO ORDERING OFF! 2 | # We just allow Salt to apply these states in order, instead of manually 3 | # specifying dependencies. 4 | 5 | {% set pyver = pillar['pyver'] %} 6 | {% set theanover = pillar['theanover'] %} 7 | {% set pyver_ints = pyver|replace('.', '') %} 8 | 9 | # Fix Grub boot issue: http://serverfault.com/a/482020 10 | 11 | /etc/default/grub: 12 | file.append: 13 | - text: GRUB_RECORDFAIL_TIMEOUT=2 14 | 15 | update-grub: 16 | cmd.run 17 | 18 | # Some scripts we'll need 19 | 20 | /root/bin: 21 | file.recurse: 22 | - source: salt://bin 23 | - template: jinja 24 | - dir_mode: '0755' 25 | - file_mode: '0755' 26 | - context: 27 | python: /usr/bin/python{{ pyver }} 28 | nb_url: http://localhost:88{{ pyver_ints }}/tree 29 | 30 | # Replace apt package cache with symlink to folder shared from host 31 | 32 | /var/cache/apt/archives: 33 | file.symlink: 34 | - target: /srv/cache/apt 35 | - force: true 36 | 37 | # Install apt packages from standard repos 38 | 39 | apt_pkgs: 40 | pkg.installed: 41 | - pkgs: 42 | {{ pillar['apt_pkgs'] }} 43 | 44 | # scipy won't build when this is enabled, but we may not need it 45 | # libsuitesparse-dev: 46 | # pkg.installed 47 | 48 | # Additional apt repositories 49 | 50 | deadsnakes: 51 | pkgrepo.managed: 52 | - ppa: fkrull/deadsnakes 53 | 54 | # FIXME the following repo is currently version-specific 55 | 56 | cran: 57 | pkgrepo.managed: 58 | - name: deb http://cran.ma.imperial.ac.uk/bin/linux/ubuntu precise/ 59 | 60 | # Workarounds for annoying dangling symlink: 61 | # https://github.com/nose-devs/nose/issues/731 62 | 63 | /usr/local/share/man: 64 | file.directory: 65 | - user: root 66 | - group: root 67 | - dir_mode: 755 68 | - follow_symlinks: True 69 | - recurse: 70 | - user 71 | - group 72 | - mode 73 | 74 | man_dir_check: 75 | cmd.run: 76 | - name: test -d /usr/local/man || ln -sf /usr/local/share/man /usr/local/man 77 | - user: root 78 | - group: root 79 | 80 | # Install Python etc. 81 | 82 | python_pkgs: 83 | pkg.installed: 84 | - pkgs: 85 | - python{{ pyver }} 86 | - python{{ pyver }}-dev 87 | - python{{ pyver }}-doc 88 | - python{{ pyver }}-tk 89 | 90 | {% set easy_install = 'easy_install-' ~ pyver %} 91 | {% set pip = 'pip' ~ pyver %} 92 | 93 | distribute: 94 | cmd.run: 95 | - name: python{{ pyver }} /root/bin/distribute_setup.py 96 | - unless: which {{ easy_install }} 97 | 98 | pip: 99 | cmd.run: 100 | - name: {{ easy_install }} pip 101 | - unless: which {{ pip }} 102 | 103 | {% set piplog = '/srv/log/pip.log' %} 104 | {% set pipcache = '/srv/cache/pip' %} 105 | {% set gitcache = '/srv/cache/src' %} 106 | 107 | {{ piplog }}: 108 | file.absent 109 | 110 | # Loop through pillar data and install all standard Python packages. 111 | # Handle them differently depending on whether they're github or pypi based. 112 | # Some of them have post-processing steps inserted after them. 113 | 114 | {% for pkg in pillar['pip_pkgs'] %} 115 | {% set name = pkg['name'] %} 116 | 117 | {% if pkg['git'] is defined %} 118 | {% set url = pkg['git'] %} 119 | {% set src = gitcache ~ '/' ~ name %} 120 | {% set dev_pkgs = '/home/vagrant/lib/dev-packages' %} 121 | 122 | # Checkout/refresh from github 123 | {{ url }}: 124 | git.latest: 125 | {% if pkg['rev'] is defined %} 126 | - rev: {{ pkg['rev'] }} 127 | {% endif %} 128 | - target: {{ src }} 129 | - force_checkout: true 130 | 131 | {% if name == 'Theano' %} 132 | # Remove invalid character from Theano -- temporary workaround 133 | {{ src }}/NEWS.txt: 134 | file.managed: 135 | - contents: "Dummy file" 136 | 137 | # Supply rc file to use correct fortran libraries 138 | /home/vagrant/.theanorc: 139 | file.managed: 140 | - source: salt://etc/theanorc 141 | - user: vagrant 142 | - group: vagrant 143 | - mode: 655 144 | {% endif %} 145 | 146 | {% if pkg.get('export', false) %} 147 | {% set install_from = dev_pkgs ~ '/' ~ name %} 148 | # Copy out of cache before installing 149 | {{ name }}_export: 150 | cmd.run: 151 | - name: git checkout-index -a -f --prefix={{ install_from }}/ 152 | - cwd: {{ src }} 153 | - user: vagrant 154 | {% else %} 155 | {% set install_from = src %} 156 | {% endif %} 157 | 158 | {% if pkg['setup'] is defined %} 159 | # Non-pip custom installation is required 160 | {{ name }}_install: 161 | cmd.run: 162 | - name: {{ pkg['setup'] }} >> "{{ piplog }}" 2>&1 163 | - cwd: {{ install_from }} 164 | {% else %} 165 | # Build and install from local copy via pip 166 | {{ name }}_install: 167 | cmd.run: 168 | - name: {{ pip }} install --log "{{ piplog }}" "{{ install_from }}" 169 | {% endif %} 170 | 171 | {% else %} 172 | 173 | {% if pkg['ver'] is defined %} 174 | {% set spec = name ~ pkg['ver'] %} 175 | {% else %} 176 | {% set spec = name %} 177 | {% endif %} 178 | 179 | # Build and install from PyPI, caching downloaded package 180 | {{ name }}_install: 181 | cmd.run: 182 | - name: {{ pip }} install --log "{{ piplog }}" --download-cache "{{ pipcache }}" "{{ spec }}" 183 | 184 | {% endif %} 185 | 186 | {% if name == 'ipython' %} 187 | # Install mathjax so we can use iPython without internet 188 | local_mathjax: 189 | cmd.run: 190 | - name: python{{ pyver }} -c "from IPython.external.mathjax import install_mathjax; install_mathjax()" 191 | {% endif %} 192 | 193 | {% endfor %} 194 | 195 | # Vowpal Wabbit 196 | 197 | https://github.com/JohnLangford/vowpal_wabbit.git: 198 | git.latest: 199 | - rev: {{ pillar['vw_rev'] }} 200 | - target: {{ gitcache }}/vowpal_wabbit 201 | - force_checkout: true 202 | 203 | vw: 204 | cmd.run: 205 | - cwd: {{ gitcache }}/vowpal_wabbit 206 | - name: make 207 | 208 | /usr/local/bin/vw: 209 | file.copy: 210 | - source: {{ gitcache }}/vowpal_wabbit/vowpalwabbit/vw 211 | - force: true 212 | 213 | # Matplotlib config for use without a GUI 214 | 215 | /home/vagrant/.config/matplotlib/matplotlibrc: 216 | file.managed: 217 | - source: salt://etc/matplotlibrc 218 | - makedirs: true 219 | - user: vagrant 220 | - group: vagrant 221 | - mode: 655 222 | 223 | # Upstart service configuration -- start on boot 224 | 225 | /etc/init/ipynb.conf: 226 | file.symlink: 227 | - target: /root/bin/ipynb.upstart 228 | 229 | # Why is this even necessary dammit! 230 | initctl reload-configuration: 231 | cmd.run 232 | 233 | ipynb: 234 | service.running: 235 | - enable: True 236 | 237 | {% if not pillar.get('slimline', false) %} 238 | 239 | # Download NLTK sample data 240 | 241 | nltk_data: 242 | cmd.run: 243 | - name: python{{ pyver }} -m nltk.downloader all 244 | - user: vagrant 245 | - group: vagrant 246 | 247 | # Install dependencies for OpenCV 248 | 249 | # Commented out for now, as it's extra bloat unless we actually need it 250 | 251 | #opencv_deps: 252 | # pkg.installed: 253 | # - pkgs: 254 | # {{ pillar['opencv_deps'] }} 255 | 256 | {% endif %} 257 | 258 | {% if pillar.get('run_tests', false) %} 259 | 260 | # Run full test suite 261 | 262 | # TODO set up runipy; currently does nothing 263 | 264 | {% endif %} 265 | 266 | 267 | -------------------------------------------------------------------------------- /FAQ.md: -------------------------------------------------------------------------------- 1 | # Snake Charmer 2 | 3 | ## Frequently Asked Questions 4 | 5 | ### Why is the virtual machine called charmed34? 6 | 7 | "34" is the 3.4 from Python 3.4, without the decimal point. And "charmed" 8 | as in snake charmers. 9 | 10 | The plan is to offer a number of different VMs with different Python versions. 11 | 12 | ### Is there a binary distribution? Building everything from source each time sucks. 13 | 14 | My plan ultimately is to release a binary image that you can download and run 15 | via Vagrant Cloud -- this should be as easy as the current install but much 16 | quicker. However this needs two things: 17 | 18 | 1. A thorough check of all the licenses of all the components to make sure 19 | there's nothing preventing us from doing that. (If there are, a 2-stage install 20 | might be the answer -- binary first, then optional extras.) 21 | 22 | 2. Somewhere to host the disk images. I don't have any funding for that yet. 23 | Volunteers welcome :-) 24 | 25 | ### Are there any ways I can speed up the install? 26 | 27 | Not the first time, but when it's up and running you can use VirtualBox's 28 | admin app to [take a snapshot](http://www.howtogeek.com/150258/how-to-save-time-by-using-snapshots-in-virtualbox/) 29 | of the machine. Then you can always restore to that, rather than having to 30 | build a new one. 31 | 32 | Snake Charmer also caches as many downloaded packages as possible, in the 33 | `.cache` subdirectory, which can help speed up future rebuilds. 34 | 35 | ### Can I share and distribute my Snake Charmer VMs via Vagrant Cloud? 36 | 37 | Absolutely! We think this would be a great way to share experimental results 38 | (reproducibly), foster international collaborations, publish data sets 39 | interactively, and facilitate training, mentoring and learning. 40 | 41 | For now, you'll have to refer to the 42 | [sharing](http://docs.vagrantup.com/v2/share/index.html) and 43 | [boxing](http://docs.vagrantup.com/v2/boxes.html) docs for more details, but 44 | watch this space for Snake Charmer-centric tutorials later. 45 | 46 | ### Why does Snake Charmer use virtual machines, rather than just installing all its components in a virtualenv? 47 | 48 | Some packages don't play well with virtualenv. Also, there are always 49 | dependencies on the underlying operating system, or on libraries installed via 50 | its native package management. These differences can't be contained within a 51 | virtualenv. 52 | 53 | One of the primary goals of Snake Charmer is reproducibility. It lets you build 54 | environments that are guaranteed to behave the same way no matter what machine 55 | you are using. 56 | 57 | Another goal is portability. A Snake Charmer VM can be duplicated, and 58 | redeployed on another machine -- or a whole cluster of machines. 59 | 60 | Virtual machines are the easiest way to achieve these goals. They also allow us 61 | to include non-Python components like R -- this would be impossible in a 62 | virtualenv. 63 | 64 | Plus you get all the benefits of Vagrant, such as being able to share machines 65 | over Vagrant Cloud, and repackage them as new boxes. 66 | 67 | Finally, VMs allow you to set resource usage limits (e.g. on RAM and CPU) that 68 | can prevent runaway processes from rendering a machine unusable. 69 | 70 | ### What are the advantages of Snake Charmer over Anaconda, Canopy or Sage? 71 | 72 | Anaconda and Canopy are Scientific Python distributions providing free 73 | editions, but they have licensing limitations and closed ecosystems. Sage is 74 | open source, run by a non-profit, and available as a VM, but is quite academic 75 | and focused largely on pure maths. 76 | 77 | None of the advantages of VMs described above apply to these distros. 78 | 79 | Also, Snake Charmer aims to provide more up-to-date software -- in fact, this 80 | is a key goal. At the time of writing, neither Sage nor Canopy support Python 3 81 | at all, and Anaconda upgraded to Python 3.4 after Snake Charmer did. Snake 82 | Charmer also uses packages from the standard Python Package Index (PyPI) rather 83 | than a proprietary package management solution. 84 | 85 | ### Are Snake Charmer VMs slow? 86 | 87 | Virtualization incurs a small overhead, but with modern hardware, this isn't 88 | usually a major factor. Make sure virtualization extensions are 89 | [enabled in your BIOS](http://www.sysprobs.com/disable-enable-virtualization-technology-bios) 90 | (most manufacturers do this by default these days). 91 | 92 | *Installation* of a new VM is currently a bit slow, due to the number of 93 | native packages that need to be compiled from source. This is an area where 94 | distros like Anaconda win out due to their binary packages. In the future we 95 | plan to supply pre-built VM images, to get you up and running instantly. 96 | 97 | ### Can Snake Charmer VMs access my GPU? 98 | 99 | **GPU computation using CUDA is not supported yet**. GPU virtualization is a 100 | feature of some 101 | [recent NVIDIA hardware](http://www.nvidia.com/object/dedicated-gpus.html) 102 | but this is not yet possible in Python. _(As far as I know anyway...)_ 103 | 104 | If you definitely need this feature, try the specialized Python distros 105 | mentioned above. 106 | 107 | ### Can I run desktop GUI apps, like IPython Qt Console, or Tkinter apps? 108 | 109 | Yes, but it's a bit fiddly. You can ssh into the VM using `vagrant ssh` and 110 | start them from the command line, and they'll run as X11 apps, as if your VM 111 | was a remote Linux server. However, Windows users (and OS X users, Mountain 112 | Lion onward) will need to install an X server such as 113 | [Xming](http://sourceforge.net/projects/xming/) or 114 | [XQuartz](http://xquartz.macosforge.org) to make this work. 115 | 116 | Plus no GUI frameworks are installed by default on the VMs -- it's a very 117 | stripped-down server-oriented edition of Ubuntu. 118 | 119 | ### How can I connect to the VM by passwordless SSH from an application or script? 120 | 121 | This may be useful if you want to set up an IPython cluster of VMs on different 122 | machines, for example. Snake Charmer automatically forwards port 22NN on the 123 | host to port 22 (SSH) on the VM, where NN is the 2-digit Python version code. 124 | e.g. the `charmed34` VM will listen for incoming ssh on port 2234. 125 | 126 | To connect to this port via ssh programmatically, use `vagrant` for the 127 | username, and use the default `insecure_private_key` that vagrant ships with. 128 | Type `vagrant ssh-config` to check where this is on your system. 129 | 130 | Basically everything on the VM happens as this user. 131 | 132 | **N.B.** As a token nod to security (see below), the SSH port forwarding on the 133 | host computer only binds to 127.0.0.1 (localhost) by default. So an incoming 134 | SSH connection from another machine would have to connect first to the host 135 | (your physical computer) and then tunnel through to the VM. In the future, 136 | there may be an option to disable this restriction, and expose the SSH port on 137 | the VM to the whole world (at your own risk...). 138 | 139 | ### How can I connect to other ports on the VM, apart from SSH and the Notebook webserver? 140 | 141 | Normally you would need to edit the Vagrantfile in order to do this, but if you 142 | can change the port that the process listens on, you can do it more easily. 143 | 144 | The Vagrantfile sets up 10 ports that are automatically forwarded from the host 145 | to the VM, numbered 90NN, 91NN, 92NN ... 99NN -- where NN is the 2-digit Python 146 | version code. e.g. the `charmed34` VM will automatically forward ports 9034, 147 | 9134, 9234 ... 9934. 148 | 149 | If you have flexibility over what port to listen on, just use one of those, and 150 | any connections to the host on that port will get forwarded automatically. 151 | 152 | For example, you might want to start a simple webserver on port 9034: 153 | 154 | ```python 155 | import SimpleHTTPServer 156 | import SocketServer 157 | 158 | PORT = 9034 159 | 160 | Handler = SimpleHTTPServer.SimpleHTTPRequestHandler 161 | 162 | httpd = SocketServer.TCPServer(("", PORT), Handler) 163 | 164 | print "serving at port", PORT 165 | httpd.serve_forever() 166 | ``` 167 | 168 | Connecting to port 9034 on the host will attach to this webserver on the VM. 169 | 170 | Unlike the SSH and Notebook ports, these ports are intentionally _not_ 171 | protected from remote machines attaching to them, as a convenience for you -- 172 | but don't expose any vulnerable or sensitive services there and then come 173 | running to us. No refunds... 174 | 175 | ### Are Snake Charmer VMs secure? 176 | 177 | #### Not especially, no. 178 | 179 | The default username and password `vagrant`/`vagrant` is the same for most 180 | Vagrant boxes, as is the insecure private key used for passwordless 181 | authentication. By default, the Notebook and SSH port forwarders only bind to 182 | your computer's `localhost` address, making it slightly harder for someone to 183 | own your VM over a network -- _in theory_, that means they'd need to be logged 184 | onto your physical computer already. But it's not an insurmountable obstacle 185 | for a dedicated attacker. Also, it's rather dependent on VirtualBox's 186 | networking configuration, which tends to change from release to release. 187 | 188 | And even if you don't put any sensitive data into your VM, 189 | there [may still be ways](http://blog.ontoillogical.com/blog/2012/10/31/breaking-in-and-out-of-vagrant/) 190 | to get access to the host from the guest. (We have closed that particular 191 | loophole but can't guarantee there aren't others.) 192 | 193 | In short, don't use it to provision publicly accessible servers or anything 194 | like that. You can always reuse the Salt config files on 'real' servers. 195 | 196 | Tightening security would be very useful, though -- can you help? 197 | 198 | ### Can I run Snake Charmer on cloud services like AWS, or other virtualization platforms like VMWare? 199 | 200 | In theory, yes, although not without some work. You'll need to understand 201 | Vagrant and rewrite the Vagrantfile to support this. In the future, we'd like 202 | to be able to make that easy out-of-the-box. 203 | 204 | -------------------------------------------------------------------------------- /CUSTOMIZING.md: -------------------------------------------------------------------------------- 1 | # Snake Charmer 2 | 3 | ## Customization guide 4 | 5 | This document covers the technology behind Snake Charmer in more detail, and 6 | discusses some ways to customize your VMs and create new ones. 7 | 8 | ### "vagrant up" -- under the covers 9 | 10 | Before you start customizing VMs, it's a good idea to understand how all the 11 | pieces fit together, and in particular, what happens when you fire up a VM for 12 | the first time. 13 | 14 | When you first perform a `vagrant up` command on a new VM, the following steps 15 | take place. The procedure is known as 'provisioning' in Vagrant terminology. 16 | 17 | #### Vagrant 18 | 19 | Firstly, Vagrant performs the following process, governed by the Vagrantfile 20 | (really just a Ruby script using Vagrant's configuration API). 21 | 22 | 1. Test for the presence of the `vagrant-vbguest` plugin, and 23 | install it automatically if it's missing. 24 | 1. Download a standard Ubuntu VM image from 25 | [Vagrant Cloud](https://vagrantcloud.com/), unless you have a local copy 26 | cached from a previous Vagrant run. 27 | 1. Create a new VM from the image, with port forwarding and folder 28 | sharing configured as described in the [README](README.md). 29 | 1. Install the Salt minion (i.e. client) service on the VM. 30 | 1. Tell the Salt minion to configure the new machine (see below). 31 | 1. Stop the Salt minion service, and disable run-on-startup behaviour. 32 | 33 | Notes: 34 | 35 | * `vagrant-vbguest` is required in order to keep VirtualBox's guest extensions 36 | package on the VM version-synced with your VirtualBox package on the host. 37 | * The VM is set up to use NAT networking, so it can see the LAN and internet 38 | but will not appear as a distinct device on the network. It is therefore only 39 | accessible from the host, via port forwarding. 40 | * The host's DNS configuration will be used to resolve hostname queries. 41 | * If your host has more than one network interface available, you'll be 42 | prompted by Vagrant to pick one. 43 | * In the initial release, only one VM is defined in the Vagrantfile 44 | (`charmed34`), but the plan is to provide multiple VMs for different versions 45 | of Python. 46 | * The `34` at the end of `charmed34` refers to Python 3.4. 47 | * Many aspects of the `Vagrantfile`, for example port numbers for forwarding, 48 | are parameterized by Python version. 49 | * Having more than one VM using the same Python version, on the same host, is 50 | not currently supported. In principle it's possible, but it would require 51 | additional work to implement naming and port forwarding correctly. 52 | * Provisioning more than one VM at the same time, from the same installation 53 | of Snake Charmer, is not supported. 54 | 55 | #### Salt 56 | 57 | The [Salt](http://www.saltstack.org)-based configuration procedure follows 58 | the following process. 59 | 60 | 1. Install a number of required packages on the VM from standard Ubuntu 61 | apt repositories. These are cached on the host in case they're needed again. 62 | 1. Enable the third-party [deadsnakes](https://launchpad.net/~fkrull/+archive/deadsnakes) 63 | repo, and install the appropriate Python version from there. 64 | 1. Install `distribute` directly from 65 | [pypi](http://pypi.python.org/packages/source/d/distribute/) in order to provide Pip. 66 | 1. One by one, install the required Python packages via Pip, either directly 67 | from PyPI or by a `git clone` of the source. As with apt packages, they are 68 | cached on the host outside the VM to save time later if they are needed again. 69 | 1. Install the IPython Notebook process as a Unix service, start it, and set it 70 | to start automatically when the VM boots. 71 | 72 | Notes: 73 | 74 | * The files used by Salt to configure the VM are in the `salt/roots/salt` 75 | subdirectory of your Snake Charmer installation directory. Most of the action 76 | happens in `init.sls`. 77 | * `.sls` files are just YAML files holding data structures understood by Salt. 78 | * Many aspects of these config files, e.g. package version numbers, 79 | are parameterized by Python version. This is what the two digits on the end of 80 | the VM name (e.g. `charmed34`) refer to. 81 | * The lists of Ubuntu and Python packages to install are stored in a file in 82 | `salt/roots/pillar` named for the Python version in use, e.g. `python34.sls`. 83 | * This lets us provide different dependencies (or different versions) for 84 | different versions of Python. 85 | * The format is described below. 86 | * The Salt log is `log//minion` on the host, in case you need it for 87 | debugging. 88 | * Likewise, the Pip log is in `log//pip.log` on the host. 89 | * The cached packages are stored in the `.cache` directory within the Snake 90 | Charmer install directory -- or `/srv/cache` within the VM. It's safe 91 | to delete this cache any time, except while you're actually performing a 92 | provisioning operation in Vagrant. 93 | 94 | ### Reprovisioning 95 | 96 | By default, the full provisioning process only takes place on first creating a 97 | VM -- since it takes so long. 98 | 99 | If you've made a configuration change, you'll need to explicitly reprovision 100 | the VM in order to apply it: 101 | 102 | vagrant reload --provision 103 | 104 | This reboots the VM and reapplies the config rules. It is usually much quicker 105 | than the initial provisioning, as in general, only new or version-changed 106 | packages will be installed. 107 | 108 | Note that this **won't** delete packages which you've removed from the package 109 | lists (see below). You'll have to do that manually via the Linux command line, 110 | if it's important. Or, just destroy and recreate the VM. 111 | 112 | ### Environment variables 113 | 114 | Snake Charmer recognizes a number of environment variables that can be used to 115 | modify the behaviour of VMs without editing config files. Set these as 116 | appropriate for your environment, then `vagrant up` or `vagrant reload`. 117 | 118 | * `CHARMER_RAM` (integer; default 2048) 119 | 120 | Amount of memory to supply to each VM, in megabytes. 1500 is probably the 121 | absolute minimum you can get away with; 2000 or more is recommended. 122 | 123 | *This will take effect on next boot, regardless of whether the `--provision` 124 | flag is used.* 125 | 126 | * `CHARMER_CPUS` (integer; default 1) 127 | 128 | Number of virtual CPUs to give each VM. Most Python programs will run happily 129 | on one CPU. If you're using multiprocessing, IPython clustering, or running the 130 | test suite (see below), raise this as high as you can -- but bear in mind 131 | you'll probably need to raise `CHARMER_RAM` somewhat too. 132 | 133 | If your CPU has [hyperthreading](http://en.wikipedia.org/wiki/Hyper-threading) 134 | (many end-user processors do these days) then don't set `CHARMER_CPUS` to more 135 | than the number of **physical cores** in your machine. This is half the number 136 | of *logical* cores that your machine reports having, if hyperthreading is 137 | enabled -- e.g. a hyperthreaded CPU that shows 8 cores really has 4 physical 138 | cores. If you try to give the VM more than the number of physical cores, it'll 139 | run **really slowly**. 140 | 141 | *This will take effect on next boot, regardless of whether the `--provision` 142 | flag is used.* 143 | 144 | * `CHARMER_SLIM` (boolean; default "false") 145 | 146 | If set to "true" this will disable downloading of some optional packages and 147 | example data files after provisioning, to speed up VM creation. Currently this 148 | disables the [NLTK downloader](http://www.nltk.org/data.html) and the Ubuntu 149 | package dependencies for OpenCV (OpenCV installation is not yet fully 150 | implemented). 151 | 152 | *This flag only has an effect during initial provisioning or reprovisioning.* 153 | 154 | * `CHARMER_NOTEBOOK_DIR` (string; default "notebooks") 155 | 156 | A custom location on the host to mount as `/home/vagrant/notebooks`, instead 157 | of the `notebooks` subdirectory of the `snake-charmer` directory. 158 | 159 | *This will take effect on next boot, regardless of whether the `--provision` 160 | flag is used.* 161 | 162 | * `CHARMER_DATA_DIR` (string; default "data") 163 | 164 | A custom location on the host to mount as `/home/vagrant/data`, instead 165 | of the `data` subdirectory of the `snake-charmer` directory. 166 | 167 | *This will take effect on next boot, regardless of whether the `--provision` 168 | flag is used.* 169 | 170 | * `CHARMER_CACHE_DIR` (string; default ".cache") 171 | 172 | A custom location on the host for cached packages and source code, instead of 173 | the `.cache` subdirectory of the `snake-charmer` directory. This is mounted as 174 | `/srv/cache` on the guest. 175 | 176 | *This will take effect on next boot, regardless of whether the `--provision` 177 | flag is used.* 178 | 179 | * `CHARMER_LOG_DIR` (string; default "log") 180 | 181 | A custom location on the host for logfiles generated during provisioning, instead of 182 | the `log` subdirectory of the `snake-charmer` directory. Within this directory, 183 | a subdirectory will be created which is named after the VM (e.g. `charmed34`). 184 | This is then mounted as `/srv/log` on the guest. 185 | 186 | *This will take effect on next boot, regardless of whether the `--provision` 187 | flag is used.* 188 | 189 | * `CHARMER_SALT_ROOT` (string; default "salt/roots/salt") and `CHARMER_PILLAR_ROOT` (string; default "salt/roots/pillar") 190 | 191 | Locations for Salt configuration files. Don't change these unless you're a 192 | Salt guru and you know what you're doing. 193 | 194 | *These will take effect on next boot, regardless of whether the `--provision` 195 | flag is used.* 196 | 197 | ### Editing the package lists 198 | 199 | The package lists are in `salt/roots/pillar/python.sls`. You are free to 200 | edit these, to add/remove packages from your VM definition. If you are creating 201 | an entirely new VM, copy one of these to get going. 202 | 203 | The `apt_pkgs` data structure describes what `.deb` packages to install via 204 | `apt-get` from the standard Ubuntu repositories (including universe & 205 | multiverse). It's a key-value map (dict) so order isn't important -- `apt-get` 206 | runs once, and figures out the correct order by looking at dependencies. 207 | 208 | The format is simply `- : `, e.g. 209 | 210 | ```yaml 211 | - libatlas-base-dev: 3.8.4-3build1 212 | ``` 213 | 214 | The `pip_pkgs` structure is used to install Python packages via Pip. We 215 | actually run Pip once for each package listed, in the order specified, letting 216 | it install dependencies for each specified package unless they've already been 217 | satisfied. 218 | 219 | To preserve order, it's a list of dicts, rather than a single dict. 220 | 221 | Packages can be specified in one of two ways. Most, simply, you can just 222 | provide a name, and a Pip-style version spec: 223 | 224 | ```yaml 225 | - name: numpy 226 | ver: ==1.8.1 227 | ``` 228 | 229 | This will install the requested version from PyPI. 230 | 231 | Or, you may need to install from a git checkout, if it's a bleeding-edge 232 | version or needs special treatment. The basic way to achieve this is by 233 | providing a name, repo URL, and revision identifier -- e.g. a commit hash 234 | or tag: 235 | 236 | ```yaml 237 | - name: Theano 238 | git: https://github.com/Theano/Theano.git 239 | rev: 46b19c24e3b04bbde3f2cf957824e07885916b9b 240 | - name: numpydoc 241 | git: https://github.com/numpy/numpydoc.git 242 | rev: latest_rc_tag 243 | ``` 244 | 245 | #### Custom installation commands 246 | 247 | If you're installing via a Git clone, you can add a custom `setup` command to 248 | use insted of Pip, which is executed (via bash) from the source directory after 249 | cloning: 250 | 251 | ```yaml 252 | - name: numba 253 | git: https://github.com/numba/numba.git 254 | rev: 4b9a36e7b344823fa2d2bb85221a14ff9e0abb03 255 | setup: pip3.4 install --install-option=build_ext --install-option="--inplace" . 256 | ``` 257 | 258 | #### Custom module names for import statements 259 | 260 | The post-install process attempts to start Python and import all of these 261 | packages, to verify that installation worked and that they can coexist. It 262 | assumes that the module name to import is just the package name in lowercase 263 | (e.g. "Cython" => `import cython`). 264 | 265 | In **either case** (PyPI or Git install), you will also need to override this 266 | behaviour with an `import` option if this isn't true, e.g. when capitalization 267 | is used in the module name, or there is some other more significant difference: 268 | 269 | ```yaml 270 | - name: beautifulsoup4 271 | ver: ==4.2.1 272 | import: bs4 273 | - name: Pyro4 274 | import: Pyro4 275 | ver: ==4.25 276 | - name: Pillow 277 | import: PIL 278 | git: https://github.com/python-imaging/Pillow.git 279 | rev: 84a701a82b33896a4d6997743c2131ab0a40c588 280 | ``` 281 | 282 | ### Hacking Vagrant 283 | 284 | For now, refer to the [Vagrant docs](http://docs.vagrantup.com/v2/) for how to 285 | perform customizations not described here, such as syncing additional folders, 286 | or spinning up VMs in AWS. 287 | 288 | ### Designing new VMs 289 | 290 | If you want to create a completely new VM template, e.g. for an OS version or 291 | Python version that isn't supplied, you'll need to modify the following files: 292 | 293 | Vagrantfile 294 | salt/roots/pillar/top.sls 295 | 296 | And copy and modify the package lists: 297 | 298 | salt/roots/pillar/python34.sls 299 | 300 | This will be covered in detail in a later release. 301 | 302 | ### Running the test suites 303 | 304 | We supply a notebook called "Snake Charmer QA" which runs the test suites for 305 | the major Python packages used. This is a big job -- we recommend using a 306 | server-grade VM with at least 12 vCPUS and 8GB of RAM, and even with that 307 | setup, the whole process will take over an hour. 308 | 309 | So, we recommend you only do this if you've made a modification to one of the 310 | fundamental underlying components, for example NumPy. Unless you have server 311 | time to spare... 312 | 313 | However, it's easy to edit that notebook to run only specific tests of 314 | interest. It's also straightforward to add new tests for packages you've added 315 | yourself. Instructions are included within the notebook. 316 | 317 | 318 | -------------------------------------------------------------------------------- /notebooks/Snake Charmer QA.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:58f9bcb2f9a59d4cc0769122b2484849040c9ebb8d21f55d18bd4814e2c4b94c" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "markdown", 13 | "metadata": {}, 14 | "source": [ 15 | "# Snake Charmer Quality Assurance Dept.\n", 16 | "## _\"All Your Eggs In One Basket\"_\n", 17 | "\n", 18 | "Welcome to the Quality Assurance Department. Here you can run tests against Snake Charmer's components, in parallel, using as many workers as you have vCPUs assigned to the VM.\n", 19 | "\n", 20 | "The results will be written to `log//test_output` (relative to your `snake-charmer` directory).\n", 21 | "\n", 22 | "**Warning:** Running the whole suite will take quite some time, even with lots of vCPUs, as some of the components' test suites take well over an hour each." 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": {}, 28 | "source": [ 29 | "### Initialization and helper functions" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "collapsed": false, 35 | "input": [ 36 | "import subprocess\n", 37 | "import functools\n", 38 | "from concurrent import futures\n", 39 | "import os\n", 40 | "import shutil\n", 41 | "import math\n", 42 | "from itertools import *\n", 43 | "import sys\n", 44 | "import datetime\n", 45 | "import re\n", 46 | "from time import asctime as now\n", 47 | "\n", 48 | "# TODO we'll need to use this for earlier versions of Python: https://pypi.python.org/pypi/futures\n", 49 | "\n", 50 | "\n", 51 | "def get_python_version():\n", 52 | " vi = sys.version_info\n", 53 | " return \"python%s.%s\" % (vi[0], vi[1])\n", 54 | "\n", 55 | " \n", 56 | "cpus = os.cpu_count()\n", 57 | "workers = max(1, cpus-1)\n", 58 | "\n", 59 | "outdir = '/srv/log/test_output'\n", 60 | "old_outdir = outdir + '_prev'\n", 61 | "\n", 62 | "python = get_python_version()\n", 63 | "\n", 64 | "shutil.rmtree(old_outdir, ignore_errors=True)\n", 65 | "if os.path.exists(outdir):\n", 66 | " shutil.move(outdir, old_outdir)\n", 67 | "os.makedirs(outdir)\n", 68 | "\n", 69 | "def log(string, f):\n", 70 | " msg = '%s %s' % (now(), string)\n", 71 | " print(msg, flush=True)\n", 72 | " print(msg, flush=True, file=f)\n", 73 | "\n", 74 | "def get_immediate_subdirectories(dir):\n", 75 | " return [name for name in os.listdir(dir)\n", 76 | " if os.path.isdir(os.path.join(dir, name))]\n", 77 | "\n", 78 | "def make_cmd_line(name, command):\n", 79 | " return '%s > %s/%s.out 2>&1' % (command.replace('$PYTHON', python), outdir, name)" 80 | ], 81 | "language": "python", 82 | "metadata": {}, 83 | "outputs": [] 84 | }, 85 | { 86 | "cell_type": "markdown", 87 | "metadata": {}, 88 | "source": [ 89 | "### Build lists of test commands and execution functions" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "collapsed": true, 95 | "input": [ 96 | "serial_test_commands = [\n", 97 | " ('ipython', '$PYTHON `which iptest3` --all -j'),\n", 98 | " ('joblib', '$PYTHON `which nosetests` joblib'),\n", 99 | " ('numba', '$PYTHON -c \"import numba; assert numba.testing.multitest()\"')\n", 100 | "]\n", 101 | "\n", 102 | "# Generate individual submodule tests for sympy, pytables, pystan and theano, as they take forever if run in one process\n", 103 | "\n", 104 | "def make_sympy_test_cmd(submod):\n", 105 | " name = 'sympy_%s' % submod\n", 106 | " cmd = '$PYTHON -c \"import sympy; assert sympy.test(\\'/%s\\')\"' % submod\n", 107 | " return (name, cmd)\n", 108 | "\n", 109 | "sympy_test_commands = (make_sympy_test_cmd(submod)\n", 110 | " for submod in get_immediate_subdirectories('/usr/local/lib/%s/dist-packages/sympy' % python)\n", 111 | " if not submod.startswith('_'))\n", 112 | "\n", 113 | "name_re = re.compile('theano.*')\n", 114 | "def make_theano_test_cmd(script, test_dir):\n", 115 | " path = '%s/%s' % (test_dir, script)\n", 116 | " match = name_re.search(path)\n", 117 | " if match is None:\n", 118 | " raise(RuntimeError('regex failed on string %s' % path))\n", 119 | " name = match.group(0).replace('/', '_').replace('.py', '')\n", 120 | " test_cache = '/tmp/' + name\n", 121 | " cmd = 'rm -rf %s && THEANO_FLAGS=\"compiledir=%s\" $PYTHON `which theano-nose` %s' % (test_cache, test_cache, path)\n", 122 | " return (name, cmd)\n", 123 | "\n", 124 | "theano_dir = '/usr/local/lib/%s/dist-packages/theano' % python\n", 125 | "theano_test_commands = [make_theano_test_cmd(script, location[0])\n", 126 | " for location in os.walk(theano_dir)\n", 127 | " for script in location[2]\n", 128 | " if script.startswith('test_')\n", 129 | " and script.endswith('.py')\n", 130 | " and '/typed_list' not in location[0]]\n", 131 | "\n", 132 | "# Make sure theano_tensor_tests_test_basic is at the start... Although this\n", 133 | "# probably isn't necessary with --heavy disabled\n", 134 | "\n", 135 | "def pred(elem):\n", 136 | " return elem[0] != 'theano_tensor_tests_test_basic'\n", 137 | "\n", 138 | "theano_test_commands_reordered = chain(dropwhile(pred, theano_test_commands),\n", 139 | " takewhile(pred, theano_test_commands))\n", 140 | "\n", 141 | "def make_pystan_test_cmd(script, test_dir):\n", 142 | " name = script.replace('test_', 'pystan_').replace('.py', '')\n", 143 | " cmd = \"(export JOBLIB_MULTIPROCESSING=0; $PYTHON `which nosetests` %s/%s)\" % (test_dir, script)\n", 144 | " return (name, cmd)\n", 145 | "\n", 146 | "pystan_test_dir = '/usr/local/lib/%s/dist-packages/pystan/tests' % python\n", 147 | "pystan_test_commands = [make_pystan_test_cmd(script, pystan_test_dir)\n", 148 | " for script in os.listdir(pystan_test_dir)\n", 149 | " if script.startswith('test_') and script.endswith('.py')]\n", 150 | "\n", 151 | "# Run all pytables tests in --heavy mode, except test_queries.py\n", 152 | "# as it literally takes hours\n", 153 | "\n", 154 | "def make_pytables_test_cmd(script, test_dir):\n", 155 | " name = script.replace('test_', 'pytables_').replace('.py', '')\n", 156 | " options = '' if name == 'pytables_queries' else '--heavy'\n", 157 | " cmd = '$PYTHON %s/%s %s' % (test_dir, script, options)\n", 158 | " return (name, cmd)\n", 159 | "\n", 160 | "pytables_test_dir = '/usr/local/lib/%s/dist-packages/tables/tests' % python\n", 161 | "pytables_test_commands = [make_pytables_test_cmd(script, pytables_test_dir)\n", 162 | " for script in os.listdir(pytables_test_dir)\n", 163 | " if script.startswith('test_') and script.endswith('.py') and script != 'test_all.py']\n", 164 | "\n", 165 | "# Make sure test_queries.py is at the start... Although this\n", 166 | "# probably isn't necessary with --heavy disabled\n", 167 | "\n", 168 | "#def pred(elem):\n", 169 | "# return elem[0] != 'pytables_queries'\n", 170 | "#\n", 171 | "#pytables_test_commands_reordered = chain(dropwhile(pred, pytables_test_commands),\n", 172 | "# takewhile(pred, pytables_test_commands))\n", 173 | "\n", 174 | "misc_test_commands = [\n", 175 | " ('cython', '(cd /srv/cache/src/Cython/ && $PYTHON runtests.py -vv)'),\n", 176 | " ('pymc', '$PYTHON -c \"import pymc; assert pymc.test(\\'full\\').wasSuccessful()\"'),\n", 177 | " ('matplotlib', '($PYTHON -c \"import matplotlib; assert matplotlib.test()\"; rm -rf /home/vagrant/notebooks/result_images)'),\n", 178 | " ('numpy', '$PYTHON -c \"import numpy; assert numpy.test(\\'full\\').wasSuccessful()\"'),\n", 179 | " ('scipy', '$PYTHON -c \"import scipy; assert scipy.test(\\'full\\').wasSuccessful()\"'),\n", 180 | " ('pandas', '$PYTHON `which nosetests` pandas'),\n", 181 | " ('sklearn', '$PYTHON `which nosetests` sklearn --exe'),\n", 182 | " ('pillow', '(cd /tmp && rm -f Images && ln -s /srv/cache/src/Pillow/Images . && $PYTHON /srv/cache/src/Pillow/selftest.py --installed)'),\n", 183 | " ('nltk', '$PYTHON /usr/local/lib/$PYTHON/dist-packages/nltk/test/runtests.py'),\n", 184 | " ('gensim', '(cd /srv/cache/src/gensim && $PYTHON setup.py test)'),\n", 185 | " ('bottleneck', '$PYTHON -c \"import bottleneck as bn; assert bn.test(\\'full\\').wasSuccessful()\"'),\n", 186 | " ('statsmodels', '$PYTHON -c \"import statsmodels as sm; assert sm.test().wasSuccessful()\"'),\n", 187 | " ('numexpr', '$PYTHON `which nosetests` numexpr'),\n", 188 | " ('patsy', '$PYTHON `which nosetests` patsy'),\n", 189 | " ('prettyplotlib', '(cd /srv/cache/src/prettyplotlib/tests && for py in $(ls test_*.py); do $PYTHON $py; done)'),\n", 190 | " ('emcee', '$PYTHON `which nosetests` emcee'),\n", 191 | " ('vowpal_wabbit', '(cd /srv/cache/src/vowpal_wabbit && make test)'),\n", 192 | " ('wabbit_wappa', '(cd /srv/cache/src/wabbit_wappa && $PYTHON `which py.test`)')\n", 193 | "]\n", 194 | "\n", 195 | "# Interleave these four command lists, to (attempt to) get good balance of CPU and IO heavy tasks\n", 196 | "\n", 197 | "parallel_test_commands = list(filter(None,\n", 198 | " chain(*zip_longest(pytables_test_commands,\n", 199 | " pystan_test_commands,\n", 200 | " theano_test_commands_reordered,\n", 201 | " misc_test_commands,\n", 202 | " sympy_test_commands))))\n", 203 | "\n", 204 | "def run_serial_tests(*args, logfile):\n", 205 | " if len(args) > 0:\n", 206 | " names = set(args)\n", 207 | " num = len(names)\n", 208 | " else:\n", 209 | " names = None\n", 210 | " num = len(serial_test_commands)\n", 211 | " log('Running %d tests\\n' % num, logfile)\n", 212 | " for name, command in serial_test_commands:\n", 213 | " if not names or name in names:\n", 214 | " cmd_line = make_cmd_line(name, command)\n", 215 | " log('%s: %s\\n' % (name, cmd_line), logfile)\n", 216 | " rc = subprocess.call(cmd_line, shell=True, executable='/bin/bash')\n", 217 | " if rc == 0:\n", 218 | " log('%s: Completed\\n' % name, logfile)\n", 219 | " else:\n", 220 | " log('%s: **** FAILED! Exited with status %d\\n' % (name, rc), logfile)\n", 221 | "\n", 222 | "def run_parallel_tests(*args, logfile):\n", 223 | " names = set(args) if len(args) > 0 else None\n", 224 | " with futures.ThreadPoolExecutor(max_workers=workers) as pool:\n", 225 | " log('Submitting tasks to pool of %d workers\\n' % workers, logfile)\n", 226 | " cmd_lines = [(name, make_cmd_line(name, command))\n", 227 | " for name, command in parallel_test_commands]\n", 228 | " \n", 229 | " task_data = []\n", 230 | " for name, cmd_line in cmd_lines:\n", 231 | " if not names or name in names:\n", 232 | " log('Submitting %s: %s\\n' % (name, cmd_line), logfile)\n", 233 | " task_data.append((pool.submit(subprocess.call,\n", 234 | " cmd_line,\n", 235 | " shell=True,\n", 236 | " executable='/bin/bash'),\n", 237 | " (name, cmd_line)))\n", 238 | " futures_map = dict(task_data)\n", 239 | " expected = len(futures_map)\n", 240 | "\n", 241 | " completed = 0\n", 242 | " for future in futures.as_completed(futures_map):\n", 243 | " completed += 1\n", 244 | " name = futures_map[future][0]\n", 245 | " if future.exception() is not None:\n", 246 | " log('%s (%d of %d): **** FAILED!\\n\\n%s\\n' %\n", 247 | " (name, completed, expected, repr(future.exception())),\n", 248 | " logfile)\n", 249 | " elif future.result() != 0:\n", 250 | " log('%s (%d of %d): **** FAILED! Exited with status %d\\n' %\n", 251 | " (name, completed, expected, future.result()),\n", 252 | " logfile)\n", 253 | " else:\n", 254 | " log('%s (%d of %d): Completed\\n' %\n", 255 | " (name, completed, expected),\n", 256 | " logfile)" 257 | ], 258 | "language": "python", 259 | "metadata": {}, 260 | "outputs": [] 261 | }, 262 | { 263 | "cell_type": "markdown", 264 | "metadata": {}, 265 | "source": [ 266 | "### Run serial tests\n", 267 | "\n", 268 | "These are tests to run in serial, as they handle their own parallelism internally. Each one will use as many vCPUs as it can get its hands on, so we let each one monopolize the VM until it's finished.\n", 269 | "\n", 270 | "When tests are started or completed, status messages appear below the next cell. If a test causes an exception or returns a non-zero exit code, check the corresponding output file in `log//test_output`. A summary of the test run will appear in `log//test_output/serial.log`.\n", 271 | "\n", 272 | "You can optionally provide one or more test names, to execute those tests only." 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "collapsed": false, 278 | "input": [ 279 | "with open(outdir + '/serial.log', 'w') as f:\n", 280 | " run_serial_tests(logfile=f)\n", 281 | "# Optionally, give a subset to run, e.g.:\n", 282 | "# run_serial_tests('joblib', 'ipython', logfile=f)" 283 | ], 284 | "language": "python", 285 | "metadata": {}, 286 | "outputs": [] 287 | }, 288 | { 289 | "cell_type": "markdown", 290 | "metadata": {}, 291 | "source": [ 292 | "### Parallel tests\n", 293 | "\n", 294 | "These are tests to run in parallel explicitly, after the serial tests have finished. Each one runs in a separate isolated Python process. They are coordinated by a thread pool within the parent process, so that no more than `workers` processes are running at once, but in some cases a test may use more than a whole vCPU, because of starting helper threads or subprocesses.\n", 295 | "\n", 296 | "In general we try to start the slower-running tests earlier.\n", 297 | "\n", 298 | "When jobs are submitted or completed, status messages appear below the next cell. If a test causes an exception or returns a non-zero exit code, check the corresponding output file in `log//test_output`. A summary of the test run will appear in `log//test_output/parallel.log`.\n", 299 | "\n", 300 | "You can optionally provide one or more test names, to execute those tests only." 301 | ] 302 | }, 303 | { 304 | "cell_type": "code", 305 | "collapsed": false, 306 | "input": [ 307 | "with open(outdir + '/parallel.log', 'w') as f:\n", 308 | " run_parallel_tests(logfile=f)\n", 309 | "# Optionally, give a subset to run, e.g.:\n", 310 | "# run_parallel_tests('pillow', 'sympy_physics', 'pytables_links', logfile=f)" 311 | ], 312 | "language": "python", 313 | "metadata": {}, 314 | "outputs": [] 315 | } 316 | ], 317 | "metadata": {} 318 | } 319 | ] 320 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Snake Charmer 2 | 3 | A portable Python workbench for data science, built with Vagrant, 4 | VirtualBox and Salt. 5 | 6 | ## Introduction 7 | 8 | Wouldn't it be great if you could magic up a local IPython Notebook server, 9 | complete with SciPy, Pandas, Matplotlib, PyMC, scikit-learn, R and Octave 10 | integration, and all the usual goodness, and running the latest version of 11 | Python, just by typing one line? 12 | 13 | vagrant up charmed34 14 | 15 | And wouldn't it be great if you could do that from pretty much any machine, 16 | and know that you'd get the exact same environment every time? 17 | 18 | Well, read on. 19 | 20 | ## What is included 21 | 22 | Snake Charmer provides an out-of-the-box workbench for data analysis, 23 | statistical modelling, machine learning, mathematical programming and 24 | visualization. 25 | 26 | It is designed to be used primarily via IPython Notebook. 27 | 28 | The environment is based on Ubuntu 12.04 and Python 3.4, with the following 29 | data science tools included. You are of course free to install any other Python 30 | or Ubuntu packages -- or anything else that fits your need. 31 | 32 | Packages marked 'alpha' or 'dev' should be considered experimental, although 33 | in many cases they are largely problem-free. We will endeavour to discover and 34 | document any known issues [here](https://github.com/andrewclegg/snake-charmer/issues). 35 | 36 | ### Data handling and processing tools 37 | 38 | * [IPython](http://ipython.org/) 2.0.0 39 | * [runipy](https://pypi.python.org/pypi/runipy) 0.0.8 40 | * [Pandas](http://pandas.pydata.org/) 0.14.0 41 | * [PyTables](http://www.pytables.org/moin) 3.1.1 42 | * [joblib](https://pythonhosted.org/joblib/) 0.8.0 alpha 3 43 | 44 | ### Graphics and visualization 45 | 46 | * [Matplotlib](http://matplotlib.org/) 1.3.1 47 | * [MPLD3](http://mpld3.github.io/) 0.2 48 | * [prettyplotlib](http://olgabot.github.io/prettyplotlib/) 0.1.7 49 | * [brewer2mpl](https://github.com/jiffyclub/brewer2mpl) 1.4 50 | * [Seaborn](http://www.stanford.edu/~mwaskom/software/seaborn/) 0.3.1 51 | * [Pillow](http://python-imaging.github.io/) 2.4.0 52 | 53 | ### Machine learning and inference 54 | 55 | * [scikit-learn](http://scikit-learn.org/) 0.15.1 56 | * [PyMC](http://pymc-devs.github.io/pymc/) 3.0 alpha 57 | * [emcee](http://dan.iel.fm/emcee/current/) 2.1.0 58 | * [Stan/PyStan](http://mc-stan.org/) 2.2.0.2 ‡ 59 | * [Theano](http://deeplearning.net/software/theano/) 0.6.0 60 | * [DEAP](https://code.google.com/p/deap/) 1.0.1 61 | * [fastcluster](http://danifold.net/fastcluster.html) 1.1.13 62 | * [Vowpal Wabbit](https://github.com/JohnLangford/vowpal_wabbit/wiki) 7.6.1 ‡ 63 | * [Wabbit Wappa](https://github.com/andrewclegg/wabbit_wappa) 0.2.0-p3 dev 64 | 65 | ### Natural language processing and text mining 66 | 67 | * [gensim](http://radimrehurek.com/gensim) 0.9.1 68 | * [NLTK](http://www.nltk.org/nltk3-alpha/) 3.0 alpha 3 (very experimental) 69 | 70 | ### Numeric and statistical computing 71 | 72 | * [NumPy](http://www.numpy.org/) 1.8.2 73 | * [SciPy](http://www.scipy.org/) 0.14.0 74 | * [Bottleneck](http://berkeleyanalytics.com/bottleneck/) 0.8.0 75 | * [Statsmodels](http://statsmodels.sourceforge.net/) 0.6.0 dev 76 | * [lifelines](https://github.com/CamDavidsonPilon/lifelines/) 0.4.0.0 77 | * [Patsy](http://patsy.readthedocs.org/en/latest/) 0.2.1 78 | * [Theano](http://deeplearning.net/software/theano/) 0.6.0 79 | * [numexpr](https://github.com/pydata/numexpr) 2.4 80 | * [SymPy](http://sympy.org/) 0.7.5 81 | * [R](http://www.r-project.org/) 2.14.1 ‡ 82 | * [r-base-dev and r-recommended](http://cran.r-project.org/bin/linux/ubuntu/) packages 83 | * [rpy2](http://rpy.sourceforge.net/rpy2.html) and [rmagic](http://ipython.org/ipython-doc/dev/config/extensions/rmagic.html) 2.3.10 84 | * [Octave](http://www.gnu.org/software/octave/) 3.2.4 ‡ 85 | * [oct2py](https://pypi.python.org/pypi/oct2py) and [octavemagic](http://nbviewer.ipython.org/github/blink1073/oct2py/blob/master/example/octavemagic_extension.ipynb) 1.3.0 86 | 87 | 88 | ### Performance optimization 89 | 90 | * [Cython](http://cython.org/) 0.21 dev 91 | * [Numba](https://github.com/numba/numba/) 0.13.1 92 | 93 | ### Connectivity and interoperability 94 | 95 | * [lxml](http://lxml.de/lxmlhtml.html) 3.3.5 96 | * [Psycopg](http://initd.org/psycopg/) 2.5.2 97 | * [pymssql](http://www.pymssql.org/) 2.1.0 98 | * [requests](http://docs.python-requests.org/en/latest/) 2.3.0 99 | 100 | *† Non-Python tools* 101 | 102 | *‡ Non-Python tools usable via Python wrapper packages* 103 | 104 | You are, of course, free to remove or upgrade these packages via `pip` or 105 | `apt-get` as usual, or experiment with additional ones. Please feel free to 106 | send pull requests when you get another package working. 107 | 108 | Coming soon: Other Python versions. Ubuntu 14.04 LTS. 109 | 110 | Potential future additions include: Parakeet, pattern, CrossCat, BayesDB, 111 | ggplot, Bokeh, Blaze, numdifftools, PuLP, CVXPY, SysCorr, bayesian, PEBL, 112 | libpgm, BayesPy, BayesOpt, mpld3, Pylearn2, nimfa, py-earth, Orange, NeuroLab, 113 | PyBrain, annoy, Zipline, Quandl, BNFinder, Alchemy API, xlrd/xlwt, NetworkX, 114 | PyMVPA, OpenCV, boto, gbq, SQLite, PyMongo, mpi4py, Jubatus, and one or 115 | more Hadoop clients. 116 | 117 | If you have suggestions for any other packages to add, please submit them by 118 | raising an [issue](https://github.com/andrewclegg/snake-charmer/issues). 119 | 120 | ## Requirements 121 | 122 | Snake Charmer runs IPython and all the associated tools in a sandboxed virtual 123 | machine. It relies on [Vagrant](http://www.vagrantup.com/) for creating and 124 | managing these, and [VirtualBox](https://www.virtualbox.org/) for running them 125 | -- so please go and install those now. 126 | 127 | *Experienced users of other virtualization platforms can edit the Vagrantfile 128 | to use one of these instead, if they prefer.* 129 | 130 | **Windows users:** you may also need to install an 131 | [ssh client](https://www.google.co.uk/search?q=ssh+windows+command+line+client) 132 | if you don't already have one on your system. Also, one Windows user 133 | [reports](https://github.com/snake-charmer-devs/snake-charmer/issues/24#issuecomment-48858182) 134 | that you might need to move `git` out of your path if Vagrant does not run correctly. 135 | 136 | Everything else is installed automatically. 137 | 138 | ### Hardware and system resources 139 | 140 | By default, the VM allocates 2048MB of RAM, and a single CPU. If your computer 141 | only *has* 2048MB, you will need to reduce this as described in the 142 | [customization guide](CUSTOMIZING.md). (If you reduce it too much, however, the 143 | VM may not build correctly. Some experimentation may be required.) 144 | 145 | It is very much recommended that you run Snake Charmer on a machine with 146 | [hardware virtualization](http://www.desktop-virtualization.com/2008/05/14/what-hardware-virtualization-really-means/) 147 | (i.e. Intel VT-x or AMD-V). It will run on systems without this, but slowly. 148 | You may need to enable it in your BIOS, but most modern systems come with it 149 | enabled by default. 150 | 151 | ## Installation 152 | 153 | Check out this git repository: 154 | 155 | # Either via ssh... 156 | git clone git@github.com:andrewclegg/snake-charmer.git 157 | # Or via https... 158 | git clone https://github.com/andrewclegg/snake-charmer.git 159 | 160 | # Then change into your new Snake Charmer directory... 161 | cd snake-charmer 162 | 163 | Start the VM: 164 | 165 | vagrant up charmed34 166 | 167 | *If you're already a Vagrant user, be aware that Snake Charmer's Vagrantfile 168 | will attempt to install the [vagrant-vbguest](https://github.com/dotless-de/vagrant-vbguest/) 169 | plugin automatically.* 170 | 171 | This command will generally take **at least an hour** to download and install 172 | all the necessary software. When this completes, it will run some tests and 173 | then display a message like this: 174 | 175 | Your VM is up and running: http://localhost:8834/tree 176 | 177 | Later rebuilds will go slightly more quickly, as downloaded package files are 178 | cached where possible. 179 | 180 | Note: you may get the following warnings after the "up and running" message: 181 | 182 | sys:1: ResourceWarning: unclosed file <_io.TextIOWrapper name='/dev/null' mode='w' encoding='ISO-8859-1'> 183 | 184 | /usr/local/lib/python3.4/dist-packages/numpy/lib/utils.py:134: DeprecationWarning: `scipy.sparse.sparsetools` is deprecated! 185 | scipy.sparse.sparsetools is a private module for scipy.sparse, and should not be used. 186 | warnings.warn(depdoc, DeprecationWarning) 187 | 188 | These can safely be ignored. 189 | 190 | ### Accessing IPython notebooks 191 | 192 | The [link](http://localhost:8834/tree) shown at the end of the installation 193 | procedure will take you to a fully-kitted-out IPython Notebook server. This can 194 | be used in exactly the same way as if it was running on your 'real' hardware 195 | (i.e. not in a VM). It runs as a Linux service which starts automatically when 196 | the VM is booted up, and will be restarted automatically if it ever crashes. 197 | 198 | The notebook server acts as if it is running in the 'notebooks' subdirectory 199 | (see **Folder structure** below). 200 | 201 | ### Testing it out 202 | 203 | Open the "Hello World" notebook to see a full list of installed packages and 204 | other system information. **N.B.** The notebook server is started with inline 205 | graphics enabled for matplotlib, but _not_ the `--pylab` option, as this is 206 | [considered harmful](http://carreau.github.io/posts/10-No-PyLab-Thanks.ipynb.html). 207 | 208 | There is also a "Snake Charmer QA" notebook supplied. This allows you to run 209 | the test suites of the major components, but **don't** run this now! It's a 210 | slow process and only needs to be performed if you've customized your VM. See 211 | the [customization guide](CUSTOMIZING.md) for more information. 212 | 213 | ## Vagrant essentials 214 | 215 | On a VM that's already been fully configured, `vagrant up` will just restart 216 | it, without going through the full install process. 217 | 218 | You can log into the server via 219 | 220 | vagrant ssh charmed34 221 | 222 | from the same directory, for full command-line control. It's an Ubuntu 12.04 223 | box, under the covers. But you can do most things through the IPython Notebook 224 | anyway, so this is rarely essential. 225 | 226 | Some more useful commands: 227 | 228 | vagrant reload charmed34 # reboot the VM (same as "vagrant up" if it's not running) 229 | vagrant halt charmed34 # shut down the VM, reclaim the memory it used 230 | vagrant destroy charmed34 # wipe it completely, reclaiming disk space too 231 | vagrant suspend charmed34 # 'hibernate' the machine, saving current state 232 | vagrant resume charmed34 # 'unhibernate' the machine 233 | 234 | See the [Vagrant docs](http://docs.vagrantup.com/v2/cli/index.html) for more 235 | details. 236 | 237 | ## Folder structure 238 | 239 | The notebook server runs from within the `notebooks` subdirectory of the 240 | current `snake-charmer` directory, and initially contains a single "Hello 241 | World" notebook. 242 | 243 | Snake Charmer uses IPython 2 so any subdirectories of `notebooks` will be 244 | visible and navigable as folders in the IPython web interface. However, you 245 | can't actually *create* directories from the web interface yet, so you'd need 246 | to log in via ssh, or just enter a shell command into IPython with `!`. 247 | 248 | Vagrant sets up a number of synced folders, which are directories visible to 249 | both the VM and the host (your computer). Files placed in these will be visible 250 | to both the VM and the host, so this is a good way to make data available to 251 | the VMs. If you create more than one VM (feature coming soon!), files in synced 252 | folders will be visible to all of them -- apart from `/srv/log` which is 253 | specific to one VM only. 254 | 255 | The paths in the left-hand column are relative to the `snake-charmer` install 256 | directory -- your local copy of this repo. 257 | 258 | Folder on your computer Folder within VM Contents 259 | ------------------------ ----------------------- -------- 260 | notebooks /home/vagrant/notebooks Any notebooks 261 | data /home/vagrant/data Data you wish to share (initially empty) 262 | .cache /srv/cache Cache for downloaded files 263 | log/charmed34 /srv/log Certain setup logs, useful for debugging only 264 | salt/roots/salt /srv/salt Config management information (ignore this) 265 | salt/roots/pillar /srv/pillar Config management information (ignore this) 266 | 267 | These are all configurable via environment variables -- see the 268 | [customization guide](CUSTOMIZING.md). 269 | 270 | ### Data persistence 271 | 272 | If you get your VM into a mess somehow, you can just type 273 | 274 | vagrant destroy charmed34 275 | vagrant up charmed34 276 | 277 | to build a new one. Files in synced folders will not be affected if you do 278 | this, so you won't lose any data or notebooks. However, any data stored on the 279 | VM but *outside* these synced folders will be lost. 280 | 281 | The virtual disk on each VM is configured with an 80GB limit -- it grows to 282 | take up real disk space on the host up to this limit, and then stops. But data 283 | stored in synced folders does not count towards this. So you will likely never 284 | reach the 80GB limit. 285 | 286 | If you want to make another folder available to the VM, for example if your 287 | datasets are stored on another disk, see the [customization guide](CUSTOMIZING.md). 288 | 289 | ### Troubleshooting 290 | 291 | The [issues list](https://github.com/snake-charmer-devs/snake-charmer/issues) 292 | is generally up to date with any current unresolved problems. 293 | 294 | #### Installation problems on Linux 295 | 296 | If you get an error during installation along the lines of: 297 | 298 | Gem::Installer::ExtensionBuildError: ERROR: Failed to build gem native extension. 299 | 300 | then you may be missing a C/C++ compiler, and/or `libxml2` and `libxslt`. 301 | 302 | On `.deb`-based Linuxes like Debian, Ubuntu and Mint, try this: 303 | 304 | sudo apt-get install libxml2-dev libxslt-dev build-essentials 305 | 306 | #### Temporary files in snake-charmer directory 307 | 308 | If you are on Linux and see lots of temporary files and directories with 309 | random names like `d20140819-2623-folctt`, `vagrant20140813-28997-ya0isv2` 310 | or `vagrant20140813-28997-ya0isv2.lock` appearing, this may be because Ruby 311 | doesn't like the permissions on your `/tmp` directory. 312 | 313 | Try this: 314 | 315 | vagrant halt 316 | chmod 1777 /tmp 317 | 318 | Then delete all the offending files, and restart the VM. 319 | 320 | This is a [known Vagrant issue](https://github.com/mitchellh/vagrant/issues/3493). 321 | 322 | #### Unexpected behaviour from a VM 323 | 324 | If a VM starts behaving strangely, the golden rule is: 325 | **Don't waste time fixing it.** 326 | 327 | This may sound strange, but the advantage of Snake Charmer is that you can 328 | create a factory-fresh VM with almost no effort at all. 329 | 330 | The first thing to try is to reboot the VM: 331 | 332 | vagrant reload 333 | 334 | Option two is reprovisioning the machine. This runs through the install 335 | process and ensures all required packages are installed. First, delete the 336 | package cache in case anything in there is messed up: 337 | 338 | # On OS X or Linux: 339 | rm -rf .cache 340 | 341 | # Or on Windows: 342 | rd /s /q .cache 343 | 344 | # N.B. Make sure you're in the snake-charmer directory first! 345 | 346 | Then reboot and reprovision: 347 | 348 | vagrant reload --provision charmed34 349 | 350 | If this doesn't fix the problem, then delete it completely, and recreate it: 351 | 352 | vagrant destroy charmed34 353 | vagrant up charmed34 354 | 355 | Finally, you could try deleting your VirtualBox machines and Vagrant configuration 356 | files: 357 | 358 | VBoxManage controlvm charmed34 poweroff 359 | VBoxManage unregistervm charmed34 --delete 360 | vagrant box remove charmed34 361 | vagrant up charmed34 362 | 363 | If this still doesn't fix it, you may have found a bug. Please open a 364 | [Github issue](https://github.com/andrewclegg/snake-charmer/issues) 365 | describing it in as much detail as possible, preferably with 366 | instructions on how to reproduce it. 367 | 368 | The VirtualBox admin GUI can of course be used to check on the status of VMs, 369 | inspect their hardware and network configuration, manually start or stop them, 370 | attach via the console, etc. 371 | 372 | #### Important reminder 373 | 374 | Only use the **host** filesystem to store data, notebooks etc. -- that is, the 375 | `data` and `notebooks` folders which are synced to the VM. If you store files 376 | in other places on a VM, they **will** be lost forever when you destroy it. 377 | 378 | *The exception is if you want to package up a new Vagrant box with the data 379 | and notebooks from your existing environment, e.g. to redistribute. In this 380 | case, all the files __must__ be stored within the VM. This is an option for 381 | Vagrant power users, primarily.* 382 | 383 | ## Sharing your VMs 384 | 385 | Snake Charmer VMs are Vagrant VMs, and Vagrant VMs can be published, shared 386 | and remotely accessed via various mechanisms. This is discussed in the 387 | [Snake Charmer FAQ](FAQ.md). 388 | 389 | ## Customizing your VMs 390 | 391 | Even if you don't know much about VirtualBox, Vagrant or Salt, you can 392 | customize your VMs in several ways -- and if you want to tinker with the 393 | configuration for these programs directly, the sky's the limit. See the 394 | separate [customization guide](CUSTOMIZING.md). 395 | 396 | ## F.A.Q. 397 | 398 | See the separate [Snake Charmer FAQ](FAQ.md). 399 | 400 | ## Credits 401 | 402 | Developed by [Andrew Clegg](https://github.com/andrewclegg) (Twitter: 403 | [@andrew_clegg](http://twitter.com/andrew_clegg)), tested at 404 | [Pearson](http://labs.pearson.com/). 405 | 406 | Thanks to the authors and contributors of all the world-class open source 407 | components included, whose hard work has made this possible. 408 | 409 | ## License 410 | 411 | Snake Charmer does *not* include bundled distributions of its components 412 | (Python, Ubuntu, Python libraries, other libraries and packages etc.). Rather, 413 | it provides a set of machine-readable instructions for obtaining these 414 | components from third-party open-source repositories. Please refer to each 415 | individual component's documentation for license details. 416 | 417 | Snake Charmer itself is distributed under the Apache License: 418 | 419 | Copyright 2014 Andrew Clegg 420 | 421 | Licensed under the Apache License, Version 2.0 (the "License"); 422 | you may not use this file except in compliance with the License. 423 | You may obtain a copy of the License at 424 | 425 | http://www.apache.org/licenses/LICENSE-2.0 426 | 427 | Unless required by applicable law or agreed to in writing, software 428 | distributed under the License is distributed on an "AS IS" BASIS, 429 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 430 | See the License for the specific language governing permissions and 431 | limitations under the License. 432 | 433 | -------------------------------------------------------------------------------- /salt/roots/salt/bin/distribute_setup.py: -------------------------------------------------------------------------------- 1 | #!python 2 | """Bootstrap distribute installation 3 | 4 | If you want to use setuptools in your package's setup.py, just include this 5 | file in the same directory with it, and add this to the top of your setup.py:: 6 | 7 | from distribute_setup import use_setuptools 8 | use_setuptools() 9 | 10 | If you want to require a specific version of setuptools, set a download 11 | mirror, or use an alternate download directory, you can do so by supplying 12 | the appropriate options to ``use_setuptools()``. 13 | 14 | This file can also be run as a script to install or upgrade setuptools. 15 | """ 16 | import os 17 | import shutil 18 | import sys 19 | import time 20 | import fnmatch 21 | import tempfile 22 | import tarfile 23 | import optparse 24 | 25 | from distutils import log 26 | 27 | try: 28 | from site import USER_SITE 29 | except ImportError: 30 | USER_SITE = None 31 | 32 | try: 33 | import subprocess 34 | 35 | def _python_cmd(*args): 36 | args = (sys.executable,) + args 37 | return subprocess.call(args) == 0 38 | 39 | except ImportError: 40 | # will be used for python 2.3 41 | def _python_cmd(*args): 42 | args = (sys.executable,) + args 43 | # quoting arguments if windows 44 | if sys.platform == 'win32': 45 | def quote(arg): 46 | if ' ' in arg: 47 | return '"%s"' % arg 48 | return arg 49 | args = [quote(arg) for arg in args] 50 | return os.spawnl(os.P_WAIT, sys.executable, *args) == 0 51 | 52 | DEFAULT_VERSION = "0.6.49" 53 | DEFAULT_URL = "http://pypi.python.org/packages/source/d/distribute/" 54 | SETUPTOOLS_FAKED_VERSION = "0.6c11" 55 | 56 | SETUPTOOLS_PKG_INFO = """\ 57 | Metadata-Version: 1.0 58 | Name: setuptools 59 | Version: %s 60 | Summary: xxxx 61 | Home-page: xxx 62 | Author: xxx 63 | Author-email: xxx 64 | License: xxx 65 | Description: xxx 66 | """ % SETUPTOOLS_FAKED_VERSION 67 | 68 | 69 | def _install(tarball, install_args=()): 70 | # extracting the tarball 71 | tmpdir = tempfile.mkdtemp() 72 | log.warn('Extracting in %s', tmpdir) 73 | old_wd = os.getcwd() 74 | try: 75 | os.chdir(tmpdir) 76 | tar = tarfile.open(tarball) 77 | _extractall(tar) 78 | tar.close() 79 | 80 | # going in the directory 81 | subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) 82 | os.chdir(subdir) 83 | log.warn('Now working in %s', subdir) 84 | 85 | # installing 86 | log.warn('Installing Distribute') 87 | if not _python_cmd('setup.py', 'install', *install_args): 88 | log.warn('Something went wrong during the installation.') 89 | log.warn('See the error message above.') 90 | # exitcode will be 2 91 | return 2 92 | finally: 93 | os.chdir(old_wd) 94 | shutil.rmtree(tmpdir) 95 | 96 | 97 | def _build_egg(egg, tarball, to_dir): 98 | # extracting the tarball 99 | tmpdir = tempfile.mkdtemp() 100 | log.warn('Extracting in %s', tmpdir) 101 | old_wd = os.getcwd() 102 | try: 103 | os.chdir(tmpdir) 104 | tar = tarfile.open(tarball) 105 | _extractall(tar) 106 | tar.close() 107 | 108 | # going in the directory 109 | subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) 110 | os.chdir(subdir) 111 | log.warn('Now working in %s', subdir) 112 | 113 | # building an egg 114 | log.warn('Building a Distribute egg in %s', to_dir) 115 | _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) 116 | 117 | finally: 118 | os.chdir(old_wd) 119 | shutil.rmtree(tmpdir) 120 | # returning the result 121 | log.warn(egg) 122 | if not os.path.exists(egg): 123 | raise IOError('Could not build the egg.') 124 | 125 | 126 | def _do_download(version, download_base, to_dir, download_delay): 127 | egg = os.path.join(to_dir, 'distribute-%s-py%d.%d.egg' 128 | % (version, sys.version_info[0], sys.version_info[1])) 129 | if not os.path.exists(egg): 130 | tarball = download_setuptools(version, download_base, 131 | to_dir, download_delay) 132 | _build_egg(egg, tarball, to_dir) 133 | sys.path.insert(0, egg) 134 | import setuptools 135 | setuptools.bootstrap_install_from = egg 136 | 137 | 138 | def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, 139 | to_dir=os.curdir, download_delay=15, no_fake=True): 140 | # making sure we use the absolute path 141 | to_dir = os.path.abspath(to_dir) 142 | was_imported = 'pkg_resources' in sys.modules or \ 143 | 'setuptools' in sys.modules 144 | try: 145 | try: 146 | import pkg_resources 147 | 148 | # Setuptools 0.7b and later is a suitable (and preferable) 149 | # substitute for any Distribute version. 150 | try: 151 | pkg_resources.require("setuptools>=0.7b") 152 | return 153 | except (pkg_resources.DistributionNotFound, 154 | pkg_resources.VersionConflict): 155 | pass 156 | 157 | if not hasattr(pkg_resources, '_distribute'): 158 | if not no_fake: 159 | _fake_setuptools() 160 | raise ImportError 161 | except ImportError: 162 | return _do_download(version, download_base, to_dir, download_delay) 163 | try: 164 | pkg_resources.require("distribute>=" + version) 165 | return 166 | except pkg_resources.VersionConflict: 167 | e = sys.exc_info()[1] 168 | if was_imported: 169 | sys.stderr.write( 170 | "The required version of distribute (>=%s) is not available,\n" 171 | "and can't be installed while this script is running. Please\n" 172 | "install a more recent version first, using\n" 173 | "'easy_install -U distribute'." 174 | "\n\n(Currently using %r)\n" % (version, e.args[0])) 175 | sys.exit(2) 176 | else: 177 | del pkg_resources, sys.modules['pkg_resources'] # reload ok 178 | return _do_download(version, download_base, to_dir, 179 | download_delay) 180 | except pkg_resources.DistributionNotFound: 181 | return _do_download(version, download_base, to_dir, 182 | download_delay) 183 | finally: 184 | if not no_fake: 185 | _create_fake_setuptools_pkg_info(to_dir) 186 | 187 | 188 | def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, 189 | to_dir=os.curdir, delay=15): 190 | """Download distribute from a specified location and return its filename 191 | 192 | `version` should be a valid distribute version number that is available 193 | as an egg for download under the `download_base` URL (which should end 194 | with a '/'). `to_dir` is the directory where the egg will be downloaded. 195 | `delay` is the number of seconds to pause before an actual download 196 | attempt. 197 | """ 198 | # making sure we use the absolute path 199 | to_dir = os.path.abspath(to_dir) 200 | try: 201 | from urllib.request import urlopen 202 | except ImportError: 203 | from urllib2 import urlopen 204 | tgz_name = "distribute-%s.tar.gz" % version 205 | url = download_base + tgz_name 206 | saveto = os.path.join(to_dir, tgz_name) 207 | src = dst = None 208 | if not os.path.exists(saveto): # Avoid repeated downloads 209 | try: 210 | log.warn("Downloading %s", url) 211 | src = urlopen(url) 212 | # Read/write all in one block, so we don't create a corrupt file 213 | # if the download is interrupted. 214 | data = src.read() 215 | dst = open(saveto, "wb") 216 | dst.write(data) 217 | finally: 218 | if src: 219 | src.close() 220 | if dst: 221 | dst.close() 222 | return os.path.realpath(saveto) 223 | 224 | 225 | def _no_sandbox(function): 226 | def __no_sandbox(*args, **kw): 227 | try: 228 | from setuptools.sandbox import DirectorySandbox 229 | if not hasattr(DirectorySandbox, '_old'): 230 | def violation(*args): 231 | pass 232 | DirectorySandbox._old = DirectorySandbox._violation 233 | DirectorySandbox._violation = violation 234 | patched = True 235 | else: 236 | patched = False 237 | except ImportError: 238 | patched = False 239 | 240 | try: 241 | return function(*args, **kw) 242 | finally: 243 | if patched: 244 | DirectorySandbox._violation = DirectorySandbox._old 245 | del DirectorySandbox._old 246 | 247 | return __no_sandbox 248 | 249 | 250 | def _patch_file(path, content): 251 | """Will backup the file then patch it""" 252 | f = open(path) 253 | existing_content = f.read() 254 | f.close() 255 | if existing_content == content: 256 | # already patched 257 | log.warn('Already patched.') 258 | return False 259 | log.warn('Patching...') 260 | _rename_path(path) 261 | f = open(path, 'w') 262 | try: 263 | f.write(content) 264 | finally: 265 | f.close() 266 | return True 267 | 268 | _patch_file = _no_sandbox(_patch_file) 269 | 270 | 271 | def _same_content(path, content): 272 | f = open(path) 273 | existing_content = f.read() 274 | f.close() 275 | return existing_content == content 276 | 277 | 278 | def _rename_path(path): 279 | new_name = path + '.OLD.%s' % time.time() 280 | log.warn('Renaming %s to %s', path, new_name) 281 | os.rename(path, new_name) 282 | return new_name 283 | 284 | 285 | def _remove_flat_installation(placeholder): 286 | if not os.path.isdir(placeholder): 287 | log.warn('Unkown installation at %s', placeholder) 288 | return False 289 | found = False 290 | for file in os.listdir(placeholder): 291 | if fnmatch.fnmatch(file, 'setuptools*.egg-info'): 292 | found = True 293 | break 294 | if not found: 295 | log.warn('Could not locate setuptools*.egg-info') 296 | return 297 | 298 | log.warn('Moving elements out of the way...') 299 | pkg_info = os.path.join(placeholder, file) 300 | if os.path.isdir(pkg_info): 301 | patched = _patch_egg_dir(pkg_info) 302 | else: 303 | patched = _patch_file(pkg_info, SETUPTOOLS_PKG_INFO) 304 | 305 | if not patched: 306 | log.warn('%s already patched.', pkg_info) 307 | return False 308 | # now let's move the files out of the way 309 | for element in ('setuptools', 'pkg_resources.py', 'site.py'): 310 | element = os.path.join(placeholder, element) 311 | if os.path.exists(element): 312 | _rename_path(element) 313 | else: 314 | log.warn('Could not find the %s element of the ' 315 | 'Setuptools distribution', element) 316 | return True 317 | 318 | _remove_flat_installation = _no_sandbox(_remove_flat_installation) 319 | 320 | 321 | def _after_install(dist): 322 | log.warn('After install bootstrap.') 323 | placeholder = dist.get_command_obj('install').install_purelib 324 | _create_fake_setuptools_pkg_info(placeholder) 325 | 326 | 327 | def _create_fake_setuptools_pkg_info(placeholder): 328 | if not placeholder or not os.path.exists(placeholder): 329 | log.warn('Could not find the install location') 330 | return 331 | pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1]) 332 | setuptools_file = 'setuptools-%s-py%s.egg-info' % \ 333 | (SETUPTOOLS_FAKED_VERSION, pyver) 334 | pkg_info = os.path.join(placeholder, setuptools_file) 335 | if os.path.exists(pkg_info): 336 | log.warn('%s already exists', pkg_info) 337 | return 338 | 339 | log.warn('Creating %s', pkg_info) 340 | try: 341 | f = open(pkg_info, 'w') 342 | except EnvironmentError: 343 | log.warn("Don't have permissions to write %s, skipping", pkg_info) 344 | return 345 | try: 346 | f.write(SETUPTOOLS_PKG_INFO) 347 | finally: 348 | f.close() 349 | 350 | pth_file = os.path.join(placeholder, 'setuptools.pth') 351 | log.warn('Creating %s', pth_file) 352 | f = open(pth_file, 'w') 353 | try: 354 | f.write(os.path.join(os.curdir, setuptools_file)) 355 | finally: 356 | f.close() 357 | 358 | _create_fake_setuptools_pkg_info = _no_sandbox( 359 | _create_fake_setuptools_pkg_info 360 | ) 361 | 362 | 363 | def _patch_egg_dir(path): 364 | # let's check if it's already patched 365 | pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') 366 | if os.path.exists(pkg_info): 367 | if _same_content(pkg_info, SETUPTOOLS_PKG_INFO): 368 | log.warn('%s already patched.', pkg_info) 369 | return False 370 | _rename_path(path) 371 | os.mkdir(path) 372 | os.mkdir(os.path.join(path, 'EGG-INFO')) 373 | pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') 374 | f = open(pkg_info, 'w') 375 | try: 376 | f.write(SETUPTOOLS_PKG_INFO) 377 | finally: 378 | f.close() 379 | return True 380 | 381 | _patch_egg_dir = _no_sandbox(_patch_egg_dir) 382 | 383 | 384 | def _before_install(): 385 | log.warn('Before install bootstrap.') 386 | _fake_setuptools() 387 | 388 | 389 | def _under_prefix(location): 390 | if 'install' not in sys.argv: 391 | return True 392 | args = sys.argv[sys.argv.index('install') + 1:] 393 | for index, arg in enumerate(args): 394 | for option in ('--root', '--prefix'): 395 | if arg.startswith('%s=' % option): 396 | top_dir = arg.split('root=')[-1] 397 | return location.startswith(top_dir) 398 | elif arg == option: 399 | if len(args) > index: 400 | top_dir = args[index + 1] 401 | return location.startswith(top_dir) 402 | if arg == '--user' and USER_SITE is not None: 403 | return location.startswith(USER_SITE) 404 | return True 405 | 406 | 407 | def _fake_setuptools(): 408 | log.warn('Scanning installed packages') 409 | try: 410 | import pkg_resources 411 | except ImportError: 412 | # we're cool 413 | log.warn('Setuptools or Distribute does not seem to be installed.') 414 | return 415 | ws = pkg_resources.working_set 416 | try: 417 | setuptools_dist = ws.find( 418 | pkg_resources.Requirement.parse('setuptools', replacement=False) 419 | ) 420 | except TypeError: 421 | # old distribute API 422 | setuptools_dist = ws.find( 423 | pkg_resources.Requirement.parse('setuptools') 424 | ) 425 | 426 | if setuptools_dist is None: 427 | log.warn('No setuptools distribution found') 428 | return 429 | # detecting if it was already faked 430 | setuptools_location = setuptools_dist.location 431 | log.warn('Setuptools installation detected at %s', setuptools_location) 432 | 433 | # if --root or --preix was provided, and if 434 | # setuptools is not located in them, we don't patch it 435 | if not _under_prefix(setuptools_location): 436 | log.warn('Not patching, --root or --prefix is installing Distribute' 437 | ' in another location') 438 | return 439 | 440 | # let's see if its an egg 441 | if not setuptools_location.endswith('.egg'): 442 | log.warn('Non-egg installation') 443 | res = _remove_flat_installation(setuptools_location) 444 | if not res: 445 | return 446 | else: 447 | log.warn('Egg installation') 448 | pkg_info = os.path.join(setuptools_location, 'EGG-INFO', 'PKG-INFO') 449 | if (os.path.exists(pkg_info) and 450 | _same_content(pkg_info, SETUPTOOLS_PKG_INFO)): 451 | log.warn('Already patched.') 452 | return 453 | log.warn('Patching...') 454 | # let's create a fake egg replacing setuptools one 455 | res = _patch_egg_dir(setuptools_location) 456 | if not res: 457 | return 458 | log.warn('Patching complete.') 459 | _relaunch() 460 | 461 | 462 | def _relaunch(): 463 | log.warn('Relaunching...') 464 | # we have to relaunch the process 465 | # pip marker to avoid a relaunch bug 466 | _cmd1 = ['-c', 'install', '--single-version-externally-managed'] 467 | _cmd2 = ['-c', 'install', '--record'] 468 | if sys.argv[:3] == _cmd1 or sys.argv[:3] == _cmd2: 469 | sys.argv[0] = 'setup.py' 470 | args = [sys.executable] + sys.argv 471 | sys.exit(subprocess.call(args)) 472 | 473 | 474 | def _extractall(self, path=".", members=None): 475 | """Extract all members from the archive to the current working 476 | directory and set owner, modification time and permissions on 477 | directories afterwards. `path' specifies a different directory 478 | to extract to. `members' is optional and must be a subset of the 479 | list returned by getmembers(). 480 | """ 481 | import copy 482 | import operator 483 | from tarfile import ExtractError 484 | directories = [] 485 | 486 | if members is None: 487 | members = self 488 | 489 | for tarinfo in members: 490 | if tarinfo.isdir(): 491 | # Extract directories with a safe mode. 492 | directories.append(tarinfo) 493 | tarinfo = copy.copy(tarinfo) 494 | tarinfo.mode = 448 # decimal for oct 0700 495 | self.extract(tarinfo, path) 496 | 497 | # Reverse sort directories. 498 | if sys.version_info < (2, 4): 499 | def sorter(dir1, dir2): 500 | return cmp(dir1.name, dir2.name) 501 | directories.sort(sorter) 502 | directories.reverse() 503 | else: 504 | directories.sort(key=operator.attrgetter('name'), reverse=True) 505 | 506 | # Set correct owner, mtime and filemode on directories. 507 | for tarinfo in directories: 508 | dirpath = os.path.join(path, tarinfo.name) 509 | try: 510 | self.chown(tarinfo, dirpath) 511 | self.utime(tarinfo, dirpath) 512 | self.chmod(tarinfo, dirpath) 513 | except ExtractError: 514 | e = sys.exc_info()[1] 515 | if self.errorlevel > 1: 516 | raise 517 | else: 518 | self._dbg(1, "tarfile: %s" % e) 519 | 520 | 521 | def _build_install_args(options): 522 | """ 523 | Build the arguments to 'python setup.py install' on the distribute package 524 | """ 525 | install_args = [] 526 | if options.user_install: 527 | if sys.version_info < (2, 6): 528 | log.warn("--user requires Python 2.6 or later") 529 | raise SystemExit(1) 530 | install_args.append('--user') 531 | return install_args 532 | 533 | def _parse_args(): 534 | """ 535 | Parse the command line for options 536 | """ 537 | parser = optparse.OptionParser() 538 | parser.add_option( 539 | '--user', dest='user_install', action='store_true', default=False, 540 | help='install in user site package (requires Python 2.6 or later)') 541 | parser.add_option( 542 | '--download-base', dest='download_base', metavar="URL", 543 | default=DEFAULT_URL, 544 | help='alternative URL from where to download the distribute package') 545 | options, args = parser.parse_args() 546 | # positional arguments are ignored 547 | return options 548 | 549 | def main(version=DEFAULT_VERSION): 550 | """Install or upgrade setuptools and EasyInstall""" 551 | options = _parse_args() 552 | tarball = download_setuptools(download_base=options.download_base) 553 | return _install(tarball, _build_install_args(options)) 554 | 555 | if __name__ == '__main__': 556 | sys.exit(main()) 557 | -------------------------------------------------------------------------------- /salt/roots/salt/etc/matplotlibrc: -------------------------------------------------------------------------------- 1 | ### MATPLOTLIBRC FORMAT 2 | 3 | # This is a sample matplotlib configuration file - you can find a copy 4 | # of it on your system in 5 | # site-packages/matplotlib/mpl-data/matplotlibrc. If you edit it 6 | # there, please note that it will be overwritten in your next install. 7 | # If you want to keep a permanent local copy that will not be 8 | # overwritten, place it in the following location: 9 | # unix/linux: 10 | # $HOME/.config/matplotlib/matplotlibrc or 11 | # $XDG_CONFIG_HOME/matplotlib/matplotlibrc (if $XDG_CONFIG_HOME is set) 12 | # other platforms: 13 | # $HOME/.matplotlib/matplotlibrc 14 | # 15 | # See http://matplotlib.org/users/customizing.html#the-matplotlibrc-file for 16 | # more details on the paths which are checked for the configuration file. 17 | # 18 | # This file is best viewed in a editor which supports python mode 19 | # syntax highlighting. Blank lines, or lines starting with a comment 20 | # symbol, are ignored, as are trailing comments. Other lines must 21 | # have the format 22 | # key : val # optional comment 23 | # 24 | # Colors: for the color values below, you can either use - a 25 | # matplotlib color string, such as r, k, or b - an rgb tuple, such as 26 | # (1.0, 0.5, 0.0) - a hex string, such as ff00ff or #ff00ff - a scalar 27 | # grayscale intensity such as 0.75 - a legal html color name, eg red, 28 | # blue, darkslategray 29 | 30 | #### CONFIGURATION BEGINS HERE 31 | 32 | # the default backend; one of GTK GTKAgg GTKCairo GTK3Agg GTK3Cairo 33 | # CocoaAgg MacOSX Qt4Agg TkAgg WX WXAgg Agg Cairo GDK PS PDF SVG 34 | # Template 35 | # You can also deploy your own backend outside of matplotlib by 36 | # referring to the module name (which must be in the PYTHONPATH) as 37 | # 'module://my_backend' 38 | backend : Agg 39 | 40 | # If you are using the Qt4Agg backend, you can choose here 41 | # to use the PyQt4 bindings or the newer PySide bindings to 42 | # the underlying Qt4 toolkit. 43 | #backend.qt4 : PyQt4 # PyQt4 | PySide 44 | 45 | # Note that this can be overridden by the environment variable 46 | # QT_API used by Enthought Tool Suite (ETS); valid values are 47 | # "pyqt" and "pyside". The "pyqt" setting has the side effect of 48 | # forcing the use of Version 2 API for QString and QVariant. 49 | 50 | # The port to use for the web server in the WebAgg backend. 51 | # webagg.port : 8888 52 | 53 | # If webagg.port is unavailable, a number of other random ports will 54 | # be tried until one that is available is found. 55 | # webagg.port_retries : 50 56 | 57 | # When True, open the webbrowser to the plot that is shown 58 | # webagg.open_in_browser : True 59 | 60 | # if you are running pyplot inside a GUI and your backend choice 61 | # conflicts, we will automatically try to find a compatible one for 62 | # you if backend_fallback is True 63 | #backend_fallback: True 64 | 65 | #interactive : False 66 | #toolbar : toolbar2 # None | toolbar2 ("classic" is deprecated) 67 | #timezone : UTC # a pytz timezone string, eg US/Central or Europe/Paris 68 | 69 | # Where your matplotlib data lives if you installed to a non-default 70 | # location. This is where the matplotlib fonts, bitmaps, etc reside 71 | #datapath : /home/jdhunter/mpldata 72 | 73 | 74 | ### LINES 75 | # See http://matplotlib.org/api/artist_api.html#module-matplotlib.lines for more 76 | # information on line properties. 77 | #lines.linewidth : 1.0 # line width in points 78 | #lines.linestyle : - # solid line 79 | #lines.color : blue # has no affect on plot(); see axes.color_cycle 80 | #lines.marker : None # the default marker 81 | #lines.markeredgewidth : 0.5 # the line width around the marker symbol 82 | #lines.markersize : 6 # markersize, in points 83 | #lines.dash_joinstyle : miter # miter|round|bevel 84 | #lines.dash_capstyle : butt # butt|round|projecting 85 | #lines.solid_joinstyle : miter # miter|round|bevel 86 | #lines.solid_capstyle : projecting # butt|round|projecting 87 | #lines.antialiased : True # render lines in antialised (no jaggies) 88 | 89 | ### PATCHES 90 | # Patches are graphical objects that fill 2D space, like polygons or 91 | # circles. See 92 | # http://matplotlib.org/api/artist_api.html#module-matplotlib.patches 93 | # information on patch properties 94 | #patch.linewidth : 1.0 # edge width in points 95 | #patch.facecolor : blue 96 | #patch.edgecolor : black 97 | #patch.antialiased : True # render patches in antialised (no jaggies) 98 | 99 | ### FONT 100 | # 101 | # font properties used by text.Text. See 102 | # http://matplotlib.org/api/font_manager_api.html for more 103 | # information on font properties. The 6 font properties used for font 104 | # matching are given below with their default values. 105 | # 106 | # The font.family property has five values: 'serif' (e.g., Times), 107 | # 'sans-serif' (e.g., Helvetica), 'cursive' (e.g., Zapf-Chancery), 108 | # 'fantasy' (e.g., Western), and 'monospace' (e.g., Courier). Each of 109 | # these font families has a default list of font names in decreasing 110 | # order of priority associated with them. When text.usetex is False, 111 | # font.family may also be one or more concrete font names. 112 | # 113 | # The font.style property has three values: normal (or roman), italic 114 | # or oblique. The oblique style will be used for italic, if it is not 115 | # present. 116 | # 117 | # The font.variant property has two values: normal or small-caps. For 118 | # TrueType fonts, which are scalable fonts, small-caps is equivalent 119 | # to using a font size of 'smaller', or about 83% of the current font 120 | # size. 121 | # 122 | # The font.weight property has effectively 13 values: normal, bold, 123 | # bolder, lighter, 100, 200, 300, ..., 900. Normal is the same as 124 | # 400, and bold is 700. bolder and lighter are relative values with 125 | # respect to the current weight. 126 | # 127 | # The font.stretch property has 11 values: ultra-condensed, 128 | # extra-condensed, condensed, semi-condensed, normal, semi-expanded, 129 | # expanded, extra-expanded, ultra-expanded, wider, and narrower. This 130 | # property is not currently implemented. 131 | # 132 | # The font.size property is the default font size for text, given in pts. 133 | # 12pt is the standard value. 134 | # 135 | #font.family : sans-serif 136 | #font.style : normal 137 | #font.variant : normal 138 | #font.weight : medium 139 | #font.stretch : normal 140 | # note that font.size controls default text sizes. To configure 141 | # special text sizes tick labels, axes, labels, title, etc, see the rc 142 | # settings for axes and ticks. Special text sizes can be defined 143 | # relative to font.size, using the following values: xx-small, x-small, 144 | # small, medium, large, x-large, xx-large, larger, or smaller 145 | #font.size : 12.0 146 | #font.serif : Bitstream Vera Serif, New Century Schoolbook, Century Schoolbook L, Utopia, ITC Bookman, Bookman, Nimbus Roman No9 L, Times New Roman, Times, Palatino, Charter, serif 147 | #font.sans-serif : Bitstream Vera Sans, Lucida Grande, Verdana, Geneva, Lucid, Arial, Helvetica, Avant Garde, sans-serif 148 | #font.cursive : Apple Chancery, Textile, Zapf Chancery, Sand, cursive 149 | #font.fantasy : Comic Sans MS, Chicago, Charcoal, Impact, Western, fantasy 150 | #font.monospace : Bitstream Vera Sans Mono, Andale Mono, Nimbus Mono L, Courier New, Courier, Fixed, Terminal, monospace 151 | 152 | ### TEXT 153 | # text properties used by text.Text. See 154 | # http://matplotlib.org/api/artist_api.html#module-matplotlib.text for more 155 | # information on text properties 156 | 157 | #text.color : black 158 | 159 | ### LaTeX customizations. See http://www.scipy.org/Wiki/Cookbook/Matplotlib/UsingTex 160 | #text.usetex : False # use latex for all text handling. The following fonts 161 | # are supported through the usual rc parameter settings: 162 | # new century schoolbook, bookman, times, palatino, 163 | # zapf chancery, charter, serif, sans-serif, helvetica, 164 | # avant garde, courier, monospace, computer modern roman, 165 | # computer modern sans serif, computer modern typewriter 166 | # If another font is desired which can loaded using the 167 | # LaTeX \usepackage command, please inquire at the 168 | # matplotlib mailing list 169 | #text.latex.unicode : False # use "ucs" and "inputenc" LaTeX packages for handling 170 | # unicode strings. 171 | #text.latex.preamble : # IMPROPER USE OF THIS FEATURE WILL LEAD TO LATEX FAILURES 172 | # AND IS THEREFORE UNSUPPORTED. PLEASE DO NOT ASK FOR HELP 173 | # IF THIS FEATURE DOES NOT DO WHAT YOU EXPECT IT TO. 174 | # preamble is a comma separated list of LaTeX statements 175 | # that are included in the LaTeX document preamble. 176 | # An example: 177 | # text.latex.preamble : \usepackage{bm},\usepackage{euler} 178 | # The following packages are always loaded with usetex, so 179 | # beware of package collisions: color, geometry, graphicx, 180 | # type1cm, textcomp. Adobe Postscript (PSSNFS) font packages 181 | # may also be loaded, depending on your font settings 182 | 183 | #text.dvipnghack : None # some versions of dvipng don't handle alpha 184 | # channel properly. Use True to correct 185 | # and flush ~/.matplotlib/tex.cache 186 | # before testing and False to force 187 | # correction off. None will try and 188 | # guess based on your dvipng version 189 | 190 | #text.hinting : auto # May be one of the following: 191 | # 'none': Perform no hinting 192 | # 'auto': Use freetype's autohinter 193 | # 'native': Use the hinting information in the 194 | # font file, if available, and if your 195 | # freetype library supports it 196 | # 'either': Use the native hinting information, 197 | # or the autohinter if none is available. 198 | # For backward compatibility, this value may also be 199 | # True === 'auto' or False === 'none'. 200 | #text.hinting_factor : 8 # Specifies the amount of softness for hinting in the 201 | # horizontal direction. A value of 1 will hint to full 202 | # pixels. A value of 2 will hint to half pixels etc. 203 | 204 | #text.antialiased : True # If True (default), the text will be antialiased. 205 | # This only affects the Agg backend. 206 | 207 | # The following settings allow you to select the fonts in math mode. 208 | # They map from a TeX font name to a fontconfig font pattern. 209 | # These settings are only used if mathtext.fontset is 'custom'. 210 | # Note that this "custom" mode is unsupported and may go away in the 211 | # future. 212 | #mathtext.cal : cursive 213 | #mathtext.rm : serif 214 | #mathtext.tt : monospace 215 | #mathtext.it : serif:italic 216 | #mathtext.bf : serif:bold 217 | #mathtext.sf : sans 218 | #mathtext.fontset : cm # Should be 'cm' (Computer Modern), 'stix', 219 | # 'stixsans' or 'custom' 220 | #mathtext.fallback_to_cm : True # When True, use symbols from the Computer Modern 221 | # fonts when a symbol can not be found in one of 222 | # the custom math fonts. 223 | 224 | #mathtext.default : it # The default font to use for math. 225 | # Can be any of the LaTeX font names, including 226 | # the special name "regular" for the same font 227 | # used in regular text. 228 | 229 | ### AXES 230 | # default face and edge color, default tick sizes, 231 | # default fontsizes for ticklabels, and so on. See 232 | # http://matplotlib.org/api/axes_api.html#module-matplotlib.axes 233 | #axes.hold : True # whether to clear the axes by default on 234 | #axes.facecolor : white # axes background color 235 | #axes.edgecolor : black # axes edge color 236 | #axes.linewidth : 1.0 # edge linewidth 237 | #axes.grid : False # display grid or not 238 | #axes.titlesize : large # fontsize of the axes title 239 | #axes.labelsize : medium # fontsize of the x any y labels 240 | #axes.labelweight : normal # weight of the x and y labels 241 | #axes.labelcolor : black 242 | #axes.axisbelow : False # whether axis gridlines and ticks are below 243 | # the axes elements (lines, text, etc) 244 | 245 | #axes.formatter.limits : -7, 7 # use scientific notation if log10 246 | # of the axis range is smaller than the 247 | # first or larger than the second 248 | #axes.formatter.use_locale : False # When True, format tick labels 249 | # according to the user's locale. 250 | # For example, use ',' as a decimal 251 | # separator in the fr_FR locale. 252 | #axes.formatter.use_mathtext : False # When True, use mathtext for scientific 253 | # notation. 254 | #axes.formatter.useoffset : True # If True, the tick label formatter 255 | # will default to labeling ticks relative 256 | # to an offset when the data range is very 257 | # small compared to the minimum absolute 258 | # value of the data. 259 | 260 | #axes.unicode_minus : True # use unicode for the minus symbol 261 | # rather than hyphen. See 262 | # http://en.wikipedia.org/wiki/Plus_and_minus_signs#Character_codes 263 | #axes.color_cycle : b, g, r, c, m, y, k # color cycle for plot lines 264 | # as list of string colorspecs: 265 | # single letter, long name, or 266 | # web-style hex 267 | #axes.xmargin : 0 # x margin. See `axes.Axes.margins` 268 | #axes.ymargin : 0 # y margin See `axes.Axes.margins` 269 | 270 | #polaraxes.grid : True # display grid on polar axes 271 | #axes3d.grid : True # display grid on 3d axes 272 | 273 | ### TICKS 274 | # see http://matplotlib.org/api/axis_api.html#matplotlib.axis.Tick 275 | #xtick.major.size : 4 # major tick size in points 276 | #xtick.minor.size : 2 # minor tick size in points 277 | #xtick.major.width : 0.5 # major tick width in points 278 | #xtick.minor.width : 0.5 # minor tick width in points 279 | #xtick.major.pad : 4 # distance to major tick label in points 280 | #xtick.minor.pad : 4 # distance to the minor tick label in points 281 | #xtick.color : k # color of the tick labels 282 | #xtick.labelsize : medium # fontsize of the tick labels 283 | #xtick.direction : in # direction: in, out, or inout 284 | 285 | #ytick.major.size : 4 # major tick size in points 286 | #ytick.minor.size : 2 # minor tick size in points 287 | #ytick.major.width : 0.5 # major tick width in points 288 | #ytick.minor.width : 0.5 # minor tick width in points 289 | #ytick.major.pad : 4 # distance to major tick label in points 290 | #ytick.minor.pad : 4 # distance to the minor tick label in points 291 | #ytick.color : k # color of the tick labels 292 | #ytick.labelsize : medium # fontsize of the tick labels 293 | #ytick.direction : in # direction: in, out, or inout 294 | 295 | 296 | ### GRIDS 297 | #grid.color : black # grid color 298 | #grid.linestyle : : # dotted 299 | #grid.linewidth : 0.5 # in points 300 | #grid.alpha : 1.0 # transparency, between 0.0 and 1.0 301 | 302 | ### Legend 303 | #legend.fancybox : False # if True, use a rounded box for the 304 | # legend, else a rectangle 305 | #legend.isaxes : True 306 | #legend.numpoints : 2 # the number of points in the legend line 307 | #legend.fontsize : large 308 | #legend.borderpad : 0.5 # border whitespace in fontsize units 309 | #legend.markerscale : 1.0 # the relative size of legend markers vs. original 310 | # the following dimensions are in axes coords 311 | #legend.labelspacing : 0.5 # the vertical space between the legend entries in fraction of fontsize 312 | #legend.handlelength : 2. # the length of the legend lines in fraction of fontsize 313 | #legend.handleheight : 0.7 # the height of the legend handle in fraction of fontsize 314 | #legend.handletextpad : 0.8 # the space between the legend line and legend text in fraction of fontsize 315 | #legend.borderaxespad : 0.5 # the border between the axes and legend edge in fraction of fontsize 316 | #legend.columnspacing : 2. # the border between the axes and legend edge in fraction of fontsize 317 | #legend.shadow : False 318 | #legend.frameon : True # whether or not to draw a frame around legend 319 | #legend.scatterpoints : 3 # number of scatter points 320 | 321 | ### FIGURE 322 | # See http://matplotlib.org/api/figure_api.html#matplotlib.figure.Figure 323 | #figure.figsize : 8, 6 # figure size in inches 324 | #figure.dpi : 80 # figure dots per inch 325 | #figure.facecolor : 0.75 # figure facecolor; 0.75 is scalar gray 326 | #figure.edgecolor : white # figure edgecolor 327 | #figure.autolayout : False # When True, automatically adjust subplot 328 | # parameters to make the plot fit the figure 329 | #figure.max_open_warning : 20 # The maximum number of figures to open through 330 | # the pyplot interface before emitting a warning. 331 | # If less than one this feature is disabled. 332 | 333 | # The figure subplot parameters. All dimensions are a fraction of the 334 | # figure width or height 335 | #figure.subplot.left : 0.125 # the left side of the subplots of the figure 336 | #figure.subplot.right : 0.9 # the right side of the subplots of the figure 337 | #figure.subplot.bottom : 0.1 # the bottom of the subplots of the figure 338 | #figure.subplot.top : 0.9 # the top of the subplots of the figure 339 | #figure.subplot.wspace : 0.2 # the amount of width reserved for blank space between subplots 340 | #figure.subplot.hspace : 0.2 # the amount of height reserved for white space between subplots 341 | 342 | ### IMAGES 343 | #image.aspect : equal # equal | auto | a number 344 | #image.interpolation : bilinear # see help(imshow) for options 345 | #image.cmap : jet # gray | jet etc... 346 | #image.lut : 256 # the size of the colormap lookup table 347 | #image.origin : upper # lower | upper 348 | #image.resample : False 349 | 350 | ### CONTOUR PLOTS 351 | #contour.negative_linestyle : dashed # dashed | solid 352 | 353 | ### Agg rendering 354 | ### Warning: experimental, 2008/10/10 355 | #agg.path.chunksize : 0 # 0 to disable; values in the range 356 | # 10000 to 100000 can improve speed slightly 357 | # and prevent an Agg rendering failure 358 | # when plotting very large data sets, 359 | # especially if they are very gappy. 360 | # It may cause minor artifacts, though. 361 | # A value of 20000 is probably a good 362 | # starting point. 363 | ### SAVING FIGURES 364 | #path.simplify : True # When True, simplify paths by removing "invisible" 365 | # points to reduce file size and increase rendering 366 | # speed 367 | #path.simplify_threshold : 0.1 # The threshold of similarity below which 368 | # vertices will be removed in the simplification 369 | # process 370 | #path.snap : True # When True, rectilinear axis-aligned paths will be snapped to 371 | # the nearest pixel when certain criteria are met. When False, 372 | # paths will never be snapped. 373 | #path.sketch : None # May be none, or a 3-tuple of the form (scale, length, 374 | # randomness). 375 | # *scale* is the amplitude of the wiggle 376 | # perpendicular to the line (in pixels). *length* 377 | # is the length of the wiggle along the line (in 378 | # pixels). *randomness* is the factor by which 379 | # the length is randomly scaled. 380 | 381 | # the default savefig params can be different from the display params 382 | # e.g., you may want a higher resolution, or to make the figure 383 | # background white 384 | #savefig.dpi : 100 # figure dots per inch 385 | #savefig.facecolor : white # figure facecolor when saving 386 | #savefig.edgecolor : white # figure edgecolor when saving 387 | #savefig.format : png # png, ps, pdf, svg 388 | #savefig.bbox : standard # 'tight' or 'standard'. 389 | # 'tight' is incompatible with pipe-based animation 390 | # backends but will workd with temporary file based ones: 391 | # e.g. setting animation.writer to ffmpeg will not work, 392 | # use ffmpeg_file instead 393 | #savefig.pad_inches : 0.1 # Padding to be used when bbox is set to 'tight' 394 | #savefig.jpeg_quality: 95 # when a jpeg is saved, the default quality parameter. 395 | #savefig.directory : ~ # default directory in savefig dialog box, 396 | # leave empty to always use current working directory 397 | #savefig.transparent : False # setting that controls whether figures are saved with a 398 | # transparent background by default 399 | 400 | # tk backend params 401 | #tk.window_focus : False # Maintain shell focus for TkAgg 402 | 403 | # ps backend params 404 | #ps.papersize : letter # auto, letter, legal, ledger, A0-A10, B0-B10 405 | #ps.useafm : False # use of afm fonts, results in small files 406 | #ps.usedistiller : False # can be: None, ghostscript or xpdf 407 | # Experimental: may produce smaller files. 408 | # xpdf intended for production of publication quality files, 409 | # but requires ghostscript, xpdf and ps2eps 410 | #ps.distiller.res : 6000 # dpi 411 | #ps.fonttype : 3 # Output Type 3 (Type3) or Type 42 (TrueType) 412 | 413 | # pdf backend params 414 | #pdf.compression : 6 # integer from 0 to 9 415 | # 0 disables compression (good for debugging) 416 | #pdf.fonttype : 3 # Output Type 3 (Type3) or Type 42 (TrueType) 417 | 418 | # svg backend params 419 | #svg.image_inline : True # write raster image data directly into the svg file 420 | #svg.image_noscale : False # suppress scaling of raster data embedded in SVG 421 | #svg.fonttype : 'path' # How to handle SVG fonts: 422 | # 'none': Assume fonts are installed on the machine where the SVG will be viewed. 423 | # 'path': Embed characters as paths -- supported by most SVG renderers 424 | # 'svgfont': Embed characters as SVG fonts -- supported only by Chrome, 425 | # Opera and Safari 426 | 427 | # docstring params 428 | #docstring.hardcopy = False # set this when you want to generate hardcopy docstring 429 | 430 | # Set the verbose flags. This controls how much information 431 | # matplotlib gives you at runtime and where it goes. The verbosity 432 | # levels are: silent, helpful, debug, debug-annoying. Any level is 433 | # inclusive of all the levels below it. If your setting is "debug", 434 | # you'll get all the debug and helpful messages. When submitting 435 | # problems to the mailing-list, please set verbose to "helpful" or "debug" 436 | # and paste the output into your report. 437 | # 438 | # The "fileo" gives the destination for any calls to verbose.report. 439 | # These objects can a filename, or a filehandle like sys.stdout. 440 | # 441 | # You can override the rc default verbosity from the command line by 442 | # giving the flags --verbose-LEVEL where LEVEL is one of the legal 443 | # levels, eg --verbose-helpful. 444 | # 445 | # You can access the verbose instance in your code 446 | # from matplotlib import verbose. 447 | #verbose.level : silent # one of silent, helpful, debug, debug-annoying 448 | #verbose.fileo : sys.stdout # a log filename, sys.stdout or sys.stderr 449 | 450 | # Event keys to interact with figures/plots via keyboard. 451 | # Customize these settings according to your needs. 452 | # Leave the field(s) empty if you don't need a key-map. (i.e., fullscreen : '') 453 | 454 | #keymap.fullscreen : f # toggling 455 | #keymap.home : h, r, home # home or reset mnemonic 456 | #keymap.back : left, c, backspace # forward / backward keys to enable 457 | #keymap.forward : right, v # left handed quick navigation 458 | #keymap.pan : p # pan mnemonic 459 | #keymap.zoom : o # zoom mnemonic 460 | #keymap.save : s # saving current figure 461 | #keymap.quit : ctrl+w, cmd+w # close the current figure 462 | #keymap.grid : g # switching on/off a grid in current axes 463 | #keymap.yscale : l # toggle scaling of y-axes ('log'/'linear') 464 | #keymap.xscale : L, k # toggle scaling of x-axes ('log'/'linear') 465 | #keymap.all_axes : a # enable all axes 466 | 467 | # Control location of examples data files 468 | #examples.directory : '' # directory to look in for custom installation 469 | 470 | ###ANIMATION settings 471 | #animation.writer : ffmpeg # MovieWriter 'backend' to use 472 | #animation.codec : mp4 # Codec to use for writing movie 473 | #animation.bitrate: -1 # Controls size/quality tradeoff for movie. 474 | # -1 implies let utility auto-determine 475 | #animation.frame_format: 'png' # Controls frame format used by temp files 476 | #animation.ffmpeg_path: 'ffmpeg' # Path to ffmpeg binary. Without full path 477 | # $PATH is searched 478 | #animation.ffmpeg_args: '' # Additional arguments to pass to ffmpeg 479 | #animation.avconv_path: 'avconv' # Path to avconv binary. Without full path 480 | # $PATH is searched 481 | #animation.avconv_args: '' # Additional arguments to pass to avconv 482 | #animation.mencoder_path: 'mencoder' 483 | # Path to mencoder binary. Without full path 484 | # $PATH is searched 485 | #animation.mencoder_args: '' # Additional arguments to pass to mencoder 486 | --------------------------------------------------------------------------------