├── test ├── Procfile ├── environment.yml └── server.py ├── bin ├── steps │ ├── hooks │ │ ├── pre_compile │ │ └── post_compile │ └── conda_compile ├── release ├── detect ├── utils └── compile ├── README.md └── LICENSE /test/Procfile: -------------------------------------------------------------------------------- 1 | web: python server.py 2 | -------------------------------------------------------------------------------- /test/environment.yml: -------------------------------------------------------------------------------- 1 | name: server 2 | dependencies: 3 | - werkzeug 4 | -------------------------------------------------------------------------------- /bin/steps/hooks/pre_compile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ -f bin/pre_compile ]; then 4 | echo "-----> Running pre-compile hook" 5 | chmod +x bin/pre_compile 6 | bin/pre_compile 7 | fi -------------------------------------------------------------------------------- /bin/steps/hooks/post_compile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ -f bin/post_compile ]; then 4 | echo "-----> Running post-compile hook" 5 | chmod +x bin/post_compile 6 | bin/post_compile 7 | fi -------------------------------------------------------------------------------- /bin/release: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # bin/release 3 | 4 | cat < 14 | 15 | BUILD_DIR=$1 16 | 17 | # Exit early if app is clearly not Python. 18 | if [ ! -f $BUILD_DIR/environment.yml ]; then 19 | if [ ! -f $BUILD_DIR/requirements.txt ] && [ ! -f $BUILD_DIR/conda-requirements.txt ] && [ ! -f $BUILD_DIR/setup.py ]; then 20 | exit 1 21 | fi 22 | fi 23 | 24 | echo Python/Conda 25 | -------------------------------------------------------------------------------- /bin/utils: -------------------------------------------------------------------------------- 1 | shopt -s extglob 2 | 3 | [ $(uname) == "Darwin" ] && SED_FLAG='-l' || SED_FLAG='-u' 4 | 5 | # Syntax sugar. 6 | indent() { 7 | RE="s/^/ /" 8 | sed $SED_FLAG "$RE" 9 | } 10 | 11 | # Clean up pip output 12 | cleanup() { 13 | sed $SED_FLAG -e 's/\.\.\.\+/.../g' | sed $SED_FLAG '/already satisfied/Id' | sed $SED_FLAG -e '/Overwriting/Id' | sed $SED_FLAG -e '/python executable/Id' | sed $SED_FLAG -e '/no previously-included files/Id' 14 | } 15 | 16 | # Buildpack Steps. 17 | function puts-step (){ 18 | echo "-----> $@" 19 | } 20 | 21 | # Buildpack Warnings. 22 | function puts-warn (){ 23 | echo " ! $@" 24 | } 25 | 26 | # Usage: $ set-env key value 27 | function set-env (){ 28 | echo "export $1=$2" >> $PROFILE_PATH 29 | } 30 | 31 | # Usage: $ set-default-env key value 32 | function set-default-env (){ 33 | echo "export $1=\${$1:-$2}" >> $PROFILE_PATH 34 | } 35 | 36 | # Usage: $ set-default-env key value 37 | function un-set-env (){ 38 | echo "unset $1" >> $PROFILE_PATH 39 | } 40 | 41 | # Does some serious copying. 42 | function deep-cp (){ 43 | find -H $1 -maxdepth 1 -name '.*' -a \( -type d -o -type f -o -type l \) -exec cp -a '{}' $2 \; 44 | cp -r $1/!(tmp) $2 45 | # echo copying $1 to $2 46 | } 47 | 48 | # Does some serious moving. 49 | function deep-mv (){ 50 | deep-cp $1 $2 51 | 52 | rm -fr $1/!(tmp) 53 | find -H $1 -maxdepth 1 -name '.*' -a \( -type d -o -type f -o -type l \) -exec rm -fr '{}' \; 54 | } 55 | -------------------------------------------------------------------------------- /bin/steps/conda_compile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | CONDA_VERSION=4.3.31 3 | puts-step "Checking for existing conda install" 4 | if [ ! -d /app/.heroku/miniconda ]; then 5 | puts-step "Preparing Python/Miniconda Environment (${CONDA_VERSION})" 6 | curl -Os https://repo.continuum.io/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh 7 | bash Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh -p /app/.heroku/miniconda/ -b | indent 8 | rm -fr Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh 9 | 10 | conda update conda --quiet --yes | indent 11 | fi 12 | 13 | (puts-step "Checking for conda env" && conda env &> /dev/null) || conda update conda --quiet --yes | indent 14 | 15 | if [ -f conda-requirements.txt ]; then 16 | puts-step "Installing dependencies using Conda" 17 | conda install --file conda-requirements.txt --yes 18 | fi 19 | 20 | if [ -f environment.yml ]; then 21 | puts-step "Creating conda environment" 22 | # TODO: Use update if its already there 23 | conda env remove --yes --quiet --name heroku-env || puts-step "No old environment to remove" 24 | conda env create --name=heroku-env --file environment.yml 25 | puts-step "Conda environment created" 26 | source activate heroku-env 27 | # conda install --yes nomkl numpy 28 | # conda remove --yes mkl mkl-service 29 | fi 30 | 31 | # Clean up the installation environment . 32 | # TODO Determine if removing packages might be bad here 33 | conda clean -pt --yes > /dev/null 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Conda Environment Buildpack 2 | =========================== 3 | 4 | This is the [Heroku Buildpack][] for [Conda][] using it's new 5 | [environment spec][]. Anything you can install with `conda install` can be 6 | installed using this, including the entire data science stack. Be careful of 7 | slug sizes, though. Heroku does have limits. 8 | 9 | ## Usage 10 | To control what gets installed, create an `environment.yml` file in the root 11 | of your repository. For example, if you wanted to install Flask, you would add 12 | this: 13 | 14 | ```yaml 15 | name: myproject # overridden at install, so this is for your use with conda env 16 | dependencies: 17 | - flask 18 | ``` 19 | 20 | Once that's created, you need to create a new Heroku app using this buildpack 21 | like this: 22 | 23 | ```console 24 | $ heroku create --buildpack https://github.com/conda/conda-buildpack.git 25 | ``` 26 | 27 | You can also add it to upcoming builds of an existing application: 28 | 29 | ```console 30 | $ heroku config:add BUILDPACK_URL=https://github.com/conda/conda-buildpack.git 31 | ``` 32 | 33 | You can test that this is running conda managed Python like this: 34 | 35 | ```console 36 | $ heroku run python 37 | Running `python` attached to terminal... up, run.7018 38 | Python 2.7.9 |Continuum Analytics, Inc.| (default, Dec 15 2014, 10:33:51) 39 | [GCC 4.4.7 20120313 (Red Hat 4.4.7-1)] on linux2 40 | Type "help", "copyright", "credits" or "license" for more information. 41 | Anaconda is brought to you by Continuum Analytics. 42 | Please check out: http://continuum.io/thanks and https://binstar.org 43 | >>> 44 | ``` 45 | 46 | 47 | ## Fair Warning 48 | 49 | Heroku limits the final application footprint (slug) size to 300MB. Start small. 50 | 51 | [Conda]: http://conda.io 52 | [environment spec]: https://github.com/conda/conda-env#environmentyml 53 | [Heroku Buildpack]: https://devcenter.heroku.com/articles/buildpacks 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | (c) 2015 Continuum Analytics, Inc. / http://continuum.io 2 | All Rights Reserved 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of Continuum Analytics, Inc. nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL CONTINUUM ANALYTICS BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | 26 | 27 | Original conda-buildpack License from Kenneth Reitz 28 | =================================================== 29 | MIT License: 30 | 31 | Copyright (C) 2013 Kenneth Reitz 32 | 33 | Permission is hereby granted, free of charge, to any person obtaining a copy 34 | of this software and associated documentation files (the "Software"), to deal 35 | in the Software without restriction, including without limitation the rights to 36 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 37 | of the Software, and to permit persons to whom the Software is furnished to do 38 | so, subject to the following conditions: 39 | 40 | The above copyright notice and this permission notice shall be included in all 41 | copies or substantial portions of the Software. 42 | 43 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 44 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 45 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 46 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 47 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 48 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 49 | -------------------------------------------------------------------------------- /bin/compile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Usage: 4 | # 5 | # $ bin/compile 6 | 7 | 8 | # Fail fast and fail hard. 9 | set -eo pipefail 10 | 11 | # Prepend proper path for virtualenv hackery. This will be deprecated soon. 12 | export PATH=:/usr/local/bin:$PATH 13 | 14 | # Paths. 15 | BIN_DIR=$(cd $(dirname $0); pwd) # absolute path 16 | ROOT_DIR=$(dirname $BIN_DIR) 17 | BUILD_DIR=$1 18 | CACHE_DIR=$2 19 | 20 | CACHED_DIRS=".heroku" 21 | 22 | # Static configurations for virtualenv caches. 23 | PROFILE_PATH="$BUILD_DIR/.profile.d/miniconda.sh" 24 | 25 | # Sanitizing environment variables. 26 | unset GIT_DIR PYTHONHOME PYTHONPATH LD_LIBRARY_PATH LIBRARY_PATH 27 | 28 | # We'll need to send these statics to other scripts we `source`. 29 | export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH 30 | 31 | # Syntax sugar. 32 | source $BIN_DIR/utils 33 | 34 | # Directory Hacks for path consistiency. 35 | APP_DIR='/app' 36 | TMP_APP_DIR=$CACHE_DIR/tmp_app_dir 37 | 38 | # Copy Anvil app dir to temporary storage... 39 | mkdir -p $TMP_APP_DIR 40 | deep-mv $APP_DIR $TMP_APP_DIR 41 | 42 | # Copy Application code in. 43 | deep-mv $BUILD_DIR $APP_DIR 44 | 45 | # Set new context. 46 | ORIG_BUILD_DIR=$BUILD_DIR 47 | BUILD_DIR=$APP_DIR 48 | 49 | # Prepend proper path buildpack use. 50 | export PATH=$BUILD_DIR/.heroku/miniconda/bin:$PATH 51 | export PYTHONUNBUFFERED=1 52 | export LANG=en_US.UTF-8 53 | 54 | # Switch to the repo's context. 55 | cd $BUILD_DIR 56 | 57 | # Experimental pre_compile hook. 58 | source $BIN_DIR/steps/hooks/pre_compile 59 | 60 | # ### The Cache 61 | mkdir -p $CACHE_DIR 62 | 63 | # Restore old artifacts from the cache. 64 | for dir in $CACHED_DIRS; do 65 | cp -R $CACHE_DIR/$dir . &> /dev/null || true 66 | done 67 | 68 | set +e 69 | # Create set-aside `.heroku` folder. 70 | mkdir .heroku &> /dev/null 71 | set -e 72 | 73 | # Make profile.d directory. 74 | mkdir -p $(dirname $PROFILE_PATH) 75 | 76 | # Actuall do the conda steps. 77 | source $BIN_DIR/steps/conda_compile 78 | 79 | # ### Finalize 80 | # 81 | 82 | # Store new artifacts in cache. 83 | for dir in $CACHED_DIRS; do 84 | rm -rf $CACHE_DIR/$dir 85 | cp -R $dir $CACHE_DIR/ 86 | done 87 | 88 | # Set context environment variables. 89 | set-env PATH '$HOME/.heroku/miniconda/bin:$PATH' 90 | PYTHONPATH='/app/.heroku/miniconda/lib/python2.7/site-packages:$PYTHONPATH' 91 | if [ -f environment.yml ]; then 92 | PYTHONPATH="/app/.heroku/miniconda/envs/heroku-env/lib/python2.7/site-packages:${PYTHONPATH}" 93 | set-default-env CONDA_DEFAULT_ENV "heroku-env" 94 | set-env PATH '$HOME/.heroku/miniconda/envs/heroku-env/bin:$PATH' 95 | fi 96 | set-default-env PYTHONPATH $PYTHONPATH 97 | set-env PYTHONUNBUFFERED true 98 | set-default-env LANG en_US.UTF-8 99 | set-default-env PYTHONHASHSEED random 100 | 101 | 102 | # Experimental post_compile hook. 103 | source $BIN_DIR/steps/hooks/post_compile 104 | 105 | # ### Fin. 106 | 107 | deep-mv $BUILD_DIR $ORIG_BUILD_DIR 108 | deep-mv $TMP_APP_DIR $APP_DIR 109 | --------------------------------------------------------------------------------