├── .ebextensions └── example_env.config ├── 01app_build.sh ├── 10stop.sh ├── 20clean.sh ├── 40config_deploy.sh ├── 40install_node.sh ├── 46update_geolitecity.sh ├── 50npm.sh ├── 55phantomjs_restart.sh ├── LICENSE.md ├── README.md ├── cron-node.sh ├── env.vars ├── harvester-phantomjs.conf ├── harvester.conf ├── harvester_front.conf ├── io-harvester.conf ├── io-server.conf ├── log_server.conf ├── logrotate.conf.elasticbeanstalk ├── web_server.conf └── z.sh /.ebextensions/example_env.config: -------------------------------------------------------------------------------- 1 | # this file is to be located in your project/.ebextensions folder 2 | 3 | #TO UPDATE NODE VERSION & ARCH 4 | # replace 0.10.24 -> 0.11.0 (for example) and x86 -> x64 in 5 | # — all *env.config 6 | # — env.vars 7 | # — in AWS EB console for each env (env configuration settings) (may be unneccessary) 8 | 9 | #TO UPDATE NPM 10 | # uncomment "rm npm updated" in 40install_node.sh 11 | 12 | #TO CHANGE NGINX VERSION 13 | # update version number in env.vars 14 | 15 | packages: 16 | yum: 17 | ImageMagick: [] 18 | git: [] 19 | mc: [] 20 | gcc: [] 21 | make: [] 22 | openssl-devel: [] 23 | tmux: [] 24 | gcc-c++: [] 25 | freetype-devel: [] 26 | fontconfig-devel: [] 27 | ruby: [] 28 | rubygems: [] 29 | pcre: [] 30 | pcre-devel: [] 31 | gperftools: [] 32 | gperftools-devel: [] 33 | rubygems: 34 | rubygems-update: [] 35 | phantom-manager: '0.0.8' 36 | 37 | option_settings: 38 | - option_name: NODE_ENV 39 | value: production 40 | - option_name: RDS_HOSTNAME 41 | value: fill_me_in 42 | - option_name: RDS_PASSWORD 43 | value: fill_me_in 44 | - option_name: RDS_USERNAME 45 | value: fill_me_in 46 | - option_name: RDS_PORT 47 | value: 3306 48 | #set this var via console to update io.log nodename 49 | - option_name: IO_LOG_NODE 50 | value: STAGE 51 | - namespace: aws:elasticbeanstalk:container:nodejs 52 | option_name: NodeVersion 53 | value: 0.10.24 54 | - namespace: aws:elasticbeanstalk:container:nodejs 55 | option_name: GzipCompression 56 | value: true 57 | 58 | files: 59 | "/opt/elasticbeanstalk/env.vars" : 60 | mode: "000775" 61 | owner: root 62 | group: users 63 | source: https://raw.github.com/kopurando/better-faster-elastic-beanstalk/master/env.vars 64 | 65 | "/opt/elasticbeanstalk/hooks/configdeploy/pre/50npm.sh" : 66 | mode: "000666" 67 | owner: root 68 | group: users 69 | content: | 70 | #no need to run npm install during configdeploy 71 | 72 | "/opt/elasticbeanstalk/hooks/appdeploy/pre/50npm.sh" : 73 | mode: "000775" 74 | owner: root 75 | group: users 76 | source: https://raw.github.com/kopurando/better-faster-elastic-beanstalk/master/50npm.sh 77 | 78 | "/opt/elasticbeanstalk/hooks/configdeploy/pre/40install_node.sh" : 79 | mode: "000775" 80 | owner: root 81 | group: users 82 | source: https://raw.github.com/kopurando/better-faster-elastic-beanstalk/master/40install_node.sh 83 | 84 | "/opt/elasticbeanstalk/hooks/appdeploy/pre/40install_node.sh" : 85 | mode: "000775" 86 | owner: root 87 | group: users 88 | source: https://raw.github.com/kopurando/better-faster-elastic-beanstalk/master/40install_node.sh 89 | 90 | "/root/.log.io/harvester.conf" : 91 | mode: "000666" 92 | owner: root 93 | group: users 94 | source: https://raw.github.com/kopurando/better-faster-elastic-beanstalk/master/harvester.conf 95 | 96 | "/root/.log.io/web_server.conf" : 97 | mode: "000666" 98 | owner: root 99 | group: users 100 | source: https://raw.github.com/kopurando/better-faster-elastic-beanstalk/master/web_server.conf 101 | 102 | "/root/.log.io/log_server.conf" : 103 | mode: "000666" 104 | owner: root 105 | group: users 106 | source: https://raw.github.com/kopurando/better-faster-elastic-beanstalk/master/log_server.conf 107 | 108 | "/root/z.sh" : 109 | mode: "000755" 110 | owner: root 111 | group: root 112 | source: https://raw.github.com/kopurando/better-faster-elastic-beanstalk/master/z.sh 113 | 114 | "/home/ec2-user/z.sh" : 115 | mode: "000755" 116 | owner: ec2-user 117 | group: ec2-user 118 | source: https://raw.github.com/kopurando/better-faster-elastic-beanstalk/master/z.sh 119 | 120 | "/root/cron-node.sh" : 121 | mode: "000755" 122 | owner: root 123 | group: root 124 | source: https://raw.github.com/kopurando/better-faster-elastic-beanstalk/master/cron-node.sh 125 | 126 | "/opt/elasticbeanstalk/#etc#nginx#conf.d#00_elastic_beanstalk_proxy.conf" : 127 | mode: "000664" 128 | owner: root 129 | group: root 130 | encoding: plain 131 | content: | 132 | #WE COME FROM env.config! 133 | 134 | upstream nodejs { 135 | server 127.0.0.1:8081; 136 | keepalive 256; 137 | } 138 | #managed by phantom-manager, do not change upstream name!!! 139 | upstream phantomjs { 140 | server 127.0.0.1:8001 fail_timeout=0; 141 | server 127.0.0.1:8002 fail_timeout=0; 142 | } 143 | gzip on; 144 | #serve static content bypassing node.js 145 | location /posters { 146 | alias /var/app/current/public/posters; 147 | } 148 | } 149 | 150 | commands: 151 | 01_install_mosh: 152 | command: yum --enablerepo=epel install -y mosh 153 | ignoreErrors: true 154 | 02_patch_ebnode_py: 155 | command: "sed -e '/rebuild/ s/^#*/#/' -i /opt/elasticbeanstalk/containerfiles/ebnode.py" 156 | ignoreErrors: true 157 | 03_inject_z_sh: 158 | command: "grep 'z.sh' /root/.bashrc || echo '. ~/z.sh' >> /root/.bashrc;grep 'z.sh' /home/ec2-user/.bashrc || echo '. ~/z.sh' >> /home/ec2-user/.bashrc" 159 | ignoreErrors: true 160 | 05_inject_io_node_name: 161 | command: IO_LOG_NODE=`grep IO_LOG_NODE /etc/init/nodejs.conf | cut --delimiter='"' --fields=2` && sed -i.bak -e s/IO_LOG_NODE/$IO_LOG_NODE/ /root/.log.io/harvester.conf 162 | ignoreErrors: true 163 | 164 | container_commands: 165 | 01_replace_nginx_eb_conf: 166 | command: "mv -f '/opt/elasticbeanstalk/#etc#nginx#conf.d#00_elastic_beanstalk_proxy.conf' '/tmp/deployment/config/#etc#nginx#conf.d#00_elastic_beanstalk_proxy.conf'" 167 | ignoreErrors: false 168 | -------------------------------------------------------------------------------- /01app_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | . /opt/elasticbeanstalk/env.vars 3 | function error_exit 4 | { 5 | eventHelper.py --msg "$1" --severity ERROR 6 | exit $2 7 | } 8 | 9 | #redirect all output to cfn-init to capture it by log.io 10 | exec >>/var/log/cfn-init.log 2>&1 11 | 12 | echo "compiling underscore templates..." 13 | cd /tmp/deployment/application && cp -r views/underscore public/templates && jade public/templates/ && find public/templates/ -name "*jade" -delete >> /var/log/cfn-init.log 14 | 15 | if [ -f "/tmp/deployment/application/public/build.js" ]; then 16 | echo ">>>> Running r.js for single build.js......" 17 | OUT=$(cd /tmp/deployment/application/public && /usr/bin/r.js -o build.js >> /var/log/cfn-init.log && mv -v /tmp/deployment/application/public/dist /tmp/deployment/application/dist && rm -rf /tmp/deployment/application/public/ && mv -v /tmp/deployment/application/dist /tmp/deployment/application/public ) || error_exit "Failed to run r.js optimizer. $OUT" $? 18 | fi 19 | 20 | if ls /tmp/deployment/application/public/build_*js &> /dev/null; then 21 | echo ">>>> Running r.js for build_desktop and build_mobile......" 22 | OUT=$(cd /tmp/deployment/application/public && r.js -o build_desktop.js >> /var/log/cfn-init.log && r.js -o build_mobile.js >> /var/log/cfn-init.log && rm -rf /tmp/deployment/application/public/ && mv -v /tmp/deployment/application/dist /tmp/deployment/application/public ) || error_exit "Failed to run r.js optimizer. $OUT" $? 23 | echo $OUT 24 | 25 | echo "computing md5 hashes...." 26 | cd /tmp/deployment/application/public/ && find . -maxdepth 3 -type f -iname '*js' -o -iname '*css' -type f | xargs md5sum | awk '{system("echo "$1" > "$2".md5")}' 27 | echo "gzipping everything!" 28 | cd /tmp/deployment/application/public/ && find . -type f -iname "*css" -o -iname "*js" -type f | while read -r x;do gzip -9 -c "$x" > "$x.gz";done 29 | echo "touching all files to make them dated with the same time (as per nginx gzip_static recommendation)" 30 | cd /tmp/deployment/application/public/ && find . -type f -exec touch {} \; 31 | fi 32 | -------------------------------------------------------------------------------- /10stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | exec >>/var/log/cfn-init.log 2>&1 3 | 4 | function error_exit 5 | { 6 | eventHelper.py --msg "$1" --severity ERROR 7 | exit $2 8 | } 9 | 10 | echo "Testing nginx config.... " 11 | #test nginx config before proceeding with restart 12 | OUT=$(/usr/sbin/nginx -tc /etc/nginx/nginx.conf && /opt/elasticbeanstalk/containerfiles/ebnode.py --action stop-all 2>&1) || error_exit "Failed to stop service daemons, CHECK NGINX CONFIG $OUT" $? 13 | echo $OUT 14 | -------------------------------------------------------------------------------- /20clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | function error_exit 3 | { 4 | eventHelper.py --msg "$1" --severity ERROR 5 | exit $2 6 | } 7 | STAMP=`date +%m%d_%H%M` 8 | mkdir /var/app_$STAMP; 9 | mv /var/app/current /var/app_$STAMP/current; 10 | rm -rf /var/app/ 11 | -------------------------------------------------------------------------------- /40config_deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #============================================================================== 3 | # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the Amazon Software License (the "License"). You may not use 6 | # this file except in compliance with the License. A copy of the License is 7 | # located at 8 | # 9 | # http://aws.amazon.com/asl/ 10 | # 11 | # or in the "license" file accompanying this file. This file is distributed on 12 | # an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or 13 | # implied. See the License for the specific language governing permissions 14 | # and limitations under the License. 15 | #============================================================================== 16 | 17 | 18 | function error_exit 19 | { 20 | eventHelper.py --msg "$1" --severity ERROR 21 | exit $2 22 | } 23 | 24 | SOURCE=/tmp/deployment/config 25 | #remove default nginx configs so they dont overwrite our own!! 26 | rm -rfv $SOURCE/#etc#nginx* 27 | for i in $(ls $SOURCE); do 28 | FILE_NAME=$(echo $i | sed -e 's/#/\//g') 29 | /bin/cp "$SOURCE/$i" "$FILE_NAME" || error_exit "Failed to copy $FILE_NAME into place." 30 | done 31 | 32 | -------------------------------------------------------------------------------- /40install_node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #source env variables including node version 3 | . /opt/elasticbeanstalk/env.vars 4 | 5 | function error_exit 6 | { 7 | eventHelper.py --msg "$1" --severity ERROR 8 | exit $2 9 | } 10 | 11 | #redirect all output to cfn-init to capture it by log.io 12 | exec >>/var/log/cfn-init.log 2>&1 13 | #download and extract desired node.js version 14 | echo "checking node..." 15 | OUT=$( [ ! -d "/opt/elasticbeanstalk/node-install" ] && echo "trying to install node.js $NODE_VER" && mkdir /opt/elasticbeanstalk/node-install ; cd /opt/elasticbeanstalk/node-install/ && \ 16 | wget -nc http://nodejs.org/dist/v$NODE_VER/node-v$NODE_VER-linux-$ARCH.tar.gz && \ 17 | tar --skip-old-files -xzpf node-v$NODE_VER-linux-$ARCH.tar.gz) || error_exit "Failed to UPDATE node version. $OUT" $?. 18 | echo $OUT 19 | 20 | 21 | #UNCOMMENT to update npm, otherwise will be updated on instance init or rebuild 22 | #rm -f /opt/elasticbeanstalk/node-install/npm_updated 23 | 24 | #download & make install desired nginx version 25 | echo "checking nginx..." 26 | 27 | #remember to add desired modules to BOTH arch-dependent commands below: 28 | case $( arch ) in 29 | ( i686 ) OUT=$([ ! -d "/root/nginx-$NGINX_VER" ] && echo "trying to install nginx $NGINX_VER" &>> /var/log/cfn-init.log && \ 30 | cd /root/ && curl --retry 10 http://nginx.org/download/nginx-$NGINX_VER.tar.gz | tar zx && cd /root/nginx-$NGINX_VER && \ 31 | ./configure --prefix=/usr/share/nginx --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log \ 32 | --http-log-path=/var/log/nginx/access.log --http-client-body-temp-path=/var/lib/nginx/tmp/client_body --http-proxy-temp-path=/var/lib/nginx/tmp/proxy \ 33 | --http-fastcgi-temp-path=/var/lib/nginx/tmp/fastcgi --http-uwsgi-temp-path=/var/lib/nginx/tmp/uwsgi --http-scgi-temp-path=/var/lib/nginx/tmp/scgi \ 34 | --pid-path=/var/run/nginx.pid --lock-path=/var/lock/subsys/nginx --user=nginx --group=nginx --with-file-aio --with-ipv6 --with-http_ssl_module \ 35 | --with-http_spdy_module --with-http_realip_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_stub_status_module \ 36 | --with-pcre --with-debug --with-ld-opt=' -Wl,-E' \ 37 | --with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m32 -march=i686 -mtune=pentium4 -fasynchronous-unwind-tables' &>> /var/log/cfn-init.log \ 38 | && make &>> /var/log/cfn-init.log && make install &>> /var/log/cfn-init.log);; 39 | 40 | ( x86_64 ) OUT=$([ ! -d "/root/nginx-$NGINX_VER" ] && echo "trying to install nginx $NGINX_VER" && \ 41 | cd /root/ && curl --retry 10 http://nginx.org/download/nginx-$NGINX_VER.tar.gz | tar zx && cd /root/nginx-$NGINX_VER &>> /var/log/cfn-init.log && \ 42 | ./configure --prefix=/usr/share/nginx --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log \ 43 | --http-log-path=/var/log/nginx/access.log --http-client-body-temp-path=/var/lib/nginx/tmp/client_body --http-proxy-temp-path=/var/lib/nginx/tmp/proxy \ 44 | --http-fastcgi-temp-path=/var/lib/nginx/tmp/fastcgi --http-uwsgi-temp-path=/var/lib/nginx/tmp/uwsgi --http-scgi-temp-path=/var/lib/nginx/tmp/scgi \ 45 | --pid-path=/var/run/nginx.pid --lock-path=/var/lock/subsys/nginx --user=nginx --group=nginx --with-file-aio --with-ipv6 --with-http_ssl_module \ 46 | --with-http_spdy_module --with-http_realip_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_stub_status_module \ 47 | --with-pcre --with-debug \ 48 | --with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic' --with-ld-opt=' -Wl,-E' &>> /var/log/cfn-init.log \ 49 | && make &>> /var/log/cfn-init.log && make install &>> /var/log/cfn-init.log );; 50 | esac 51 | 52 | echo $OUT 53 | 54 | #make sure node binaries can be found globally 55 | if [ ! -L /usr/bin/node ]; then 56 | ln -s /opt/elasticbeanstalk/node-install/node-v$NODE_VER-linux-$ARCH/bin/node /usr/bin/node 57 | fi 58 | 59 | if [ ! -L /usr/bin/npm ]; then 60 | ln -s /opt/elasticbeanstalk/node-install/node-v$NODE_VER-linux-$ARCH/bin/npm /usr/bin/npm 61 | fi 62 | 63 | echo "checking npm..." 64 | if [ ! -f "/opt/elasticbeanstalk/node-install/npm_updated" ]; then 65 | cd /opt/elasticbeanstalk/node-install/node-v$NODE_VER-linux-$ARCH/bin/ && /opt/elasticbeanstalk/node-install/node-v$NODE_VER-linux-$ARCH/bin/npm update npm -g 66 | touch /opt/elasticbeanstalk/node-install/npm_updated 67 | echo "YAY! Updated global NPM version to `npm -v`" 68 | else 69 | echo "Skipping NPM -g version update. To update, please uncomment 40install_node.sh:12" 70 | fi 71 | 72 | -------------------------------------------------------------------------------- /46update_geolitecity.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | function error_exit 3 | { 4 | eventHelper.py --msg "$1" --severity ERROR 5 | exit $2 6 | } 7 | exec >>/var/log/cfn-init.log 2>&1 8 | [ ! -f "/var/GeoLiteCity.dat" ] && echo "GeoLiteCity.dat not found, fetching new one..." && cd /var/ && curl --retry 10 http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz | zcat > /var/GeoLiteCity.new && mv -vf /var/GeoLiteCity.new /var/GeoLiteCity.dat 9 | 10 | OUT=$(cp -vf /var/GeoLiteCity.dat /var/app/current/geolite_data/GeoLiteCity.dat) || error_exit "Failed to fetch/1update GeoLiteCity.dat $OUT" $? 11 | echo $OUT 12 | -------------------------------------------------------------------------------- /50npm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | . /opt/elasticbeanstalk/env.vars 3 | function error_exit 4 | { 5 | eventHelper.py --msg "$1" --severity ERROR 6 | exit $2 7 | } 8 | 9 | #redirect all output to cfn-init to capture it by log.io 10 | exec >>/var/log/cfn-init.log 2>&1 11 | echo "------------------------------ — Logger hiccup NOW! — ---------------------------------------" 12 | /sbin/stop io-server 13 | /sbin/stop io-harvester 14 | 15 | #avoid long NPM fetch hangups 16 | npm config set fetch-retry-maxtimeout 15000 17 | #if log.io is not installed, install it and forever.js 18 | # do not install forever, as we moved services to /etc/init to decrease RAM footprint 19 | # type -P forever && echo "... found, skipping install" || npm install -g --production forever --user 'root' 20 | type -P log.io-server && echo "... found, skipping install" || npm install -g --production log.io --user 'root' 21 | 22 | if [ -f "/etc/init/nodejs.conf" ]; then 23 | IO_LOG_NODE=`grep IO_LOG_NODE /etc/init/nodejs.conf | cut --delimiter='"' --fields=2` && sed -i.bak -e s/IO_LOG_NODE/$IO_LOG_NODE/ /root/.log.io/harvester.conf 24 | fi 25 | 26 | if [[ ! `pgrep -f log.io-server` ]]; then 27 | /sbin/start io-server 28 | sleep 2 29 | /sbin/start io-harvester 30 | echo "done" 31 | fi 32 | 33 | #install other global stuff 34 | type -P phantomjs && echo "... found, skipping install" || { 35 | npm install -g --production phantomjs@">=1.9.6 <2.0.0" --user 'root' 36 | #npm install -g --production casperjs --user 'root' 37 | } 38 | type -P r.js && echo "... found, skipping install" || npm install -g --production requirejs@">=2.1.11 <3.0.0" --user 'root' 39 | type -P jade && echo "... found, skipping install" || npm install -g --production jade@">=1.3.1 <2.0.0" --user 'root' 40 | 41 | #install not-installed yet app node_modules 42 | if [ ! -d "/var/node_modules" ]; then 43 | mkdir /var/node_modules ; 44 | fi 45 | if [ -d /tmp/deployment/application ]; then 46 | ln -s /var/node_modules /tmp/deployment/application/ 47 | fi 48 | 49 | echo "------------------------------ — Installing/updating NPM modules, it might take a while, go take a leak or have a healthy snack... — -----------------------------------" 50 | OUT=$([ -d "/tmp/deployment/application" ] && cd /tmp/deployment/application && /opt/elasticbeanstalk/node-install/node-v$NODE_VER-linux-$ARCH/bin/npm install --production) || error_exit "Failed to run npm install. $OUT" $? 51 | echo $OUT 52 | 53 | chmod -R o+r /var/node_modules 54 | -------------------------------------------------------------------------------- /55phantomjs_restart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #if instance is not suited for phantom, dont run it 3 | exec >>/var/log/cfn-init.log 2>&1 4 | 5 | if [ -f "/etc/init/nodejs.conf" ]; then 6 | IO_LOG_NODE=`grep IO_LOG_NODE /etc/init/nodejs.conf | cut --delimiter='"' --fields=2` 7 | fi 8 | 9 | if [[ $IO_LOG_NODE != *phanto* ]] ; then echo "Not a PHANTOMJS intsance, doing nothing, bye." && exit 0 ; fi ; 10 | function error_exit 11 | { 12 | eventHelper.py --msg "$1" --severity ERROR 13 | exit $2 14 | } 15 | OUT=$(/var/app/current/phantomjs/phmon_restart.sh) || error_exit "Failed to restart PhantomJS. $OUT" $? 16 | echo $OUT 17 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 kopurando 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | better-faster-elastic-beanstalk 2 | =============================== 3 | 4 | # 2017 UPDATE 5 | This project has not been updated for 3 years now (as we switched to docker), but as the problem still exists (AWS, wtf?), please refer to one of recent forks, or this project: 6 | https://github.com/mixmaxhq/eb-fix-npm/ 7 | (not a fork, but rather inspired by BFEB) 8 | 9 | Peace. 10 | ------ 11 | 12 | # Intro 13 | 14 | This collection of hooks for AWS Elastic Beanstalk node.js containers that was designed to dramatically reduce deployment times, avoid unnecessary rebuilding of node_modules, install any version of node.js, install global NPM modules and more. 15 | 16 | This project demonstrates several tweaks to the standard node.js Elastic Beanstalk configuration to significantly reduce deployment times by avoiding unnecessary rebuilds of node_modules. 17 | 18 | ## Main features 19 | 20 | 1. Install ANY desired node.js version as per your env.config (including 21 | the most recent ones that are not yet supported by AWS EB) — control your node.js version to avoid surprises with sudden AWS updates or compatibility issues. 22 | 2. Avoid rebuilding existing node modules, including in-app 23 | node_modules dir — dramatically speed up deployment. 24 | 3. Install node.js globally (and any desired module as well), if your use-case requires so. 25 | 26 | Please see the original discussion on StackOverflow to grasp the concept: 27 | http://stackoverflow.com/questions/21200251/avoid-rebuilding-node-modules-in-elastic-beanstalk 28 | 29 | __NOTE__: this respo is way ahead of the StackOverflow thread, and heavily customised for my own use-case. It still works as a charm though and saves a lot of time on each deployment. Feel free to look around, be inspired and find solutions for your problems in my code, but it is essential to fork this repo and adapt it for your own needs. 30 | 31 | ## Usage 32 | 33 | Put `.ebextensions/example_env.config` into your project `.ebextensions` dir and read through the file to get a grasp of its mechanics. 34 | example_env.config from this repo does following: 35 | - installs bunch of packages (such as ImageMagick, git, tmux, mosh etc.); 36 | - fills option settings with default values (I strongly recommend NOT to put any passwords here, even in a private repo, and instead set them via instances profiles jsons, see docs here http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/create_deploy_nodejs.container.html); 37 | - fetches all *.sh hooks from this _public_ repo. You can (and should) replace them with your own hooks from your private repos of course; 38 | - sets up simple PhantomJS proxy via nginx config modification; 39 | - injects z.sh (for admin comfort); 40 | - sets up logging io.server... 41 | 42 | ... and much much more. We used these hooks in our production and it worked bulletproof (unless github went down :). Chances are, you won't need 90% of functionality we developed, so please refer to the original SO thread for more ideas and less sophisticated config for generic use-cases: 43 | 44 | http://stackoverflow.com/questions/21200251/avoid-rebuilding-node-modules-in-elastic-beanstalk 45 | 46 | ## Principle of action 47 | 48 | We use `.ebextensions/example_env.config` to replace default AWS deploy & config hooks with customized ones (all *.sh files, see detais below). Also, in a default EB container setup some env variables are missing (`$HOME` for example) and `node-gyp` sometimes fails during rebuild because of that (took me 2 hours of googling and reinstalling libxmljs to resolve it). 49 | 50 | Below are the files to be included along with your build. You can inject them via env.config as inline code or via `source: URL` (as in `example_env.config` from this repo) 51 | 52 | **`env.vars`**: desired node version & arch are included here and in `env.config, see below; 53 | 54 | **`40install_node.sh`**: fetch and ungzip desired node.js version, make global symlinks, update global npm version, 55 | 56 | **`50npm.sh`**: creates /var/node_modules, symlinks it to app dir and runs npm install. You can install any module globally from here, they will land in /root/.npm. 57 | 58 | *you can add more *.sh hooks for your needs*: see `55phantomjs_restart.sh` and `46update_geolitecity.sh` for examples. 59 | 60 | **`env.config`**: note node version here too, and to be safe, put desired node version in env config in your AWS console as well. I'm not 100% certain which of these settings will take precedence. 61 | 62 | There you have it: on t1.micro instance deployment now takes 20-30 secs instead of 10-15 minutes! If you deploy 10 times a day, this tweak will save you 3 (three) weeks in a year. 63 | Hope it helps and special thanks to AWS EB staff for my lost weekend :) 64 | 65 | 66 | 67 | ##todo 68 | - remove (rename) and rebuild node_modules in case NodeJS or npm version gets changed 69 | -------------------------------------------------------------------------------- /cron-node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #get env vars from EB app container and inject them here. Run as cron-node.sh cron-app.js 3 | eval "$(grep -E '^env [A-Za-z0-9_-]+="[^"]+"$' /etc/init/nodejs.conf | sed 's/env /export /g')" 4 | export CRON_NODE_COMMAND="node $1" 5 | cd /var/app/current 6 | exec su -s /bin/sh -c 'PATH=$PATH:$NODE_HOME/bin $CRON_NODE_COMMAND 2>&1' nodejs >> /var/log/cron 7 | -------------------------------------------------------------------------------- /env.vars: -------------------------------------------------------------------------------- 1 | export HOME=/root 2 | export NPM_CONFIG_LOGLEVEL=warn 3 | export NPM_CONFIG_COLOR=false 4 | export NPM_CONFIG_PRODUCTION=true 5 | #note node vesion in .elasticbeanstalk/*.jsons 6 | export NODE_VER=0.12.0 7 | case $( arch ) in 8 | ( i686 ) export ARCH=x86;; 9 | ( x86_64 ) export ARCH=x64;; 10 | esac 11 | export NGINX_VER=1.5.12 12 | export PATH="$PATH:/opt/elasticbeanstalk/node-install/node-v$NODE_VER-linux-$ARCH/bin/:/root/.npm:/opt/elasticbeanstalk/lib/ruby/bin/" 13 | -------------------------------------------------------------------------------- /harvester-phantomjs.conf: -------------------------------------------------------------------------------- 1 | exports.config = { 2 | nodeName: "IO_LOG_NODE", 3 | logStreams: { 4 | deploy: [ 5 | "/var/log/cfn-init.log", 6 | "/var/log/eb-commandprocessor.log" 7 | ], 8 | systemCron: [ 9 | "/var/log/cron", 10 | "/var/log/secure" 11 | ], 12 | PhantomJS:[ 13 | "/var/log/phantomjs/phantomjs.log" 14 | ], 15 | PhantomMonitor:[ 16 | "/var/log/phantomjs/phantom_monitor.log" 17 | ], 18 | nginx:[ 19 | "/var/log/nginx/access_phantomjs.log", 20 | "/var/log/nginx/access.log", 21 | "/var/log/nginx/error.log" 22 | ]}, 23 | server: { 24 | host: '0.0.0.0', 25 | port: 28777 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /harvester.conf: -------------------------------------------------------------------------------- 1 | exports.config = { 2 | nodeName: "IO_LOG_NODE", 3 | logStreams: { 4 | deploy: [ 5 | "/var/log/cfn-init.log", 6 | "/var/log/eb-commandprocessor.log" 7 | ], 8 | systemCron: [ 9 | "/var/log/cron", 10 | "/var/log/secure" 11 | ], 12 | nodejs:[ 13 | "/var/log/nodejs/nodejs.log" 14 | ], 15 | nginx:[ 16 | "/var/log/nginx/access.log", 17 | "/var/log/nginx/error.log" 18 | ]}, 19 | server: { 20 | host: '0.0.0.0', 21 | port: 28777 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /harvester_front.conf: -------------------------------------------------------------------------------- 1 | exports.config = { 2 | nodeName: "IO_LOG_NODE", 3 | logStreams: { 4 | deploy: [ 5 | "/var/log/cfn-init.log", 6 | "/var/log/eb-tools.log" 7 | ], 8 | systemCron: [ 9 | "/var/log/cron", 10 | "/var/log/secure" 11 | ], 12 | PhantomJS:[ 13 | "/var/log/phantomjs/phantomjs.log" 14 | ], 15 | PhantomMonitor:[ 16 | "/var/log/phantomjs/phantom_monitor.log" 17 | ], 18 | nodejs:[ 19 | "/var/log/nodejs/nodejs.log" 20 | ], 21 | nginxDirect:[ 22 | "/var/log/nginx/access_.log", 23 | "/var/log/nginx/error.log" 24 | ], 25 | nginxNode:[ 26 | "/var/log/nginx/access_nodejs.log", 27 | ], 28 | nginxMobile:[ 29 | "/var/log/nginx/access_mobile.log", 30 | ], 31 | nginxPhantom:[ 32 | "/var/log/nginx/access_phantomjs.log", 33 | ], 34 | nginxLocalPhantom:[ 35 | "/var/log/nginx/access_localphantomjs.log", 36 | ]}, 37 | server: { 38 | host: '0.0.0.0', 39 | port: 28777 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /io-harvester.conf: -------------------------------------------------------------------------------- 1 | # log.io harvester 2 | # 3 | 4 | description "log.io harvester" 5 | stop on runlevel [!2345] 6 | 7 | respawn 8 | respawn limit 30 60 9 | 10 | exec su -s /bin/sh -c '/usr/bin/log.io-harvester 2>&1' root > /dev/null 11 | -------------------------------------------------------------------------------- /io-server.conf: -------------------------------------------------------------------------------- 1 | # log.io server 2 | # 3 | 4 | description "log.io server" 5 | stop on runlevel [!2345] 6 | 7 | respawn 8 | respawn limit 30 60 9 | 10 | exec su -s /bin/sh -c '/usr/bin/log.io-server 2>&1' root > /dev/null 11 | -------------------------------------------------------------------------------- /log_server.conf: -------------------------------------------------------------------------------- 1 | exports.config = { 2 | host: '127.0.0.1', 3 | port: 28777 4 | } 5 | -------------------------------------------------------------------------------- /logrotate.conf.elasticbeanstalk: -------------------------------------------------------------------------------- 1 | /var/log/nodejs/nodejs.log { 2 | daily 3 | size 10M 4 | missingok 5 | notifempty 6 | rotate 30 7 | sharedscripts 8 | compress 9 | copytruncate 10 | dateext 11 | } 12 | /var/log/phantomjs/*log { 13 | daily 14 | size 10M 15 | missingok 16 | notifempty 17 | rotate 30 18 | sharedscripts 19 | compress 20 | copytruncate 21 | dateext 22 | } 23 | -------------------------------------------------------------------------------- /web_server.conf: -------------------------------------------------------------------------------- 1 | exports.config = { 2 | host: '0.0.0.0', 3 | port: 28778, 4 | // Enable HTTP Basic Authentication 5 | auth: { 6 | user: "adminka", 7 | pass: "balerinka" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /z.sh: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2009 rupa deadwyler under the WTFPL license 2 | 3 | # maintains a jump-list of the directories you actually use 4 | # 5 | # INSTALL: 6 | # * put something like this in your .bashrc/.zshrc: 7 | # . /path/to/z.sh 8 | # * cd around for a while to build up the db 9 | # * PROFIT!! 10 | # * optionally: 11 | # set $_Z_CMD in .bashrc/.zshrc to change the command (default z). 12 | # set $_Z_DATA in .bashrc/.zshrc to change the datafile (default ~/.z). 13 | # set $_Z_NO_RESOLVE_SYMLINKS to prevent symlink resolution. 14 | # set $_Z_NO_PROMPT_COMMAND if you're handling PROMPT_COMMAND yourself. 15 | # set $_Z_EXCLUDE_DIRS to an array of directories to exclude. 16 | # 17 | # USE: 18 | # * z foo # cd to most frecent dir matching foo 19 | # * z foo bar # cd to most frecent dir matching foo and bar 20 | # * z -r foo # cd to highest ranked dir matching foo 21 | # * z -t foo # cd to most recently accessed dir matching foo 22 | # * z -l foo # list matches instead of cd 23 | # * z -c foo # restrict matches to subdirs of $PWD 24 | 25 | [ -d "${_Z_DATA:-$HOME/.z}" ] && { 26 | echo "ERROR: z.sh's datafile (${_Z_DATA:-$HOME/.z}) is a directory." 27 | } 28 | 29 | _z() { 30 | 31 | local datafile="${_Z_DATA:-$HOME/.z}" 32 | 33 | # bail if we don't own ~/.z (we're another user but our ENV is still set) 34 | [ -f "$datafile" -a ! -O "$datafile" ] && return 35 | 36 | # add entries 37 | if [ "$1" = "--add" ]; then 38 | shift 39 | 40 | # $HOME isn't worth matching 41 | [ "$*" = "$HOME" ] && return 42 | 43 | # don't track excluded dirs 44 | local exclude 45 | for exclude in "${_Z_EXCLUDE_DIRS[@]}"; do 46 | [ "$*" = "$exclude" ] && return 47 | done 48 | 49 | # maintain the data file 50 | local tempfile="$datafile.$RANDOM" 51 | while read line; do 52 | # only count directories 53 | [ -d "${line%%\|*}" ] && echo $line 54 | done < "$datafile" | awk -v path="$*" -v now="$(date +%s)" -F"|" ' 55 | BEGIN { 56 | rank[path] = 1 57 | time[path] = now 58 | } 59 | $2 >= 1 { 60 | # drop ranks below 1 61 | if( $1 == path ) { 62 | rank[$1] = $2 + 1 63 | time[$1] = now 64 | } else { 65 | rank[$1] = $2 66 | time[$1] = $3 67 | } 68 | count += $2 69 | } 70 | END { 71 | if( count > 6000 ) { 72 | # aging 73 | for( x in rank ) print x "|" 0.99*rank[x] "|" time[x] 74 | } else for( x in rank ) print x "|" rank[x] "|" time[x] 75 | } 76 | ' 2>/dev/null >| "$tempfile" 77 | # do our best to avoid clobbering the datafile in a race condition 78 | if [ $? -ne 0 -a -f "$datafile" ]; then 79 | env rm -f "$tempfile" 80 | else 81 | env mv -f "$tempfile" "$datafile" || env rm -f "$tmpfile" 82 | fi 83 | 84 | # tab completion 85 | elif [ "$1" = "--complete" ]; then 86 | while read line; do 87 | [ -d "${line%%\|*}" ] && echo $line 88 | done < "$datafile" | awk -v q="$2" -F"|" ' 89 | BEGIN { 90 | if( q == tolower(q) ) imatch = 1 91 | split(substr(q, 3), fnd, " ") 92 | } 93 | { 94 | if( imatch ) { 95 | for( x in fnd ) tolower($1) !~ tolower(fnd[x]) && $1 = "" 96 | } else { 97 | for( x in fnd ) $1 !~ fnd[x] && $1 = "" 98 | } 99 | if( $1 ) print $1 100 | } 101 | ' 2>/dev/null 102 | 103 | else 104 | # list/go 105 | while [ "$1" ]; do case "$1" in 106 | --) while [ "$1" ]; do shift; local fnd="$fnd $1";done;; 107 | -*) local opt=${1:1}; while [ "$opt" ]; do case ${opt:0:1} in 108 | c) local fnd="^$PWD $fnd";; 109 | h) echo "${_Z_CMD:-z} [-chlrtx] args" >&2; return;; 110 | x) sed -i "\:^${PWD}|.*:d" "$datafile";; 111 | l) local list=1;; 112 | r) local typ="rank";; 113 | t) local typ="recent";; 114 | esac; opt=${opt:1}; done;; 115 | *) local fnd="$fnd $1";; 116 | esac; local last=$1; shift; done 117 | [ "$fnd" -a "$fnd" != "^$PWD " ] || local list=1 118 | 119 | # if we hit enter on a completion just go there 120 | case "$last" in 121 | # completions will always start with / 122 | /*) [ -z "$list" -a -d "$last" ] && cd "$last" && return;; 123 | esac 124 | 125 | # no file yet 126 | [ -f "$datafile" ] || return 127 | 128 | local cd 129 | cd="$(while read line; do 130 | [ -d "${line%%\|*}" ] && echo $line 131 | done < "$datafile" | awk -v t="$(date +%s)" -v list="$list" -v typ="$typ" -v q="$fnd" -F"|" ' 132 | function frecent(rank, time) { 133 | # relate frequency and time 134 | dx = t - time 135 | if( dx < 3600 ) return rank * 4 136 | if( dx < 86400 ) return rank * 2 137 | if( dx < 604800 ) return rank / 2 138 | return rank / 4 139 | } 140 | function output(files, out, common) { 141 | # list or return the desired directory 142 | if( list ) { 143 | cmd = "sort -n >&2" 144 | for( x in files ) { 145 | if( files[x] ) printf "%-10s %s\n", files[x], x | cmd 146 | } 147 | if( common ) { 148 | printf "%-10s %s\n", "common:", common > "/dev/stderr" 149 | } 150 | } else { 151 | if( common ) out = common 152 | print out 153 | } 154 | } 155 | function common(matches) { 156 | # find the common root of a list of matches, if it exists 157 | for( x in matches ) { 158 | if( matches[x] && (!short || length(x) < length(short)) ) { 159 | short = x 160 | } 161 | } 162 | if( short == "/" ) return 163 | # use a copy to escape special characters, as we want to return 164 | # the original. yeah, this escaping is awful. 165 | clean_short = short 166 | gsub(/[\(\)\[\]\|]/, "\\\\&", clean_short) 167 | for( x in matches ) if( matches[x] && x !~ clean_short ) return 168 | return short 169 | } 170 | BEGIN { split(q, words, " "); hi_rank = ihi_rank = -9999999999 } 171 | { 172 | if( typ == "rank" ) { 173 | rank = $2 174 | } else if( typ == "recent" ) { 175 | rank = $3 - t 176 | } else rank = frecent($2, $3) 177 | matches[$1] = imatches[$1] = rank 178 | for( x in words ) { 179 | if( $1 !~ words[x] ) delete matches[$1] 180 | if( tolower($1) !~ tolower(words[x]) ) delete imatches[$1] 181 | } 182 | if( matches[$1] && matches[$1] > hi_rank ) { 183 | best_match = $1 184 | hi_rank = matches[$1] 185 | } else if( imatches[$1] && imatches[$1] > ihi_rank ) { 186 | ibest_match = $1 187 | ihi_rank = imatches[$1] 188 | } 189 | } 190 | END { 191 | # prefer case sensitive 192 | if( best_match ) { 193 | output(matches, best_match, common(matches)) 194 | } else if( ibest_match ) { 195 | output(imatches, ibest_match, common(imatches)) 196 | } 197 | } 198 | ')" 199 | [ $? -gt 0 ] && return 200 | [ "$cd" ] && cd "$cd" 201 | fi 202 | } 203 | 204 | alias ${_Z_CMD:-z}='_z 2>&1' 205 | 206 | [ "$_Z_NO_RESOLVE_SYMLINKS" ] || _Z_RESOLVE_SYMLINKS="-P" 207 | 208 | if compctl >/dev/null 2>&1; then 209 | # zsh 210 | [ "$_Z_NO_PROMPT_COMMAND" ] || { 211 | # populate directory list, avoid clobbering any other precmds. 212 | if [ "$_Z_NO_RESOLVE_SYMLINKS" ]; then 213 | _z_precmd() { 214 | _z --add "${PWD:a}" 215 | } 216 | else 217 | _z_precmd() { 218 | _z --add "${PWD:A}" 219 | } 220 | fi 221 | [[ -n "${precmd_functions[(r)_z_precmd]}" ]] || { 222 | precmd_functions+=(_z_precmd) 223 | } 224 | } 225 | _z_zsh_tab_completion() { 226 | # tab completion 227 | local compl 228 | read -l compl 229 | reply=(${(f)"$(_z --complete "$compl")"}) 230 | } 231 | compctl -U -K _z_zsh_tab_completion _z 232 | elif complete >/dev/null 2>&1; then 233 | # bash 234 | # tab completion 235 | complete -o filenames -C '_z --complete "$COMP_LINE"' ${_Z_CMD:-z} 236 | [ "$_Z_NO_PROMPT_COMMAND" ] || { 237 | # populate directory list. avoid clobbering other PROMPT_COMMANDs. 238 | grep "_z --add" <<< "$PROMPT_COMMAND" >/dev/null || { 239 | PROMPT_COMMAND="$PROMPT_COMMAND"$'\n''_z --add "$(pwd '$_Z_RESOLVE_SYMLINKS' 2>/dev/null)" 2>/dev/null;' 240 | } 241 | } 242 | fi 243 | --------------------------------------------------------------------------------