├── requirements.txt ├── support └── build │ ├── _docker │ ├── env.default │ ├── cedar-14.Dockerfile │ ├── heroku-16.Dockerfile │ └── README.md │ ├── extensions │ ├── no-debug-non-zts-20151012 │ │ ├── v8js-bare-2.0.0 │ │ ├── v8js-2.0.0 │ │ ├── v8js │ │ └── v8js-bare │ ├── no-debug-non-zts-20160303 │ │ ├── v8js-bare-2.0.0 │ │ └── v8js-2.0.0 │ └── no-debug-non-zts-20170718 │ │ ├── v8js-bare-2.0.0 │ │ └── v8js-2.0.0 │ ├── _util │ ├── include │ │ ├── manifest.sh │ │ └── manifest.py │ ├── deploy.sh │ ├── mkrepo.sh │ ├── remove.sh │ └── sync.sh │ ├── utils │ └── chrpath │ └── libraries │ └── v8 ├── auto-updater ├── 00-clean-env ├── 10-v8-version ├── 99-ship-it ├── 20-php-versions └── 50-update-pecl-php7 └── README.md /requirements.txt: -------------------------------------------------------------------------------- 1 | bob-builder>=0.0.11 2 | s3cmd>=1.6.0 3 | -------------------------------------------------------------------------------- /support/build/_docker/env.default: -------------------------------------------------------------------------------- 1 | AWS_ACCESS_KEY_ID 2 | AWS_SECRET_ACCESS_KEY 3 | 4 | S3_BUCKET=lang-php 5 | S3_PREFIX=dist-cedar-14-develop/ 6 | S3_REGION=s3 7 | 8 | STACK=cedar-14 9 | -------------------------------------------------------------------------------- /support/build/extensions/no-debug-non-zts-20151012/v8js-bare-2.0.0: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Build Path: /app/.heroku/php/ 3 | # Build Deps: php-7.0.27, libraries/v8 4 | 5 | source $(dirname $0)/v8js-bare 6 | -------------------------------------------------------------------------------- /auto-updater/00-clean-env: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | git update-index -q --ignore-submodules --refresh 4 | git diff-index --quiet HEAD -- || { 5 | echo >&2 "Uncommitted changes, please commit or stash them" 6 | exit 1 7 | } 8 | -------------------------------------------------------------------------------- /support/build/extensions/no-debug-non-zts-20160303/v8js-bare-2.0.0: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Build Path: /app/.heroku/php/ 3 | # Build Deps: php-7.1.13, libraries/v8 4 | 5 | source $(dirname $0)/../no-debug-non-zts-20151012/v8js-bare 6 | -------------------------------------------------------------------------------- /support/build/extensions/no-debug-non-zts-20170718/v8js-bare-2.0.0: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Build Path: /app/.heroku/php/ 3 | # Build Deps: php-7.2.1, libraries/v8 4 | 5 | source $(dirname $0)/../no-debug-non-zts-20151012/v8js-bare 6 | -------------------------------------------------------------------------------- /support/build/extensions/no-debug-non-zts-20151012/v8js-2.0.0: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Build Path: /app/.heroku/php/ 3 | # Build Deps: extensions/no-debug-non-zts-20151012/v8js-bare-2.0.0, libraries/v8 4 | 5 | # That's it - we're combining the dep build and the extension build here 6 | 7 | source $(dirname $0)/v8js 8 | -------------------------------------------------------------------------------- /support/build/extensions/no-debug-non-zts-20160303/v8js-2.0.0: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Build Path: /app/.heroku/php/ 3 | # Build Deps: extensions/no-debug-non-zts-20160303/v8js-bare-2.0.0, libraries/v8 4 | 5 | # That's it - we're combining the dep build and the extension build here 6 | 7 | source $(dirname $0)/../no-debug-non-zts-20151012/v8js 8 | -------------------------------------------------------------------------------- /support/build/extensions/no-debug-non-zts-20170718/v8js-2.0.0: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Build Path: /app/.heroku/php/ 3 | # Build Deps: extensions/no-debug-non-zts-20170718/v8js-bare-2.0.0, libraries/v8 4 | 5 | # That's it - we're combining the dep build and the extension build here 6 | 7 | source $(dirname $0)/../no-debug-non-zts-20151012/v8js 8 | -------------------------------------------------------------------------------- /support/build/_docker/cedar-14.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM heroku/cedar:14 2 | 3 | WORKDIR /app 4 | ENV WORKSPACE_DIR=/app/support/build 5 | ENV PATH=/app/support/build/_util:$PATH 6 | 7 | RUN apt-get update && apt-get install -y python-pip 8 | 9 | COPY requirements.txt /app/requirements.txt 10 | 11 | RUN pip install -r /app/requirements.txt 12 | 13 | COPY . /app 14 | -------------------------------------------------------------------------------- /support/build/_docker/heroku-16.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM heroku/heroku:16-build 2 | 3 | WORKDIR /app 4 | ENV WORKSPACE_DIR=/app/support/build 5 | ENV PATH=/app/support/build/_util:$PATH 6 | 7 | RUN apt-get update && apt-get install -y python-pip 8 | 9 | COPY requirements.txt /app/requirements.txt 10 | 11 | RUN pip install -r /app/requirements.txt 12 | 13 | COPY . /app 14 | -------------------------------------------------------------------------------- /support/build/_util/include/manifest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | print_or_export_manifest_cmd() { 4 | if [[ "${MANIFEST_CMD:-}" ]]; then 5 | echo "$1" > $MANIFEST_CMD 6 | else 7 | echo "-----> Done. Run '$1' to upload manifest." 8 | fi 9 | } 10 | 11 | generate_manifest_cmd() { 12 | echo "s3cmd --ssl${AWS_ACCESS_KEY_ID+" --access_key=\$AWS_ACCESS_KEY_ID"}${AWS_SECRET_ACCESS_KEY+" --secret_key=\$AWS_SECRET_ACCESS_KEY"} --acl-public -m application/json put $(pwd)/${1} s3://${S3_BUCKET}/${S3_PREFIX}${1}" 13 | } 14 | -------------------------------------------------------------------------------- /auto-updater/10-v8-version: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | tmp=`mktemp` 4 | wget -O$tmp 'https://omahaproxy.appspot.com/all?os=linux&channel=beta' 5 | 6 | latest_version=$(awk -F, '{ if ( $11 != "v8_version" ) { print $11 } }' < $tmp) 7 | 8 | rm -f $tmp 9 | echo "latest version: $latest_version" 10 | 11 | sed -e "s/^\(DEFAULT_VERSION=\).*/\1\"$latest_version\"/" -i support/build/libraries/v8 12 | 13 | git update-index -q --ignore-submodules --refresh 14 | git diff-index --quiet HEAD -- && { 15 | echo "" 16 | echo "Nothing changed." 17 | exit 0 18 | } 19 | 20 | git commit -a -m "Update latest PHP versions" 21 | -------------------------------------------------------------------------------- /support/build/utils/chrpath: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Build Path: /app/.heroku/php/ 3 | 4 | OUT_PREFIX=$1 5 | 6 | # fail hard 7 | set -o pipefail 8 | # fail harder 9 | set -eux 10 | 11 | DEFAULT_VERSION="0.13" 12 | dep_version=${VERSION:-$DEFAULT_VERSION} 13 | dep_dirname=chrpath-${dep_version} 14 | dep_archive_name=${dep_dirname}.tar.gz 15 | dep_url=http://ftp.tux.org/pub/X-Windows/ftp.hungry.com/chrpath/chrpath-0.13.tar.gz 16 | 17 | echo "-----> Building chrpath ${dep_version}..." 18 | 19 | curl -L ${dep_url} | tar xz 20 | 21 | pushd ${dep_dirname} 22 | ./configure --prefix=${OUT_PREFIX} 23 | make -s -j 9 24 | make install -s 25 | popd 26 | 27 | echo "-----> Done." 28 | -------------------------------------------------------------------------------- /auto-updater/99-ship-it: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -xeo pipefail 3 | 4 | for stack in heroku-16 cedar-14; do 5 | docker build --tag heroku-php-build-$stack --file $(pwd)/support/build/_docker/$stack.Dockerfile . 6 | 7 | dokku_run() { 8 | docker run --env-file=env-file -e STACK=$stack -e S3_PREFIX=dist-$stack-develop/ -e UPSTREAM_S3_PREFIX=dist-$stack-stable/ -it heroku-php-build-$stack $* 9 | } 10 | 11 | shipit() { 12 | dokku_run support/build/_util/deploy.sh --overwrite $* 13 | } 14 | 15 | shipit libraries/v8 16 | 17 | for dir in no-debug-non-zts-20151012 no-debug-non-zts-20160303 no-debug-non-zts-20170718; do 18 | shipit `cd support/build/; ls -1 extensions/$dir/v8js-bare-2.*` 19 | shipit `cd support/build/; ls -1 extensions/$dir/v8js-2.*` 20 | done 21 | 22 | dokku_run support/build/_util/mkrepo.sh --upload 23 | dokku_run support/build/_util/sync.sh heroku-v8js dist-$stack-stable/ 24 | done 25 | -------------------------------------------------------------------------------- /support/build/libraries/v8: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Build Path: /app/.heroku/php/ 3 | 4 | OUT_PREFIX=$1 5 | 6 | # fail hard 7 | set -o pipefail 8 | # fail harder 9 | set -eux 10 | 11 | DEFAULT_VERSION="6.4.388.19" 12 | dep_version=${VERSION:-$DEFAULT_VERSION} 13 | dep_dirname=v8 14 | 15 | echo "-----> Building v8 ${dep_version}..." 16 | 17 | git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git 18 | export PATH=`pwd`/depot_tools:"$PATH" 19 | fetch v8 20 | 21 | pushd ${dep_dirname} 22 | git checkout ${dep_version} 23 | gclient sync 24 | 25 | tools/dev/v8gen.py -vv x64.release -- is_component_build=true 26 | 27 | ninja -C out.gn/x64.release/ 28 | 29 | 30 | mkdir -p ${OUT_PREFIX}/lib ${OUT_PREFIX}/include 31 | cp out.gn/x64.release/lib*.so ${OUT_PREFIX}/lib/ 32 | cp out.gn/x64.release/*_blob.bin ${OUT_PREFIX}/lib/ 33 | cp out.gn/x64.release/icudtl.dat ${OUT_PREFIX}/lib/ 34 | cp -R include/* ${OUT_PREFIX}/include/ 35 | popd 36 | 37 | echo "-----> Done." 38 | -------------------------------------------------------------------------------- /auto-updater/20-php-versions: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | tmp=`mktemp` 4 | wget -O$tmp 'https://s3.amazonaws.com/lang-php/?prefix=dist-cedar-14-stable/php-' 5 | 6 | for php_dist in 7.0 7.1 7.2; do 7 | echo "Processing PHP version ----> $php_dist" 8 | latest_version=$( 9 | ( 10 | xmllint --shell $tmp < 5 else {} 4 | stack=re.match("^([^-]+)(?:-([0-9]+))?$", os.getenv("STACK", "cedar-14")) 5 | require["heroku-sys/"+stack.group(1)] = "^{}.0.0".format(stack.group(2) or "1") 6 | require["heroku/installer-plugin"] = "^1.2.0" 7 | 8 | manifest = { 9 | "type": sys.argv[1], 10 | "name": sys.argv[2], 11 | "version": sys.argv[3], 12 | "dist": { 13 | "type": "heroku-sys-tar", 14 | "url": "https://"+os.getenv("S3_BUCKET")+"."+os.getenv("S3_REGION", "s3")+".amazonaws.com/"+os.getenv("S3_PREFIX")+sys.argv[4] 15 | }, 16 | "require": require, 17 | "conflict": json.loads(sys.argv[6]) if len(sys.argv) > 6 else {}, 18 | "replace": json.loads(sys.argv[7]) if len(sys.argv) > 7 else {}, 19 | "extra": json.loads(sys.argv[8]) if len(sys.argv) > 8 else {}, 20 | "time": datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") 21 | } 22 | 23 | json.dump(manifest, sys.stdout, sort_keys=True) 24 | -------------------------------------------------------------------------------- /auto-updater/50-update-pecl-php7: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -xeo pipefail 3 | 4 | tmp=`mktemp` 5 | wget -O$tmp https://pecl.php.net/package/v8js 6 | 7 | latest_version=$( 8 | perl -ne 'if(m/v8js-(2\.\d+\.\d+)\.tgz/) { print "$1\n"; }' < $tmp \ 9 | | sort --version-sort \ 10 | | tail -n 1) 11 | echo "Latest PHP V8Js version ----> $latest_version" 12 | 13 | for dir in support/build/extensions/no-debug-non-zts-20151012/ support/build/extensions/no-debug-non-zts-20160303/; do 14 | pushd $dir 15 | old_name="$(ls -1 v8js-bare-*)" 16 | old_version="${old_name#v8js-bare-}" 17 | 18 | if [ "$old_version" != "$latest_version" ]; then 19 | git mv "v8js-$old_version" "v8js-$latest_version" 20 | git mv "v8js-bare-$old_version" "v8js-bare-$latest_version" 21 | 22 | sed -e "s/v8js-bare-$old_version/v8js-bare-$latest_version/g" -i "v8js-$latest_version" 23 | fi 24 | popd 25 | done 26 | 27 | rm -f $tmp 28 | 29 | git update-index -q --ignore-submodules --refresh 30 | git diff-index --quiet HEAD -- && { 31 | echo "" 32 | echo "Nothing changed." 33 | exit 0 34 | } 35 | 36 | git commit -a -m "Update latest PECL v8js versions (php7 branch)" 37 | -------------------------------------------------------------------------------- /support/build/extensions/no-debug-non-zts-20151012/v8js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Build Path: /app/.heroku/php/ 3 | 4 | # fail hard 5 | set -o pipefail 6 | # fail harder 7 | set -eu 8 | 9 | source $(dirname $BASH_SOURCE)/../../_util/include/manifest.sh 10 | 11 | OUT_PREFIX=$1 12 | 13 | ZEND_MODULE_API_VERSION=$(basename $(dirname $0)) 14 | ZEND_MODULE_API_VERSION=${ZEND_MODULE_API_VERSION#no-debug-non-zts-} 15 | 16 | dep_formula=${0#$WORKSPACE_DIR/} 17 | dep_name=$(basename $BASH_SOURCE) 18 | dep_version=${dep_formula##*"/${dep_name}-"} 19 | dep_package=ext-${dep_name}-${dep_version} 20 | case ${ZEND_MODULE_API_VERSION} in 21 | 20151012) 22 | series=7.0 23 | ;; 24 | 20160303) 25 | series=7.1 26 | ;; 27 | 20170718) 28 | series=7.2 29 | ;; 30 | *) 31 | echo unknown zend module api version: ${ZEND_MODULE_API_VERSION} 32 | exit 2 33 | esac 34 | dep_manifest=${dep_package}_php-$series.composer.json 35 | 36 | echo "-----> Merging dependencies..." 37 | 38 | mkdir -p ${OUT_PREFIX}/etc/php/conf.d 39 | cat > ${OUT_PREFIX}/etc/php/conf.d/v8js.ini-dist <<'EOF' 40 | extension = v8js.so 41 | EOF 42 | 43 | MANIFEST_REQUIRE="${MANIFEST_REQUIRE:-"{\"heroku-sys/php\":\"${series}.*\"}"}" 44 | MANIFEST_CONFLICT="${MANIFEST_CONFLICT:-"{\"heroku-sys/hhvm\":\"*\"}"}" 45 | MANIFEST_EXTRA="${MANIFEST_EXTRA:-"{\"config\":\"etc/php/conf.d/v8js.ini-dist\"}"}" 46 | MANIFEST_REPLACE="${MANIFEST_REPLACE:-"{}"}" 47 | 48 | python $(dirname $BASH_SOURCE)/../../_util/include/manifest.py "heroku-sys-php-extension" "heroku-sys/ext-${dep_name}" "$dep_version" "${dep_formula}.tar.gz" "$MANIFEST_REQUIRE" "$MANIFEST_CONFLICT" "$MANIFEST_REPLACE" "$MANIFEST_EXTRA" > $dep_manifest 49 | 50 | print_or_export_manifest_cmd "$(generate_manifest_cmd "$dep_manifest")" 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # heroku-v8js 2 | 3 | Custom platform packages of V8Js for Heroku. 4 | 5 | See [Heroku documentation](https://github.com/heroku/heroku-buildpack-php/blob/master/support/build/README.md) 6 | for details on what this is all about. 7 | 8 | ## Usage 9 | 10 | In order to use V8Js on Heroku you can (meanwhile) stick with the normal PHP buildpack. 11 | You just have to set the `HEROKU_PHP_PLATFORM_REPOSITORIES` environment variable on your app, 12 | depending on the stack you've chosen. 13 | 14 | This will cause Heroku to fetch V8Js from "heroku-v8js" S3 bucket which is managed by 15 | [@stesie](https://github.com/stesie/) (current maintainer of V8Js). 16 | 17 | ### Configuration via Heroku Web Frontend 18 | 19 | Within Heroku portal, choose your app and go to the Settings tab. In the section labeled 20 | Info you'll see which stack is currently activated, this should be either *heroku-16* or *cedar-14*. 21 | 22 | Then click the button *Reveal Config Vars* from the Config Variables section, add a new record 23 | with key *HEROKU_PHP_PLATFORM_REPOSITORIES* and 24 | 25 | * for heroku-16 stack: https://heroku-v8js.s3.amazonaws.com/dist-heroku-16-stable/packages.json as value 26 | * for cedar-14 stack: https://heroku-v8js.s3.amazonaws.com/dist-cedar-14-stable/packages.json 27 | 28 | ### Configuration via Heroku CLI 29 | 30 | To find out which stack is currently active, use `heroku apps:info`. 31 | 32 | Then if you're using the heroku-16 stack: 33 | 34 | ``` 35 | heroku config:set HEROKU_PHP_PLATFORM_REPOSITORIES="https://heroku-v8js.s3.amazonaws.com/dist-heroku-16-stable/packages.json" 36 | ``` 37 | 38 | Otherwise on cedar-14 type this: 39 | 40 | ``` 41 | heroku config:set HEROKU_PHP_PLATFORM_REPOSITORIES="https://heroku-v8js.s3.amazonaws.com/dist-cedar-14-stable/packages.json" 42 | ``` 43 | -------------------------------------------------------------------------------- /support/build/_docker/README.md: -------------------------------------------------------------------------------- 1 | # Building Platform Packages using Docker 2 | 3 | ## Building the Image 4 | 5 | **After every change to your formulae, perform the following** from the root of the Git repository (not from `support/build/_docker/`): 6 | 7 | $ docker build --tag heroku-php-build-cedar-14 --file $(pwd)/support/build/_docker/cedar-14.Dockerfile . 8 | 9 | ## Configuration 10 | 11 | File `env.default` contains a list of required env vars, some with default values. You can modify it with the values you desire, or pass them to `docker run` using `--env`. 12 | 13 | Out of the box, you'll likely want to change `S3_BUCKET` and `S3_PREFIX` to match your info. Instead of setting `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in that file, it is recommended to pass them to `docker run` through the environment, or explicitly using `--env`, in order to prevent accidental commits of credentials. 14 | 15 | ## Using the Image 16 | 17 | From the root of the Git repository (not from `support/build/_docker/`): 18 | 19 | docker run --rm --tty --interactive --env-file=support/build/_docker/env.default heroku-php-build-cedar-14 /bin/bash 20 | 21 | That runs with values from `env.default`; if you need to pass e.g. `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` because they are not already in your environment, do either: 22 | 23 | AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... docker run --rm --tty --interactive --env-file=support/build/_docker/env.default heroku-php-build-cedar-14 /bin/bash 24 | 25 | or 26 | 27 | docker run --rm --tty --interactive --env-file=support/build/_docker/env.default -e AWS_ACCESS_KEY_ID=... -e AWS_SECRET_ACCESS_KEY=... heroku-php-build-cedar-14 /bin/bash 28 | 29 | You then have a shell where you can run `bob build`, `deploy.sh` and so forth. 30 | 31 | The `support/build/_util/` directory is on `$PATH` in the image. 32 | -------------------------------------------------------------------------------- /support/build/extensions/no-debug-non-zts-20151012/v8js-bare: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Build Path: /app/.heroku/php/ 3 | 4 | # fail hard 5 | set -o pipefail 6 | # fail harder 7 | set -eux 8 | 9 | source $(dirname $BASH_SOURCE)/../../_util/include/manifest.sh 10 | 11 | OUT_PREFIX=$1 12 | 13 | export PATH=${OUT_PREFIX}/bin:${PATH} 14 | 15 | dep_formula=${0#$WORKSPACE_DIR/} 16 | dep_name=$(basename $BASH_SOURCE) 17 | dep_version=${dep_formula##*"/${dep_name}-"} 18 | dep_package=pkg-ext-${dep_name}-${dep_version} 19 | dep_dirname=v8js-${dep_version} 20 | dep_archive_name=${dep_dirname}.tgz 21 | dep_url=https://pecl.php.net/get/${dep_archive_name} 22 | series=$(php-config --version | cut -d. -f1,2) # get "5.5", "5.6", "7.0" etc for the php requirement in the manifest 23 | dep_manifest=${dep_package}_php-$series.composer.json 24 | 25 | echo "-----> Building ${dep_package}..." 26 | 27 | curl -L ${dep_url} | tar xz 28 | 29 | pushd ${dep_dirname} 30 | phpize 31 | ./configure \ 32 | --prefix=${OUT_PREFIX} \ 33 | --with-v8js=${OUT_PREFIX} 34 | 35 | # link against libstdc++ also; required on Ubuntu Trusty environment 36 | sed -e 's/^V8JS_SHARED_LIBADD.*/& -lstdc++/' -i Makefile 37 | 38 | make -s -j 9 39 | # php was a build dep, and it's in $OUT_PREFIX. nuke that, then make install so all we're left with is the extension 40 | 41 | # run tests 42 | export NO_INTERACTION=1 43 | export REPORT_EXIT_STATUS=1 44 | make test 45 | 46 | rm -rf ${OUT_PREFIX}/* 47 | make install -s 48 | strip --strip-unneeded ${OUT_PREFIX}/lib/php/extensions/*/*.so 49 | popd 50 | 51 | MANIFEST_REQUIRE="${MANIFEST_REQUIRE:-"{\"heroku-sys/php\":\"${series}.*\"}"}" 52 | MANIFEST_CONFLICT="${MANIFEST_CONFLICT:-"{\"heroku-sys/hhvm\":\"*\"}"}" 53 | MANIFEST_EXTRA="${MANIFEST_EXTRA:-"{}"}" 54 | MANIFEST_REPLACE="${MANIFEST_REPLACE:-"{}"}" 55 | 56 | python $(dirname $BASH_SOURCE)/../../_util/include/manifest.py "heroku-sys-package" "heroku-sys/pkg-ext-${dep_name}" "$dep_version" "${dep_formula}.tar.gz" "$MANIFEST_REQUIRE" "$MANIFEST_CONFLICT" "$MANIFEST_REPLACE" "$MANIFEST_EXTRA" > $dep_manifest 57 | 58 | print_or_export_manifest_cmd "$(generate_manifest_cmd "$dep_manifest")" 59 | -------------------------------------------------------------------------------- /support/build/_util/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # fail hard 4 | set -o pipefail 5 | # fail harder 6 | set -eu 7 | 8 | publish=false 9 | 10 | # process flags 11 | optstring=":-:" 12 | while getopts "$optstring" opt; do 13 | case $opt in 14 | -) 15 | case "$OPTARG" in 16 | publish) 17 | publish=true 18 | break 19 | ;; 20 | *) 21 | OPTIND=1 22 | break 23 | ;; 24 | esac 25 | esac 26 | done 27 | # clear processed "publish" argument 28 | shift $((OPTIND-1)) 29 | 30 | if [[ $# -lt 1 ]]; then 31 | echo "Usage: $(basename $0) [--publish] FORMULA-VERSION [--overwrite]" >&2 32 | echo " If --publish is given, mkrepo.sh will be invoked after a successful deploy to" >&2 33 | echo " re-generate the repo. CAUTION: this will cause all manifests in the bucket to" >&2 34 | echo " be included in the repo, including potentially currently unpublished ones." >&2 35 | echo " All additional arguments, including --overwrite, are passed through to 'bob'." >&2 36 | exit 2 37 | fi 38 | 39 | if [[ -z "${AWS_ACCESS_KEY_ID:-}" || -z "${AWS_SECRET_ACCESS_KEY:-}" ]]; then 40 | echo '$AWS_ACCESS_KEY_ID or $AWS_SECRET_ACCESS_KEY not set!' >&2 41 | exit 2 42 | fi 43 | 44 | # a helper (print_or_export_manifest_cmd) called in the script invoked by Bob will write to this if set 45 | export MANIFEST_CMD=$(mktemp -t "manifest.XXXXX") 46 | trap 'rm -rf $MANIFEST_CMD;' EXIT 47 | 48 | # make sure we start cleanly 49 | rm -rf /app/.heroku/php 50 | 51 | # pass through args (so users can pass --overwrite etc) 52 | # but modify any path by stripping $WORKSPACE_DIR from the front, if it's there 53 | # so that one can also pass in the full path to the formula relative to the root, and not just relative to $WORKSPACE_DIR 54 | # that allows for simpler mass build loops using wildcards without having to worry about the relative location of other references such as an --env-file, like: 55 | # for f in support/build/php-{5,7}.* support/build/extensions/no-debug-non-zts-201*/{redis,blackfire,imagick}-*; do docker run --rm --tty --interactive --env-file=../dockerenv.cedar-14 heroku-php-builder-cedar-14 deploy.sh $f; done 56 | args=() 57 | for var in "$@"; do 58 | expanded="$(pwd)/$var" 59 | if [[ -f $expanded ]]; then 60 | var="${expanded#$WORKSPACE_DIR/}" 61 | fi 62 | args+=("$var") 63 | done 64 | 65 | bob deploy "${args[@]}" 66 | 67 | # invoke manifest upload 68 | echo "" 69 | echo "Uploading manifest..." 70 | . $MANIFEST_CMD 71 | 72 | if $publish; then 73 | echo "Updating repository..." 74 | $(dirname $BASH_SOURCE)/mkrepo.sh --upload "$S3_BUCKET" "${S3_PREFIX}" 75 | fi 76 | -------------------------------------------------------------------------------- /support/build/_util/mkrepo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # fail hard 4 | set -o pipefail 5 | # fail harder 6 | set -eu 7 | 8 | function s3cmd_get_progress() { 9 | len=0 10 | while read line; do 11 | if [[ "$len" -gt 0 ]]; then 12 | # repeat a backspace $len times 13 | # need to use seq; {1..$len} doesn't work 14 | printf '%0.s\b' $(seq 1 $len) 15 | fi 16 | echo -n "$line" 17 | len=${#line} 18 | done < <(grep --line-buffered -o -E '\[[0-9]+ of [0-9]+\]') # filter only the "[1 of 99]" bits from 's3cmd get' output 19 | } 20 | 21 | upload=false 22 | 23 | # process flags 24 | optstring=":-:" 25 | while getopts "$optstring" opt; do 26 | case $opt in 27 | -) 28 | case "$OPTARG" in 29 | upload) 30 | upload=true 31 | ;; 32 | *) 33 | echo "Invalid option: --$OPTARG" >&2 34 | exit 2 35 | ;; 36 | esac 37 | esac 38 | done 39 | # clear processed arguments 40 | shift $((OPTIND-1)) 41 | 42 | if [[ $# == "1" ]]; then 43 | echo "Usage: $(basename $0) [--upload] [S3_BUCKET S3_PREFIX [MANIFEST...]]" >&2 44 | echo " S3_BUCKET: S3 bucket name for packages.json upload; default: '\$S3_BUCKET'." >&2 45 | echo " S3_PREFIX: S3 prefix, e.g. '' or 'dist-stable/'; default: '\${S3_PREFIX}'." >&2 46 | echo " If MANIFEST arguments are given, those are used to build the repo; otherwise," >&2 47 | echo " all manifests from given or default S3_BUCKET+S3_PREFIX are downloaded." >&2 48 | echo " A --upload flag triggers immediate upload, otherwise instructions are printed." >&2 49 | echo " If --upload, or if stdout is a terminal, packages.json will be written to cwd." >&2 50 | echo " If no --upload, and if stdout is a pipe, repo JSON will be echo'd to stdout." >&2 51 | exit 2 52 | fi 53 | 54 | here=$(cd $(dirname $0); pwd) 55 | 56 | if [[ $# != "0" ]]; then 57 | S3_BUCKET=$1; shift 58 | S3_PREFIX=$1; shift 59 | fi 60 | 61 | if [[ $# == "0" ]]; then 62 | manifests_tmp=$(mktemp -d -t "dst-repo.XXXXX") 63 | trap 'rm -rf $manifests_tmp;' EXIT 64 | echo -n "-----> Fetching manifests... " >&2 65 | ( 66 | cd $manifests_tmp 67 | s3cmd --ssl --progress get s3://${S3_BUCKET}/${S3_PREFIX}*.composer.json 2>&1 | tee download.log | s3cmd_get_progress >&2 || { echo -e "failed! Error:\n$(cat download.log)" >&2; exit 1; } 68 | rm download.log 69 | ) 70 | echo "" >&2 71 | manifests=$manifests_tmp/*.composer.json 72 | else 73 | manifests="$@" 74 | fi 75 | 76 | echo "-----> Generating packages.json..." >&2 77 | redir=false 78 | if $upload || [[ -t 1 ]]; then 79 | redir=true 80 | # if stdout is a terminal or if we're uploading; we write a "packages.json" instead of echoing 81 | # this is so other programs can pipe our output and capture the generated repo from stdout 82 | # also back up stdout so we restore it to the right thing (tty or pipe) later 83 | exec 3>&1 1>packages.json 84 | fi 85 | 86 | # sort so that packages with the same name and version (e.g. ext-memcached 2.2.0) show up with their hhvm or php requirement in descending order - otherwise a Composer limitation means that a simple "ext-memcached: * + php: ^5.5.17" request would install 5.5.latest and not 5.6.latest, as it finds the 5.5.* requirement extension first and sticks to that instead of 5.6. For packages with identical names and versions (but different e.g. requirements), Composer basically treats them as equal and picks as a winner whatever it finds first. The requirements have to be written like "x.y.*" for this to work of course. 87 | python -c 'import sys, json; from distutils import version; json.dump({"packages": [ sorted([json.load(open(item)) for item in sys.argv[1:] if json.load(open(item)).get("type", "") != "heroku-sys-package"], key=lambda package: version.LooseVersion(package.get("require", {}).get("heroku-sys/hhvm", package.get("require", {}).get("heroku-sys/php", "0.0.0"))), reverse=True) ] }, sys.stdout, sort_keys=True)' $manifests 88 | 89 | # restore stdout 90 | # note that 'exec >$(tty)' does not work as FD 1 may have been a pipe originally and not a tty 91 | if $redir; then 92 | exec 1>&3 3>&- 93 | fi 94 | 95 | cmd="s3cmd --ssl${AWS_ACCESS_KEY_ID+" --access_key=\$AWS_ACCESS_KEY_ID"}${AWS_SECRET_ACCESS_KEY+" --secret_key=\$AWS_SECRET_ACCESS_KEY"} --acl-public -m application/json put packages.json s3://${S3_BUCKET}/${S3_PREFIX}packages.json" 96 | if $upload; then 97 | echo "-----> Uploading packages.json..." >&2 98 | eval "$cmd 1>&2" 99 | echo "-----> Done." >&2 100 | elif [[ -t 1 ]]; then 101 | echo "-----> Done. Run '$cmd' to upload repository." >&2 102 | fi 103 | -------------------------------------------------------------------------------- /support/build/_util/remove.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # fail hard 4 | set -o pipefail 5 | # fail harder 6 | set -eu 7 | 8 | publish=true 9 | 10 | # process flags 11 | optstring=":-:" 12 | while getopts "$optstring" opt; do 13 | case $opt in 14 | -) 15 | case "$OPTARG" in 16 | no-publish) 17 | publish=false 18 | ;; 19 | *) 20 | echo "Invalid option: --$OPTARG" >&2 21 | exit 2 22 | ;; 23 | esac 24 | esac 25 | done 26 | # clear processed arguments 27 | shift $((OPTIND-1)) 28 | 29 | if [[ $# -lt "1" ]]; then 30 | echo "Usage: $(basename $0) [--no-publish] MANIFEST..." >&2 31 | echo " MANIFEST: name of manifest file, e.g. 'ext-event-2.0.0_php-5.6.composer.json'" >&2 32 | echo " If --no-publish is given, mkrepo.sh will NOT be invoked after removal to" >&2 33 | echo " re-generate the repo." >&2 34 | echo " CAUTION: re-generating the repo will cause all manifests in the bucket" >&2 35 | echo " to be included in the repo, including potentially currently unpublished ones." >&2 36 | echo " CAUTION: using --no-publish means the repo will point to non-existing packages" >&2 37 | echo " until 'mkrepo.sh --upload' is run!" >&2 38 | echo " Bucket name and prefix will be read from '\$S3_BUCKET' and '\$S3_PREFIX'." >&2 39 | exit 2 40 | fi 41 | 42 | S3_PREFIX=${S3_PREFIX:-} 43 | 44 | here=$(cd $(dirname $0); pwd) 45 | 46 | manifests=("$@") 47 | 48 | indices="${!manifests[@]}" 49 | for index in $indices; do 50 | manifests[$index]="s3://${S3_BUCKET}/${S3_PREFIX}${manifests[$index]%.composer.json}.composer.json" 51 | done 52 | 53 | manifests_tmp=$(mktemp -d -t "dst-repo.XXXXX") 54 | trap 'rm -rf $manifests_tmp;' EXIT 55 | echo "-----> Fetching manifests..." >&2 56 | ( 57 | cd $manifests_tmp 58 | s3cmd --ssl get "${manifests[@]}" 1>&2 59 | ) 60 | 61 | echo " 62 | WARNING: POTENTIALLY DESTRUCTIVE ACTION! 63 | 64 | The following packages will be REMOVED 65 | from s3://${S3_BUCKET}/${S3_PREFIX}: 66 | $(IFS=$'\n'; echo "${manifests[*]:-(none)}" | xargs -n1 basename | sed -e 's/^/ - /' -e 's/.composer.json$//') 67 | " >&2 68 | 69 | if $publish; then 70 | echo "NOTICE: You have selected to publish the repo after removal of packages." >&2 71 | echo "This means the repo will be re-generated based on the current bucket contents!" >&2 72 | regenmsg="& regenerate packages.json" 73 | else 74 | regenmsg="without updating the repo" 75 | echo "WARNING: You have selected to NOT publish the repo after removal of packages." >&2 76 | echo "This means the repo will point to non-existing packages until mkrepo.sh is run!" >&2 77 | fi 78 | echo "" >&2 79 | 80 | read -p "Are you sure you want to remove the packages $regenmsg? [yN] " proceed 81 | 82 | [[ ! $proceed =~ [yY](es)* ]] && exit 83 | 84 | echo "" >&2 85 | 86 | remove_files=() 87 | for manifest in "${manifests[@]}"; do 88 | echo "Removing $(basename $manifest ".composer.json"):" >&2 89 | if filename=$(cat $manifests_tmp/$(basename $manifest) | python <(cat <<-'PYTHON' # beware of single quotes in body 90 | import sys, json, re; 91 | manifest=json.load(sys.stdin) 92 | url=manifest.get("dist",{}).get("url","").partition("https://"+sys.argv[1]+"."+sys.argv[2]+".amazonaws.com/"+sys.argv[3]) 93 | if url[0]: 94 | # dist URL does not match https://${dst_bucket}.${dst_region}.amazonaws.com/${dst_prefix} 95 | print(url[0]) 96 | sys.exit(1) 97 | else: 98 | print(url[2]) 99 | PYTHON 100 | ) $S3_BUCKET ${S3_REGION:-s3} ${S3_PREFIX}) 101 | then 102 | echo " - queued '$filename' for removal." >&2 103 | remove_files+=("$filename") 104 | else 105 | # the dist URL points somewhere else, so we are not touching that 106 | echo " - WARNING: not removing '$filename' (in manifest 'dist.url')!" >&2 107 | fi 108 | echo -n " - removing manifest file '$(basename $manifest)'... " >&2 109 | out=$(s3cmd rm ${AWS_ACCESS_KEY_ID+"--access_key=$AWS_ACCESS_KEY_ID"} ${AWS_SECRET_ACCESS_KEY+"--secret_key=$AWS_SECRET_ACCESS_KEY"} --ssl "$manifest" 2>&1) || { echo -e "failed! Error:\n$out" >&2; exit 1; } 110 | rm $manifests_tmp/$(basename $manifest) 111 | echo "done." >&2 112 | done 113 | 114 | echo "" >&2 115 | 116 | if $publish; then 117 | echo -n "Generating and uploading packages.json... " >&2 118 | out=$(cd $manifests_tmp; $here/mkrepo.sh --upload 2>&1) || { echo -e "failed! Error:\n$out" >&2; exit 1; } 119 | cat >&2 <<-EOF 120 | done! 121 | $(echo "$out" | grep -E '^Public URL' | sed 's/^Public URL of the object is: http:/Public URL of the repository is: https:/') 122 | 123 | EOF 124 | fi 125 | 126 | if [[ "${#remove_files[@]}" != "0" ]]; then 127 | echo "Removing files queued for deletion from bucket:" >&2 128 | for filename in "${remove_files[@]}"; do 129 | echo -n " - removing '$filename'... " >&2 130 | out=$(s3cmd rm ${AWS_ACCESS_KEY_ID+"--access_key=$AWS_ACCESS_KEY_ID"} ${AWS_SECRET_ACCESS_KEY+"--secret_key=$AWS_SECRET_ACCESS_KEY"} --ssl s3://${S3_BUCKET}/${S3_PREFIX}${filename} 2>&1) && echo "done." >&2 || echo -e "failed! Error:\n$out" >&2 131 | done 132 | echo "" >&2 133 | fi 134 | 135 | echo "Removal complete. 136 | " >&2 137 | 138 | if ! $publish; then 139 | cat >&2 <<-EOF 140 | WARNING: repo has not been re-generated. It may currently be in a broken state. 141 | There may be packages still listed in the repo that have just been removed. 142 | Run 'mkrepo.sh --upload' at once to return repository into a consistent state. 143 | 144 | EOF 145 | fi 146 | -------------------------------------------------------------------------------- /support/build/_util/sync.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | set -o pipefail 5 | 6 | function s3cmd_get_progress() { 7 | len=0 8 | while read line; do 9 | if [[ "$len" -gt 0 ]]; then 10 | # repeat a backspace $len times 11 | # need to use seq; {1..$len} doesn't work 12 | printf '%0.s\b' $(seq 1 $len) 13 | fi 14 | echo -n "$line" 15 | len=${#line} 16 | done < <(grep --line-buffered -o -E '\[[0-9]+ of [0-9]+\]') # filter only the "[1 of 99]" bits from 's3cmd get' output 17 | } 18 | 19 | remove=true 20 | 21 | # process flags 22 | optstring=":-:" 23 | while getopts "$optstring" opt; do 24 | case $opt in 25 | -) 26 | case "$OPTARG" in 27 | no-remove) 28 | remove=false 29 | ;; 30 | *) 31 | echo "Invalid option: --$OPTARG" >&2 32 | exit 2 33 | ;; 34 | esac 35 | esac 36 | done 37 | # clear processed arguments 38 | shift $((OPTIND-1)) 39 | 40 | if [[ $# -lt "2" || $# -gt "6" ]]; then 41 | echo "Usage: $(basename $0) [--no-remove] DEST_BUCKET DEST_PREFIX [DEST_REGION [SOURCE_BUCKET SOURCE_PREFIX [SOURCE_REGION]]]" >&2 42 | echo " DEST_BUCKET: destination S3 bucket name." >&2 43 | echo " DEST_REGION: destination bucket region, e.g. us-west-1; default: 's3'." >&2 44 | echo " DEST_PREFIX: destination prefix, e.g. '' or 'dist-stable/'." >&2 45 | echo " SOURCE_BUCKET: source S3 bucket name; default: '\$S3_BUCKET'." >&2 46 | echo " SOURCE_REGION: source bucket region; default: '\$S3_REGION' or 's3'." >&2 47 | echo " SOURCE_PREFIX: source prefix; default: '\${S3_PREFIX}'." >&2 48 | echo " --no-remove: no removal of destination packages that are not in source bucket." >&2 49 | exit 2 50 | fi 51 | 52 | dst_bucket=$1; shift 53 | dst_prefix=$1; shift 54 | if [[ $# -gt 2 ]]; then 55 | # region name given 56 | dst_region=$1; shift 57 | else 58 | dst_region="s3" 59 | fi 60 | echo "$@" 61 | src_bucket=${1:-$S3_BUCKET}; shift || true 62 | src_prefix=${1:-$S3_PREFIX}; shift || true 63 | if [[ $# == "1" ]]; then 64 | # region name given 65 | src_region=$1; shift 66 | else 67 | src_region=${S3_REGION:-"s3"} 68 | fi 69 | 70 | src_tmp=$(mktemp -d -t "src-repo.XXXXX") 71 | dst_tmp=$(mktemp -d -t "dst-repo.XXXXX") 72 | here=$(cd $(dirname $0); pwd) 73 | 74 | # clean up at the end 75 | trap 'rm -rf $src_tmp $dst_tmp;' EXIT 76 | 77 | echo -n "Fetching source's manifests from s3://${src_bucket}/${src_prefix}... " >&2 78 | ( 79 | cd $src_tmp 80 | out=$(s3cmd --ssl get s3://${src_bucket}/${src_prefix}packages.json 2>&1) || { echo -e "No packages.json in source repo:\n$out" >&2; exit 1; } 81 | s3cmd --ssl --progress get s3://${src_bucket}/${src_prefix}*.composer.json 2>&1 | tee download.log | s3cmd_get_progress >&2 || { echo -e "failed! Error:\n$(cat download.log)" >&2; exit 1; } 82 | ls *.composer.json 2>/dev/null 1>&2 || { echo "failed; no manifests found!" >&2; exit 1; } 83 | rm download.log 84 | ) 85 | echo "" >&2 86 | 87 | # this mkrepo.sh call won't actually download, but use the given *.composer.json, and echo a generated packages.json 88 | # we use this to compare to the downloaded packages.json 89 | $here/mkrepo.sh $src_bucket $src_prefix ${src_tmp}/*.composer.json 2>/dev/null | python -c 'import sys, json; sys.exit(abs(cmp(json.load(open(sys.argv[1])), json.load(sys.stdin))))' ${src_tmp}/packages.json || { 90 | echo "WARNING: packages.json from source does not match its list of manifests!" >&2 91 | echo " You should run 'mkrepo.sh' to update, or ask the bucket maintainers to do so." >&2 92 | read -p "Would you like to abort this operation? [Yn] " proceed 93 | [[ ! $proceed =~ [nN]o* ]] && exit 1 # yes is the default so doing yes | sync.sh won't do something stupid 94 | } 95 | 96 | echo -n "Fetching destination's manifests from s3://${dst_bucket}/${dst_prefix}... " >&2 97 | ( 98 | cd $dst_tmp 99 | s3cmd --ssl --progress get s3://${dst_bucket}/${dst_prefix}*.composer.json 2>&1 | tee download.log | s3cmd_get_progress >&2 || { echo -e "failed! Error:\n$(cat download.log)" >&2; exit 1; } 100 | rm download.log 101 | ) 102 | echo "" >&2 103 | 104 | comm=$(comm <(cd $src_tmp; ls -1 *.composer.json) <(cd $dst_tmp; ls -1 *.composer.json 2> /dev/null)) # comm produces three columns of output: entries only in left file, entries only in right file, entries in both 105 | add_manifests=$(echo "$comm" | grep '^\S' || true) # no tabs means output in col 1 = files only in src 106 | remove_manifests=$(echo "$comm" | grep '^\s\S' | cut -c2- || true) # one tab means output in col 2 = files only in dst 107 | common=$(echo "$comm" | grep '^\s\s' | cut -c3- || true) # two tabs means output in col 3 = files in both 108 | update_manifests=() 109 | ignore_manifests=() 110 | for filename in $common; do 111 | result=0 112 | python <(cat <<-'PYTHON' # beware of single quotes in body 113 | from __future__ import print_function 114 | import sys, json, os, datetime 115 | # for python 2+3 compat 116 | def stderrprint(*args, **kwargs): 117 | print(*args, file=sys.stderr, **kwargs) 118 | src_manifest = json.load(open(sys.argv[1])) 119 | dst_manifest = json.load(open(sys.argv[2])) 120 | # remove URLs so they don't interfere with comparison 121 | src_manifest.get("dist", {}).pop("url", None) 122 | dst_manifest.get("dist", {}).pop("url", None) 123 | # same for times, but we'll look at them 124 | try: 125 | src_time = datetime.datetime.strptime(src_manifest.pop("time"), "%Y-%m-%d %H:%M:%S") # UTC 126 | except KeyError, ValueError: 127 | src_time = datetime.datetime.utcfromtimestamp(os.path.getmtime(sys.argv[1])) 128 | stderrprint("WARNING: source manifest {} has invalid time entry, using mtime: {}".format(os.path.basename(sys.argv[1]), src_time.isoformat())) 129 | try: 130 | dst_time = datetime.datetime.strptime(dst_manifest.pop("time"), "%Y-%m-%d %H:%M:%S") # UTC 131 | except KeyError, ValueError: 132 | dst_time = datetime.datetime.utcfromtimestamp(os.path.getmtime(sys.argv[2])) 133 | stderrprint("WARNING: destination manifest {} has invalid time entry, using mtime: {}".format(os.path.basename(sys.argv[2]), dst_time.isoformat())) 134 | # a newer source time means we will copy 135 | if src_time > dst_time: 136 | sys.exit(0) 137 | else: 138 | # 1 = content identical, src_time = dst_time (up to date) 139 | # 3 = content different, src_time = dst_time (weird) 140 | # 5 = content identical, src_time < dst_time (probably needs sync the other way) 141 | # 7 = content different, src_time < dst_time (probably needs sync the other way) 142 | ret = 1 143 | ret = ret | abs(cmp(src_manifest, dst_manifest))<<1 144 | ret = ret | (src_time < dst_time)<<2 145 | sys.exit(ret) 146 | PYTHON 147 | ) $src_tmp/$filename $dst_tmp/$filename || result=$? 148 | if [[ $result -eq 0 ]]; then 149 | update_manifests+=($filename) 150 | elif [[ $result != "1" ]]; then 151 | case $result in 152 | 3) 153 | ignore_manifests+=("$filename (contents differ, time fields identical!?)") 154 | ;; 155 | 5) 156 | ignore_manifests+=("$filename (contents match, destination manifest newer)") 157 | ;; 158 | 7) 159 | ignore_manifests+=("$filename (contents differ, destination manifest newer)") 160 | ;; 161 | esac 162 | fi 163 | done 164 | 165 | echo " 166 | WARNING: POTENTIALLY DESTRUCTIVE ACTION! 167 | 168 | The following packages will be IGNORED: 169 | $(IFS=$'\n'; echo "${ignore_manifests[*]:-(none)}" | sed -e 's/^/ - /' -e 's/.composer.json//') 170 | 171 | The following packages will be ADDED 172 | from s3://${src_bucket}/${src_prefix} 173 | to s3://${dst_bucket}/${dst_prefix}: 174 | $(echo "${add_manifests:-(none)}" | sed -e 's/^/ - /' -e 's/.composer.json$//') 175 | 176 | The following packages will be UPDATED (source manifest is newer) 177 | from s3://${src_bucket}/${src_prefix} 178 | to s3://${dst_bucket}/${dst_prefix}: 179 | $(IFS=$'\n'; echo "${update_manifests[*]:-(none)}" | sed -e 's/^/ - /' -e 's/.composer.json$//') 180 | 181 | The following packages will $($remove || echo -n "NOT ")be REMOVED 182 | from s3://${dst_bucket}/${dst_prefix}$($remove && echo -n ":")$($remove || echo -ne "\n because '--no-remove' was given:") 183 | $(echo "${remove_manifests:-(none)}" | sed -e 's/^/ - /' -e 's/.composer.json$//') 184 | " >&2 185 | 186 | # clear remove_manifests if --no-remove given 187 | $remove || remove_manifests= 188 | 189 | if [[ ! "$add_manifests" && ! "$remove_manifests" && "${#update_manifests[@]}" -eq 0 ]]; then 190 | echo "Nothing to do. Aborting." >&2 191 | exit 192 | fi 193 | 194 | read -p "Are you sure you want to sync to destination & regenerate packages.json? [yN] " proceed 195 | 196 | [[ ! $proceed =~ [yY](es)* ]] && exit 197 | 198 | echo "" >&2 199 | 200 | copied_files=() 201 | for manifest in $add_manifests ${update_manifests[@]:-}; do 202 | echo "Copying ${manifest%.composer.json}:" >&2 203 | if filename=$(cat ${src_tmp}/${manifest} | python <(cat <<-'PYTHON' # beware of single quotes in body 204 | import sys, json; 205 | manifest=json.load(sys.stdin) 206 | url=manifest.get("dist",{}).get("url","").partition("https://"+sys.argv[1]+"."+sys.argv[2]+".amazonaws.com/"+sys.argv[3]) 207 | if url[0]: 208 | # dist URL does not match https://${src_bucket}.${src_region}.amazonaws.com/${src_prefix} 209 | print(url[0]) 210 | sys.exit(1) 211 | else: 212 | # rewrite dist URL in manifest to destination bucket 213 | manifest["dist"]["url"] = "https://"+sys.argv[4]+"."+sys.argv[5]+".amazonaws.com/"+sys.argv[6]+url[2] 214 | json.dump(manifest, open(sys.argv[7], "w"), sort_keys=True) 215 | print(url[2]) 216 | PYTHON 217 | ) $src_bucket $src_region $src_prefix $dst_bucket $dst_region $dst_prefix ${dst_tmp}/${manifest}) 218 | then 219 | # the dist URL in the source's manifest points to the source bucket, so we copy the file to the dest bucket 220 | echo -n " - copying '$filename'... " >&2 221 | out=$(s3cmd ${AWS_ACCESS_KEY_ID+"--access_key=$AWS_ACCESS_KEY_ID"} ${AWS_SECRET_ACCESS_KEY+"--secret_key=$AWS_SECRET_ACCESS_KEY"} --ssl --acl-public cp s3://${src_bucket}/${src_prefix}${filename} s3://${dst_bucket}/${dst_prefix}${filename} 2>&1) || { echo -e "failed! Error:\n$out" >&2; exit 1; } 222 | copied_files+=("$filename") 223 | echo "done." >&2 224 | else 225 | # the dist URL points somewhere else, so we are not touching that 226 | echo " - WARNING: not copying '$filename' (in manifest 'dist.url')!" >&2 227 | # just copy over the manifest (in the above branch, the Python script in the if expression already took care of that) 228 | cp ${src_tmp}/${manifest} ${dst_tmp}/${manifest} 229 | fi 230 | echo -n " - copying manifest file '$manifest'... " >&2 231 | out=$(s3cmd ${AWS_ACCESS_KEY_ID+"--access_key=$AWS_ACCESS_KEY_ID"} ${AWS_SECRET_ACCESS_KEY+"--secret_key=$AWS_SECRET_ACCESS_KEY"} --ssl --acl-public -m application/json put ${dst_tmp}/${manifest} s3://${dst_bucket}/${dst_prefix}${manifest} 2>&1) || { echo -e "failed! Error:\n$out" >&2; exit 1; } 232 | echo "done." >&2 233 | done 234 | 235 | remove_files=() 236 | for manifest in $remove_manifests; do 237 | echo "Removing ${manifest%.composer.json}:" >&2 238 | if filename=$(cat ${dst_tmp}/${manifest} | python <(cat <<-'PYTHON' # beware of single quotes in body 239 | import sys, json; 240 | manifest=json.load(sys.stdin) 241 | url=manifest.get("dist",{}).get("url","").partition("https://"+sys.argv[1]+"."+sys.argv[2]+".amazonaws.com/"+sys.argv[3]) 242 | if url[0]: 243 | # dist URL does not match https://${dst_bucket}.${dst_region}.amazonaws.com/${dst_prefix} 244 | print(url[0]) 245 | sys.exit(1) 246 | else: 247 | print(url[2]) 248 | PYTHON 249 | ) $dst_bucket $dst_region $dst_prefix) 250 | then 251 | # the dist URL in the destination manifest points to the destination bucket, so we remove that file at the end of the script... 252 | if [[ " ${copied_files[@]:-} " =~ " $filename " ]]; then 253 | # ...unless it was copied earlier (may happen if a new/updated manifest points to the same file name that this to-be-removed one is using) 254 | echo " - NOTICE: keeping newly copied '$filename'!" >&2 255 | else 256 | echo " - queued '$filename' for removal." >&2 257 | remove_files+=("$filename") 258 | fi 259 | else 260 | # the dist URL points somewhere else, so we are not touching that 261 | echo " - WARNING: not removing '$filename' (in manifest 'dist.url')!" >&2 262 | fi 263 | echo -n " - removing manifest file '$manifest'... " >&2 264 | out=$(s3cmd rm ${AWS_ACCESS_KEY_ID+"--access_key=$AWS_ACCESS_KEY_ID"} ${AWS_SECRET_ACCESS_KEY+"--secret_key=$AWS_SECRET_ACCESS_KEY"} --ssl s3://${dst_bucket}/${dst_prefix}${manifest} 2>&1) || { echo -e "failed! Error:\n$out" >&2; exit 1; } 265 | rm ${dst_tmp}/${manifest} 266 | echo "done." >&2 267 | done 268 | 269 | echo "" >&2 270 | 271 | echo -n "Generating and uploading packages.json... " >&2 272 | out=$(cd $dst_tmp; $here/mkrepo.sh --upload $dst_bucket $dst_prefix *.composer.json 2>&1) || { echo -e "failed! Error:\n$out" >&2; exit 1; } 273 | echo "done! 274 | $(echo "$out" | grep -E '^Public URL' | sed 's/^Public URL of the object is: http:/Public URL of the repository is: https:/') 275 | " >&2 276 | 277 | if [[ "${#remove_files[@]}" != "0" ]]; then 278 | echo "Removing files queued for deletion from destination:" >&2 279 | for filename in "${remove_files[@]}"; do 280 | echo -n " - removing '$filename'... " >&2 281 | out=$(s3cmd rm ${AWS_ACCESS_KEY_ID+"--access_key=$AWS_ACCESS_KEY_ID"} ${AWS_SECRET_ACCESS_KEY+"--secret_key=$AWS_SECRET_ACCESS_KEY"} --ssl s3://${dst_bucket}/${dst_prefix}${filename} 2>&1) && echo "done." >&2 || echo -e "failed! Error:\n$out" >&2 282 | done 283 | echo "" >&2 284 | fi 285 | 286 | echo "Sync complete. 287 | " >&2 288 | --------------------------------------------------------------------------------