├── tile_generator ├── __init__.py ├── templates │ ├── config │ │ ├── blobs.yml │ │ └── final.yml │ ├── jobs │ │ ├── charts_to_disk.sh.erb │ │ ├── monit │ │ ├── docker-bosh.sh.erb │ │ ├── delete-charts.sh.erb │ │ ├── opsmgr.env.erb │ │ ├── deploy-charts.sh.erb │ │ ├── spec │ │ └── delete-all.sh.erb │ ├── gitignore │ ├── src │ │ ├── templates │ │ │ └── all_open.json │ │ └── common │ │ │ └── utils.sh │ ├── tile │ │ ├── migration.js │ │ └── metadata.yml │ ├── packages │ │ ├── spec │ │ └── packaging │ └── tile.yml ├── version.py ├── build_unittest.py ├── tile_unittest.py ├── template_unittest.py ├── helm.py ├── tile.py ├── bosh_unittest.py ├── pcf_unittest.py ├── build.py ├── package_flags_unittest.py ├── helm_unittest.py ├── util.py ├── erb.py ├── tile_metadata_unittest.py ├── package_definitions.py ├── bosh.py └── template.py ├── sample ├── no-op-release │ ├── src │ │ └── .gitkeep │ ├── jobs │ │ └── no-op │ │ │ ├── monit │ │ │ ├── templates │ │ │ └── bin │ │ │ │ └── run │ │ │ └── spec │ ├── packages │ │ └── .gitkeep │ ├── config │ │ ├── blobs.yml │ │ └── final.yml │ ├── .gitignore │ └── build.sh ├── example-chart │ ├── .helmignore │ ├── plans │ │ └── default.yaml │ ├── plans.yaml │ ├── Chart.yaml │ ├── images │ │ └── spacebears.tgz │ ├── values.yaml │ └── templates │ │ ├── _helpers.tpl │ │ ├── secrets.yaml │ │ ├── svc.yaml │ │ └── db.yaml ├── runtime-test-release │ ├── src │ │ └── .gitkeep │ ├── packages │ │ └── .gitkeep │ ├── config │ │ ├── blobs.yml │ │ └── final.yml │ ├── jobs │ │ └── hello │ │ │ ├── monit │ │ │ ├── templates │ │ │ ├── index.html │ │ │ ├── stop.erb │ │ │ └── start.erb │ │ │ └── spec │ ├── .gitignore │ ├── build.sh │ └── manifests │ │ └── lite.yml ├── src │ ├── app │ │ ├── runtime.txt │ │ ├── requirements.txt │ │ └── app.py │ ├── binary-app │ │ ├── Procfile │ │ └── app │ ├── Dockerfile │ ├── buildpack │ │ └── bin │ │ │ ├── detect │ │ │ ├── release │ │ │ └── compile │ └── build.sh ├── resources │ ├── icon.png │ ├── redis-13.1.2.tgz │ ├── no-op-release.tgz │ ├── runtime-test-release.tgz │ ├── standalone-release.tgz │ └── cfplatformeng-docker-tile-example.tgz ├── .gitignore ├── diff-versions ├── diff_tile_metadata_yaml.py └── missing-properties.yml ├── pyinstaller ├── pcf_entrypoint.py ├── tile_entrypoint.py ├── README ├── pcf.spec ├── tile.spec └── build-binaries.sh ├── MANIFEST.in ├── requirements.txt ├── examples ├── make-pivnet-manifest └── README.md ├── Makefile ├── .gitignore ├── scripts ├── pcf ├── tile ├── tile_generator.sh └── run_local_tests.sh ├── ci ├── generate_pipeline_yml.py ├── docker-tile-generator │ └── Dockerfile ├── scripts │ ├── run-unittests.sh │ ├── run-deploymenttests.sh │ ├── setup-venv.sh │ ├── pcf-backup.sh │ ├── tile-build.sh │ ├── run-acceptancetests.sh │ ├── pcf-restore.sh │ ├── run-removetile.sh │ └── run-deploytile.sh ├── deployment-tests │ ├── runtime_config_deploymenttest.py │ ├── app3_deploymenttest.py │ ├── app4_deploymenttest.py │ ├── app1_deploymenttest.py │ ├── broker2_deploymenttest.py │ └── app2_deploymenttest.py ├── pipeline-helm.yml ├── pie-cleanup.yml └── acceptance-tests │ ├── bosh_acceptancetest.py │ └── tile_acceptancetest.py ├── install-git-hook.sh ├── NOTICE ├── setup.py ├── README.md └── CODE-OF-CONDUCT.md /tile_generator/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sample/no-op-release/src/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sample/no-op-release/jobs/no-op/monit: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sample/no-op-release/packages/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sample/example-chart/.helmignore: -------------------------------------------------------------------------------- 1 | images 2 | -------------------------------------------------------------------------------- /sample/example-chart/plans/default.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sample/runtime-test-release/src/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sample/src/app/runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.x 2 | -------------------------------------------------------------------------------- /sample/src/binary-app/Procfile: -------------------------------------------------------------------------------- 1 | web: ./app -------------------------------------------------------------------------------- /sample/no-op-release/config/blobs.yml: -------------------------------------------------------------------------------- 1 | --- {} 2 | -------------------------------------------------------------------------------- /sample/runtime-test-release/packages/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tile_generator/templates/config/blobs.yml: -------------------------------------------------------------------------------- 1 | --- {} 2 | -------------------------------------------------------------------------------- /tile_generator/templates/jobs/charts_to_disk.sh.erb: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sample/runtime-test-release/config/blobs.yml: -------------------------------------------------------------------------------- 1 | --- {} 2 | -------------------------------------------------------------------------------- /sample/no-op-release/config/final.yml: -------------------------------------------------------------------------------- 1 | name: no-op-release 2 | -------------------------------------------------------------------------------- /tile_generator/templates/gitignore: -------------------------------------------------------------------------------- 1 | product/ 2 | release/ 3 | -------------------------------------------------------------------------------- /tile_generator/version.py: -------------------------------------------------------------------------------- 1 | version_string = '15.1.0.dev1' 2 | -------------------------------------------------------------------------------- /sample/no-op-release/jobs/no-op/templates/bin/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | -------------------------------------------------------------------------------- /sample/src/app/requirements.txt: -------------------------------------------------------------------------------- 1 | flask>=0.12.3 2 | requests>=2.9.1 3 | -------------------------------------------------------------------------------- /sample/runtime-test-release/config/final.yml: -------------------------------------------------------------------------------- 1 | name: runtime-test-release 2 | -------------------------------------------------------------------------------- /pyinstaller/pcf_entrypoint.py: -------------------------------------------------------------------------------- 1 | from tile_generator import pcf 2 | pcf.main() 3 | -------------------------------------------------------------------------------- /pyinstaller/tile_entrypoint.py: -------------------------------------------------------------------------------- 1 | from tile_generator import tile 2 | tile.main() 3 | -------------------------------------------------------------------------------- /sample/resources/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cf-platform-eng/tile-generator/HEAD/sample/resources/icon.png -------------------------------------------------------------------------------- /sample/example-chart/plans.yaml: -------------------------------------------------------------------------------- 1 | - name: "default" 2 | description: "default plan for spacebears" 3 | file: "default.yaml" 4 | -------------------------------------------------------------------------------- /sample/resources/redis-13.1.2.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cf-platform-eng/tile-generator/HEAD/sample/resources/redis-13.1.2.tgz -------------------------------------------------------------------------------- /sample/resources/no-op-release.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cf-platform-eng/tile-generator/HEAD/sample/resources/no-op-release.tgz -------------------------------------------------------------------------------- /sample/example-chart/Chart.yaml: -------------------------------------------------------------------------------- 1 | name: spacebears 2 | description: spacebears service and spacebears broker helm chart 3 | version: 0.0.1 4 | -------------------------------------------------------------------------------- /sample/no-op-release/jobs/no-op/spec: -------------------------------------------------------------------------------- 1 | --- 2 | name: no-op 3 | 4 | templates: {bin/run: bin/run} 5 | 6 | packages: [] 7 | 8 | properties: {} 9 | -------------------------------------------------------------------------------- /sample/resources/runtime-test-release.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cf-platform-eng/tile-generator/HEAD/sample/resources/runtime-test-release.tgz -------------------------------------------------------------------------------- /sample/resources/standalone-release.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cf-platform-eng/tile-generator/HEAD/sample/resources/standalone-release.tgz -------------------------------------------------------------------------------- /sample/example-chart/images/spacebears.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cf-platform-eng/tile-generator/HEAD/sample/example-chart/images/spacebears.tgz -------------------------------------------------------------------------------- /tile_generator/templates/src/templates/all_open.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "destination": "0.0.0.0-255.255.255.255", 4 | "protocol": "all" 5 | } 6 | ] 7 | -------------------------------------------------------------------------------- /tile_generator/templates/config/final.yml: -------------------------------------------------------------------------------- 1 | --- 2 | blobstore: 3 | provider: local 4 | options: 5 | blobstore_path: /tmp/{{ name }}-blobs 6 | final_name: {{ name }} -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include tile_generator/templates * 2 | include LICENSE 3 | include NOTICE 4 | include README.md 5 | include requirements.txt 6 | include version.txt 7 | -------------------------------------------------------------------------------- /sample/resources/cfplatformeng-docker-tile-example.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cf-platform-eng/tile-generator/HEAD/sample/resources/cfplatformeng-docker-tile-example.tgz -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Cerberus>=1.1 2 | click>=6.2 3 | Jinja2>=3.1 4 | PyYAML>=3.1 5 | docker-py>=1.6.0 6 | requests>2.11 7 | requests-toolbelt 8 | mock>=2.0.0 9 | pexpect>=4.2.1 10 | -------------------------------------------------------------------------------- /sample/src/binary-app/app: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | response='Hello, world!' 4 | while true; do 5 | echo -e "HTTP/1.1 200\r\nContent-Type: text/html\r\n\r\n${response}" | nc -lvN "${PORT:-8080}"; 6 | done -------------------------------------------------------------------------------- /tile_generator/templates/tile/migration.js: -------------------------------------------------------------------------------- 1 | exports.migrate = function(properties) { 2 | {% if migration %} 3 | {{ migration | render | indent(2,false) }} 4 | {% endif %} 5 | return properties; 6 | }; 7 | -------------------------------------------------------------------------------- /tile_generator/templates/packages/spec: -------------------------------------------------------------------------------- 1 | --- 2 | name: {{ package.name }} 3 | 4 | dependencies: [] 5 | 6 | files: 7 | {% for file in files %} 8 | - {{ package.name }}/{{ file.name }} 9 | {% endfor %} 10 | -------------------------------------------------------------------------------- /sample/.gitignore: -------------------------------------------------------------------------------- 1 | release/ 2 | product/ 3 | tile-history.yml 4 | vendor/ 5 | resources/*.zip 6 | resources/*.tgz 7 | docker-cache/ 8 | cache/ 9 | 10 | !resources/no-op-release.tgz 11 | !resources/runtime-test-release.tgz 12 | -------------------------------------------------------------------------------- /sample/runtime-test-release/jobs/hello/monit: -------------------------------------------------------------------------------- 1 | check process hello 2 | with pidfile /var/vcap/sys/run/hello/hello.pid 3 | start program /var/vcap/jobs/hello/bin/start 4 | stop program /var/vcap/jobs/hello/bin/stop 5 | group vcap 6 | -------------------------------------------------------------------------------- /sample/runtime-test-release/jobs/hello/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Runtime Test Release 4 | 5 | 6 |

7 | Runtime Test Release: Success 8 |

9 | 10 | 11 | -------------------------------------------------------------------------------- /examples/make-pivnet-manifest: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eu 4 | 5 | cat < 5 | 3. pip install pyinstaller 6 | 4. pyinstaller (pcf/tile).spec 7 | The resulting binary can be found in ./dist/ 8 | -------------------------------------------------------------------------------- /sample/example-chart/values.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # image source is https://github.com/cf-platform-eng/pcf-examples/tree/master/src/docker 3 | image: cfplatformeng/spacebears 4 | imageTag: latest 5 | 6 | spacebears: 7 | username: admin_db 8 | password: monkey123 9 | 10 | service: 11 | type: LoadBalancer -------------------------------------------------------------------------------- /sample/runtime-test-release/jobs/hello/spec: -------------------------------------------------------------------------------- 1 | --- 2 | name: hello 3 | 4 | templates: 5 | start.erb: bin/start 6 | stop.erb: bin/stop 7 | index.html: data/index.html 8 | 9 | packages: [] 10 | 11 | properties: 12 | hello.port: 13 | description: Port to listen on 14 | default: 8118 15 | -------------------------------------------------------------------------------- /scripts/pcf: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | MY_DIR="$( cd "$( dirname "$0" )" && pwd )" 4 | 5 | echo "Tile generator is now pip-installable" 6 | echo "Please follow the updated installation instructions at:" 7 | echo "http://cf-platform-eng.github.io/isv-portal/tile-generator" 8 | echo "and remove ${MY_DIR} from your \$PATH" 9 | exit 1 -------------------------------------------------------------------------------- /scripts/tile: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | MY_DIR="$( cd "$( dirname "$0" )" && pwd )" 4 | 5 | echo "Tile generator is now pip-installable" 6 | echo "Please follow the updated installation instructions at:" 7 | echo "http://cf-platform-eng.github.io/isv-portal/tile-generator" 8 | echo "and remove ${MY_DIR} from your \$PATH" 9 | exit 1 -------------------------------------------------------------------------------- /sample/runtime-test-release/jobs/hello/templates/stop.erb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | PIDFILE="/var/vcap/sys/run/hello/hello.pid" 5 | 6 | /sbin/start-stop-daemon \ 7 | --pidfile "$PIDFILE" \ 8 | --retry TERM/20/QUIT/1/KILL \ 9 | --oknodo \ 10 | --stop 11 | 12 | rm /var/vcap/sys/run/hello/hello.pid 13 | -------------------------------------------------------------------------------- /sample/no-op-release/.gitignore: -------------------------------------------------------------------------------- 1 | config/private.yml 2 | config/dev.yml 3 | blobs 4 | dev_releases 5 | releases/*.tgz 6 | releases/**/*.tgz 7 | .dev_builds 8 | .final_builds/jobs/**/*.tgz 9 | .final_builds/packages/**/*.tgz 10 | .DS_Store 11 | .idea 12 | *.swp 13 | *~ 14 | *# 15 | #* 16 | 17 | tmp/**/* 18 | !tmp/.gitkeep 19 | 20 | -------------------------------------------------------------------------------- /sample/runtime-test-release/.gitignore: -------------------------------------------------------------------------------- 1 | config/private.yml 2 | config/dev.yml 3 | blobs 4 | dev_releases 5 | releases/*.tgz 6 | releases/**/*.tgz 7 | .dev_builds 8 | .final_builds/jobs/**/*.tgz 9 | .final_builds/packages/**/*.tgz 10 | .DS_Store 11 | .idea 12 | *.swp 13 | *~ 14 | *# 15 | #* 16 | 17 | tmp/**/* 18 | !tmp/.gitkeep 19 | 20 | -------------------------------------------------------------------------------- /tile_generator/templates/jobs/monit: -------------------------------------------------------------------------------- 1 | {% if not errand %} 2 | check process {{ job_name }} 3 | with pidfile /var/vcap/sys/run/{{ job_name }}/{{ job_name }}.pid 4 | start program "/var/vcap/jobs/{{ job_name }}/bin/{{ job_name }}_ctl start" 5 | stop program "/var/vcap/jobs/{{ job_name }}/bin/{{ job_name }}_ctl stop" 6 | group vcap 7 | {% endif %} 8 | -------------------------------------------------------------------------------- /sample/example-chart/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | 3 | {{/* 4 | Create a default fully qualified app name. 5 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 6 | */}} 7 | {{- define "fullname" -}} 8 | {{- printf "%s" .Release.Name | trunc 63 | trimSuffix "-" -}} 9 | {{- end -}} 10 | -------------------------------------------------------------------------------- /sample/example-chart/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: {{ template "fullname" . }} 5 | labels: 6 | app: {{ template "fullname" . }} 7 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 8 | release: "{{ .Release.Name }}" 9 | type: Opaque 10 | data: 11 | admin-username: {{ .Values.spacebears.username | b64enc | quote }} 12 | admin-password: {{ .Values.spacebears.password | b64enc | quote }} 13 | -------------------------------------------------------------------------------- /sample/example-chart/templates/svc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ template "fullname" . }} 5 | labels: 6 | app: {{ template "fullname" . }} 7 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 8 | release: "{{ .Release.Name }}" 9 | spec: 10 | type: {{ .Values.service.type }} 11 | ports: 12 | - name: db 13 | port: 9000 14 | targetPort: 9000 15 | selector: 16 | app: {{ template "fullname" . }} 17 | -------------------------------------------------------------------------------- /sample/no-op-release/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | echo "Be sure to increment version if re-using an environment" 6 | version=0.1.4 7 | 8 | rm -rf dev_releases/no-op-release 9 | 10 | # pcf-1-11 didn't like the bosh release build with bosh2 11 | # bosh2 version 12 | bosh create-release --force --version=$version --tarball=../resources/no-op-release.tgz 13 | 14 | # bosh1 version 15 | #bosh create release --force --version=$version --with-tarball 16 | #mv dev_releases/no-op-release/no-op-release-$version.tgz ../resources/no-op-release.tgz 17 | -------------------------------------------------------------------------------- /sample/runtime-test-release/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | echo "Be sure to increment version if re-using an environment" 6 | version=0.2.0 7 | 8 | rm -rf dev_releases/runtime-test-release 9 | 10 | # pcf-1-11 didn't like the bosh release build with bosh2 11 | # bosh2 version 12 | bosh create-release --force --version=$version --tarball=../resources/runtime-test-release.tgz 13 | 14 | # bosh1 version 15 | # bosh create release --force --version=$version --with-tarball 16 | # mv dev_releases/runtime-test-release/runtime-test-release-0.1.5.tgz ../resources/runtime-test-release.tgz 17 | -------------------------------------------------------------------------------- /ci/generate_pipeline_yml.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from jinja2 import Template 5 | 6 | clusters = ['_four', '-6_0', '_7_prerelease', '-4_0-lite', '-6_0-lite', '-10_0-lite'] 7 | # Commenting out this as we only have one example and it breaks 8 | 9 | tiles = [] # [d for d in os.listdir('../examples') if os.path.isdir(os.path.join('../examples', d))] 10 | 11 | with open('pipeline.yml.jinja2', 'r') as f: 12 | t = Template(f.read()) 13 | 14 | with open('pipeline.yml', 'w') as f: 15 | f.write(t.render(clusters=clusters, tiles=tiles)) 16 | 17 | print("Successfully generated pipeline.yml") 18 | -------------------------------------------------------------------------------- /install-git-hook.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" 4 | HOOKS_DIR="${SCRIPT_DIR}"/.git/hooks 5 | 6 | echo 'Installing pre-push hook...' 7 | 8 | mkdir -p "${HOOKS_DIR}" 9 | echo '#!/bin/bash -e 10 | remote="$1" 11 | url="$2" 12 | 13 | if ! git diff-index --quiet HEAD --; then 14 | echo "You have uncommited changes" 15 | exit 1 16 | fi 17 | 18 | if [[ $url = "git@github.com:cf-platform-eng/tile-generator.git" ]]; then 19 | ./scripts/run_local_tests.sh withcache 20 | fi 21 | 22 | exit 0' > "${HOOKS_DIR}"/pre-push 23 | 24 | chmod +x "${HOOKS_DIR}"/pre-push 25 | 26 | echo 'Done!' 27 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | tile-generator 2 | 3 | Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. -------------------------------------------------------------------------------- /tile_generator/templates/packages/packaging: -------------------------------------------------------------------------------- 1 | # abort script on any command that exit with a non zero value 2 | set -e 3 | 4 | {% for file in files %} 5 | {% if file.unzip %} 6 | unzip -d ${BOSH_INSTALL_TARGET} {{ package.name }}/{{ file.name }} 7 | {% elif file.untar %} 8 | tar -C ${BOSH_INSTALL_TARGET} -vxf {{ package.name }}/{{ file.name }} 9 | {% else %} 10 | cp {{ package.name }}/{{ file.name }} ${BOSH_INSTALL_TARGET} 11 | {% endif %} 12 | {% if file.chmod %} 13 | {% if file.unzip %} 14 | chmod -R {{ file.chmod }} ${BOSH_INSTALL_TARGET} 15 | {% else %} 16 | chmod {{ file.chmod }} ${BOSH_INSTALL_TARGET}/{{ file.name }} 17 | {% endif %} 18 | {% endif %} 19 | {% endfor %} 20 | -------------------------------------------------------------------------------- /sample/runtime-test-release/jobs/hello/templates/start.erb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | LOG_DIR="/var/vcap/sys/log/hello" 6 | RUN_DIR="/var/vcap/sys/run/hello" 7 | DATA_DIR="/var/vcap/store/hello" 8 | PIDFILE="$RUN_DIR/hello.pid" 9 | 10 | export PORT=<%= p("hello.port", "8118") %> 11 | 12 | mkdir -p "$LOG_DIR" "$RUN_DIR" "$DATA_DIR" 13 | chown -R vcap:vcap "$LOG_DIR" "$RUN_DIR" "$DATA_DIR" 14 | 15 | cp /var/vcap/jobs/hello/data/* $DATA_DIR 16 | chmod +r $DATA_DIR/* 17 | 18 | /sbin/start-stop-daemon \ 19 | --pidfile "$PIDFILE" \ 20 | --make-pidfile \ 21 | --chuid vcap:vcap \ 22 | --chdir $DATA_DIR \ 23 | --start \ 24 | --exec /usr/bin/python3 \ 25 | -- -m http.server $PORT \ 26 | >> "$LOG_DIR/hello.out.log" \ 27 | 2>> "$LOG_DIR/hello.err.log" 28 | -------------------------------------------------------------------------------- /sample/src/buildpack/bin/detect: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # bin/detect 3 | 4 | # tile-generator 5 | # 6 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | 20 | exit 1 21 | -------------------------------------------------------------------------------- /sample/src/buildpack/bin/release: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # bin/release 3 | 4 | # tile-generator 5 | # 6 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | 20 | exit 1 21 | -------------------------------------------------------------------------------- /sample/runtime-test-release/manifests/lite.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | name: runtime-test 4 | 5 | releases: 6 | - name: runtime-test-release 7 | version: latest 8 | 9 | stemcells: 10 | - alias: bosh-warden-boshlite-ubuntu-trusty-go_agent 11 | os: ubuntu-trusty 12 | version: latest 13 | 14 | update: 15 | canaries: 1 16 | max_in_flight: 10 17 | canary_watch_time: 1000-30000 18 | update_watch_time: 1000-30000 19 | 20 | instance_groups: 21 | - name: runtime-test 22 | instances: 1 23 | azs: [z1] 24 | jobs: 25 | - name: hello 26 | release: runtime-test-release 27 | properties: 28 | hello: 29 | port: 8119 30 | vm_type: minimal 31 | cloud_properties: 32 | tags: 33 | - allow-ssh 34 | stemcell: bosh-warden-boshlite-ubuntu-trusty-go_agent 35 | networks: 36 | - name: default 37 | -------------------------------------------------------------------------------- /sample/src/buildpack/bin/compile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # bin/compile 3 | 4 | # tile-generator 5 | # 6 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | 20 | exit 1 21 | -------------------------------------------------------------------------------- /ci/docker-tile-generator/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG python3=mirror.gcr.io/python:3-slim 2 | 3 | FROM ${python3} 4 | 5 | RUN apt-get update && \ 6 | apt-get install --yes git wget zip && \ 7 | rm -rf /var/lib/apt/lists/* 8 | 9 | RUN wget -q -O- https://api.github.com/repos/cloudfoundry/bosh-cli/releases/latest | grep 'browser_download_url.*linux-amd64' | cut -d '"' -f 4 | xargs wget -q 10 | RUN mv bosh-cli-*-linux-amd64 /usr/local/bin/bosh 11 | RUN chmod 755 /usr/local/bin/bosh 12 | 13 | RUN wget -q -O /tmp/cf.tgz https://packages.cloudfoundry.org/stable?release=linux64-binary 14 | RUN tar -C /tmp -zxf /tmp/cf.tgz 15 | RUN mv /tmp/cf8 /usr/local/bin/cf8 16 | RUN mv /tmp/cf /usr/local/bin/cf 17 | 18 | COPY bundle*.tar.gz . 19 | RUN tar -C /tmp -zxf bundle*.tar.gz 20 | RUN mv /tmp/dist-*/tile_linux-64bit /usr/local/bin/tile 21 | RUN mv /tmp/dist-*/pcf_linux-64bit /usr/local/bin/pcf 22 | -------------------------------------------------------------------------------- /sample/diff-versions: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | # Filter the output of a recursive diff between samples. 4 | # Gets rid of the noise when comparing two runs on the sample. 5 | 6 | use strict; 7 | use warnings; 8 | 9 | if (scalar(@ARGV) != 2) { 10 | print STDERR "$0: show the relevant difference between two sample tile directories.\n"; 11 | print STDERR "Usage: $0 \n"; 12 | exit(1); 13 | } 14 | 15 | my $a = shift; 16 | my $b = shift; 17 | 18 | my @ignore_fields = qw( [a-z0-f-]+:$ blobstore_id: commit_hash: fingerprint: latest_release_filename: object_id: sha: sha1: version: ); 19 | my $fields = join('|', @ignore_fields); 20 | 21 | my @difflines = `diff --recursive $a $b`; 22 | for (@difflines) { 23 | # The funky-looking one here is diff's line number/operation notation. 24 | print unless m/^[<>]\s+($fields)|^(\d+,)?\d+[acd]\d+(,\d+)?$|^---$|^Binary files |_noop.js$|\.blobs:/ 25 | } 26 | -------------------------------------------------------------------------------- /ci/scripts/run-unittests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | REPO_DIR="$( cd "$( dirname "$0" )/../.." && pwd )" 20 | 21 | . "${REPO_DIR}/ci/scripts/setup-venv.sh" 22 | 23 | cd ${REPO_DIR} 24 | python -m unittest discover -v -p '*_unittest.py' 25 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Tile Generator Examples 2 | This directory contains examples of various types of tiles. 3 | 4 | ## Adding Examples 5 | Each example should have a `build.sh` script that does whatever needs 6 | to be done, and then calls `tile build`, passing through any 7 | parameters. The Tile Generator CI pipeline expects this interface. 8 | 9 | We plan to incrementally split aparat the large, one-of-everything 10 | [sample tile](../sample) into several smaller example tiles that look 11 | more like real product tiles. 12 | 13 | ## Naming Convention 14 | - To make it clear in Ops Manager that these are test tiles, not to 15 | beconfused with real Pivotal or partner products, please prefix the 16 | `label` field in the tile.yml with "Test". 17 | 18 | - To keep them from cluttering up Tile Dashboard, please prefix the 19 | `name` field in tile.yml with "z-test". 20 | 21 | - Even though these tiles will only ever show up for admins of PivNet, 22 | please use prepend "Z" to the product name on PivNet. 23 | -------------------------------------------------------------------------------- /sample/example-chart/templates/db.yaml: -------------------------------------------------------------------------------- 1 | kind: Deployment 2 | apiVersion: extensions/v1beta1 3 | metadata: 4 | name: {{ template "fullname" . }} 5 | labels: 6 | app: {{ template "fullname" . }} 7 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 8 | release: "{{ .Release.Name }}" 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | app: {{ template "fullname" . }} 14 | template: 15 | metadata: 16 | labels: 17 | app: {{ template "fullname" . }} 18 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 19 | release: "{{ .Release.Name }}" 20 | heritage: "{{ .Release.Service }}" 21 | spec: 22 | containers: 23 | - name: spacebears-db-node 24 | image: {{ .Values.image }} 25 | imagePullPolicy: Always 26 | ports: 27 | - containerPort: 9000 28 | env: 29 | - name: ADMIN_USERNAME 30 | value: {{ .Values.spacebears.username }} 31 | - name: ADMIN_PASSWORD 32 | value: {{ .Values.spacebears.password }} 33 | -------------------------------------------------------------------------------- /ci/scripts/run-deploymenttests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | TILE_DIR="$( cd "$1" && pwd )" 20 | POOL_DIR="$( cd "$2" && pwd )" 21 | 22 | MY_DIR="$( cd "$( dirname "$0" )" && pwd )" 23 | REPO_DIR="$( cd "${MY_DIR}/../.." && pwd )" 24 | BASE_DIR="$( cd "${REPO_DIR}/.." && pwd )" 25 | TEST_DIR="$( cd "${REPO_DIR}/ci/deployment-tests" && pwd )" 26 | 27 | . "${REPO_DIR}/ci/scripts/setup-venv.sh" 28 | 29 | cd ${POOL_DIR} 30 | 31 | python -m unittest discover -v -s ${TEST_DIR} -p '*_deploymenttest.py' 32 | -------------------------------------------------------------------------------- /tile_generator/build_unittest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import unittest 19 | from . import build 20 | import sys 21 | from contextlib import contextmanager 22 | from io import StringIO 23 | 24 | @contextmanager 25 | def capture_output(): 26 | new_out, new_err = StringIO(), StringIO() 27 | old_out, old_err = sys.stdout, sys.stderr 28 | try: 29 | sys.stdout, sys.stderr = new_out, new_err 30 | yield sys.stdout, sys.stderr 31 | finally: 32 | sys.stdout, sys.stderr = old_out, old_err 33 | 34 | if __name__ == '__main__': 35 | unittest.main() 36 | -------------------------------------------------------------------------------- /sample/diff_tile_metadata_yaml.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import dictdiffer 4 | import sys 5 | import yaml 6 | 7 | 8 | if len(sys.argv) != 3: 9 | print(('Usage: %s FILE_A FILE_B\nSort tile metadata yaml files and compare FILE_A to FILE_B.' % sys.argv[0].split('/')[-1])) 10 | sys.exit(1) 11 | 12 | with open(sys.argv[1], 'r') as f: 13 | a = f.read() 14 | with open(sys.argv[2], 'r') as f: 15 | b = f.read() 16 | 17 | aa = yaml.load(a, yaml.Loader) 18 | bb = yaml.load(b, yaml.Loader) 19 | 20 | for conf in [aa, bb]: 21 | for release in conf['releases']: 22 | if release.get('name') == 'test-tile': 23 | release.pop('version') 24 | release.pop('file') 25 | conf.pop('product_version') 26 | conf.pop('provides_product_versions') 27 | conf['property_blueprints'] = sorted(conf['property_blueprints'], key=lambda k: k['name']) 28 | conf['job_types'] = sorted(conf['job_types'], key=lambda k: k['name']) 29 | for x in conf['job_types']: x['manifest'] = sorted(x['manifest'].items()) if type(x['manifest']) is dict else sorted(yaml.load(x['manifest'], yaml.Loader).items()) 30 | 31 | from pprint import pprint 32 | for x in list(dictdiffer.diff(aa,bb)): pprint(x); print('') 33 | 34 | -------------------------------------------------------------------------------- /pyinstaller/pcf.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python -*- 2 | 3 | import os 4 | import platform 5 | import sys 6 | 7 | block_cipher = None 8 | arch = '64bit' if sys.maxsize > 2**32 else '32bit' 9 | 10 | dirs = [os.path.abspath(d[0]) for d in os.walk(os.path.join('.', 'tile_generator', 'templates'))] 11 | files = [(os.path.join(d,'*'), os.path.relpath(d)) for d in dirs] 12 | 13 | a = Analysis(['pcf_entrypoint.py'], 14 | pathex=['.'], 15 | binaries=[], 16 | datas=files, 17 | # work-around for https://github.com/pypa/setuptools/issues/1963 18 | hiddenimports=['pkg_resources.py2_warn'], 19 | hookspath=[], 20 | runtime_hooks=[], 21 | excludes=[], 22 | win_no_prefer_redirects=False, 23 | win_private_assemblies=False, 24 | cipher=block_cipher) 25 | pyz = PYZ(a.pure, a.zipped_data, 26 | cipher=block_cipher) 27 | exe = EXE(pyz, 28 | a.scripts, 29 | a.binaries, 30 | a.zipfiles, 31 | a.datas, 32 | name='pcf_' + platform.system().lower() + '-' + arch, 33 | debug=False, 34 | strip=False, 35 | upx=False, 36 | runtime_tmpdir=None, 37 | console=True ) 38 | -------------------------------------------------------------------------------- /pyinstaller/tile.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python -*- 2 | 3 | import os 4 | import platform 5 | import sys 6 | 7 | block_cipher = None 8 | arch = '64bit' if sys.maxsize > 2**32 else '32bit' 9 | 10 | dirs = [os.path.abspath(d[0]) for d in os.walk(os.path.join('.', 'tile_generator', 'templates'))] 11 | files = [(os.path.join(d,'*'), os.path.relpath(d)) for d in dirs] 12 | 13 | a = Analysis(['tile_entrypoint.py'], 14 | pathex=['.'], 15 | binaries=[], 16 | datas=files, 17 | # work-around for https://github.com/pypa/setuptools/issues/1963 18 | hiddenimports=['pkg_resources.py2_warn'], 19 | hookspath=[], 20 | runtime_hooks=[], 21 | excludes=[], 22 | win_no_prefer_redirects=False, 23 | win_private_assemblies=False, 24 | cipher=block_cipher) 25 | pyz = PYZ(a.pure, a.zipped_data, 26 | cipher=block_cipher) 27 | exe = EXE(pyz, 28 | a.scripts, 29 | a.binaries, 30 | a.zipfiles, 31 | a.datas, 32 | name='tile_' + platform.system().lower() + '-' + arch, 33 | debug=False, 34 | strip=False, 35 | upx=False, 36 | runtime_tmpdir=None, 37 | console=True ) 38 | -------------------------------------------------------------------------------- /tile_generator/templates/jobs/docker-bosh.sh.erb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e # exit immediately if a simple command exits with a non-zero status 4 | set -u # report the usage of uninitialized variables 5 | 6 | # Setup env vars and folders 7 | RUN_DIR=/var/vcap/sys/run/{{ job_name }} 8 | LOG_DIR=/var/vcap/sys/log/{{ job_name }} 9 | PACKAGES_DIR=/var/vcap/packages/{{package.name}} 10 | 11 | PIDFILE=$RUN_DIR/{{ job_name }}.pid 12 | 13 | source /var/vcap/packages/common/utils.sh 14 | 15 | case $1 in 16 | 17 | start) 18 | pid_guard $PIDFILE {{ job_name }} 19 | 20 | mkdir -p $RUN_DIR 21 | mkdir -p $LOG_DIR 22 | 23 | chown vcap:vcap $RUN_DIR 24 | chown -R vcap:vcap $LOG_DIR 25 | 26 | echo $$ > $PIDFILE 27 | chown vcap:vcap $PIDFILE 28 | 29 | mkdir -p /var/vcap/data/certs 30 | echo "<%=p('tls_cacert')%>" > /var/vcap/data/certs/opsman-ca.pem 31 | 32 | for image in $PACKAGES_DIR/*.tgz; do 33 | /var/vcap/packages/docker/bin/docker \ 34 | --host unix:///var/vcap/sys/run/docker/docker.sock \ 35 | load -i $image \ 36 | >>$LOG_DIR/{{ job_name }}.stdout.log \ 37 | 2>>$LOG_DIR/{{ job_name }}.stderr.log 38 | done 39 | 40 | # do nothing forever 41 | exec tail -f /dev/null 42 | ;; 43 | 44 | stop) 45 | kill_and_wait $PIDFILE 46 | 47 | ;; 48 | *) 49 | echo "Usage: {{ job_name }}_ctl {start|stop}" 50 | 51 | ;; 52 | 53 | esac 54 | exit 0 55 | -------------------------------------------------------------------------------- /sample/src/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | BUILD_DIR="$1" 20 | MY_DIR="$( cd "$( dirname "$0" )" && pwd )" 21 | SAMPLE_DIR="$( cd "${MY_DIR}/.." && pwd )" 22 | 23 | if [ -n "${BUILD_DIR}" ]; then 24 | cp -r "${SAMPLE_DIR}"/* "${BUILD_DIR}" 25 | SAMPLE_DIR="${BUILD_DIR}" 26 | fi 27 | 28 | SRC_DIR="$( cd "${SAMPLE_DIR}/src" && pwd )" 29 | APP_DIR="$( cd "${SRC_DIR}/app" && pwd )" 30 | BUILDPACK_DIR="$( cd "${SRC_DIR}/buildpack" && pwd )" 31 | RESOURCES_DIR="$( cd "${SAMPLE_DIR}/resources" && pwd )" 32 | 33 | cd "${APP_DIR}" 34 | mkdir -p vendor 35 | pip download --no-binary :all: --dest vendor -r requirements.txt 36 | zip -r "${RESOURCES_DIR}/app.zip" * 37 | 38 | cd "${BUILDPACK_DIR}" 39 | zip -r "${RESOURCES_DIR}/buildpack.zip" bin/detect bin/compile bin/release -------------------------------------------------------------------------------- /ci/scripts/setup-venv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | _VENV="/tmp/tile-generator-env" 20 | 21 | if [ -z "${REPO_DIR}" ]; then 22 | echo "REPO_DIR env variable is not set. Needs to point to tile-generator repo dir." 23 | exit 1 24 | fi 25 | 26 | deactivate >/dev/null 2>&1 || echo '' 27 | 28 | # If there is no venv dir or anything passed to this script 29 | if [ ! -d "${_VENV}" ] || [ "$1" = "recreate" ]; then 30 | rm -rf "${_VENV}" 31 | echo "Creating a new virtual environment..." 32 | virtualenv -p python3 "${_VENV}" 33 | . "${_VENV}/bin/activate" 34 | cd "${REPO_DIR}" 35 | pwd 36 | pip install -e . 37 | echo 'sdf' 38 | else 39 | echo "Using virtual environment located at ${_VENV}..." 40 | fi 41 | 42 | . "$_VENV/bin/activate" 43 | command -v tile >/dev/null 2>&1 || { echo "Command 'tile' not found. Virtualenv at ${_VENV} must be corrupt." >&2; exit 1; } 44 | -------------------------------------------------------------------------------- /tile_generator/templates/jobs/delete-charts.sh.erb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # TBD - Delete all helm charts deployed by this release 4 | 5 | set -e 6 | set -x 7 | 8 | export PATH="/var/vcap/packages/pks_cli:$PATH" 9 | export PATH="/var/vcap/packages/helm_cli/linux-amd64:$PATH" 10 | export PATH="/var/vcap/packages/kubectl_cli:$PATH" 11 | 12 | export PKS_HOST=$(grep \\.pivotal-container-service\\.infrastructure\\.pivotal-container-service-.\*\\.bosh /etc/hosts | cut -d ' ' -f 1) 13 | export PKS_USERNAME=<%= Shellwords.escape(p('pks_username')) %> 14 | export PKS_PASSWORD=<%= Shellwords.escape(p('pks_password')) %> 15 | export PKS_CLUSTER_NAME=<%= Shellwords.escape(p('pks_cluster')) %> 16 | 17 | export BOSH_DEPLOYMENT=<%= spec.deployment %> 18 | 19 | # Use PKS to connect kubectl to the named cluster 20 | # 21 | pks login --api "$PKS_HOST" --username "$PKS_USERNAME" --password "$PKS_PASSWORD" --skip-ssl-verification # FIXME --ca-cert /path/to/cert 22 | pks get-credentials "$PKS_CLUSTER_NAME" 23 | 24 | # Check if tiller is there. If not, there's nothing to do 25 | # 26 | if ! helm list 2>/dev/null; then 27 | echo "tiller is not installed in this cluster, nothing to remove" 28 | exit 0 29 | fi 30 | 31 | # We do not remove tiller itself, as it may be shared with other tiles 32 | 33 | # FIXME Remove any cached docker images that were pushed by this release 34 | 35 | # FIXME - This only works for a single helm chart 36 | HELM_NAME="${BOSH_DEPLOYMENT}" 37 | HELM_RELEASE=$(helm list "^${HELM_NAME}\$") 38 | if [ -z "$HELM_RELEASE" ]; then 39 | echo "${HELM_NAME} is not deployed" 40 | else 41 | helm delete "${HELM_NAME}" --purge 42 | fi 43 | -------------------------------------------------------------------------------- /ci/scripts/pcf-backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | POOL_DIR="$( cd "$1" && pwd )" 20 | BACKUP_DIR="$( cd "$2" && pwd )" 21 | 22 | MY_DIR="$( cd "$( dirname "$0" )" && pwd )" 23 | REPO_DIR="$( cd "${MY_DIR}/../.." && pwd )" 24 | BASE_DIR="$( cd "${REPO_DIR}/.." && pwd )" 25 | 26 | PCF="pcf" 27 | 28 | PCF_NAME=`cat "${POOL_DIR}/name"` 29 | if [ -z "${PCF_NAME}" ]; then 30 | echo "No pcf environment has been claimed" 31 | exit 1 32 | fi 33 | 34 | FLAG_FILE="pcf-backup-${PCF_NAME}-0.0.0.yml" 35 | BACKUP_FILE="pcf-backup-${PCF_NAME}-0.0.1.yml" 36 | 37 | cd "${POOL_DIR}" 38 | 39 | if [ "$3" -eq "init" ]; then 40 | echo "Initializing ${PCF_NAME} for backup" 41 | touch "${BACKUP_DIR}/${FLAG_FILE}" 42 | echo 43 | exit 0 44 | fi 45 | 46 | if [ -f "${BACKUP_DIR}/${BACKUP_FILE}" ]; then 47 | echo "Backup ${BACKUP_FILE} already exists" 48 | echo 49 | exit 0 50 | fi 51 | 52 | echo "Backing up to ${BACKUP_FILE}" 53 | $PCF backup "${BACKUP_DIR}/${BACKUP_FILE}" 54 | echo 55 | -------------------------------------------------------------------------------- /ci/scripts/tile-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | SOURCE_DIR=$1 20 | HISTORY_DIR=$2 21 | TARGET_DIR=$3 22 | DOCKER_DIR=$4 23 | VERSION_DIR=$5 24 | 25 | MY_DIR="$( cd "$( dirname "$0" )" && pwd )" 26 | REPO_DIR="$( cd "${MY_DIR}/../.." && pwd )" 27 | BASE_DIR="$( cd "${REPO_DIR}/.." && pwd )" 28 | 29 | TILE="tile" 30 | 31 | cd ${BASE_DIR} 32 | 33 | HISTORY="${HISTORY_DIR}/tile-history-*.yml" 34 | if [ -e "${HISTORY}" ]; then 35 | cp ${HISTORY} ${SOURCE_DIR}/tile-history.yml 36 | fi 37 | 38 | if [ -n "${DOCKER_DIR}" ]; then 39 | DOCKER_DIR="--cache $BASE_DIR/$DOCKER_DIR" 40 | fi 41 | 42 | VERSION="" 43 | if [ -n "${VERSION_DIR}" ]; then 44 | VERSION=`cat ${VERSION_DIR}/version` 45 | fi 46 | 47 | (cd ${SOURCE_DIR}; $TILE build $DOCKER_DIR $VERSION) 48 | 49 | VERSION=`grep '^version:' ${SOURCE_DIR}/tile-history.yml | sed 's/^version: //'` 50 | HISTORY="tile-history-${VERSION}.yml" 51 | 52 | cp ${SOURCE_DIR}/product/*.pivotal ${TARGET_DIR} 53 | cp ${SOURCE_DIR}/tile-history.yml ${TARGET_DIR}/${HISTORY} 54 | -------------------------------------------------------------------------------- /sample/missing-properties.yml: -------------------------------------------------------------------------------- 1 | --- 2 | customer_name: Jimmy's Johnnys 3 | street_address: Cartaway Alley 4 | city: New Jersey 5 | country: country_us 6 | username: SpongeBob 7 | password: { 'secret': SquarePants } 8 | my_creds: 9 | identity: username 10 | password: p4ssw0rd 11 | example_collection: 12 | - album: Chronicles 13 | artist: Rush 14 | explicit: false 15 | - album: Greatest Hits 16 | artist: Queen 17 | explicit: true 18 | example_selector: Filet Mignon 19 | example_selector.filet_mignon_option.rarity_dropdown: medium 20 | example_selector.filet_mignon_option.steak_creds: 21 | identity: steakeater 22 | password: deadbeef 23 | custom_dynamic_service_plan_1: 24 | - name: plan3 25 | guid: plan3-guid 26 | description1: Plan for license key 3 27 | license_key1: 001002003004 28 | num_seats1: 30 29 | - name: plan4 30 | guid: plan4-guid 31 | description1: Plan for license key 4 32 | license_key1: 005006007008 33 | num_seats1: 40 34 | custom_dynamic_service_plan_2: 35 | - name: plan5 36 | guid: plan5-guid 37 | description2: Plan for license key 5 38 | api_key2: api123 39 | num_of_license2: 50 40 | - name: plan6 41 | guid: plan6-guid 42 | description2: Plan for license key 6 43 | api_key2: api456 44 | num_of_license2: 60 45 | canary_watch_timeout: 100001 46 | update_watch_timeout: 100002 47 | is_feature_enabled: feature_disabled 48 | planets: 49 | - Earth 50 | 51 | # Example of job-specific configuration, supported in PCF 1.8+. 52 | jobs: 53 | redis: 54 | # Resource configuration is passed in like this: 55 | resource_config: 56 | persistent_disk: 57 | size_mb: "10240" 58 | # Job-specific properties can be passed in here as well, like this: 59 | # property_name: property_value 60 | -------------------------------------------------------------------------------- /tile_generator/templates/tile/metadata.yml: -------------------------------------------------------------------------------- 1 | --- 2 | {{ tile_metadata.base | yaml }} 3 | 4 | releases: 5 | {% for release in releases.values() %} 6 | {% if release.file %} 7 | - file: {{ release.file }} 8 | name: {{ release.release_name }} 9 | version: '{{ release.version }}' 10 | {% endif %} 11 | {% endfor %} 12 | 13 | {{ tile_metadata.stemcell_criteria | yaml }} 14 | 15 | property_blueprints: 16 | {{ tile_metadata.property_blueprints | yaml }} 17 | 18 | form_types: {% if not tile_metadata.form_types %}[]{% else %} 19 | 20 | {{ tile_metadata.form_types | yaml }} 21 | {% endif %} 22 | 23 | job_types: {% if not tile_metadata.job_types %}[]{% else %} 24 | 25 | {{ tile_metadata.job_types | yaml }} 26 | {% endif %} 27 | 28 | {% if tile_metadata.runtime_configs %} 29 | runtime_configs: 30 | {% for cfg in tile_metadata.runtime_configs %} 31 | - name: {{ cfg.name }} 32 | runtime_config: {{ cfg.runtime_config | yaml_literal | indent }} 33 | {% endfor %} 34 | {% endif %} 35 | 36 | post_deploy_errands: {% if not tile_metadata.post_deploy_errands %}[]{% else %} 37 | 38 | {{ tile_metadata.post_deploy_errands | yaml }} 39 | {% endif %} 40 | 41 | pre_delete_errands: {% if not tile_metadata.pre_delete_errands %}[]{% else %} 42 | 43 | {{ tile_metadata.pre_delete_errands | yaml }} 44 | {% endif %} 45 | 46 | update: 47 | canaries: {{ ( update and update.canaries ) or 1 }} 48 | canary_watch_time: {{ ( update and update.canary_watch_time ) or '10000-100000' }} 49 | max_in_flight: {{ ( update and update.max_in_flight ) or 1 }} 50 | update_watch_time: {{ ( update and update.update_watch_time ) or '10000-100000' }} 51 | 52 | provides_product_versions: 53 | - name: {{ tile_metadata.base.name }} 54 | version: '{{ tile_metadata.base.product_version }}' 55 | -------------------------------------------------------------------------------- /scripts/tile_generator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | set -e 20 | 21 | SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" 22 | VENV="$SCRIPT_DIR/tile-generator-env" 23 | 24 | function show_help { 25 | echo "Usage: `basename "$0"` [OPTIONS] COMMAND [ARGS]... 26 | 27 | Options: 28 | --help Show this message and exit. 29 | --clear Clear out the virtual environment and start from scratch. 30 | 31 | Commands: 32 | tile 33 | pcf 34 | " 35 | } 36 | 37 | function create_venv { 38 | deactivate >/dev/null 2>&1 || echo '' 39 | rm -rf $VENV 40 | echo "Creating a new virtual environment..." 41 | virtualenv -q -p python3 $VENV 42 | source $VENV/bin/activate 43 | pip install tile-generator 44 | source $VENV/bin/activate 45 | } 46 | 47 | if [ ! -d $VENV ]; then 48 | create_venv 49 | fi 50 | 51 | if [ ! -n "$1" ]; then 52 | show_help 53 | fi 54 | 55 | while [ -n "$1" ]; do 56 | case "$1" in 57 | -h) show_help ;; 58 | --help) show_help ;; 59 | --clear) create_venv ;; 60 | tile) shift; source $VENV/bin/activate; tile "$@"; break ;; 61 | pcf) shift; source $VENV/bin/activate; pcf "$@"; break ;; 62 | *) echo "Command or option not recognized '$1'"; show_help ;; 63 | esac 64 | shift 65 | done 66 | 67 | -------------------------------------------------------------------------------- /ci/deployment-tests/runtime_config_deploymenttest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import re 18 | import unittest 19 | 20 | import requests 21 | 22 | from tile_generator import opsmgr 23 | 24 | 25 | class VerifyRuntimeConfig(unittest.TestCase): 26 | def setUp(self): 27 | self.cfinfo = opsmgr.get_cfinfo() 28 | self.schema_version = self.cfinfo['schema_version'] 29 | 30 | def test_yes(self): 31 | if re.match(r'1\.+', self.schema_version): 32 | print("OpsMan is too old to support runtime config, skipping test") 33 | return 34 | 35 | hostname = 'runtime-conf-yes.' + self.cfinfo['system_domain'] 36 | url = 'http://' + hostname 37 | response = requests.get(url) 38 | 39 | self.assertEqual(response.status_code, 200) 40 | self.assertRegex(response.text, r'.*Runtime Test Release: Success.*') 41 | 42 | def test_no(self): 43 | if re.match(r'1\.+', self.schema_version): 44 | print("OpsMan is too old to support runtime config, skipping test") 45 | return 46 | 47 | hostname = 'runtime-conf-no.' + self.cfinfo['system_domain'] 48 | url = 'http://' + hostname 49 | response = requests.get(url) 50 | 51 | self.assertEqual(response.status_code, 502) 52 | -------------------------------------------------------------------------------- /ci/scripts/run-acceptancetests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | TILE_DIR=$1 20 | 21 | # Cleanup previous run 22 | rm -rf ${TILE_DIR}/product 23 | rm -rf ${TILE_DIR}/release 24 | 25 | MY_DIR="$( cd "$( dirname "$0" )" && pwd )" 26 | REPO_DIR="$( cd "${MY_DIR}/../.." && pwd )" 27 | BASE_DIR="$( cd "${REPO_DIR}/.." && pwd )" 28 | TEST_DIR="$( cd "${REPO_DIR}/ci/acceptance-tests" && pwd )" 29 | 30 | . "${REPO_DIR}/ci/scripts/setup-venv.sh" 31 | 32 | cd ${TILE_DIR} 33 | 34 | TILE_FILE=`ls *.pivotal` 35 | if [ -z "${TILE_FILE}" ]; then 36 | echo "No match for *.pivotal" 37 | exit 1 38 | fi 39 | 40 | mkdir product && ( cd product && unzip "../${TILE_FILE}" ) 41 | 42 | BOSH_FILE=`ls product/releases/test-tile-*.tgz` 43 | if [ -z "${BOSH_FILE}" ]; then 44 | echo "No match for releases/test-tile-*.tgz" 45 | exit 1 46 | fi 47 | 48 | mkdir release && ( cd release && gunzip -c "../${BOSH_FILE}" | tar xf - ) 49 | 50 | for TGZ_FILE in release/*/*.tgz 51 | do 52 | TGZ_DIR=`echo "${TGZ_FILE}" | sed "s/\.tgz\$//"` 53 | mkdir -p "${TGZ_DIR}" 54 | ( cd "${TGZ_DIR}"; gunzip -c "../../../${TGZ_FILE}" | tar xf - ) 55 | # Remove tar file so we don't get double counts in tests 56 | rm "${TGZ_FILE}" 57 | done 58 | 59 | python -m unittest discover -v -s ${TEST_DIR} -p '*_acceptancetest.py' 60 | -------------------------------------------------------------------------------- /tile_generator/templates/jobs/opsmgr.env.erb: -------------------------------------------------------------------------------- 1 | SCHEME=https 2 | ADMIN_USER=<%= properties.cf.admin_user %> 3 | ADMIN_PASSWORD=<%= properties.cf.admin_password %> 4 | DOMAIN=<%= properties.domain %> 5 | APP_DOMAIN=<%= properties.app_domains[0] %> 6 | CF_ORG=<%= properties.org %> 7 | CF_SPACE=<%= properties.space %> 8 | CF_TARGET=https://api.<%= properties.app_domains[0] %> 9 | CF_SKIP_SSL=<%= properties.ssl.skip_cert_verify %> 10 | SECURITY_USER_NAME=<%= properties.security.user %> 11 | SECURITY_USER_PASSWORD=<%= properties.security.password %> 12 | APPLY_OPEN_SECURITY_GROUP=<%= properties.apply_open_security_group %> 13 | ALLOW_PAID_SERVICE_PLANS=<%= properties.allow_paid_service_plans %> 14 | UAA_ADMIN_CLIENT=<%= properties.uaa.admin_client %> 15 | UAA_ADMIN_CLIENT_SECRET=<%= properties.uaa.admin_client_secret %> 16 | 17 | {% for service_plan_form in context.service_plan_forms %} 18 | {{ service_plan_form | plans_json(false, false) }} 19 | {% endfor %} 20 | 21 | {% for property in context.all_properties %} 22 | {{ property | env_variable(false, false) }} 23 | {% endfor %} 24 | 25 | <% empty_dict = {} %> 26 | <% empty_list = [] %> 27 | {% for release in context.releases.values() if release.consumes %} 28 | {% for link_name, link in release.consumes.items() %} 29 | <% if_link('{{ link_name }}') do |link| %> 30 | <% hosts = link.instances.map { |instance| instance.address } %> 31 | {{ link_name | shell_variable_name }}_HOST=<%= link.instances.empty? ? "" : link.instances[0].address %> 32 | {{ link_name | shell_variable_name }}_HOSTS=<%= Shellwords.escape(hosts.to_json) %> 33 | {{ link_name | shell_variable_name }}_PROPERTIES=<% Shellwords.escape(link.properties.to_json) %> 34 | <% end.else do %> 35 | {{ link_name | shell_variable_name }}_HOST=<%= Shellwords.escape(empty_list.to_json) %> 36 | {{ link_name | shell_variable_name }}_HOSTS=<%= Shellwords.escape(empty_list.to_json) %> 37 | {{ link_name | shell_variable_name }}_PROPERTIES=<% Shellwords.escape(empty_dict.to_json) %> 38 | <% end %> 39 | {% endfor %} 40 | {% endfor %} 41 | -------------------------------------------------------------------------------- /scripts/run_local_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | REPO_DIR="$( cd "$( dirname "$0" )/.." && pwd )" 20 | LOCAL_TEST_DIR="${REPO_DIR}/temp_local_test" 21 | 22 | "${REPO_DIR}/ci/scripts/run-unittests.sh" 23 | 24 | rm -rf "${LOCAL_TEST_DIR}" 25 | mkdir "${LOCAL_TEST_DIR}" 26 | 27 | . "${REPO_DIR}/ci/scripts/setup-venv.sh" 28 | "${REPO_DIR}/sample/src/build.sh" 29 | 30 | if [ "$1" = "withcache" ]; then 31 | echo "########################################################" 32 | echo "# Building a new .pivotal file using cache option. #" 33 | echo "########################################################" 34 | pushd "${REPO_DIR}/sample" 35 | mkdir -p cache 36 | tile build --cache cache 37 | cp product/*.pivotal "${LOCAL_TEST_DIR}" 38 | cp tile-history*.yml "${LOCAL_TEST_DIR}" 39 | popd 40 | else 41 | echo "#############################################################" 42 | echo "# Creating a new .pivotal file. Use 'withcache' argument to #" 43 | echo "# build using the cache option for 'tile build'. #" 44 | echo "#############################################################" 45 | "${REPO_DIR}"/ci/scripts/tile-build.sh "${REPO_DIR}"/sample "${REPO_DIR}"/sample "${LOCAL_TEST_DIR}" 46 | fi 47 | 48 | "${REPO_DIR}"/ci/scripts/run-acceptancetests.sh "${LOCAL_TEST_DIR}" 49 | 50 | rm -rf "${LOCAL_TEST_DIR}" 51 | -------------------------------------------------------------------------------- /tile_generator/tile_unittest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import unittest 19 | from click.testing import CliRunner 20 | import os 21 | import shutil 22 | import tempfile 23 | from . import tile 24 | 25 | class TestTileInit(unittest.TestCase): 26 | # tile.init() changes the working directory. In normal usage, 27 | # this is fine, but for unit tests this changes the local 28 | # state, so we should restore it. 29 | def setUp(self): 30 | self.cwd = os.getcwd() 31 | 32 | def tearDown(self): 33 | os.chdir(self.cwd) 34 | 35 | def test_tile_init_works(self): 36 | tmpdir = tempfile.mkdtemp() 37 | try: 38 | tiledir = os.path.join(tmpdir, 'my-tile') 39 | runner = CliRunner() 40 | result = runner.invoke(tile.init_cmd, [tiledir]) 41 | self.assertEqual(result.exit_code, 0) 42 | self.assertTrue(os.path.isfile(os.path.join(tiledir, 'tile.yml'))) 43 | finally: 44 | shutil.rmtree(tmpdir) 45 | 46 | # Encountered unicode in `pcf apply-changes` output. Stashing 47 | # the next two tests to document the problem and solution, as 48 | # we may encounter this pattern elsewhere. 49 | ## This passes (i.e., raises error as expected) in my 50 | ## terminal, but not in CI pipeline. 51 | # def test_print_unicode_works(self): 52 | # with self.assertRaises(UnicodeEncodeError): 53 | # print(u'foo\xb1bar') 54 | 55 | def test_print_utf8_works(self): 56 | print('foo\xb1bar'.encode('utf-8')) 57 | self.assertTrue(True) 58 | 59 | if __name__ == '__main__': 60 | unittest.main() 61 | -------------------------------------------------------------------------------- /ci/scripts/pcf-restore.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | TILE_DIR="$( cd "$1" && pwd )" 20 | POOL_DIR="$( cd "$2" && pwd )" 21 | BACKUP_DIR="$( cd "$3" && pwd )" 22 | 23 | MY_DIR="$( cd "$( dirname "$0" )" && pwd )" 24 | REPO_DIR="$( cd "${MY_DIR}/../.." && pwd )" 25 | BASE_DIR="$( cd "${REPO_DIR}/.." && pwd )" 26 | 27 | PCF="pcf" 28 | 29 | PCF_NAME=`cat "${POOL_DIR}/name"` 30 | if [ -z "${PCF_NAME}" ]; then 31 | echo "No pcf environment has been claimed" 32 | exit 1 33 | fi 34 | 35 | BACKUP_FILE="pcf-backup-${PCF_NAME}-0.0.1.yml" 36 | 37 | TILE_FILE=`cd "${TILE_DIR}"; ls *.pivotal` 38 | if [ -z "${TILE_FILE}" ]; then 39 | echo "No files matching ${TILE_DIR}/*.pivotal" 40 | ls -lR "${TILE_DIR}" 41 | exit 1 42 | fi 43 | 44 | PRODUCT=`echo "${TILE_FILE}" | sed "s/-[^-]*$//"` 45 | VERSION=`echo "${TILE_FILE}" | sed "s/.*-//" | sed "s/\.pivotal\$//"` 46 | 47 | cd "${POOL_DIR}" 48 | 49 | echo "Available products:" 50 | $PCF products 51 | echo 52 | 53 | if $PCF is-available "${PRODUCT}" ; then 54 | echo "Deleting unused products" 55 | $PCF delete-unused-products 56 | echo 57 | 58 | echo "Available products:" 59 | $PCF products 60 | echo 61 | fi 62 | 63 | if ! $PCF is-installed "${PRODUCT}" ; then 64 | echo "It appears that ${PRODUCT} was successfully removed - skipping restore" 65 | echo 66 | exit 0 67 | fi 68 | 69 | echo "Restoring from ${BACKUP_FILE}" 70 | $PCF restore "${BACKUP_DIR}/${BACKUP_FILE}" 71 | echo 72 | 73 | echo "Available products:" 74 | $PCF products 75 | echo 76 | -------------------------------------------------------------------------------- /tile_generator/template_unittest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import unittest 19 | from . import template 20 | 21 | class TestPropertyFilter(unittest.TestCase): 22 | def test_property_filter_simple_property(self): 23 | property = { 24 | 'name': 'prop', 25 | 'type': 'string', 26 | } 27 | result = template.render_property(property) 28 | self.assertEqual(result, {'prop': '(( .properties.prop.value ))' }) 29 | 30 | def test_property_filter_complex_property(self): 31 | property = { 32 | 'name': 'prop', 33 | 'type': 'simple_credentials', 34 | } 35 | result = template.render_property(property) 36 | self.assertEqual(result, { 37 | 'prop': { 38 | 'identity': '(( .properties.prop.identity ))', 39 | 'password': '(( .properties.prop.password ))', 40 | } 41 | }) 42 | 43 | class TestVariableNameFilter(unittest.TestCase): 44 | def test_no_change(self): 45 | self.assertEqual(template.render_shell_variable_name('FOO'), 'FOO') 46 | self.assertEqual(template.render_shell_variable_name('F00'), 'F00') 47 | self.assertEqual(template.render_shell_variable_name('FOO_BAR'), 'FOO_BAR') 48 | 49 | def test_hyphen_to_underscore(self): 50 | self.assertEqual(template.render_shell_variable_name('FOO-BAR'), 'FOO_BAR') 51 | self.assertEqual(template.render_shell_variable_name('FOO--BAR'), 'FOO_BAR') 52 | 53 | def test_uppercases_letters(self): 54 | self.assertEqual(template.render_shell_variable_name('foo'), 'FOO') 55 | self.assertEqual(template.render_shell_variable_name('Foo'), 'FOO') 56 | -------------------------------------------------------------------------------- /tile_generator/helm.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import yaml 4 | import json 5 | 6 | import requests 7 | 8 | def find_required_images(values): 9 | images = [] 10 | values = values is not None and { k.lower():v for k,v in values.items() } or {} 11 | for key, value in values.items(): 12 | if key in [ 'image', 'repository' ]: 13 | if isinstance(value, dict): 14 | image = value.get('repository', value.get('name', value.get('image'))) 15 | tag = value.get('tag', value.get('imagetag')) 16 | if image is None: 17 | if tag is None: 18 | images += find_required_images(value) 19 | continue 20 | image = tag 21 | tag = None 22 | else: 23 | image = value 24 | tag = values.get('tag', values.get('imagetag')) 25 | if tag is not None: 26 | image += ':' + str(tag) 27 | images += [ image ] 28 | else: 29 | if isinstance(value, dict): 30 | images += find_required_images(value) 31 | return images 32 | 33 | def get_chart_info(chart_dir): 34 | chart_file = os.path.join(chart_dir, 'Chart.yaml') 35 | with open(chart_file) as f: 36 | chart = yaml.safe_load(f) 37 | values_file = os.path.join(chart_dir, 'values.yaml') 38 | with open(values_file) as f: 39 | chart_values = yaml.safe_load(f) 40 | 41 | return { 42 | 'name': chart.get('name', chart.get('Name')), 43 | 'version': chart.get('version', chart.get('Version')), 44 | 'required_images': find_required_images(chart_values), 45 | } 46 | 47 | def get_latest_release_tag(): 48 | result = requests.get('https://api.github.com/repos/kubernetes/helm/releases/latest') 49 | result.raise_for_status() 50 | release = result.json() 51 | return release['tag_name'] 52 | 53 | def get_latest_kubectl_tag(): 54 | result = requests.get('https://storage.googleapis.com/kubernetes-release/release/stable.txt') 55 | result.raise_for_status() 56 | release = result.text.strip() 57 | return release 58 | 59 | if __name__ == '__main__': 60 | for chart in sys.argv[1:]: 61 | print(json.dumps(get_chart_info(chart), indent=4)) 62 | -------------------------------------------------------------------------------- /ci/scripts/run-removetile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | TILE_DIR="$( cd "$1" && pwd )" 20 | POOL_DIR="$( cd "$2" && pwd )" 21 | 22 | MY_DIR="$( cd "$( dirname "$0" )" && pwd )" 23 | REPO_DIR="$( cd "${MY_DIR}/../.." && pwd )" 24 | BASE_DIR="$( cd "${REPO_DIR}/.." && pwd )" 25 | 26 | PCF="pcf" 27 | 28 | TILE_FILE=`cd "${TILE_DIR}"; ls *.pivotal` 29 | if [ -z "${TILE_FILE}" ]; then 30 | echo "No files matching ${TILE_DIR}/*.pivotal" 31 | ls -lR "${TILE_DIR}" 32 | exit 1 33 | fi 34 | 35 | PRODUCT=`echo "${TILE_FILE}" | sed "s/-[0-9].*$//"` 36 | VERSION=`echo "${TILE_FILE}" | sed "s/${PRODUCT}-//" | sed "s/\.pivotal\$//"` 37 | 38 | cd "${POOL_DIR}" 39 | 40 | echo "Available products:" 41 | $PCF products 42 | echo 43 | 44 | if ! $PCF is-installed "${PRODUCT}" ; then 45 | echo "${PRODUCT} not installed - skipping removal" 46 | exit 0 47 | fi 48 | 49 | echo "Uninstalling ${PRODUCT}" 50 | $PCF uninstall "${PRODUCT}" 51 | echo 52 | 53 | echo "Applying Changes" 54 | $PCF apply-changes --product "${PRODUCT}" --delete-errands delete-all,delete-meta-buildpack --deploy-errands '' 55 | echo 56 | 57 | echo "Available products:" 58 | $PCF products 59 | echo 60 | 61 | if $PCF is-installed "${PRODUCT}" ; then 62 | echo "${PRODUCT} remains installed - remove failed" 63 | exit 1 64 | fi 65 | 66 | exit 0 # We do not want to remove unused products other than our own 67 | 68 | # echo "Deleting unused products" 69 | # $PCF delete-unused-products 70 | # echo 71 | 72 | # echo "Available products:" 73 | # $PCF products 74 | # echo 75 | 76 | # if $PCF is-available "${PRODUCT}" ; then 77 | # echo "${PRODUCT} remains available - remove failed" 78 | # exit 1 79 | # fi 80 | -------------------------------------------------------------------------------- /pyinstaller/build-binaries.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | set -e 20 | 21 | SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" 22 | VENV="$SCRIPT_DIR/build-env" 23 | 24 | function show_help { 25 | echo "Usage: `basename "$0"` [OPTIONS] 26 | 27 | Options: 28 | --help Show this message and exit. 29 | --recreate-venv Recreate the virtual environment and start from scratch. 30 | --statix Use staticx to ship a fully static package. 31 | " 32 | } 33 | 34 | function create_venv { 35 | deactivate >/dev/null 2>&1 || echo '' 36 | rm -rf $VENV 37 | echo "Creating a new virtual environment..." 38 | virtualenv -q -p python3 $VENV 39 | source $VENV/bin/activate 40 | # Build for current project. Assumes tile-generator src is up a dir 41 | pip install -e $SCRIPT_DIR/../ 42 | #https://github.com/pypa/pip/issues/6163#issuecomment-456772043 43 | pip install pyinstaller --no-use-pep517 44 | pip install staticx 45 | } 46 | 47 | function create_binaries { 48 | source $VENV/bin/activate 49 | pyinstaller -y $SCRIPT_DIR/pcf.spec 50 | pyinstaller -y $SCRIPT_DIR/tile.spec 51 | } 52 | 53 | function statix_all { 54 | create_binaries 55 | staticx dist/tile* dist/tile* 56 | staticx dist/pcf* dist/pcf* 57 | } 58 | 59 | if [ ! -d $VENV ]; then 60 | create_venv 61 | fi 62 | 63 | if [ ! -n "$1" ]; then 64 | # No options passed. Do the build. 65 | create_binaries 66 | fi 67 | 68 | while [ -n "$1" ]; do 69 | case "$1" in 70 | -h) show_help ;; 71 | --help) show_help ;; 72 | --recreate-venv) create_venv ;; 73 | --staticx) statix_all ;; 74 | *) echo "Option not recognized '$1'"; show_help ;; 75 | esac 76 | shift 77 | done 78 | 79 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | import os 21 | from setuptools import setup 22 | from tile_generator.version import version_string 23 | 24 | # read the contents of your README file 25 | from os import path 26 | this_directory = path.abspath(path.dirname(__file__)) 27 | with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: 28 | long_description = f.read() 29 | 30 | def get_version(): 31 | return version_string 32 | 33 | 34 | with open('requirements.txt') as f: 35 | requirements = f.read().splitlines() 36 | 37 | setup( 38 | name = "tile-generator", 39 | version = get_version(), 40 | description = 'Tools supporting development of Pivotal Cloud Foundry services and add-ons.', 41 | long_description=long_description, 42 | long_description_content_type='text/markdown', 43 | url = 'https://github.com/cf-platform-eng/tile-generator', 44 | author = 'Pivotal Cloud Foundry Platform Engineering', 45 | license = 'Apache 2.0', 46 | classifiers = [ 47 | 'Development Status :: 4 - Beta', 48 | 'Environment :: Console', 49 | 'Intended Audience :: Developers', 50 | 'License :: OSI Approved :: Apache Software License', 51 | 'Programming Language :: Python :: 3 :: Only', 52 | 'Topic :: Software Development', 53 | 'Topic :: Software Development :: Code Generators', 54 | ], 55 | keywords = [ 56 | 'pivotal cloud foundry', 57 | 'tile', 58 | 'generator' 59 | ], 60 | packages = [ 'tile_generator' ], 61 | install_requires=requirements, 62 | include_package_data = True, 63 | entry_points = { 64 | 'console_scripts': [ 65 | 'tile = tile_generator.tile:main', 66 | 'pcf = tile_generator.pcf:main', 67 | ] 68 | } 69 | ) 70 | -------------------------------------------------------------------------------- /ci/scripts/run-deploytile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | TILE_DIR="$( cd "$1" && pwd )" 20 | POOL_DIR="$( cd "$2" && pwd )" 21 | 22 | MY_DIR="$( cd "$( dirname "$0" )" && pwd )" 23 | REPO_DIR="$( cd "${MY_DIR}/../.." && pwd )" 24 | BASE_DIR="$( cd "${REPO_DIR}/.." && pwd )" 25 | 26 | PCF="pcf" 27 | 28 | NETWORK="$(jq -r .ert_subnet $POOL_DIR/metadata)" 29 | if [ -n "$NETWORK" ]; then 30 | NETWORK="--network $NETWORK" 31 | fi 32 | 33 | TILE_FILE=`cd "${TILE_DIR}"; ls *.pivotal` 34 | if [ -z "${TILE_FILE}" ]; then 35 | echo "No files matching ${TILE_DIR}/*.pivotal" 36 | ls -lR "${TILE_DIR}" 37 | exit 1 38 | fi 39 | 40 | PRODUCT=`echo "${TILE_FILE}" | sed "s/-[0-9].*$//"` 41 | VERSION=`echo "${TILE_FILE}" | sed "s/${PRODUCT}-//" | sed "s/\.pivotal\$//"` 42 | 43 | cd "${POOL_DIR}" 44 | 45 | echo "Enable diego_docker:" 46 | $PCF target -o system -s system 47 | cf enable-feature-flag diego_docker 48 | 49 | echo "Available products:" 50 | $PCF products 51 | echo 52 | 53 | echo "Uploading ${TILE_FILE}" 54 | $PCF import "${TILE_DIR}/${TILE_FILE}" 55 | echo 56 | 57 | echo "Available products:" 58 | $PCF products 59 | $PCF is-available "${PRODUCT}" "${VERSION}" 60 | echo 61 | 62 | echo "Installing product ${PRODUCT} version ${VERSION}" 63 | $PCF install "${PRODUCT}" "${VERSION}" 64 | echo 65 | 66 | echo "Available products:" 67 | $PCF products 68 | $PCF is-installed "${PRODUCT}" "${VERSION}" 69 | echo 70 | 71 | echo "Configuring product ${PRODUCT}" 72 | $PCF configure --skip-validation "${PRODUCT}" ${NETWORK} "${REPO_DIR}/sample/missing-properties.yml" 73 | echo 74 | 75 | echo "Applying Changes" 76 | $PCF apply-changes --product "${PRODUCT}" --deploy-errands deploy-meta-buildpack,deploy-all,acceptance-tests --delete-errands '' 77 | echo 78 | -------------------------------------------------------------------------------- /tile_generator/templates/jobs/deploy-charts.sh.erb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | 6 | export PATH="/var/vcap/packages/pks_cli:$PATH" 7 | export PATH="/var/vcap/packages/helm_cli/linux-amd64:$PATH" 8 | export PATH="/var/vcap/packages/kubectl_cli:$PATH" 9 | 10 | export PKS_HOST=$(grep \\.pivotal-container-service\\.infrastructure\\.pivotal-container-service-.\*\\.bosh /etc/hosts | cut -d ' ' -f 1) 11 | export PKS_USERNAME=<%= Shellwords.escape(p('pks_username')) %> 12 | export PKS_PASSWORD=<%= Shellwords.escape(p('pks_password')) %> 13 | export PKS_CLUSTER_NAME=<%= Shellwords.escape(p('pks_cluster')) %> 14 | 15 | export BOSH_DEPLOYMENT=<%= spec.deployment %> 16 | 17 | # Use PKS to connect kubectl to the named cluster 18 | # 19 | pks login --api "$PKS_HOST" --username "$PKS_USERNAME" --password "$PKS_PASSWORD" --skip-ssl-verification # FIXME --ca-cert /path/to/cert 20 | pks get-credentials "$PKS_CLUSTER_NAME" 21 | 22 | # Install Tiller 23 | # 24 | helm init 25 | 26 | # Create a service account with the right permissions for tiller to do it's thing with RBAC enabled 27 | # 28 | kubectl create serviceaccount --namespace kube-system tiller || true 29 | kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller || true 30 | kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}' || true 31 | 32 | # Wait for tiller to become ready 33 | # 34 | echo -n "waiting for tiller" 35 | while ! helm list 2>/dev/null; do 36 | echo -n . 37 | sleep 1 38 | done 39 | echo 40 | 41 | # Maybe run a docker registry in the cluster to use? https://stackoverflow.com/a/42564191 42 | # Or use Harbor if it's there? 43 | for image in /var/vcap/packages/*-images/*; do 44 | echo "FIXME: use the packaged image $image" 45 | done 46 | 47 | # Unzip any zipped helm charts 48 | # 49 | for zip in /var/vcap/packages/*/*.zip; do 50 | if unzip -l ${zip} \*/Chart.yaml >/dev/null; then 51 | (cd $(dirname ${zip}); unzip ${zip}) 52 | fi 53 | done 54 | 55 | # Now install all helm charts 56 | # 57 | for chart in /var/vcap/packages/*/*/Chart.yaml; do 58 | # FIXME - This only works for a single helm chart 59 | HELM_NAME="${BOSH_DEPLOYMENT}" 60 | HELM_RELEASE=$(helm list "^${HELM_NAME}\$") 61 | if [ -z "$HELM_RELEASE" ]; then 62 | helm install -n "${HELM_NAME}" $(dirname $chart) 63 | else 64 | echo "${HELM_NAME} is already deployed" 65 | fi 66 | done 67 | -------------------------------------------------------------------------------- /tile_generator/templates/src/common/utils.sh: -------------------------------------------------------------------------------- 1 | 2 | mkdir -p /var/vcap/sys/log 3 | 4 | exec > >(tee -a >(logger -p user.info -t vcap.$(basename $0).stdout) | awk -W interactive '{ gsub(/\\n/, ""); system("echo -n [$(date +\"%Y-%m-%d %H:%M:%S%z\")]"); print " " $0 }' >>/var/vcap/sys/log/$(basename $0).log) 5 | exec 2> >(tee -a >(logger -p user.error -t vcap.$(basename $0).stderr) | awk -W interactive '{ gsub(/\\n/, ""); system("echo -n [$(date +\"%Y-%m-%d %H:%M:%S%z\")]"); print " " $0 }' >>/var/vcap/sys/log/$(basename $0).err.log) 6 | 7 | pid_guard() { 8 | echo "------------ STARTING `basename $0` at `date` --------------" | tee /dev/stderr 9 | pidfile=$1 10 | name=$2 11 | 12 | if [ -f "$pidfile" ]; then 13 | pid=$(head -1 "$pidfile") 14 | 15 | if [ -n "$pid" ] && [ -e /proc/$pid ]; then 16 | echo "$name is already running, please stop it first" 17 | exit 1 18 | fi 19 | 20 | echo "Removing stale pidfile..." 21 | rm $pidfile 22 | fi 23 | } 24 | 25 | wait_pidfile() { 26 | pidfile=$1 27 | try_kill=$2 28 | timeout=${3:-0} 29 | force=${4:-0} 30 | countdown=$(( $timeout * 10 )) 31 | 32 | if [ -f "$pidfile" ]; then 33 | pid=$(head -1 "$pidfile") 34 | 35 | if [ -z "$pid" ]; then 36 | echo "Unable to get pid from $pidfile" 37 | exit 1 38 | fi 39 | 40 | if [ -e /proc/$pid ]; then 41 | if [ "$try_kill" = "1" ]; then 42 | echo "Killing $pidfile: $pid " 43 | kill $pid 44 | fi 45 | while [ -e /proc/$pid ]; do 46 | sleep 0.1 47 | [ "$countdown" != '0' -a $(( $countdown % 10 )) = '0' ] && echo -n . 48 | if [ $timeout -gt 0 ]; then 49 | if [ $countdown -eq 0 ]; then 50 | if [ "$force" = "1" ]; then 51 | echo -ne "\nKill timed out, using kill -9 on $pid... " 52 | kill -9 $pid 53 | sleep 0.5 54 | fi 55 | break 56 | else 57 | countdown=$(( $countdown - 1 )) 58 | fi 59 | fi 60 | done 61 | if [ -e /proc/$pid ]; then 62 | echo "Timed Out" 63 | else 64 | echo "Stopped" 65 | fi 66 | else 67 | echo "Process $pid is not running" 68 | fi 69 | 70 | rm -f $pidfile 71 | else 72 | echo "Pidfile $pidfile doesn't exist" 73 | fi 74 | } 75 | 76 | kill_and_wait() { 77 | pidfile=$1 78 | # Monit default timeout for start/stop is 30s 79 | # Append 'with timeout {n} seconds' to monit start/stop program configs 80 | timeout=${2:-25} 81 | force=${3:-1} 82 | 83 | wait_pidfile $pidfile 1 $timeout $force 84 | } 85 | 86 | running_in_container() { 87 | # look for a non-root cgroup 88 | grep --quiet --invert-match ':/$' /proc/self/cgroup 89 | } 90 | 91 | -------------------------------------------------------------------------------- /ci/pipeline-helm.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | resources: 4 | 5 | - name: base-pipeline-docker-image 6 | type: docker-image 7 | source: 8 | repository: cfplatformeng/tile-generator-pipeline 9 | email: {{docker-hub-email}} 10 | username: {{docker-hub-username}} 11 | password: {{docker-hub-password}} 12 | 13 | - name: tile-generator-repo 14 | type: git 15 | source: 16 | ignore_paths: [ "ci/docker-tile-generator", "ci/docker-tile-pipeline" ] 17 | branch: helm 18 | uri: http://github.com/cf-platform-eng/tile-generator.git 19 | 20 | - name: tile-generator-dockerfile-repo 21 | type: git 22 | source: 23 | paths: [ "ci/docker-tile-generator" ] 24 | branch: {{github-branch}} 25 | uri: http://github.com/cf-platform-eng/tile-generator.git 26 | 27 | - name: tile-generator-docker-image 28 | type: docker-image 29 | source: 30 | repository: cfplatformeng/tile-generator-helm 31 | email: {{docker-hub-email}} 32 | username: {{docker-hub-username}} 33 | password: {{docker-hub-password}} 34 | 35 | jobs: 36 | 37 | - name: unit-tests 38 | plan: 39 | - aggregate: 40 | - get: base-pipeline-docker-image 41 | - get: tile-generator-repo 42 | trigger: true 43 | - task: run-unit-tests 44 | image: base-pipeline-docker-image 45 | config: 46 | platform: linux 47 | inputs: 48 | - name: tile-generator-repo 49 | run: 50 | path: tile-generator-repo/ci/scripts/run-unittests.sh 51 | args: [ "tile-generator-repo/tile_generator" ] 52 | 53 | - name: package-tile-generator 54 | plan: 55 | - aggregate: 56 | - get: base-pipeline-docker-image 57 | passed: [ unit-tests ] 58 | trigger: true 59 | - get: tile-generator-repo 60 | passed: [ unit-tests ] 61 | trigger: true 62 | - get: tile-generator-dockerfile-repo 63 | trigger: true 64 | - task: build-package 65 | image: base-pipeline-docker-image 66 | config: 67 | platform: linux 68 | inputs: 69 | - name: tile-generator-repo 70 | outputs: 71 | - name: tile-generator-package 72 | run: 73 | path: sh 74 | args: 75 | - -exc 76 | - | 77 | cd tile-generator-repo 78 | echo "0.9" >version.txt 79 | python setup.py sdist 80 | cp dist/tile-generator-*.tar.gz ../tile-generator-package/tile-generator-helm-0.9.tar.gz 81 | - task: prepare-docker-build 82 | image: base-pipeline-docker-image 83 | config: 84 | platform: linux 85 | inputs: 86 | - name: tile-generator-dockerfile-repo 87 | - name: tile-generator-package 88 | outputs: 89 | - name: docker-build-dir 90 | run: 91 | path: sh 92 | args: 93 | - -c 94 | - cp tile-generator-package/* docker-build-dir/ && cp tile-generator-dockerfile-repo/ci/docker-tile-generator/* docker-build-dir/ 95 | - put: tile-generator-docker-image 96 | params: 97 | build: docker-build-dir 98 | -------------------------------------------------------------------------------- /tile_generator/tile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | import click 21 | import sys 22 | import yaml 23 | import os 24 | import types 25 | from . import build 26 | from . import template 27 | from . import config 28 | from .config import Config 29 | from.version import version_string 30 | 31 | @click.group() 32 | @click.version_option(version_string, '-v', '--version', message='%(prog)s version %(version)s') 33 | def cli(): 34 | pass 35 | 36 | @cli.command('init') 37 | @click.argument('name', nargs=1, required=False) 38 | def init_cmd(name): 39 | dir = '.' 40 | if name is not None: 41 | dir = name 42 | os.mkdir(name) 43 | os.chdir(name) 44 | if os.path.isfile(config.CONFIG_FILE): 45 | print('Already initialized.', file=sys.stderr) 46 | sys.exit(0) 47 | name = os.path.basename(os.getcwd()) 48 | template.render(config.CONFIG_FILE, config.CONFIG_FILE, { 'name': name }) 49 | template.render('.gitignore', 'gitignore', {}) 50 | print('Generated initial tile files; begin by customizing "{}/tile.yml"'.format(dir)) 51 | 52 | @cli.command('build') 53 | @click.argument('version', required=False) 54 | @click.option('--verbose', is_flag=True) 55 | @click.option('--sha1', is_flag=True) 56 | @click.option('--cache', type=str, default=None) 57 | def build_cmd(version, verbose, sha1, cache): 58 | cfg = Config().read() 59 | 60 | cfg.set_version(version) 61 | cfg.set_verbose(verbose) 62 | cfg.set_sha1(sha1) 63 | cfg.set_cache(cache) 64 | print('name:', cfg.get('name', '')) 65 | print('label:', cfg.get('label', '')) 66 | print('description:', cfg.get('description', '')) 67 | print('version:', cfg.get('version', '')) 68 | print('sha1:', cfg.get('sha1', '')) 69 | stemcell = cfg.get('stemcell_criteria', {}) 70 | print('stemcell:', stemcell.get('os', ''), stemcell.get('version', '')) 71 | print() 72 | build.build(cfg) 73 | cfg.save_history() 74 | 75 | @cli.command('expand') 76 | @click.argument('version', required=False) 77 | def expand_cmd(version): 78 | cfg = Config().read() 79 | cfg.set_version(version) 80 | config.write_yaml(sys.stdout, dict(cfg)) 81 | 82 | def main(): 83 | try: 84 | cli(prog_name='tile') 85 | except Exception as e: 86 | click.echo(e, err=True) 87 | sys.exit(1) 88 | 89 | 90 | if __name__ == '__main__': 91 | main() -------------------------------------------------------------------------------- /ci/pie-cleanup.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | resources: 4 | 5 | resources: 6 | - name: timer 7 | type: time 8 | source: 9 | interval: 20h 10 | start: 12:00 AM -0700 11 | stop: 4:00 AM -0700 12 | 13 | - name: tile-generator-docker-image 14 | type: docker-image 15 | source: 16 | repository: cfplatformeng/tile-generator 17 | email: {{docker-hub-email}} 18 | username: {{docker-hub-username}} 19 | password: {{docker-hub-password}} 20 | 21 | - name: pcf-environment-1-6 22 | type: pool 23 | source: 24 | uri: git@github.com:cf-platform-eng/pipeline-resource-pool-v2 25 | branch: master 26 | pool: pcf1_6 27 | private_key: {{github-pool-private-key}} 28 | 29 | - name: pcf-environment-1-7 30 | type: pool 31 | source: 32 | uri: git@github.com:cf-platform-eng/pipeline-resource-pool-v2 33 | branch: master 34 | pool: pcf1_7 35 | private_key: {{github-pool-private-key}} 36 | 37 | - name: pcf-environment-1-8 38 | type: pool 39 | source: 40 | uri: git@github.com:cf-platform-eng/pipeline-resource-pool-v2 41 | branch: master 42 | pool: pcf 43 | private_key: {{github-pool-private-key}} 44 | 45 | jobs: 46 | 47 | - name: reboot-1-6 48 | plan: 49 | - aggregate: 50 | - get: timer 51 | trigger: true 52 | - get: tile-generator-docker-image 53 | - put: pcf-environment-1-6 54 | params: 55 | acquire: true 56 | - task: reboot 57 | image: tile-generator-docker-image 58 | config: 59 | platform: linux 60 | inputs: 61 | - name: pcf-environment-1-6 62 | run: 63 | path: sh 64 | args: 65 | - -exc 66 | - | 67 | cd pcf-environment-1-6 68 | pcf reboot -y 69 | ensure: 70 | put: pcf-environment-1-6 71 | params: 72 | release: pcf-environment-1-6 73 | 74 | - name: reboot-1-7 75 | plan: 76 | - aggregate: 77 | - get: timer 78 | trigger: true 79 | - get: tile-generator-docker-image 80 | - put: pcf-environment-1-7 81 | params: 82 | acquire: true 83 | - task: reboot 84 | image: tile-generator-docker-image 85 | config: 86 | platform: linux 87 | inputs: 88 | - name: pcf-environment-1-7 89 | run: 90 | path: sh 91 | args: 92 | - -exc 93 | - | 94 | cd pcf-environment-1-7 95 | pcf reboot -y 96 | ensure: 97 | put: pcf-environment-1-7 98 | params: 99 | release: pcf-environment-1-7 100 | 101 | - name: reboot-1-8 102 | plan: 103 | - aggregate: 104 | - get: timer 105 | trigger: true 106 | - get: tile-generator-docker-image 107 | - put: pcf-environment-1-8 108 | params: 109 | acquire: true 110 | - task: reboot 111 | image: tile-generator-docker-image 112 | config: 113 | platform: linux 114 | inputs: 115 | - name: pcf-environment-1-8 116 | run: 117 | path: sh 118 | args: 119 | - -exc 120 | - | 121 | cd pcf-environment-1-8 122 | pcf reboot -y 123 | ensure: 124 | put: pcf-environment-1-8 125 | params: 126 | release: pcf-environment-1-8 127 | -------------------------------------------------------------------------------- /tile_generator/bosh_unittest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import sys 18 | import tarfile 19 | import tempfile 20 | import yaml 21 | from contextlib import contextmanager 22 | from io import StringIO, BytesIO 23 | 24 | import unittest 25 | from . import bosh 26 | import mock 27 | 28 | @contextmanager 29 | def capture_output(): 30 | new_out, new_err = StringIO(), StringIO() 31 | old_out, old_err = sys.stdout, sys.stderr 32 | try: 33 | sys.stdout, sys.stderr = new_out, new_err 34 | yield sys.stdout, sys.stderr 35 | finally: 36 | sys.stdout, sys.stderr = old_out, old_err 37 | 38 | 39 | class TestBoshCheck(unittest.TestCase): 40 | @mock.patch('distutils.spawn.find_executable') 41 | @mock.patch('subprocess.check_output') 42 | @mock.patch('sys.exit') 43 | def test_passes_when_on_path(self, mock_sys_exit, mock_output, mock_find_executable): 44 | mock_find_executable.return_value = b'Some' 45 | mock_output.return_value = b'version 2.' 46 | 47 | bosh.ensure_bosh() 48 | 49 | 50 | mock_sys_exit.assert_not_called() 51 | 52 | 53 | @mock.patch('distutils.spawn.find_executable') 54 | @mock.patch('sys.exit') 55 | def test_exits_when_not_present(self, mock_sys_exit, mock_find_executable): 56 | mock_find_executable.return_value = None 57 | 58 | bosh.ensure_bosh() 59 | 60 | mock_sys_exit.assert_called() 61 | 62 | class TestManifest(unittest.TestCase): 63 | def test_get_manifest_finds_bare_release_mf(self): 64 | with tempfile.NamedTemporaryFile() as tf: 65 | tar = tarfile.open(tf.name, mode='w') 66 | manifest_text = 'manifest' 67 | manifest_yaml = yaml.safe_dump(manifest_text, encoding='utf-8') 68 | info = tarfile.TarInfo('release.MF') 69 | info.size = len(manifest_yaml) 70 | tar.addfile(tarinfo=info, fileobj=BytesIO(manifest_yaml)) 71 | tar.close() 72 | br = bosh.BoshRelease({'name': 'my-release'}, None) 73 | actual = br.get_manifest(tf.name) 74 | self.assertEqual(actual, manifest_text) 75 | 76 | def test_get_manifest_finds_dot_slash_release_mf(self): 77 | with tempfile.NamedTemporaryFile() as tf: 78 | tar = tarfile.open(tf.name, mode='w') 79 | manifest_text = 'manifest' 80 | manifest_yaml = yaml.safe_dump(manifest_text, encoding='utf-8') 81 | info = tarfile.TarInfo('./release.MF') 82 | info.size = len(manifest_yaml) 83 | tar.addfile(tarinfo=info, fileobj=BytesIO(manifest_yaml)) 84 | tar.close() 85 | br = bosh.BoshRelease({'name': 'my-release'}, None) 86 | actual = br.get_manifest(tf.name) 87 | self.assertEqual(actual, manifest_text) 88 | 89 | def test_get_manifest_throws_when_no_manifest(self): 90 | with tempfile.NamedTemporaryFile() as tf: 91 | tar = tarfile.open(tf.name, mode='w') 92 | tar.close() 93 | br = bosh.BoshRelease({'name': 'my-release'}, None) 94 | with self.assertRaises(Exception): 95 | actual = br.get_manifest(tf.name) 96 | 97 | if __name__ == '__main__': 98 | unittest.main() 99 | -------------------------------------------------------------------------------- /tile_generator/pcf_unittest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import unittest 19 | import mock 20 | import requests 21 | from io import BytesIO 22 | from .pcf import bosh_env_cmd 23 | from click.testing import CliRunner 24 | 25 | 26 | def build_response(body, encoding='application/json', status_code=200): 27 | response = requests.Response() 28 | response.status_code = status_code 29 | response.encoding = encoding 30 | response.raw = BytesIO(body) 31 | response.request = requests.Request() 32 | response.request.url = 'https://example.com/' 33 | return response 34 | 35 | 36 | @mock.patch('tile_generator.opsmgr.get') 37 | class TestBoshEnvCmd(unittest.TestCase): 38 | def test_pcf_2_2_uses_jobs(self, mock_get): 39 | mock_get.side_effect = [ 40 | build_response(opsmgr_responses['/api/v0/deployed/director/credentials/director_credentials']), 41 | build_response(opsmgr_responses['2.2 /api/v0/deployed/director/manifest']) 42 | ] 43 | 44 | # Since bosh_env_cmd is wrapped by a @cli.command, we need to invoke it this way 45 | runner = CliRunner() 46 | result = runner.invoke(bosh_env_cmd, []) 47 | 48 | self.assertIn('BOSH_ENVIRONMENT="10.0.0.5"', result.output) 49 | self.assertIn('BOSH_CA_CERT="/var/tempest/workspaces/default/root_ca_certificate"', result.output) 50 | self.assertIn('BOSH USERNAME=director', result.output) 51 | self.assertIn('BOSH PASSWORD=super-secret-password', result.output) 52 | 53 | def test_pcf_2_3_uses_instance_groups(self, mock_get): 54 | mock_get.side_effect = [ 55 | build_response(opsmgr_responses['/api/v0/deployed/director/credentials/director_credentials']), 56 | build_response(opsmgr_responses['2.3 /api/v0/deployed/director/manifest']) 57 | ] 58 | 59 | # Since bosh_env_cmd is wrapped by a @cli.command, we need to invoke it this way 60 | runner = CliRunner() 61 | result = runner.invoke(bosh_env_cmd, []) 62 | 63 | self.assertIn('BOSH_ENVIRONMENT="10.0.0.6"', result.output) 64 | self.assertIn('BOSH_CA_CERT="/var/tempest/workspaces/default/root_ca_certificate"', result.output) 65 | self.assertIn('BOSH USERNAME=director', result.output) 66 | self.assertIn('BOSH PASSWORD=super-secret-password', result.output) 67 | 68 | 69 | if __name__ == '__main__': 70 | unittest.main() 71 | 72 | opsmgr_responses = {} 73 | opsmgr_responses['/api/v0/deployed/director/credentials/director_credentials'] = b"""{ 74 | "credential": { 75 | "type": "simple_credentials", 76 | "value": { 77 | "password": "super-secret-password", 78 | "identity": "director" 79 | } 80 | } 81 | }""" 82 | opsmgr_responses['2.2 /api/v0/deployed/director/manifest'] = b"""{ 83 | "jobs": [ { "properties": { "director": { "address": "10.0.0.5" } } } ] 84 | }""" 85 | opsmgr_responses['2.3 /api/v0/deployed/director/manifest'] = b"""{ 86 | "instance_groups": [ { "properties": { "director": { "address": "10.0.0.6" } } } ] 87 | }""" 88 | -------------------------------------------------------------------------------- /ci/deployment-tests/app3_deploymenttest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import unittest 18 | import json 19 | import sys 20 | import os 21 | import requests 22 | from tile_generator import opsmgr 23 | 24 | class VerifyApp3(unittest.TestCase): 25 | 26 | def setUp(self): 27 | self.cfinfo = opsmgr.get_cfinfo() 28 | self.hostname = 'tg-test-app3.' + self.cfinfo['apps_domain'] 29 | self.url = 'http://' + self.hostname 30 | 31 | def test_responds_to_hello(self): 32 | headers = { 'Accept': 'application/json' } 33 | response = requests.get(self.url + '/hello', headers=headers) 34 | response.raise_for_status() 35 | 36 | def test_receives_custom_properties(self): 37 | headers = { 'Accept': 'application/json' } 38 | response = requests.get(self.url + '/env', headers=headers) 39 | response.raise_for_status() 40 | env = response.json() 41 | self.assertEqual(env.get('AUTHOR'), 'Tile Ninja') 42 | self.assertEqual(env.get('CUSTOMER_NAME'), "Jimmy's Johnnys") 43 | self.assertEqual(env.get('STREET_ADDRESS'), 'Cartaway Alley') 44 | self.assertEqual(env.get('CITY'), 'New Jersey') 45 | self.assertEqual(env.get('ZIP_CODE'), '90310') 46 | self.assertEqual(env.get('COUNTRY'), 'country_us') 47 | 48 | def test_receives_expected_services(self): 49 | headers = { 'Accept': 'application/json' } 50 | response = requests.get(self.url + '/env', headers=headers) 51 | response.raise_for_status() 52 | env = response.json() 53 | vcap_services = json.loads(env.get('VCAP_SERVICES')) 54 | broker1_service = vcap_services.get('tg-test-broker1-service', None) 55 | self.assertTrue(broker1_service is not None) 56 | self.assertEqual(len(broker1_service), 1) 57 | self.assertEqual(broker1_service[0].get('plan'), 'second-plan') 58 | 59 | def test_has_versioned_name(self): 60 | headers = { 'Accept': 'application/json' } 61 | response = requests.get(self.url + '/env', headers=headers) 62 | response.raise_for_status() 63 | env = response.json() 64 | vcap_application = json.loads(env.get('VCAP_APPLICATION')) 65 | name = vcap_application.get('application_name') 66 | self.assertTrue(name.startswith('tg-test-app3-')) 67 | 68 | def test_is_in_correct_space(self): 69 | headers = { 'Accept': 'application/json' } 70 | response = requests.get(self.url + '/env', headers=headers) 71 | response.raise_for_status() 72 | env = response.json() 73 | vcap_application = json.loads(env.get('VCAP_APPLICATION')) 74 | space= vcap_application.get('space_name') 75 | self.assertEqual(space, 'test-tile-space') 76 | 77 | def test_does_not_receive_admin_credentials(self): 78 | headers = { 'Accept': 'application/json' } 79 | response = requests.get(self.url + '/env', headers=headers) 80 | response.raise_for_status() 81 | env = response.json() 82 | user = env.get('CF_ADMIN_USER') 83 | username = env.get('CF_ADMIN_USERNAME') 84 | password = env.get('CF_ADMIN_PASSWORD') 85 | self.assertTrue(user is None) 86 | self.assertTrue(username is None) 87 | self.assertTrue(password is None) 88 | 89 | if __name__ == '__main__': 90 | unittest.main() 91 | -------------------------------------------------------------------------------- /ci/deployment-tests/app4_deploymenttest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import unittest 18 | import json 19 | import sys 20 | import os 21 | import requests 22 | from tile_generator import opsmgr 23 | 24 | class VerifyApp4Proxy(unittest.TestCase): 25 | 26 | def getEnv(self, url): 27 | headers = { 'Accept': 'application/json' } 28 | response = requests.get(url + '/env', headers=headers) 29 | response.raise_for_status() 30 | return response.json() 31 | 32 | def setUp(self): 33 | self.cfinfo = opsmgr.get_cfinfo() 34 | self.proxy = 'http://tg-test-app1.' + self.cfinfo['apps_domain'] 35 | self.env = self.getEnv(self.proxy) 36 | self.host = self.env.get('DOCKER_TCP_HOST') 37 | if self.host is not None: 38 | self.url = self.proxy + '/proxy?url=http://' + self.host + ':8080' 39 | 40 | def skipIfNoHost(self): 41 | if self.host is None: 42 | self.skipTest("Proxy did not receive app4 host address") 43 | 44 | def test_responds_to_hello(self): 45 | self.skipIfNoHost() 46 | headers = { 'Accept': 'application/json' } 47 | response = requests.get(self.url + '/hello', headers=headers) 48 | response.raise_for_status() 49 | 50 | class VerifyApp4(unittest.TestCase): 51 | 52 | def setUp(self): 53 | self.cfinfo = opsmgr.get_cfinfo() 54 | self.hostname = 'my-route-tg-test-app4.' + self.cfinfo['system_domain'] 55 | self.url = 'http://' + self.hostname 56 | 57 | def test_responds_to_hello(self): 58 | headers = { 'Accept': 'application/json' } 59 | response = requests.get(self.url + '/hello', headers=headers) 60 | response.raise_for_status() 61 | 62 | def test_receives_custom_properties(self): 63 | headers = { 'Accept': 'application/json' } 64 | response = requests.get(self.url + '/env', headers=headers) 65 | response.raise_for_status() 66 | env = response.json() 67 | self.assertEqual(env.get('AUTHOR'), 'Tile Ninja') 68 | self.assertEqual(env.get('CUSTOMER_NAME'), "Jimmy's Johnnys") 69 | self.assertEqual(env.get('STREET_ADDRESS'), 'Cartaway Alley') 70 | self.assertEqual(env.get('CITY'), 'New Jersey') 71 | self.assertEqual(env.get('ZIP_CODE'), '90310') 72 | self.assertEqual(env.get('COUNTRY'), 'country_us') 73 | 74 | def test_receives_expected_collection(self): 75 | headers = { 'Accept': 'application/json' } 76 | response = requests.get(self.url + '/env', headers=headers) 77 | response.raise_for_status() 78 | env = response.json() 79 | example_collection = json.loads(env.get('EXAMPLE_COLLECTION')) 80 | self.assertTrue(isinstance(example_collection, list)) 81 | self.assertEqual(len(example_collection), 2) 82 | 83 | def test_receives_expected_selector(self): 84 | headers = { 'Accept': 'application/json' } 85 | response = requests.get(self.url + '/env', headers=headers) 86 | response.raise_for_status() 87 | env = response.json() 88 | example_selector = json.loads(env.get('EXAMPLE_SELECTOR')) 89 | self.assertTrue(isinstance(example_selector, dict)) 90 | self.assertEqual(example_selector['value'], 'Filet Mignon') 91 | # self.assertEquals(example_selector['selected_option']['rarity_dropdown'], 'medium') 92 | 93 | if __name__ == '__main__': 94 | unittest.main() 95 | -------------------------------------------------------------------------------- /ci/deployment-tests/app1_deploymenttest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import unittest 18 | import json 19 | import sys 20 | import os 21 | import requests 22 | from tile_generator import opsmgr 23 | 24 | class VerifyApp1(unittest.TestCase): 25 | 26 | def setUp(self): 27 | self.cfinfo = opsmgr.get_cfinfo() 28 | self.hostname = 'tg-test-app1.' + self.cfinfo['apps_domain'] 29 | self.url = 'http://' + self.hostname 30 | 31 | def test_responds_to_hello(self): 32 | headers = { 'Accept': 'application/json' } 33 | response = requests.get(self.url + '/hello', headers=headers) 34 | response.raise_for_status() 35 | 36 | def test_receives_custom_properties(self): 37 | headers = { 'Accept': 'application/json' } 38 | response = requests.get(self.url + '/env', headers=headers) 39 | response.raise_for_status() 40 | env = response.json() 41 | self.assertEqual(env.get('AUTHOR'), 'Tile Ninja') 42 | self.assertEqual(env.get('CUSTOMER_NAME'), "Jimmy's Johnnys") 43 | self.assertEqual(env.get('STREET_ADDRESS'), 'Cartaway Alley') 44 | self.assertEqual(env.get('CITY'), 'New Jersey') 45 | self.assertEqual(env.get('ZIP_CODE'), '90310') 46 | self.assertEqual(env.get('COUNTRY'), 'country_us') 47 | 48 | def test_receives_expected_services(self): 49 | headers = { 'Accept': 'application/json' } 50 | response = requests.get(self.url + '/env', headers=headers) 51 | response.raise_for_status() 52 | env = response.json() 53 | vcap_services = json.loads(env.get('VCAP_SERVICES')) 54 | 55 | def test_receives_expected_collection(self): 56 | headers = { 'Accept': 'application/json' } 57 | response = requests.get(self.url + '/env', headers=headers) 58 | response.raise_for_status() 59 | env = response.json() 60 | example_collection = json.loads(env.get('EXAMPLE_COLLECTION')) 61 | self.assertTrue(isinstance(example_collection, list)) 62 | self.assertEqual(len(example_collection), 2) 63 | 64 | def test_receives_expected_selector(self): 65 | headers = { 'Accept': 'application/json' } 66 | response = requests.get(self.url + '/env', headers=headers) 67 | response.raise_for_status() 68 | env = response.json() 69 | example_selector = json.loads(env.get('EXAMPLE_SELECTOR')) 70 | self.assertTrue(isinstance(example_selector, dict)) 71 | self.assertEqual(example_selector['value'], 'Filet Mignon') 72 | # self.assertEquals(example_selector['selected_option']['rarity_dropdown'], 'medium') 73 | 74 | def test_has_versioned_name(self): 75 | headers = { 'Accept': 'application/json' } 76 | response = requests.get(self.url + '/env', headers=headers) 77 | response.raise_for_status() 78 | env = response.json() 79 | vcap_application = json.loads(env.get('VCAP_APPLICATION')) 80 | name = vcap_application.get('application_name') 81 | self.assertTrue(name.startswith('tg-test-app1-')) 82 | 83 | def test_is_in_correct_space(self): 84 | headers = { 'Accept': 'application/json' } 85 | response = requests.get(self.url + '/env', headers=headers) 86 | response.raise_for_status() 87 | env = response.json() 88 | vcap_application = json.loads(env.get('VCAP_APPLICATION')) 89 | space= vcap_application.get('space_name') 90 | self.assertEqual(space, 'test-tile-space') 91 | 92 | if __name__ == '__main__': 93 | unittest.main() 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PCF Tile Generator 2 | 3 | PCF Tile Generator is a suite of tools to help you develop, package, test, 4 | and deploy services and other add-ons to Pivotal Cloud Foundry. The tile generator 5 | uses templates and patterns that are based on years of experience integrating 6 | third-party services into Cloud Foundry, and eliminates much of the need for 7 | you to have intimate knowledge of all the tools involved. 8 | 9 | - Documentation: [Pivotal Documentation](http://docs.pivotal.io/tiledev/tile-generator.html) 10 | - PCF Tile Developers Guide: [Pivotal Documentation](http://docs.pivotal.io/tiledev/index.html) 11 | - Roadmap: [Github Issues](https://github.com/cf-platform-eng/tile-generator/issues) 12 | - CI Pipeline: [TPE Concourse](https://tpe-concourse-rock.acc.broadcom.net/teams/ppe-isv/pipelines/tile-generator/) 13 | 14 | ## Continuous Integration 15 | 16 | The master branch of this repository is being monitored by 17 | [this TPE Concourse pipeline](https://tpe-concourse-rock.acc.broadcom.net/teams/ppe-isv/pipelines/tile-generator/). 18 | The pipeline verifies that: 19 | 20 | - The tile generator passes all unit tests in `tile_generator/*_unittest.py` 21 | - The tile generator successfully builds the sample tile in `sample` 22 | - The generated tile passes all acceptance tests in `ci/acceptance-tests` 23 | - The generated tile successfully deploys to a current version of PCF 24 | - The deployed tile passes all deployment tests in `ci/deployment-tests` 25 | 26 | ## Updating pipeline.yml 27 | - After updating the pipeline template file - pipeline.yml.jinja2, please run the below script to generate the pipeline file from inside the ci directory 28 | ``` 29 | python3 generate_pipeline_yml.py 30 | ``` 31 | 32 | To target the pipeline run the following command 33 | 34 | ``` 35 | fly login -t -c https://tpe-concourse-rock.acc.broadcom.net -n ppe-isv 36 | ``` 37 | 38 | You need to be a member of the ppe-isv team for the above command to work. 39 | 40 | ## Contributing to the Tile Generator 41 | 42 | We welcome comments, questions, and contributions from community members. Please consider 43 | the following ways to contribute: 44 | 45 | - File Github issues for questions, bugs and new features and comment and vote on the ones that you are interested in. 46 | - If you want to contribute code, please make your code changes on a fork of this repository and submit a 47 | pull request to the master branch of tile-generator. We strongly suggest that you first file an issue to 48 | let us know of your intent, or comment on the issue you are planning to address. 49 | 50 | ### Development 51 | 52 | For development, it is useful to install the tile-generator package in 53 | *editable* mode. That is, you can install the tile-generator package 54 | in a way that points to your local repository, so that your code 55 | changes are immediately available through the `tile` or `pcf` 56 | commands. To do this, run this command in your tile-generator 57 | repository directory: 58 | 59 | ``` 60 | ./install-git-hook.sh 61 | pip install -e . 62 | ``` 63 | 64 | To avoid downloading dependencies on every `tile build`: 65 | 1. `cd sample` 66 | 2. `mkdir cache` 67 | 3. `tile build --cache cache` 68 | 69 | To verify if there are any lint issues: 70 | ``` 71 | python -m tabnanny filename.py 72 | ``` 73 | 74 | Run indiv 75 | 76 | Before executing `./scripts/run_local_tests.sh` install virtualenv with `pip install virtualenv` 77 | 78 | Then to execute all test using the cache from the project root use: 79 | `./scripts/run_local_tests.sh withcache` 80 | 81 | ### Note: Mac Binaries are no longer supported 82 | 83 | Tile generator cli no longer releases new Mac binaries. Check the [commit](https://github.com/cf-platform-eng/tile-generator/commit/1e8db6fb25f1c0e499965df0a113818188548d5b) that removed support for Mac support. MacStadium account has been cancelled and we currently don't have a way to test tile generator cli Mac binaries. 84 | 85 | -------------------------------------------------------------------------------- /ci/deployment-tests/broker2_deploymenttest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import unittest 18 | import json 19 | import sys 20 | import os 21 | import requests 22 | from tile_generator import opsmgr 23 | 24 | class VerifyApp2(unittest.TestCase): 25 | 26 | def setUp(self): 27 | self.cfinfo = opsmgr.get_cfinfo() 28 | self.hostname = 'tg-test-broker2.' + self.cfinfo['apps_domain'] 29 | self.url = 'http://' + self.hostname 30 | 31 | def test_responds_to_hello(self): 32 | headers = { 'Accept': 'application/json' } 33 | response = requests.get(self.url + '/hello', headers=headers) 34 | response.raise_for_status() 35 | 36 | def test_receives_custom_properties(self): 37 | headers = { 'Accept': 'application/json' } 38 | response = requests.get(self.url + '/env', headers=headers) 39 | response.raise_for_status() 40 | env = response.json() 41 | self.assertEqual(env.get('AUTHOR'), 'Tile Ninja') 42 | self.assertEqual(env.get('CUSTOMER_NAME'), "Jimmy's Johnnys") 43 | self.assertEqual(env.get('STREET_ADDRESS'), 'Cartaway Alley') 44 | self.assertEqual(env.get('CITY'), 'New Jersey') 45 | self.assertEqual(env.get('ZIP_CODE'), '90310') 46 | self.assertEqual(env.get('COUNTRY'), 'country_us') 47 | 48 | def test_receives_link_hosts(self): 49 | headers = { 'Accept': 'application/json' } 50 | response = requests.get(self.url + '/env', headers=headers) 51 | response.raise_for_status() 52 | env = response.json() 53 | self.assertTrue(env.get('REDIS_HOST') is not None) 54 | self.assertTrue(env.get('REDIS_HOSTS') is not None) 55 | self.assertTrue(env.get('NATS_HOST') is not None) 56 | self.assertTrue(env.get('NATS_HOSTS') is not None) 57 | 58 | def test_receives_link_properties(self): 59 | headers = { 'Accept': 'application/json' } 60 | response = requests.get(self.url + '/env', headers=headers) 61 | response.raise_for_status() 62 | env = response.json() 63 | self.assertIsNotNone(json.loads(env.get('REDIS_PROPERTIES'))) 64 | self.assertIsNotNone(json.loads(env.get('NATS_PROPERTIES'))) 65 | 66 | def test_has_versioned_name(self): 67 | headers = { 'Accept': 'application/json' } 68 | response = requests.get(self.url + '/env', headers=headers) 69 | response.raise_for_status() 70 | env = response.json() 71 | vcap_application = json.loads(env.get('VCAP_APPLICATION')) 72 | name = vcap_application.get('application_name') 73 | self.assertTrue(name.startswith('tg-test-broker2-')) 74 | 75 | def test_is_in_correct_space(self): 76 | headers = { 'Accept': 'application/json' } 77 | response = requests.get(self.url + '/env', headers=headers) 78 | response.raise_for_status() 79 | env = response.json() 80 | vcap_application = json.loads(env.get('VCAP_APPLICATION')) 81 | space= vcap_application.get('space_name') 82 | self.assertEqual(space, 'test-tile-space') 83 | 84 | def test_receives_broker_credentials(self): 85 | headers = { 'Accept': 'application/json' } 86 | response = requests.get(self.url + '/env', headers=headers) 87 | response.raise_for_status() 88 | env = response.json() 89 | security_user_name = env.get('SECURITY_USER_NAME') 90 | security_user_password = env.get('SECURITY_USER_PASSWORD') 91 | self.assertIsNotNone(security_user_name) 92 | self.assertIsNotNone(security_user_password) 93 | 94 | if __name__ == '__main__': 95 | unittest.main() 96 | -------------------------------------------------------------------------------- /tile_generator/templates/jobs/spec: -------------------------------------------------------------------------------- 1 | --- 2 | name: {{ job_type }} 3 | templates: 4 | {% if errand %} 5 | {{ job_type }}.sh.erb: bin/run 6 | opsmgr.env.erb: bin/opsmgr.env 7 | {% else %} 8 | {{ job_type }}.sh.erb: bin/{{ job_type }}_ctl 9 | opsmgr.env.erb: bin/opsmgr.env 10 | {% endif %} 11 | packages: 12 | {% if not package %} 13 | {% for package in context.packages if not package.is_bosh_release %} 14 | - {{ package.name }} 15 | {% endfor %} 16 | {% for package in packages %} 17 | - {{ package.name }} 18 | {% endfor %} 19 | {% else %} 20 | {% if not package.is_docker_app %} 21 | - {{ package.name }} 22 | {% endif %} 23 | {% if not errand %} 24 | - common 25 | {% endif %} 26 | {% endif %} 27 | properties: 28 | domain: 29 | description: 'CloudFoundry system domain' 30 | app_domains: 31 | description: 'CloudFoundry application domains' 32 | org: 33 | description: 'Org for the Application' 34 | default: '' 35 | space: 36 | description: 'Space for the Application' 37 | default: '' 38 | ssl.skip_cert_verify: 39 | description: 'Whether to verify SSL certs when making web requests' 40 | cf.admin_user: 41 | description: 'Username of the CF admin user' 42 | cf.admin_password: 43 | description: 'Password of the CF admin user' 44 | uaa.admin_client: 45 | description: 'Name of the UAA Admin Client' 46 | uaa.admin_client_secret: 47 | description: 'Password of the UAA Admin Client' 48 | apply_open_security_group: 49 | description: 'Open security group for the app to access outside' 50 | default: false 51 | allow_paid_service_plans: 52 | description: 'Allow use of paid service plans' 53 | default: false 54 | security.user: 55 | description: 'Basic auth user' 56 | security.password: 57 | description: 'Basic auth password' 58 | tls_cacert: 59 | description: "Ops Manager's CA certificate" 60 | opsman_ca: 61 | description: "Ops Manager's CA certificate" 62 | pks_username: 63 | description: 'Username for the PKS API' 64 | pks_password: 65 | description: 'Password for the PKS API' 66 | {% for property in context.all_properties %} 67 | {{ property.name }}: 68 | description: "{{ property.description or property.name }}" 69 | {% endfor %} 70 | {% for service_plan_form in context.service_plan_forms %} 71 | {{ service_plan_form.name }}: 72 | description: '{{ service_plan_form.name }}' 73 | {% endfor %} 74 | {% for package in context.packages %} 75 | {{ package.name }}.name: 76 | description: 'Name of package' 77 | {% if package.is_external_broker %} 78 | {{ package.name }}.url: 79 | description: 'URL to access service' 80 | {{ package.name }}.user: 81 | description: 'Basic auth username to access service' 82 | {{ package.name }}.password: 83 | description: 'Basic auth password to access service' 84 | {% elif package.is_broker %} 85 | {{ package.name }}.user: 86 | description: 'Basic auth username' 87 | {{ package.name }}.password: 88 | description: 'Basic auth password' 89 | {% endif %} 90 | {% if package.is_app %} 91 | {{ package.name }}.app_manifest: 92 | description: 'Application manifest' 93 | {{ package.name }}.auto_services: 94 | description: 'Services to bind' 95 | default: [] 96 | {% endif %} 97 | {% if package.is_broker %} 98 | {{ package.name }}.enable_global_access_to_plans: 99 | description: 'Access to all service plans' 100 | default: false 101 | {% endif %} 102 | {% if package.is_buildpack %} 103 | {{ package.name }}.buildpack_order: 104 | description: 'Order of the Buildpack' 105 | {% endif %} 106 | {% endfor %} 107 | 108 | {% if job_type == 'deploy-all' or job_type == 'delete-all' %} 109 | consumes: 110 | {% for release in context.releases.values() if release.consumes %} 111 | {% for link_name, link in release.consumes.items() %} 112 | - name: {{ link_name }} 113 | type: {{ link.type or link_name }} 114 | {% if link.optional is defined and link.optional %} 115 | optional: true 116 | {% endif %} 117 | {% endfor %} 118 | {% endfor %} 119 | {% endif %} 120 | -------------------------------------------------------------------------------- /sample/src/app/app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | import os 20 | import sys 21 | import traceback 22 | import json 23 | import requests 24 | import re 25 | 26 | from flask import Flask 27 | from flask import request 28 | app = Flask(__name__) 29 | 30 | vcap_application = json.loads(os.getenv('VCAP_APPLICATION','{ "name": "none", "application_uris": [ "http://localhost:8080" ] }')) 31 | host = vcap_application['application_uris'][0] 32 | name = re.sub('-[0-9].*\\Z', '', vcap_application['name']) 33 | 34 | def route(url): 35 | return host + url 36 | 37 | catalog = { 38 | "services": [{ 39 | "id": name + '-id', 40 | "name": name + '-service', 41 | "description": "A generated test service", 42 | "bindable": True, 43 | "plans": [{ 44 | "id": name + '-plan-1-id', 45 | "name": "first-plan", 46 | "description": "A first, free, service plan" 47 | },{ 48 | "id": name + '-plan-2-id', 49 | "name": "second-plan", 50 | "description": "A second, for-a-fee, service plan", 51 | "free": False 52 | }], 53 | "dashboard_client": { 54 | "id": name + '-client-id', 55 | "secret": "secret", 56 | "redirect_uri": "http://" + route('/dashboard') 57 | } 58 | }] 59 | } 60 | 61 | @app.route("/hello") 62 | def hello(): 63 | return 'hello' 64 | 65 | @app.route("/env") 66 | def environment(): 67 | return json.dumps(dict(os.environ), indent=4) 68 | 69 | @app.route("/ls", methods=['POST']) 70 | def ls(): 71 | if not request.json or not 'directory' in request.json: 72 | return 'Bad request: needs "directory" key in json', 500 73 | directory = request.json['directory'] 74 | if not os.path.isdir(directory): 75 | return '{} is not a directory'.format(directory), 500 76 | return json.dumps(os.listdir(directory)) 77 | 78 | @app.route("/proxy") 79 | def proxy(): 80 | url = request.args.get('url') 81 | if url is None: 82 | return json.dumps('expected url parameter'), 404 83 | response = requests.get(url, verify=False) 84 | return response.content, response.status_code 85 | 86 | @app.route("/v2/catalog") 87 | def broker_catalog(): 88 | return json.dumps(catalog, indent=4) 89 | 90 | @app.route("/v2/service_instances/", methods=['PUT']) 91 | def broker_provision_instance(instance_id): 92 | return json.dumps({ 'dashboard_url': route('/dashboard/' + instance_id) }, indent=4), 201 93 | 94 | @app.route("/v2/service_instances/", methods=['DELETE']) 95 | def broker_deprovision_instance(instance_id): 96 | return json.dumps({}, indent=4), 200 97 | 98 | @app.route("/v2/service_instances//service_bindings/", methods=['PUT']) 99 | def broker_bind_instance(instance_id, binding_id): 100 | return json.dumps({ 'credentials': { 'uri': route('/service/' + instance_id) }}, indent=4), 201 101 | 102 | @app.route("/v2/service_instances//service_bindings/", methods=['DELETE']) 103 | def broker_unbind_instance(instance_id, binding_id): 104 | return json.dumps({}, indent=4), 200 105 | 106 | @app.errorhandler(500) 107 | def internal_error(error): 108 | print(error) 109 | return "Internal server error", 500 110 | 111 | if __name__ == "__main__": 112 | try: 113 | app.run(host='0.0.0.0', port=int(os.getenv('PORT', '8080'))) 114 | print(" * Exited normally", file=sys.stderr) 115 | except: 116 | print(" * Exited with exception", file=sys.stderr) 117 | traceback.print_exc() 118 | -------------------------------------------------------------------------------- /ci/acceptance-tests/bosh_acceptancetest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import unittest 18 | import sys 19 | import os 20 | import glob 21 | import yaml 22 | 23 | class VerifyBoshRelease(unittest.TestCase): 24 | 25 | def test_has_manifest(self): 26 | self.assertTrue(os.path.exists('release/release.MF')) 27 | 28 | def test_all_jobs_have_monit(self): 29 | self.assertEqual(len(glob.glob('release/jobs/*/monit')), len(glob.glob('release/jobs/*'))) 30 | 31 | def test_errands_have_empty_monit(self): 32 | for monit in glob.glob('release/jobs/*/monit'): 33 | if not monit.startswith('release/jobs/docker-bosh-'): 34 | self.assertTrue(os.path.exists(monit), monit) 35 | self.assertEqual(os.path.getsize(monit), 0, monit) 36 | 37 | def test_non_errands_have_nonempty_monit(self): 38 | for monit in glob.glob('release/jobs/*'): 39 | if monit.startswith('release/jobs/docker-bosh-'): 40 | self.assertTrue(os.path.exists(monit), monit) 41 | self.assertNotEqual(os.path.getsize(monit), 0, monit) 42 | 43 | def test_all_jobs_have_manifest(self): 44 | self.assertEqual(len(glob.glob('release/jobs/*/job.MF')), len(glob.glob('release/jobs/*'))) 45 | 46 | def test_form_properties_exist_in_deploy_all(self): 47 | deploy_all = read_file('release/jobs/deploy-all/templates/deploy-all.sh.erb') 48 | self.assertIn(b'export AUTHOR=', deploy_all) 49 | self.assertIn(b'export NATS_HOST=', deploy_all) 50 | self.assertIn(b'export NATS_HOSTS=', deploy_all) 51 | self.assertIn(b'export NATS_PROPERTIES=', deploy_all) 52 | 53 | def test_form_properties_exist_in_delete_all(self): 54 | delete_all = read_file('release/jobs/delete-all/templates/delete-all.sh.erb') 55 | self.assertIn(b'export AUTHOR=', delete_all) 56 | self.assertIn(b'export NATS_HOST=', delete_all) 57 | self.assertIn(b'export NATS_HOSTS=', delete_all) 58 | self.assertIn(b'export NATS_PROPERTIES=', delete_all) 59 | 60 | # def test_cf_errand_manifest_has_cf_cli_package(self): 61 | # for manifest in glob.glob('release/jobs/*/job.MF'): 62 | # if not manifest.startswith('release/jobs/docker-bosh-'): 63 | # self.assertTrue('cf_cli' in read_yaml(manifest).get('packages', []), manifest) 64 | 65 | # def test_bosh_job_spec_has_no_cf_cli_package(self): 66 | # for manifest in glob.glob('release/jobs/*/job.MF'): 67 | # if manifest.startswith('release/jobs/docker-bosh-'): 68 | # self.assertFalse('cf_cli' in read_yaml(manifest).get('packages', []), manifest) 69 | 70 | def test_all_jobs_have_template(self): 71 | # 2 templates for each job: one to run the job, and one with the opsmgr env vars. 72 | self.assertEqual(len(glob.glob('release/jobs/*/templates/*.erb')), 2 * len(glob.glob('release/jobs/*'))) 73 | 74 | # def test_has_complete_cf_cli_package(self): 75 | # self.assertEqual(len(glob.glob('release/packages/cf_cli')), 1) 76 | # self.assertEqual(len(glob.glob('release/packages/cf_cli/cf_cli/cf-linux-amd64.tgz')), 1) 77 | # self.assertEqual(len(glob.glob('release/packages/cf_cli/packaging')), 1) 78 | # self.assertTrue('cf-linux-amd64.tgz' in read_file('release/packages/cf_cli/packaging')) 79 | 80 | def test_has_complete_common_package(self): 81 | self.assertEqual(len(glob.glob('release/packages/common')), 1) 82 | self.assertEqual(len(glob.glob('release/packages/common/common/utils.sh')), 1) 83 | 84 | def read_yaml(filename): 85 | with open(filename, 'rb') as file: 86 | return yaml.safe_load(file) 87 | 88 | def read_file(filename): 89 | with open(filename, 'rb') as file: 90 | return file.read() 91 | 92 | if __name__ == '__main__': 93 | unittest.main() 94 | -------------------------------------------------------------------------------- /tile_generator/build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | import os 21 | import sys 22 | import errno 23 | import requests 24 | import shutil 25 | import subprocess 26 | import tarfile 27 | from . import template 28 | try: 29 | # Python 3 30 | from urllib.request import urlretrieve 31 | except ImportError: 32 | # Python 2 33 | from urllib.request import urlretrieve 34 | import zipfile 35 | import yaml 36 | import datetime 37 | 38 | from .tile_metadata import TileMetadata 39 | from .bosh import * 40 | from .util import * 41 | from .version import version_string 42 | 43 | LIB_PATH = os.path.dirname(os.path.realpath(__file__)) 44 | REPO_PATH = os.path.realpath(os.path.join(LIB_PATH, '..')) 45 | DOCKER_BOSHRELEASE_VERSION = '23' 46 | 47 | def build(config): 48 | build_bosh_releases(config) 49 | build_tile(config) 50 | 51 | def build_bosh_releases(config): 52 | mkdir_p('release', clobber=True) 53 | for release in config.get('releases', {}).values(): 54 | release_name = release['name'] 55 | bosh_release = BoshRelease(release, config) 56 | tarball = bosh_release.get_tarball() 57 | metadata = bosh_release.get_metadata() 58 | release.update(metadata) 59 | print() 60 | 61 | def build_tile_metadata(context): 62 | tile_metadata = TileMetadata(context) 63 | return tile_metadata.build() 64 | 65 | def build_tile(context): 66 | mkdir_p('product', clobber=True) 67 | mkdir_p('product/releases') 68 | mkdir_p('product/tile-generator') 69 | tile_name = context['name'] 70 | tile_version = context['version'] 71 | print('tile generate metadata') 72 | context['tile_metadata'] = build_tile_metadata(context) 73 | template.render('product/metadata/' + tile_name + '.yml', 'tile/metadata.yml', context) 74 | print('tile generate migrations') 75 | migrations = 'product/migrations/v1/' + datetime.datetime.now().strftime('%Y%m%d%H%M') + '_noop.js' 76 | template.render(migrations, 'tile/migration.js', context) 77 | print('tile generate package') 78 | pivotal_file = os.path.join('product', tile_name + '-' + tile_version + '.pivotal') 79 | print('include tile generator version and inputs') 80 | with open(os.path.join('product', 'tile-generator', 'version'), 'w') as f: 81 | f.write(version_string) 82 | shutil.copy('tile.yml', os.path.join('product', 'tile-generator', 'tile.yml')) 83 | with zipfile.ZipFile(pivotal_file, 'w', allowZip64=True) as f: 84 | for release in context.get('releases', {}).values(): 85 | print('tile include release', release['release_name'] + '-' + release['version']) 86 | shutil.copy(release['tarball'], os.path.join('product/releases', release['file'])) 87 | f.write( 88 | os.path.join('product/releases', release['file']), 89 | os.path.join('releases', release['file'])) 90 | f.write( 91 | os.path.join('product/metadata', tile_name + '.yml'), 92 | os.path.join('metadata', tile_name + '.yml')) 93 | f.write(migrations, migrations.replace('product/', '', 1)) 94 | f.write( 95 | os.path.join('product/tile-generator', 'tile.yml'), 96 | os.path.join('tile-generator', 'tile.yml')) 97 | f.write( 98 | os.path.join('product/tile-generator', 'version'), 99 | os.path.join('tile-generator', 'version')) 100 | 101 | print('created tile', pivotal_file) 102 | -------------------------------------------------------------------------------- /tile_generator/package_flags_unittest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import os 18 | import unittest 19 | import sys 20 | import tempfile 21 | import shutil 22 | 23 | from contextlib import contextmanager 24 | from io import StringIO 25 | 26 | from .package_flags import get_disk_size_for_chart 27 | 28 | 29 | @contextmanager 30 | def capture_output(): 31 | new_out, new_err = StringIO(), StringIO() 32 | old_out, old_err = sys.stdout, sys.stderr 33 | try: 34 | sys.stdout, sys.stderr = new_out, new_err 35 | yield sys.stdout, sys.stderr 36 | finally: 37 | sys.stdout, sys.stderr = old_out, old_err 38 | 39 | 40 | class TestGetDiskSpaceForChart(unittest.TestCase): 41 | chart_directory = None 42 | other_chart_directory = None 43 | 44 | def setUp(self): 45 | self.chart_directory = tempfile.mkdtemp() 46 | self.other_chart_directory = tempfile.mkdtemp() 47 | 48 | def tearDown(self): 49 | if self.chart_directory != None: 50 | shutil.rmtree(self.chart_directory) 51 | if self.other_chart_directory != None: 52 | shutil.rmtree(self.other_chart_directory) 53 | 54 | def make_file_with_size(self, filepath, mb): 55 | f = open(filepath, "w+") 56 | f.write("*" * (mb * 1024 * 1024)) 57 | f.close() 58 | 59 | def test_missing_directory_is_none(self): 60 | size = get_disk_size_for_chart(None) 61 | self.assertEqual(size, 4096) 62 | 63 | def test_missing_directory_is_empty(self): 64 | size = get_disk_size_for_chart("/does/not/exist") 65 | self.assertEqual(size, 4096) 66 | 67 | def test_empty_directory_is_empty(self): 68 | size = get_disk_size_for_chart(self.chart_directory) 69 | self.assertEqual(size, 4096) 70 | 71 | def test_directory_with_file_shows_size(self): 72 | self.make_file_with_size(os.path.join(self.chart_directory, "a_file"), 1) 73 | size = get_disk_size_for_chart(self.chart_directory) 74 | self.assertEqual(size, 4097) 75 | 76 | def test_directory_with_empty_subdirs(self): 77 | os.mkdir(os.path.join(self.chart_directory, "directory1")) 78 | os.mkdir(os.path.join(self.chart_directory, "directory2")) 79 | 80 | size = get_disk_size_for_chart(self.chart_directory) 81 | self.assertEqual(size, 4096) 82 | 83 | def test_directory_with_subdirs_with_files_shows_size(self): 84 | os.mkdir(os.path.join(self.chart_directory, "directory1")) 85 | os.mkdir(os.path.join(self.chart_directory, "directory2")) 86 | self.make_file_with_size(os.path.join(self.chart_directory, "directory1", "file1"), 1) 87 | self.make_file_with_size(os.path.join(self.chart_directory, "directory2", "file2"), 1) 88 | 89 | size = get_disk_size_for_chart(self.chart_directory) 90 | self.assertEqual(size, 4098) 91 | 92 | def test_two_directories(self): 93 | self.make_file_with_size(os.path.join(self.chart_directory, "file1"), 1) 94 | self.make_file_with_size(os.path.join(self.other_chart_directory, "file2"), 1) 95 | 96 | size = get_disk_size_for_chart(self.chart_directory, self.other_chart_directory) 97 | self.assertEqual(size, 4098) 98 | 99 | def test_two_directories_one_empty(self): 100 | self.make_file_with_size(os.path.join(self.chart_directory, "file1"), 1) 101 | 102 | size = get_disk_size_for_chart(self.chart_directory, self.other_chart_directory) 103 | self.assertEqual(size, 4097) 104 | 105 | def test_two_directories_one_undefined(self): 106 | self.make_file_with_size(os.path.join(self.chart_directory, "file1"), 1) 107 | 108 | size = get_disk_size_for_chart(self.chart_directory, None) 109 | self.assertEqual(size, 4097) 110 | 111 | if __name__ == '__main__': 112 | unittest.main() 113 | -------------------------------------------------------------------------------- /ci/deployment-tests/app2_deploymenttest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import unittest 18 | import json 19 | import sys 20 | import os 21 | import requests 22 | from tile_generator import opsmgr 23 | 24 | class VerifyApp2(unittest.TestCase): 25 | 26 | def setUp(self): 27 | self.cfinfo = opsmgr.get_cfinfo() 28 | self.hostname = 'tg-test-app2-hostname.' + self.cfinfo['apps_domain'] 29 | self.url = 'http://' + self.hostname 30 | 31 | def test_responds_to_hello(self): 32 | headers = { 'Accept': 'application/json' } 33 | response = requests.get(self.url + '/hello', headers=headers) 34 | response.raise_for_status() 35 | 36 | def test_receives_custom_properties(self): 37 | headers = { 'Accept': 'application/json' } 38 | response = requests.get(self.url + '/env', headers=headers) 39 | response.raise_for_status() 40 | env = response.json() 41 | self.assertEqual(env.get('AUTHOR'), 'Tile Ninja') 42 | self.assertEqual(env.get('CUSTOMER_NAME'), "Jimmy's Johnnys") 43 | self.assertEqual(env.get('STREET_ADDRESS'), 'Cartaway Alley') 44 | self.assertEqual(env.get('CITY'), 'New Jersey') 45 | self.assertEqual(env.get('ZIP_CODE'), '90310') 46 | self.assertEqual(env.get('COUNTRY'), 'country_us') 47 | 48 | def test_receives_link_hosts(self): 49 | headers = { 'Accept': 'application/json' } 50 | response = requests.get(self.url + '/env', headers=headers) 51 | response.raise_for_status() 52 | env = response.json() 53 | self.assertTrue(env.get('REDIS_HOST') is not None) 54 | self.assertTrue(env.get('REDIS_HOSTS') is not None) 55 | self.assertTrue(env.get('NATS_HOST') is not None) 56 | self.assertTrue(env.get('NATS_HOSTS') is not None) 57 | 58 | def test_receives_link_properties(self): 59 | headers = { 'Accept': 'application/json' } 60 | response = requests.get(self.url + '/env', headers=headers) 61 | response.raise_for_status() 62 | env = response.json() 63 | self.assertIsNotNone(json.loads(env.get('REDIS_PROPERTIES'))) 64 | self.assertIsNotNone(json.loads(env.get('NATS_PROPERTIES'))) 65 | 66 | def test_receives_expected_services(self): 67 | headers = { 'Accept': 'application/json' } 68 | response = requests.get(self.url + '/env', headers=headers) 69 | response.raise_for_status() 70 | env = response.json() 71 | vcap_services = json.loads(env.get('VCAP_SERVICES')) 72 | broker1_service = vcap_services.get('tg-test-broker1-service', None) 73 | self.assertTrue(broker1_service is not None) 74 | self.assertEqual(len(broker1_service), 1) 75 | self.assertEqual(broker1_service[0].get('plan'), 'first-plan') 76 | 77 | def test_has_versioned_name(self): 78 | headers = { 'Accept': 'application/json' } 79 | response = requests.get(self.url + '/env', headers=headers) 80 | response.raise_for_status() 81 | env = response.json() 82 | vcap_application = json.loads(env.get('VCAP_APPLICATION')) 83 | name = vcap_application.get('application_name') 84 | self.assertTrue(name.startswith('tg-test-app2-')) 85 | 86 | def test_is_in_correct_space(self): 87 | headers = { 'Accept': 'application/json' } 88 | response = requests.get(self.url + '/env', headers=headers) 89 | response.raise_for_status() 90 | env = response.json() 91 | vcap_application = json.loads(env.get('VCAP_APPLICATION')) 92 | space= vcap_application.get('space_name') 93 | self.assertEqual(space, 'test-tile-space') 94 | 95 | def test_receives_expected_admin_credentials(self): 96 | headers = { 'Accept': 'application/json' } 97 | response = requests.get(self.url + '/env', headers=headers) 98 | response.raise_for_status() 99 | env = response.json() 100 | user = env.get('CF_ADMIN_USER') 101 | username = env.get('CF_ADMIN_USERNAME') 102 | password = env.get('CF_ADMIN_PASSWORD') 103 | self.assertEqual(user, username) 104 | self.assertEqual(user, 'system_services') 105 | self.assertFalse(password is None) 106 | 107 | if __name__ == '__main__': 108 | unittest.main() 109 | -------------------------------------------------------------------------------- /tile_generator/helm_unittest.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from . import helm 3 | 4 | class TestImageFinder(unittest.TestCase): 5 | 6 | def test_finds_top_level_image(self): 7 | values = { 8 | 'image': 'foo/bar', 9 | 'tag': 1.2 10 | } 11 | images = helm.find_required_images(values) 12 | self.assertEqual(len(images), 1) 13 | self.assertEqual(images[0], "foo/bar:1.2") 14 | 15 | def test_finds_top_level_image_uppercase(self): 16 | values = { 17 | 'Image': 'foo/bar', 18 | 'Tag': 1.2 19 | } 20 | images = helm.find_required_images(values) 21 | self.assertEqual(len(images), 1) 22 | self.assertEqual(images[0], "foo/bar:1.2") 23 | 24 | def test_finds_top_level_image_using_repository(self): 25 | values = { 26 | 'repository': 'foo/bar', 27 | 'tag': 1.2 28 | } 29 | images = helm.find_required_images(values) 30 | self.assertEqual(len(images), 1) 31 | self.assertEqual(images[0], "foo/bar:1.2") 32 | 33 | def test_finds_top_level_image_using_imagetag(self): 34 | values = { 35 | 'image': 'foo/bar', 36 | 'imagetag': 1.2 37 | } 38 | images = helm.find_required_images(values) 39 | self.assertEqual(len(images), 1) 40 | self.assertEqual(images[0], "foo/bar:1.2") 41 | 42 | def test_finds_nested_image(self): 43 | values = { 44 | 'level1': { 45 | 'level2': { 46 | 'image': 'foo/bar', 47 | 'tag': 1.2 48 | } 49 | } 50 | } 51 | images = helm.find_required_images(values) 52 | self.assertEqual(len(images), 1) 53 | self.assertEqual(images[0], "foo/bar:1.2") 54 | 55 | def test_finds_nested_image_uppercase(self): 56 | values = { 57 | 'level1': { 58 | 'level2': { 59 | 'Image': 'foo/bar', 60 | 'Tag': 1.2 61 | } 62 | } 63 | } 64 | images = helm.find_required_images(values) 65 | self.assertEqual(len(images), 1) 66 | self.assertEqual(images[0], "foo/bar:1.2") 67 | 68 | def test_finds_nested_image_using_repository(self): 69 | values = { 70 | 'level1': { 71 | 'level2': { 72 | 'repository': 'foo/bar', 73 | 'tag': 1.2 74 | } 75 | } 76 | } 77 | images = helm.find_required_images(values) 78 | self.assertEqual(len(images), 1) 79 | self.assertEqual(images[0], "foo/bar:1.2") 80 | 81 | def test_finds_nested_image_using_imagetag(self): 82 | values = { 83 | 'level1': { 84 | 'level2': { 85 | 'image': 'foo/bar', 86 | 'imagetag': 1.2 87 | } 88 | } 89 | } 90 | images = helm.find_required_images(values) 91 | self.assertEqual(len(images), 1) 92 | self.assertEqual(images[0], "foo/bar:1.2") 93 | 94 | def test_finds_nested_image_using_imagetag(self): 95 | values = { 96 | 'level1': { 97 | 'level2': { 98 | 'image': 'foo/bar', 99 | 'imagetag': 1.2 100 | } 101 | } 102 | } 103 | images = helm.find_required_images(values) 104 | self.assertEqual(len(images), 1) 105 | self.assertEqual(images[0], "foo/bar:1.2") 106 | 107 | def test_finds_nested_image_in_image(self): # Case found in fluent-bit helm chart 108 | values = { 109 | 'image': { 110 | 'image': 'foo/bar', 111 | 'imagetag': 1.2 112 | } 113 | } 114 | images = helm.find_required_images(values) 115 | self.assertEqual(len(images), 1) 116 | self.assertEqual(images[0], "foo/bar:1.2") 117 | 118 | def test_handles_empty_values(self): # Case found in weave-cloud helm chart 119 | values = None 120 | images = helm.find_required_images(values) 121 | self.assertEqual(len(images), 0) 122 | 123 | def test_handles_tag_only(self): # Case found in anchore helm chart 124 | values = { 125 | 'image': { 126 | 'tag': 'foo/bar:1.2' 127 | } 128 | } 129 | images = helm.find_required_images(values) 130 | self.assertEqual(len(images), 1) 131 | self.assertEqual(images[0], "foo/bar:1.2") 132 | -------------------------------------------------------------------------------- /tile_generator/templates/tile.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # The high-level description of your tile. 3 | # Replace these properties with real values. 4 | # 5 | name: {{ name }} # By convention lowercase with dashes 6 | icon_file: resources/icon.png 7 | label: Brief Text for the Tile Icon 8 | description: Longer description of the tile's purpose 9 | # metadata_version: 1.8 # Optional, defaults to 1.8 10 | 11 | # Global defaults (all optional) 12 | # 13 | # org: test-org # Name of org to create for your apps 14 | # space: test-space # Name of space to create for your apps 15 | # apply_open_security_group: true # Apply open security group, default: false 16 | # standalone: false # Suppress default CF properties in property_blueprints metadata, default: false 17 | 18 | # Specify the packages to be included in your tile. 19 | # The format of the section to include depends on the type 20 | # of package you are describing. For fragments of each type 21 | # that you can copy & paste, see: 22 | # 23 | # http://docs.pivotal.io/tiledev/tile-generator.html 24 | # 25 | packages: 26 | - name: my-application 27 | type: app 28 | # label: My fabulous appplication # Package name for use in human-readable labels in OpsManager 29 | manifest: 30 | path: resources/my-application-1.0.0.jar 31 | buildpack: java_buildpack_offline 32 | # command: python app.py 33 | # memory: 256M 34 | 35 | # New bosh release package with standalone job 36 | # - name: my-bosh-release 37 | # type: bosh-release 38 | # path: resources/standalone-release.tgz 39 | # jobs: 40 | # - name: my-job 41 | # instances: 1 42 | # dynamic_ip: 1 43 | # type: standalone # Don't include default CF properties in generated job manifest 44 | # templates: 45 | # - name: my-job 46 | # release: standalone-release 47 | 48 | # Include stemcell criteria if you don't want to accept the default. 49 | # Since this stemcell is only used to run pre and post errands, we 50 | # strongly recommend you leave this alone so that your tile always 51 | # runs with the latest stemcell. 52 | # 53 | # stemcell_criteria: 54 | # os: ubuntu-trusty 55 | # requires_cpi: false 56 | # version: '3062' 57 | 58 | # Add properties you want to pass to your applications. 59 | # Properties specified here will not be configurable by the user. 60 | # 61 | # properties: 62 | # - name: example_property 63 | # type: string 64 | # default: specify a value 65 | # label: Label for the field on the GUI 66 | # description: Longer description of the field's purpose 67 | 68 | # Uncomment this section if you want to display forms with configurable 69 | # properties in Ops Manager. These properties will be passed to your 70 | # applications as environment variables. You can also refer to them 71 | # elsewhere in this template by using: 72 | # (( .properties. )) 73 | # 74 | # forms: 75 | # - name: buildpack_properties 76 | # label: Buildpack 77 | # description: Buildpack_properties 78 | # properties: 79 | # - name: buildpack_rank 80 | # type: integer 81 | # default: 0 82 | # label: Buildpack rank for this buildpack_rank 83 | # description: Ranking of this buildpack relative to others 84 | 85 | # Add any dependencies your tile has on other installed products. 86 | # This is often appropriate when using automatic service provisioning 87 | # for any of your packages above, with services provided by other 88 | # products. 89 | # 90 | # requires_product_versions: 91 | # - name: p-mysql 92 | # version: '>= 1.7' 93 | 94 | # Customize upgrade parameters if the defaults don't meet your needs. 95 | # 96 | # update: 97 | # canaries: 1 98 | # canary_watch_time: 10000-100000 99 | # max_in_flight: 1 100 | # update_watch_time: 10000-100000 101 | 102 | # If prior versions of your tile are installed and configured, their 103 | # property values will not be overwritten by new defaults provided 104 | # in this file. If that is your intent, you must do the overwriting 105 | # explicitly using JavaScript migration functions, like so: 106 | # 107 | # migration: | 108 | # properties['properties']['.properties.org']['value'] = 'system'; 109 | 110 | # Bosh add-ons can be configured by adding runtime_configs to the 111 | # tile.yml 112 | # 113 | # runtime_configs: 114 | # - name: MY-RUNTIME-CONFIG 115 | # runtime_config: 116 | # releases: 117 | # - name: os-conf 118 | # version: 15 119 | # addons: 120 | # - name: MY-ADDON-NAME 121 | # jobs: 122 | # - name: MY-RUNTIME-CONFIG-JOB 123 | # release: os-conf 124 | # properties: 125 | # MY-ADDON-NAME: 126 | # ... 127 | # 128 | # Bosh releases referenced by add-ons must be either already present, 129 | # or added to the tile separately using a package of type bosh-release. 130 | -------------------------------------------------------------------------------- /tile_generator/templates/jobs/delete-all.sh.erb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export PATH="/var/vcap/packages/cf-cli-6-linux/bin:$PATH" 4 | export CF=`which cf` 5 | 6 | function cf() { 7 | echo cf "$@" 8 | output=$($CF "$@" 2>&1) 9 | result="$?" 10 | if [ "$result" -ne "0" ]; then 11 | echo "$output" 12 | # No exit here - best effort continuation on failure for the delete script 13 | fi 14 | } 15 | 16 | function is_true() { 17 | equals_ignore_case "$1" "true" 18 | } 19 | 20 | function equals_ignore_case() { 21 | echo "$1" | grep -i "^$2\$" >/dev/null 22 | } 23 | 24 | function import_opsmgr_variables() { 25 | export SCHEME=https 26 | export ADMIN_USER={{ 'properties.cf.admin_user' | shell_string }} 27 | export ADMIN_PASSWORD={{ 'properties.cf.admin_password' | shell_string }} 28 | export DOMAIN={{ 'properties.domain' | shell_string }} 29 | export APP_DOMAIN={{ 'properties.app_domains[0]' | shell_string }} 30 | export CF_ORG={{ 'properties.org' | shell_string }} 31 | export CF_SPACE={{ 'properties.space' | shell_string }} 32 | export CF_TARGET=$SCHEME://api.${DOMAIN} 33 | export CF_SKIP_SSL={{ 'properties.ssl.skip_cert_verify' | shell_string }} 34 | 35 | {% for property in context.all_properties %} 36 | {% if not property.skip_export %} 37 | {{ property | env_variable | indent }}`` 38 | {% endif %} 39 | {% endfor %} 40 | <% empty_dict = {} %> 41 | <% empty_list = [] %> 42 | {% for release in context.releases.values() if release.consumes %} 43 | {% for link in release.consumes %} 44 | <% if_link('{{ link }}') do |link| %> 45 | <% hosts = link.instances.map { |instance| instance.address } %> 46 | export {{ link | shell_variable_name }}_HOST="<%= link.instances.empty? ? "" : link.instances[0].address %>" 47 | export {{ link | shell_variable_name }}_HOSTS=<%= Shellwords.escape(hosts.to_json) %> 48 | export {{ link | shell_variable_name }}_PROPERTIES=<%= Shellwords.escape(link.properties.to_json) %> 49 | <% end.else do %> 50 | export {{ link | shell_variable_name }}_HOST=<%= Shellwords.escape(empty_list.to_json) %> 51 | export {{ link | shell_variable_name }}_HOSTS=<%= Shellwords.escape(empty_list.to_json) %> 52 | export {{ link | shell_variable_name }}_PROPERTIES=<%= Shellwords.escape(empty_dict.to_json) %> 53 | <% end %> 54 | {% endfor %} 55 | {% endfor %} 56 | } 57 | 58 | function write_opsman_cert { 59 | mkdir -p /var/vcap/data/certs 60 | echo "<%=p('opsman_ca')%>" > /var/vcap/data/certs/opsman-ca.pem 61 | } 62 | 63 | function authenticate() { 64 | $CF --version 65 | if is_true "$CF_SKIP_SSL"; then 66 | cf api $CF_TARGET --skip-ssl-validation 67 | else 68 | cf api $CF_TARGET 69 | fi 70 | cf auth $ADMIN_USER $ADMIN_PASSWORD 71 | } 72 | 73 | function target_org() { 74 | if [ -z "$CF_ORG" ]; then 75 | CF_ORG={{ context.name }}-org 76 | fi 77 | cf target -o $CF_ORG >/dev/null 78 | } 79 | 80 | function target_space() { 81 | if [ -z "$CF_SPACE" ]; then 82 | CF_SPACE={{ context.name }}-space 83 | fi 84 | cf target -s $CF_SPACE >/dev/null 85 | } 86 | 87 | function delete_empty_space() { 88 | output=`$CF apps | tail -1` 89 | if [ "$output" == "No apps found" ]; then 90 | cf delete-space -f $CF_SPACE 91 | fi 92 | } 93 | 94 | function delete_empty_org() { 95 | output=`$CF spaces | tail -1` 96 | if [ "$output" == "No spaces found" ]; then 97 | cf delete-org -f $CF_ORG 98 | fi 99 | } 100 | 101 | function delete_all_versions() { 102 | $CF apps | grep "^$1-" | while read app rest; do 103 | cf delete -r -f $app 104 | done 105 | } 106 | 107 | function delete_service_broker() { 108 | {% if context.purge_service_brokers %} 109 | services=`$CF service-access | sed -n "/^broker: $1$/,/^broker: /p" | grep -v -e "^[[:space:]]*$\|^broker:\|service *plan *access *orgs" | tr -s ' ' | cut -d ' ' -f 2 | uniq` 110 | for service in $services; do 111 | cf purge-service-offering -f "$service" 112 | done 113 | {% endif %} 114 | cf delete-service-broker -f $1 115 | } 116 | 117 | import_opsmgr_variables 118 | write_opsman_cert 119 | authenticate 120 | if target_org; then 121 | if target_space; then 122 | 123 | {% for package in context.packages | reverse %} 124 | # Delete package {{ package.name }} 125 | # 126 | {% if package.pre_delete %} 127 | {{ package.pre_delete | render }} 128 | {% endif %} 129 | 130 | {% if package.delete %} 131 | {{ package.delete | render }} 132 | {% else %} 133 | 134 | {% if package.is_broker %} 135 | delete_service_broker {{ package.name | hyphens }} 136 | {% endif %} 137 | {% if package.is_app %} 138 | delete_all_versions {{ package.name | hyphens }} 139 | {% endif %} 140 | {% if package.is_buildpack %} 141 | cf delete-buildpack -f {{ package.name }} 142 | {% endif %} 143 | 144 | {% endif %} 145 | 146 | {% if package.post_delete %} 147 | {{ package.post_delete | render }} 148 | {% endif %} 149 | 150 | {% endfor %} 151 | delete_empty_space 152 | fi 153 | delete_empty_org 154 | fi 155 | -------------------------------------------------------------------------------- /tile_generator/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | import errno 21 | import os 22 | import os.path 23 | import requests 24 | import shutil 25 | import sys 26 | import re 27 | import zipfile 28 | try: 29 | # Python 3 30 | from urllib.request import urlretrieve 31 | except ImportError: 32 | # Python 2 33 | from urllib.request import urlretrieve 34 | 35 | def mkdir_p(dir, clobber=False): 36 | if clobber and os.path.isdir(dir): 37 | shutil.rmtree(dir) 38 | try: 39 | os.makedirs(dir) 40 | return dir 41 | except os.error as e: 42 | if e.errno != errno.EEXIST: 43 | raise 44 | 45 | def download(url, filename, cache=None): 46 | if cache is not None: 47 | basename = os.path.basename(filename) 48 | cachename = os.path.join(cache, basename) 49 | if os.path.isfile(cachename): 50 | print('- using cached version of', basename) 51 | shutil.copy(cachename, filename) 52 | return 53 | # Special url to find a file associated with a github release. 54 | # github://cf-platform-eng/meta-buildpack/meta-buildpack.tgz 55 | # will find the file named meta-buildpack-0.0.3.tgz in the latest 56 | # release for https://github.com/cf-platform-eng/meta-buildpack 57 | if url.startswith("github:"): 58 | repo_name = url.replace('github:', '', 1).lstrip("/") 59 | file_name = os.path.basename(repo_name) 60 | repo_name = os.path.dirname(repo_name) 61 | url = "https://api.github.com/repos/" + repo_name + "/releases/latest" 62 | headers = {} 63 | if os.getenv('GITHUB_API_TOKEN'): 64 | headers['Authorization'] = 'token ' + os.environ['GITHUB_API_TOKEN'] 65 | response = requests.get(url, headers=headers, stream=True) 66 | response.raise_for_status() 67 | release = response.json() 68 | assets = release.get('assets', []) 69 | url = None 70 | pattern = re.compile('.*\\.'.join(file_name.rsplit('.', 1))+'\\Z') 71 | for asset in assets: 72 | if pattern.match(asset['name']) is not None: 73 | url = asset['browser_download_url'] 74 | break 75 | if url is None: 76 | print('no matching asset found for repo', repo_name, 'file', file_name, file=sys.stderr) 77 | sys.exit(1) 78 | # Fallthrough intentional, we now proceed to download the URL we found 79 | if url.startswith("http:") or url.startswith("https"): 80 | # [mboldt:20160908] Using urllib.urlretrieve gave an "Access 81 | # Denied" page when trying to download docker boshrelease. 82 | # I don't know why. requests.get works. Do what works. 83 | response = requests.get(url, stream=True) 84 | response.raise_for_status() 85 | with open(filename, 'wb') as file: 86 | for chunk in response.iter_content(chunk_size=1024): 87 | if chunk: 88 | file.write(chunk) 89 | elif url.startswith("docker:"): 90 | docker_image = url.replace('docker:', '', 1) 91 | try: 92 | from docker.client import Client 93 | docker_cli = Client.from_env() 94 | docker_cli.pull(docker_image) 95 | image = docker_cli.get_image(docker_image) 96 | image_tar = open(filename,'wb') 97 | image_tar.write(image.data) 98 | image_tar.close() 99 | except KeyError as e: 100 | print('docker not configured on this machine (or environment variables are not properly set)', file=sys.stderr) 101 | sys.exit(1) 102 | except Exception as e: 103 | print(e) 104 | print(docker_image, 'not found on local machine', file=sys.stderr) 105 | print('you must either pull the image, or download it and use the --cache option', file=sys.stderr) 106 | sys.exit(1) 107 | elif os.path.isdir(url): 108 | shutil.copytree(url, filename) 109 | else: 110 | shutil.copy(url, filename) 111 | if cache: 112 | if os.path.isdir(filename): 113 | basename = os.path.basename(filename) 114 | cachedir = os.path.join(cache, basename) 115 | if os.path.exists(cachedir): 116 | shutil.rmtree(cachedir) 117 | shutil.copytree(filename, os.path.join(cache, basename)) 118 | elif os.path.isfile(filename): 119 | shutil.copy(filename, cache) 120 | else: 121 | print(filename, 'is not a file or directory. Cannot cache.', file=sys.stderr) 122 | 123 | def zip_dir(zipfilename, dirname): 124 | with zipfile.ZipFile(zipfilename, 'w', allowZip64=True) as packagezip: 125 | if os.path.isdir(dirname): 126 | for root, dirs, files in os.walk(dirname): 127 | for file in files: 128 | abspath = os.path.join(root, file) 129 | relpath = abspath[len(dirname)+1:] # +1 for trailing slash. 130 | packagezip.write(abspath, relpath) 131 | elif os.path.isfile(dirname): 132 | packagezip.write(dirname, os.path.basename(dirname)) 133 | -------------------------------------------------------------------------------- /tile_generator/erb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | import os 21 | import sys 22 | import errno 23 | import yaml 24 | import json 25 | from . import opsmgr 26 | import random 27 | import string 28 | from . import build 29 | 30 | from jinja2 import Environment, FileSystemLoader, exceptions, pass_context 31 | 32 | PATH = os.path.dirname(os.path.realpath(__file__)) 33 | TEMPLATE_PATH = os.path.realpath(os.path.join(PATH, '..', 'templates')) 34 | 35 | 36 | def render_hyphens(input): 37 | return input.replace('_', '-') 38 | 39 | 40 | @pass_context 41 | def render_shell_string(context, input): 42 | expression = context.environment.compile_expression(input, undefined_to_none=False) 43 | return "'" + str(expression(context)).replace("'", "'\\''") + "'" 44 | 45 | 46 | @pass_context 47 | def render_plans_json(context, input): 48 | name = 'missing.' + input 49 | form = context.environment.compile_expression(name, undefined_to_none=False)(context) 50 | plans = {} 51 | for p in form: 52 | plans[p['name']] = p 53 | return input.upper() + "='" + json.dumps(plans).replace("'", "'\\''") + "'" 54 | 55 | 56 | TEMPLATE_ENVIRONMENT = Environment( 57 | trim_blocks=True, lstrip_blocks=True, 58 | comment_start_string='<%', comment_end_string='%>', # To completely ignore Ruby code blocks 59 | ) 60 | TEMPLATE_ENVIRONMENT.loader = FileSystemLoader(TEMPLATE_PATH) 61 | TEMPLATE_ENVIRONMENT.filters['hyphens'] = render_hyphens 62 | TEMPLATE_ENVIRONMENT.filters['shell_string'] = render_shell_string 63 | TEMPLATE_ENVIRONMENT.filters['plans_json'] = render_plans_json 64 | 65 | 66 | def render(errand_name, config_dir): 67 | template_file = os.path.join('jobs', errand_name + '.sh.erb') 68 | config = compile_config(config_dir) 69 | target_path = errand_name + '.sh' 70 | with open(target_path, 'wb') as target: 71 | target.write(TEMPLATE_ENVIRONMENT.get_template(template_file).render(config)) 72 | return target_path 73 | 74 | 75 | def mkdir_p(dir): 76 | try: 77 | os.makedirs(dir) 78 | except os.error as e: 79 | if e.errno != errno.EEXIST: 80 | raise 81 | 82 | 83 | def get_file_properties(filename): 84 | try: 85 | with open(filename) as f: 86 | properties = yaml.safe_load(f) 87 | if properties is None: 88 | return {} 89 | else: 90 | return properties 91 | except IOError as e: 92 | print(filename, 'not found', file=sys.stderr) 93 | sys.exit(1) 94 | 95 | 96 | def get_cf_properties(): 97 | cf = opsmgr.get_cfinfo() 98 | properties = {} 99 | properties['cf'] = { 100 | 'admin_user': cf['system_services_username'], 101 | 'admin_password': cf['system_services_password'], 102 | } 103 | properties['domain'] = cf['system_domain'] 104 | properties['app_domains'] = [cf['apps_domain']] 105 | properties['ssl'] = {'skip_cert_verify': True} 106 | properties['security'] = { 107 | 'user': 'admin', 108 | 'password': ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(12)) 109 | } 110 | return properties 111 | 112 | 113 | def merge_properties(properties, new_properties): 114 | for p in new_properties: 115 | if properties.get(p, None) is None: 116 | properties[p] = new_properties[p] 117 | 118 | 119 | def merge_property_array(properties, new_properties): 120 | for p in new_properties: 121 | name = p['name'] 122 | if properties.get(name, None) is None: 123 | value = p.get('value', p.get('default', None)) 124 | if value is not None: 125 | properties[name] = value 126 | 127 | 128 | def compile_config(config_dir): 129 | context = get_file_properties(os.path.join(config_dir, 'tile.yml')) 130 | build.validate_config(context) 131 | build.add_defaults(context) 132 | build.upgrade_config(context) 133 | 134 | properties = {} 135 | missing = get_file_properties(os.path.join(config_dir, 'missing-properties.yml')) 136 | merge_properties(properties, get_cf_properties()) 137 | merge_properties(properties, context) 138 | merge_property_array(properties, context.get('properties', [])) 139 | for form in context.get('forms', []): 140 | merge_property_array(properties, form.get('properties', [])) 141 | for package in context.get('packages', []): 142 | merge_properties(package, missing.get(package['name'], {})) 143 | merge_properties(package, properties['security']) 144 | merge_properties(properties, {package['name'].replace('-', '_'): package}) 145 | merge_properties(properties, missing) 146 | 147 | properties['org'] = properties.get('org', properties['name'] + '-org') 148 | properties['space'] = properties.get('space', properties['name'] + '-space') 149 | 150 | return { 151 | 'context': context, 152 | 'properties': properties, 153 | 'missing': missing, 154 | } 155 | -------------------------------------------------------------------------------- /CODE-OF-CONDUCT.md: -------------------------------------------------------------------------------- 1 | 2 | # Contributor Covenant Code of Conduct 3 | 4 | ## Our Pledge 5 | 6 | We as members, contributors, and leaders pledge to make participation in our 7 | community a harassment-free experience for everyone, regardless of age, body 8 | size, visible or invisible disability, ethnicity, sex characteristics, gender 9 | identity and expression, level of experience, education, socio-economic status, 10 | nationality, personal appearance, race, religion, or sexual identity 11 | and orientation. 12 | 13 | We pledge to act and interact in ways that contribute to an open, welcoming, 14 | diverse, inclusive, and healthy community. 15 | 16 | ## Our Standards 17 | 18 | Examples of behavior that contributes to a positive environment for our 19 | community include: 20 | 21 | * Demonstrating empathy and kindness toward other people 22 | * Being respectful of differing opinions, viewpoints, and experiences 23 | * Giving and gracefully accepting constructive feedback 24 | * Accepting responsibility and apologizing to those affected by our mistakes, 25 | and learning from the experience 26 | * Focusing on what is best not just for us as individuals, but for the 27 | overall community 28 | 29 | Examples of unacceptable behavior include: 30 | 31 | * The use of sexualized language or imagery, and sexual attention or 32 | advances of any kind 33 | * Trolling, insulting or derogatory comments, and personal or political attacks 34 | * Public or private harassment 35 | * Publishing others' private information, such as a physical or email 36 | address, without their explicit permission 37 | * Other conduct which could reasonably be considered inappropriate in a 38 | professional setting 39 | 40 | ## Enforcement Responsibilities 41 | 42 | Community leaders are responsible for clarifying and enforcing our standards of 43 | acceptable behavior and will take appropriate and fair corrective action in 44 | response to any behavior that they deem inappropriate, threatening, offensive, 45 | or harmful. 46 | 47 | Community leaders have the right and responsibility to remove, edit, or reject 48 | comments, commits, code, wiki edits, issues, and other contributions that are 49 | not aligned to this Code of Conduct, and will communicate reasons for moderation 50 | decisions when appropriate. 51 | 52 | ## Scope 53 | 54 | This Code of Conduct applies within all community spaces, and also applies when 55 | an individual is officially representing the community in public spaces. 56 | Examples of representing our community include using an official e-mail address, 57 | posting via an official social media account, or acting as an appointed 58 | representative at an online or offline event. 59 | 60 | ## Enforcement 61 | 62 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 63 | reported to the community leaders responsible for enforcement at 64 | [INSERT CONTACT METHOD]. 65 | All complaints will be reviewed and investigated promptly and fairly. 66 | 67 | All community leaders are obligated to respect the privacy and security of the 68 | reporter of any incident. 69 | 70 | ## Enforcement Guidelines 71 | 72 | Community leaders will follow these Community Impact Guidelines in determining 73 | the consequences for any action they deem in violation of this Code of Conduct: 74 | 75 | ### 1. Correction 76 | 77 | **Community Impact**: Use of inappropriate language or other behavior deemed 78 | unprofessional or unwelcome in the community. 79 | 80 | **Consequence**: A private, written warning from community leaders, providing 81 | clarity around the nature of the violation and an explanation of why the 82 | behavior was inappropriate. A public apology may be requested. 83 | 84 | ### 2. Warning 85 | 86 | **Community Impact**: A violation through a single incident or series 87 | of actions. 88 | 89 | **Consequence**: A warning with consequences for continued behavior. No 90 | interaction with the people involved, including unsolicited interaction with 91 | those enforcing the Code of Conduct, for a specified period of time. This 92 | includes avoiding interactions in community spaces as well as external channels 93 | like social media. Violating these terms may lead to a temporary or 94 | permanent ban. 95 | 96 | ### 3. Temporary Ban 97 | 98 | **Community Impact**: A serious violation of community standards, including 99 | sustained inappropriate behavior. 100 | 101 | **Consequence**: A temporary ban from any sort of interaction or public 102 | communication with the community for a specified period of time. No public or 103 | private interaction with the people involved, including unsolicited interaction 104 | with those enforcing the Code of Conduct, is allowed during this period. 105 | Violating these terms may lead to a permanent ban. 106 | 107 | ### 4. Permanent Ban 108 | 109 | **Community Impact**: Demonstrating a pattern of violation of community 110 | standards, including sustained inappropriate behavior, harassment of an 111 | individual, or aggression toward or disparagement of classes of individuals. 112 | 113 | **Consequence**: A permanent ban from any sort of public interaction within 114 | the community. 115 | 116 | ## Attribution 117 | 118 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 119 | version 2.0, available at 120 | [https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0]. 121 | 122 | Community Impact Guidelines were inspired by 123 | [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. 124 | 125 | For answers to common questions about this code of conduct, see the FAQ at 126 | [https://www.contributor-covenant.org/faq][FAQ]. Translations are available 127 | at [https://www.contributor-covenant.org/translations][translations]. 128 | 129 | [homepage]: https://www.contributor-covenant.org 130 | [v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html 131 | [Mozilla CoC]: https://github.com/mozilla/diversity 132 | [FAQ]: https://www.contributor-covenant.org/faq 133 | [translations]: https://www.contributor-covenant.org/translations 134 | 135 | -------------------------------------------------------------------------------- /tile_generator/tile_metadata_unittest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import unittest 19 | from .tile_metadata import TileMetadata 20 | from .config import Config 21 | from collections import OrderedDict 22 | 23 | def find_by_field(field_name, field_value, list_of_dict): 24 | for d in list_of_dict: 25 | if field_name in d and field_value == d[field_name]: 26 | return d 27 | 28 | class TestTileMetadata(unittest.TestCase): 29 | 30 | def test_non_configurable_selector_fields_not_in_property_inputs(self): 31 | forms_with_non_configurable_selector_option = [ 32 | { 33 | 'name': 'form_with_unconfigurable_selector_option', 34 | 'label': 'Form With Unconfigurable Selector Option', 35 | 'description': 'Form With Unconfigurable Selector Option', 36 | 'properties': [ 37 | { 38 | 'name': 'selector_with_unconfigurable_option', 39 | 'label': 'Selector with unconfigurable option', 40 | 'description': 'This selector has an unconfigurable option', 41 | 'default': 'unconfigurable_option', 42 | 'configurable': True, 43 | 'type': 'selector', 44 | 'option_templates': [ 45 | { 46 | 'name': 'unconfigurable_option', 47 | 'select_value': 'unconfigurable', 48 | 'property_blueprints': [ 49 | { 50 | 'name': 'property_name', 51 | 'default': 'unconfigurable_property_value', 52 | 'type': 'string', 53 | 'label': 'This is not configurable', 54 | 'configurable': False, 55 | }, 56 | ], 57 | }, 58 | { 59 | 'name': 'configurable_option', 60 | 'select_value': 'configurable', 61 | 'property_blueprints': [ 62 | { 63 | 'name': 'property_name', 64 | 'default': 'configurable_property_value', 65 | 'type': 'string', 66 | 'label': 'This is configurable', 67 | 'configurable': True, 68 | }, 69 | ], 70 | }, 71 | ], 72 | }, 73 | ], 74 | }, 75 | ] 76 | config = Config(name='validname', icon_file='/dev/null', label='some_label', description='This is required', forms=forms_with_non_configurable_selector_option) 77 | metadata = TileMetadata(config) 78 | metadata._build_form_types() 79 | selector_property_inputs = metadata.tile_metadata['form_types'][0]['property_inputs'][0]['selector_property_inputs'] 80 | inputs_with_no_configurable_fields = find_by_field('label', 'unconfigurable', selector_property_inputs) 81 | self.assertIsNone(inputs_with_no_configurable_fields.get('property_inputs')) 82 | 83 | def test_standalone_tile_excludes_default_cf_blueprints(self): 84 | config = Config( 85 | name='validname', 86 | icon_file='/dev/null', 87 | label='some_label', 88 | description='This is required', 89 | service_plan_forms=[], 90 | org='Test Org', 91 | space='Test Space', 92 | apply_open_security_group=False, 93 | allow_paid_service_plans=False, 94 | releases=OrderedDict(), 95 | all_properties=[], 96 | standalone=True) 97 | metadata = TileMetadata(config) 98 | metadata._build_property_blueprints() 99 | 100 | expected_blueprints = [] 101 | self.assertEqual(metadata.tile_metadata['property_blueprints'], expected_blueprints) 102 | 103 | def test_non_standalone_tile_includes_default_cf_blueprints(self): 104 | config = Config( 105 | name='validname', 106 | icon_file='/dev/null', 107 | label='some_label', 108 | description='This is required', 109 | service_plan_forms=[], 110 | org='Test Org', 111 | space='Test Space', 112 | apply_open_security_group=False, 113 | allow_paid_service_plans=False, 114 | releases=OrderedDict(), 115 | all_properties=[]) 116 | metadata = TileMetadata(config) 117 | metadata._build_property_blueprints() 118 | 119 | expected_blueprints = [ 120 | { 121 | 'configurable': True, 122 | 'default': 'Test Org', 123 | 'name': 'org', 124 | 'type': 'string' 125 | }, 126 | { 127 | 'configurable': True, 128 | 'default': 'Test Space', 129 | 'name': 'space', 130 | 'type': 'string' 131 | }, 132 | { 133 | 'configurable': True, 134 | 'default': False, 135 | 'name': 'apply_open_security_group', 136 | 'type': 'boolean' 137 | }, 138 | { 139 | 'configurable': True, 140 | 'default': False, 141 | 'name': 'allow_paid_service_plans', 142 | 'type': 'boolean' 143 | }, 144 | ] 145 | 146 | self.assertEqual(metadata.tile_metadata['property_blueprints'], expected_blueprints) 147 | 148 | if __name__ == '__main__': 149 | unittest.main() 150 | -------------------------------------------------------------------------------- /tile_generator/package_definitions.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import os 3 | import sys 4 | import yaml 5 | 6 | from . import package_flags as flag 7 | 8 | # Inspired by https://gist.github.com/angstwad/bf22d1822c38a92ec0a9 9 | def merge_dict(dct, merge_dct): 10 | for k, v in merge_dct.items(): 11 | if k in dct and isinstance(dct[k], dict) and isinstance(merge_dct[k], dict): 12 | merge_dict(dct[k], merge_dct[k]) 13 | else: 14 | dct[k] = copy.deepcopy(merge_dct[k]) 15 | 16 | # This coerce belongs to PackageDockerBosh, but needs 17 | # to be reachable by BasePackage 18 | def _to_yaml(manifest): 19 | try: 20 | return yaml.safe_load(manifest) 21 | except: 22 | print('docker-bosh manifest must be valid yaml') 23 | sys.exit(1) 24 | 25 | 26 | class BasePackage(object): 27 | _schema = { 28 | # NOTE: Is the '-' to '_' conversion really needed? 29 | 'name': {'type': 'string', 'required': True, 'regex': '[a-z][a-z0-9]*(_[a-z0-9]+)*$','coerce': lambda v: v.lower().replace('-','_')}, 30 | } 31 | 32 | @classmethod 33 | def schema(self): 34 | # Start by defining the package-type of the child most class 35 | schema = {'package-type': {'type': 'string', 'required': True, 'allowed': [self.package_type]}} 36 | for s in self.__mro__[::-1]: 37 | try: 38 | merge_dict(schema, s._schema) 39 | except AttributeError: 40 | pass 41 | return schema 42 | 43 | @classmethod 44 | def normalize_file_lists(self, package): 45 | files = package.get('files', []) 46 | path = package.get('path') 47 | if path is not None: 48 | files += [ { 'path': path } ] 49 | package['path'] = os.path.basename(path) 50 | manifest = package.get('manifest', {}) 51 | manifest_path = manifest.get('path', None) 52 | if manifest_path is not None: 53 | files += [ { 'path': manifest_path } ] 54 | package['manifest']['path'] = os.path.basename(manifest_path) 55 | for docker_image in package.get('docker_images', []): 56 | filename = docker_image.lower().replace('/','-').replace(':','-') + '.tgz' 57 | files += [ { 'path': 'docker:' + docker_image, 'name': filename } ] 58 | for file in files: 59 | file['name'] = file.get('name', os.path.basename(file['path'])) 60 | 61 | package['files'] = files 62 | 63 | 64 | class PackageBoshRelease(BasePackage): 65 | package_type = 'bosh-release' 66 | flags = [flag.BoshRelease] 67 | _schema = { 68 | 'jobs': {'type': 'list', 'required': False, 'schema':{ 69 | 'type': 'dict', 'schema': { 70 | 'name': {'type': 'string', 'required': True, 'regex': '^[a-zA-Z0-9\-\_]*$'}, 71 | 'memory': {'required': False, 'type': 'number'}, 72 | 'varname': {'type': 'string', 'default_setter': lambda doc: doc['name'].lower().replace('-','_')} 73 | } 74 | }}, 75 | } 76 | 77 | @classmethod 78 | def normalize_file_lists(self, package): 79 | pass 80 | 81 | 82 | class PackageDockerBosh(BasePackage): 83 | package_type = 'docker-bosh' 84 | flags = [flag.DockerBosh] # flag.Docker 85 | _schema = { 86 | # TODO: Remove the dependency on this in templates 87 | 'is_docker': {'type': 'boolean', 'default': True}, 88 | 'docker_images': {'required': False}, 89 | 'memory': {'required': False, 'type': 'number'}, 90 | 'routes': {'required': False, 'type': 'list', 'schema': {'type': 'dict', 'schema': { 91 | 'prefix': {'required': True}, 92 | 'port': {'required': True},}}}, 93 | 'manifest': {'type': 'dict', 'allow_unknown': True, 'required': True, 'coerce': _to_yaml, 94 | 'keyschema': {'forbidden': ['path']}, 'schema': { 95 | 'containers': {'type': 'list', 'schema': {'type': 'dict', 'schema': { 96 | 'name': {'type': 'string', 'required': True, 'regex': '^[a-z][a-zA-Z0-9_]*$'}, 97 | 'env_file': {'type': 'list', 'default': [], 'schema': { 98 | 'type': 'string'}}}}}}}, 99 | } 100 | 101 | 102 | class BaseAppAndAppBroker(BasePackage): 103 | flags = [flag.Cf, flag.App] 104 | _schema = { 105 | 'manifest': {'type': 'dict', 'allow_unknown': True, 'required': True, 106 | 'keyschema': {'forbidden': ['random-route']}, 107 | 'schema': {'buildpack': {'required': True}}}, 108 | } 109 | 110 | 111 | class PackageApp(BaseAppAndAppBroker): 112 | package_type = 'app' 113 | flags = BaseAppAndAppBroker.flags 114 | 115 | 116 | class PackageAppBroker(BaseAppAndAppBroker): 117 | package_type = 'app-broker' 118 | flags = BaseAppAndAppBroker.flags + [flag.Broker] # flag.BrokerApp 119 | 120 | 121 | class PackageBuildpack(BasePackage): 122 | package_type = 'buildpack' 123 | flags = [flag.Cf, flag.Buildpack] 124 | 125 | 126 | class PackageDecorator(BasePackage): 127 | package_type = 'decorator' 128 | flags = [flag.Cf, flag.Buildpack, flag.Decorator] 129 | 130 | 131 | class PackageExternalBroker(BasePackage): 132 | package_type = 'external-broker' 133 | flags = [flag.Cf, flag.Broker, flag.ExternalBroker] 134 | 135 | 136 | class PackageDockerApp(BasePackage): 137 | package_type = 'docker-app' 138 | flags = [flag.Cf, flag.App] # flag.Docker, flag.DockerApp 139 | _schema = { 140 | # TODO: Remove the dependency on this in templates 141 | 'is_docker': {'type': 'boolean', 'default': True}, 142 | 'is_docker_app': {'type': 'boolean', 'default': True}, 143 | 'manifest': {'type': 'dict', 'allow_unknown': True, 'required': True, 144 | 'keyschema': {'forbidden': ['path']}, 'schema': {}}, 145 | } 146 | 147 | 148 | class PackageDockerAppBroker(BasePackage): 149 | package_type = 'docker-app-broker' 150 | flags = [flag.Cf, flag.App, flag.Broker] # flag.Docker flag.BrokerApp, flag.DockerApp, 151 | _schema = { 152 | # TODO: Remove the dependency on this in templates 153 | 'is_docker': {'type': 'boolean', 'default': True}, 154 | 'is_docker_app': {'type': 'boolean', 'default': True}, 155 | 'manifest': {'type': 'dict', 'allow_unknown': True, 'required': True, 156 | 'keyschema': {'forbidden': ['path']}, 'schema': {}}, 157 | } 158 | 159 | 160 | class PackageBlob(BasePackage): 161 | package_type = 'blob' 162 | flags = [flag.Cf] # flag.Blob 163 | 164 | 165 | class PackageHelm(BasePackage): 166 | package_type = 'helm' 167 | flags = [flag.Helm] 168 | _schema = { 169 | 'path': {'type': 'string', 'required': True }, 170 | 'zip_if_needed': {'type': 'boolean', 'default': True} 171 | } -------------------------------------------------------------------------------- /ci/acceptance-tests/tile_acceptancetest.py: -------------------------------------------------------------------------------- 1 | # tile-generator 2 | # 3 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import unittest 18 | import sys 19 | import os 20 | import glob 21 | import yaml 22 | 23 | class VerifyTile(unittest.TestCase): 24 | 25 | def test_has_valid_migrations(self): 26 | self.assertTrue(os.path.exists('product/migrations/v1')) 27 | files = glob.glob('product/migrations/v1/*.js') 28 | self.assertEqual(len(files), 1) 29 | 30 | def test_has_valid_metadata(self): 31 | self.assertTrue(os.path.exists('product/metadata')) 32 | files = glob.glob('product/metadata/*.yml') 33 | self.assertEqual(len(files), 1) 34 | read_yaml(files[0]) # Ensure corrent yaml syntax 35 | 36 | def test_contains_tile_yml(self): 37 | self.assertTrue(os.path.exists('product/tile-generator')) 38 | files = glob.glob('product/tile-generator/tile.yml') 39 | self.assertEqual(len(files), 1) 40 | read_yaml(files[0]) 41 | 42 | class VerifyMetadata(unittest.TestCase): 43 | 44 | def setUp(self): 45 | self.assertTrue(os.path.exists('product/metadata')) 46 | files = glob.glob('product/metadata/*.yml') 47 | self.assertEqual(len(files), 1) 48 | self.metadata = read_yaml(files[0]) 49 | 50 | def test_has_expected_name(self): 51 | self.assertEqual(self.metadata['name'], 'test-tile') 52 | 53 | def test_has_expected_label(self): 54 | self.assertEqual(self.metadata['label'], 'Generated Test Tile') 55 | 56 | def test_has_expected_description(self): 57 | self.assertEqual(self.metadata['description'], 'A sample tile generated by tile-generator') 58 | 59 | def test_product_version_is_string(self): 60 | self.assertIsInstance(self.metadata['product_version'], str) 61 | 62 | def test_metadata_version_is_string(self): 63 | self.assertIsInstance(self.metadata['metadata_version'], str) 64 | 65 | def test_has_expected_minimum_version_for_upgrade(self): 66 | self.assertEqual(self.metadata['minimum_version_for_upgrade'], '0.0.1') 67 | 68 | def test_has_rank(self): 69 | self.assertEqual(self.metadata['rank'], 1) 70 | 71 | def test_has_expected_serial(self): 72 | self.assertTrue(self.metadata['serial']) 73 | 74 | def test_has_service_broker_flag(self): 75 | self.assertFalse(self.metadata['service_broker']) 76 | 77 | 78 | class VerifyProperties(unittest.TestCase): 79 | 80 | def setUp(self): 81 | self.assertTrue(os.path.exists('product/metadata')) 82 | files = glob.glob('product/metadata/*.yml') 83 | self.assertEqual(len(files), 1) 84 | self.metadata = read_yaml(files[0]) 85 | 86 | def test_optional(self): 87 | blueprints = self.metadata['property_blueprints'] 88 | self.assertFalse(find_by_name(blueprints, 'author')['optional']) 89 | self.assertTrue(find_by_name(blueprints, 'customer_name')['optional']) 90 | self.assertFalse(find_by_name(blueprints, 'street_address')['optional']) 91 | 92 | def test_bosh_release_has_properties(self): 93 | job = find_by_name(self.metadata['job_types'], 'redis') 94 | self.assertIn('author', job['manifest']) 95 | 96 | def test_default_internet_connected(self): 97 | job = find_by_name(self.metadata['job_types'], 'redis') 98 | self.assertIn('default_internet_connected', job) 99 | self.assertFalse(job['default_internet_connected']) 100 | 101 | def test_run_errand_default(self): 102 | job = find_by_name(self.metadata['job_types'], 'sanity-tests') 103 | self.assertEqual(job['run_post_deploy_errand_default'], 'when-changed') 104 | 105 | def test_bosh_release_properties_merged(self): 106 | job = find_by_name(self.metadata['job_types'], 'sanity-tests') 107 | manifest = yaml.safe_load(job['manifest']) 108 | cf = manifest['cf'] 109 | self.assertIn('some', cf) # Property defined in tile.yml. 110 | self.assertIn('admin_user', cf) # Auto-included property. 111 | 112 | def test_deploy_all_has_broker_user_and_password(self): 113 | job = find_by_name(self.metadata['job_types'], 'deploy-all') 114 | manifest = yaml.safe_load(job['manifest']) 115 | broker = manifest['tg_test_broker1'] 116 | self.assertIn('user', broker) 117 | self.assertIn('password', broker) 118 | 119 | def test_cross_deployment_link_in_metadata(self): 120 | deploy_all_job = find_by_name(self.metadata['job_types'], 'deploy-all') 121 | deploy_all_template = find_by_name(deploy_all_job['templates'], 'deploy-all') 122 | self.assertIn('consumes', deploy_all_template) 123 | consumes = yaml.safe_load(deploy_all_template['consumes']) 124 | self.assertIn('nats', consumes) 125 | self.assertIn('from', consumes['nats']) 126 | self.assertEqual(consumes['nats'].get('deployment'), '(( ..cf.deployment_name ))') 127 | self.assertEqual(consumes['nats'].get('from'), 'nats') 128 | 129 | class VerifyConstraints(unittest.TestCase): 130 | 131 | def setUp(self): 132 | self.assertTrue(os.path.exists('product/metadata')) 133 | files = glob.glob('product/metadata/*.yml') 134 | self.assertEqual(len(files), 1) 135 | self.metadata = read_yaml(files[0]) 136 | 137 | def test_resource_constraints(self): 138 | job = find_by_name(self.metadata['job_types'], 'sanity-tests') 139 | resource_defs = job['resource_definitions'] 140 | self.assertEqual(find_by_name(resource_defs, 'cpu')['constraints']['min'], 2) 141 | self.assertEqual(find_by_name(resource_defs, 'ephemeral_disk')['constraints']['min'], 4096) 142 | self.assertEqual(find_by_name(resource_defs, 'persistent_disk')['constraints']['min'], 0) 143 | self.assertEqual(find_by_name(resource_defs, 'ram')['constraints']['min'], 512) 144 | 145 | class VerifyForms(unittest.TestCase): 146 | 147 | def setUp(self): 148 | self.assertTrue(os.path.exists('product/metadata')) 149 | files = glob.glob('product/metadata/*.yml') 150 | self.assertEqual(len(files), 1) 151 | self.metadata = read_yaml(files[0]) 152 | 153 | def test_collection_form_does_not_contain_uuid_property(self): 154 | form = find_by_name(self.metadata['form_types'], 'albums_form') 155 | collection_ref = form['property_inputs'][0] 156 | for property_ref in collection_ref['property_inputs']: 157 | self.assertNotEqual(property_ref['reference'], 'id') 158 | 159 | class VerifyJobs(unittest.TestCase): 160 | 161 | def test_cross_deployment_link_in_deploy_all_job(self): 162 | deploy_all_sh_file = 'release/jobs/deploy-all/templates/deploy-all.sh.erb' 163 | self.assertTrue(os.path.exists(deploy_all_sh_file)) 164 | deploy_all_sh = read_file(deploy_all_sh_file) 165 | self.assertIn(b'NATS_HOST=', deploy_all_sh) 166 | self.assertIn(b'NATS_HOSTS=', deploy_all_sh) 167 | self.assertIn(b'cf set-env $1 NATS_HOST ', deploy_all_sh) 168 | self.assertIn(b'cf set-env $1 NATS_HOSTS ', deploy_all_sh) 169 | 170 | def test_in_deployment_link_in_deploy_all_job(self): 171 | deploy_all_sh_file = 'release/jobs/deploy-all/templates/deploy-all.sh.erb' 172 | self.assertTrue(os.path.exists(deploy_all_sh_file)) 173 | deploy_all_sh = read_file(deploy_all_sh_file) 174 | self.assertIn(b'REDIS_HOST=', deploy_all_sh) 175 | self.assertIn(b'REDIS_HOSTS=', deploy_all_sh) 176 | self.assertIn(b'cf set-env $1 REDIS_HOST ', deploy_all_sh) 177 | self.assertIn(b'cf set-env $1 REDIS_HOSTS ', deploy_all_sh) 178 | 179 | def test_docker_bosh_link_in_deploy_all_job(self): 180 | deploy_all_sh_file = 'release/jobs/deploy-all/templates/deploy-all.sh.erb' 181 | self.assertTrue(os.path.exists(deploy_all_sh_file)) 182 | deploy_all_sh = read_file(deploy_all_sh_file) 183 | self.assertIn(b'DOCKER_TCP_HOST=', deploy_all_sh) 184 | self.assertIn(b'DOCKER_TCP_HOSTS=', deploy_all_sh) 185 | self.assertIn(b'cf set-env $1 DOCKER_TCP_HOST ', deploy_all_sh) 186 | self.assertIn(b'cf set-env $1 DOCKER_TCP_HOSTS ', deploy_all_sh) 187 | 188 | def test_link_properties_in_deploy_all_job(self): 189 | deploy_all_sh_file = 'release/jobs/deploy-all/templates/deploy-all.sh.erb' 190 | self.assertTrue(os.path.exists(deploy_all_sh_file)) 191 | deploy_all_sh = read_file(deploy_all_sh_file) 192 | self.assertIn(b'NATS_PROPERTIES=', deploy_all_sh) 193 | self.assertIn(b'REDIS_PROPERTIES=', deploy_all_sh) 194 | self.assertIn(b'cf set-env $1 NATS_PROPERTIES ', deploy_all_sh) 195 | self.assertIn(b'cf set-env $1 REDIS_PROPERTIES ', deploy_all_sh) 196 | 197 | def test_consumes_links_in_deploy_all_spec(self): 198 | deploy_all_spec_file = 'release/jobs/deploy-all/job.MF' 199 | self.assertTrue(os.path.exists(deploy_all_spec_file)) 200 | spec = read_yaml(deploy_all_spec_file) 201 | self.assertIn('consumes', spec) 202 | self.assertIsNotNone(find_by_name(spec['consumes'], 'redis')) 203 | self.assertIsNotNone(find_by_name(spec['consumes'], 'nats')) 204 | self.assertIsNotNone(find_by_name(spec['consumes'], 'docker-tcp')) 205 | 206 | class VerifyRuntimeConfig(unittest.TestCase): 207 | 208 | def setUp(self): 209 | self.assertTrue(os.path.exists('product/metadata')) 210 | files = glob.glob('product/metadata/*.yml') 211 | self.assertEqual(len(files), 1) 212 | self.metadata = read_yaml(files[0]) 213 | 214 | def test_runtime_config_is_present(self): 215 | expected = yaml.safe_load( 216 | ''' 217 | - name: example-runtime-config 218 | runtime_config: | 219 | addons: 220 | - include: 221 | jobs: 222 | - name: no-op 223 | release: no-op-release 224 | jobs: 225 | - name: hello 226 | release: runtime-test-release 227 | name: runtime-test-release 228 | properties: 229 | hello: 230 | port: 8119 231 | releases: 232 | - name: runtime-test-release 233 | version: 0.2.0 234 | ''' 235 | ) 236 | self.assertEqual(self.metadata.get('runtime_configs'), expected) 237 | 238 | 239 | def find_by_name(lst, name): 240 | return next(x for x in lst if x.get('name', None) == name) 241 | 242 | def read_yaml(filename): 243 | with open(filename, 'rb') as file: 244 | return yaml.safe_load(file) 245 | 246 | def read_file(filename): 247 | with open(filename, 'rb') as file: 248 | return file.read() 249 | 250 | if __name__ == '__main__': 251 | unittest.main() 252 | -------------------------------------------------------------------------------- /tile_generator/bosh.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | import os 21 | import sys 22 | import errno 23 | import requests 24 | import shutil 25 | import subprocess 26 | import tarfile 27 | import tempfile 28 | from distutils import spawn 29 | from . import template 30 | try: 31 | # Python 3 32 | from urllib.request import urlretrieve 33 | except ImportError: 34 | # Python 2 35 | from urllib.request import urlretrieve 36 | import zipfile 37 | import yaml 38 | import re 39 | import datetime 40 | 41 | from .util import * 42 | 43 | class BoshRelease: 44 | 45 | def __init__(self, release, context): 46 | self.name = release['name'] 47 | self.release_dir = os.path.join('release', self.name) 48 | self.path = release.get('path', None) 49 | self.jobs = release.get('jobs', []) 50 | self.packages = release.get('packages', []) 51 | self.context = context 52 | self.config = release 53 | self.tarball = None 54 | 55 | def get_metadata(self): 56 | tarball = self.get_tarball() 57 | manifest = self.get_manifest(tarball) 58 | return { 59 | 'release_name': manifest['name'], 60 | 'version': manifest['version'], 61 | 'tarball': tarball, 62 | 'file': os.path.basename(tarball), 63 | } 64 | 65 | def get_manifest(self, tarball): 66 | with tarfile.open(tarball) as tar: 67 | if './release.MF' in tar.getnames(): 68 | manifest_file = tar.extractfile('./release.MF') 69 | elif 'release.MF' in tar.getnames(): 70 | manifest_file = tar.extractfile('release.MF') 71 | else: 72 | raise Exception('No release manifest found in ' + tarball) 73 | manifest = yaml.safe_load(manifest_file) 74 | manifest_file.close() 75 | return manifest 76 | 77 | def get_tarball(self): 78 | if self.tarball is not None and os.path.isfile(self.tarball): 79 | return self.tarball 80 | if self.path is not None: 81 | print('download bosh release', self.name) 82 | return self.download_tarball() 83 | return self.build_tarball() 84 | 85 | def download_tarball(self): 86 | def semver_x_greater_or_equal_to_y(x, y): 87 | # split the semver into major minor patch 88 | x = [int(d) for d in x.split('.')] 89 | y = [int(d) for d in y.split('.')] 90 | 91 | return x >= y 92 | 93 | mkdir_p(self.release_dir) 94 | tarball = os.path.join(self.release_dir, self.name + '.tgz') 95 | download(self.path, tarball, self.context.get('cache')) 96 | manifest = self.get_manifest(tarball) 97 | if manifest['name'] == 'cf-cli': 98 | # Enforce at least version 1.15 as prior versions have a CVE 99 | # https://docs.google.com/document/d/177QPJHKXMld1AD-GNHeildVfTGCWrGP-GSSlDmJY9eI/edit?ts=5ccd96fa 100 | if not semver_x_greater_or_equal_to_y(manifest['version'], '1.15.0'): 101 | raise RuntimeError('The cf-cli bosh release should be version 1.15.0 or higher. Detected %s' % manifest['version']) 102 | self.tarball = os.path.join(self.release_dir, manifest['name'] + '-' + manifest['version'] + '.tgz') 103 | os.rename(tarball, self.tarball) 104 | return self.tarball 105 | 106 | def build_tarball(self): 107 | mkdir_p(self.release_dir) 108 | 109 | self.__bosh('init-release') 110 | template.render( 111 | os.path.join(self.release_dir, 'config/final.yml'), 112 | 'config/final.yml', 113 | self.context) 114 | 115 | for package in self.packages: 116 | self.add_package(package) 117 | 118 | for job in self.jobs: 119 | self.add_job(job) 120 | self.__bosh('upload-blobs') 121 | filename=self.name + '-' + self.context['version'] + '.tgz' 122 | 123 | args = ['create-release', '--force','--final', '--tarball', filename, '--version', self.context['version']] 124 | if self.context.get('sha1'): 125 | args.insert(3, '--sha2') 126 | 127 | self.tarball = self.__bosh(*args, capture='Release tarball') 128 | self.tarball = os.path.join(self.release_dir,filename) 129 | return self.tarball 130 | 131 | def add_job(self, job): 132 | job_name = job['name'] 133 | job_type = job.get('type', job_name) 134 | job_template = job.get('template', job_type) 135 | is_errand = job.get('lifecycle', None) == 'errand' 136 | package = job.get('package', None) 137 | packages = job.get('packages', []) 138 | self.__bosh('generate-job', job_type) 139 | job_context = { 140 | 'job_name': job_name, 141 | 'job_type': job_type, 142 | 'context': self.context, 143 | 'package': package, 144 | 'packages': packages, 145 | 'errand': is_errand, 146 | } 147 | 148 | template.render( 149 | os.path.join(self.release_dir, 'jobs', job_type, 'spec'), 150 | os.path.join('jobs', 'spec'), 151 | job_context 152 | ) 153 | template.render( 154 | os.path.join(self.release_dir, 'jobs', job_type, 'templates', job_type + '.sh.erb'), 155 | os.path.join('jobs', job_template + '.sh.erb'), 156 | job_context 157 | ) 158 | template.render( 159 | os.path.join(self.release_dir, 'jobs', job_type, 'templates', 'opsmgr.env.erb'), 160 | os.path.join('jobs', 'opsmgr.env.erb'), 161 | job_context 162 | ) 163 | template.render( 164 | os.path.join(self.release_dir, 'jobs', job_type, 'monit'), 165 | os.path.join('jobs', 'monit'), 166 | job_context 167 | ) 168 | 169 | def needs_zip(self, package): 170 | # Only zip package types that require single files 171 | if not package.get('is_cf', False) and not package.get('zip_if_needed', False): 172 | return False 173 | files = package['files'] 174 | # ...if it has more than one file... 175 | if len(files) > 1: 176 | return True 177 | # ...or a single file is not already a zip. 178 | elif len(files) == 1: 179 | return not zipfile.is_zipfile(files[0]['path']) 180 | return False 181 | 182 | def add_blob(self,package): 183 | for file in package ['files']: 184 | self.__bosh('add-blob',os.path.realpath(file['path']),file['name']) 185 | 186 | def add_package(self, package): 187 | name = package['name'] 188 | dir = package.get('dir', 'blobs') 189 | self.__bosh('generate-package', name) 190 | target_dir = os.path.realpath(os.path.join(self.release_dir, dir, name)) 191 | package_dir = os.path.realpath(os.path.join(self.release_dir, 'packages', name)) 192 | mkdir_p(target_dir) 193 | template_dir = 'packages' 194 | # Download files for package 195 | if self.needs_zip(package): 196 | staging_dir = tempfile.mkdtemp() 197 | file_options = dict() 198 | for file in package.get('files', []): 199 | for key in [k for k in file.keys() if k not in ['name', 'path']]: 200 | file_options[key] = file[key] 201 | download(file['path'], os.path.join(staging_dir, file['name']), cache=self.context.get('cache', None)) 202 | path = package.get('manifest', {}).get('path', '') 203 | dir_to_zip = os.path.join(staging_dir, path) if path else staging_dir 204 | zipfilename = os.path.realpath(os.path.join(target_dir, package['name'] + '.zip')) 205 | zip_dir(zipfilename, dir_to_zip) 206 | shutil.rmtree(staging_dir) 207 | newpath = os.path.basename(zipfilename) 208 | for job in self.jobs: 209 | if job.get('manifest', {}).get(package['name'], {}).get('app_manifest'): 210 | job['manifest'][package['name']]['app_manifest']['path'] = newpath 211 | result = { 'path': zipfilename, 'name': os.path.basename(zipfilename) } 212 | result.update(file_options) 213 | package['files'] = [result] 214 | self.__bosh('add-blob',zipfilename,os.path.join(name,os.path.basename(zipfilename))) 215 | else: 216 | for file in package.get('files', []): 217 | download(file['path'], os.path.join(target_dir, file['name']), cache=self.context.get('cache', None)) 218 | self.__bosh('add-blob',os.path.join(target_dir, file['name']),os.path.join(name,file['name'])) 219 | # Construct context for template rendering 220 | package_context = { 221 | 'context': self.context, 222 | 'package': package, 223 | 'files': package.get('files', []), 224 | } 225 | template.render( 226 | os.path.join(package_dir, 'spec'), 227 | os.path.join(template_dir, 'spec'), 228 | package_context 229 | ) 230 | template.render( 231 | os.path.join(package_dir, 'packaging'), 232 | os.path.join(template_dir, 'packaging'), 233 | package_context 234 | ) 235 | 236 | def __bosh(self, *argv, **kw): 237 | return run_bosh(self.release_dir, *argv, **kw) 238 | 239 | 240 | def ensure_bosh(): 241 | bosh_exec = spawn.find_executable('bosh') 242 | if not bosh_exec: 243 | print("'bosh' command should be on the path. See https://bosh.io for installation instructions") 244 | sys.exit(1) 245 | 246 | if bosh_exec: 247 | output = subprocess.check_output(["bosh", "--version"], stderr=subprocess.STDOUT, cwd=".") 248 | if output.startswith(b"version 1."): 249 | print("You are running an older version of bosh. Please upgrade to the latest version. See https://bosh.io/docs/cli-v2.html for installation instructions") 250 | sys.exit(1) 251 | 252 | def run_bosh(working_dir, *argv, **kw): 253 | ensure_bosh() 254 | 255 | # Ensure that the working_dir is a git repo, needed for bosh's create-release. 256 | # This is used to avoid this bug https://www.pivotaltracker.com/story/show/159156765 257 | if 'create-release' in argv: 258 | print(working_dir) 259 | cmd = 'if ! git rev-parse --git-dir 2> /dev/null; then git init; fi' 260 | subprocess.call(cmd, shell=True, cwd=working_dir) 261 | 262 | # Change the commands 263 | argv = list(argv) 264 | print('bosh', ' '.join(argv)) 265 | command = ['bosh', '--no-color', '--non-interactive'] + argv 266 | capture = kw.get('capture', None) 267 | try: 268 | output = subprocess.check_output(command, stderr=subprocess.STDOUT, cwd=working_dir) 269 | if capture is not None: 270 | for l in output.split(b'\n'): 271 | if l.startswith(bytes(capture, 'utf-8')): 272 | output = l.split(':', 1)[-1].strip() 273 | break 274 | return output 275 | except subprocess.CalledProcessError as e: 276 | if argv[0] == 'init' and argv[1] == 'release' and 'Release already initialized' in e.output: 277 | return e.output 278 | if argv[0] == 'generate' and 'already exists' in e.output: 279 | return e.output 280 | print(e.output) 281 | sys.exit(e.returncode) 282 | -------------------------------------------------------------------------------- /tile_generator/template.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # tile-generator 4 | # 5 | # Copyright (c) 2015-Present Pivotal Software, Inc. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | import os 21 | import re 22 | import sys 23 | import errno 24 | import yaml 25 | 26 | from jinja2 import Template, Environment, FileSystemLoader, exceptions, pass_context 27 | 28 | PATH = os.path.dirname(os.path.realpath(__file__)) 29 | TEMPLATE_PATH = os.path.realpath(os.path.join(PATH, 'templates')) 30 | 31 | PROPERTY_FIELDS = { 32 | 'simple_credentials': ['identity', 'password'], 33 | 'rsa_cert_credentials': ['private_key_pem', 'cert_pem', 'public_key_pem', 'cert_and_private_key_pems'], 34 | 'rsa_pkey_credentials': ['private_key_pem', 'public_key_pem', 'public_key_openssh', 'public_key_fingerprint'], 35 | 'salted_credentials': ['salt', 'identity', 'password'], 36 | 'selector': ['value', ('selected_option', 'selected_option.parsed_manifest(manifest_snippet)')], 37 | } 38 | 39 | 40 | def render_hyphens(input): 41 | return input.replace('_', '-') 42 | 43 | 44 | def expand_selector(input): 45 | if input.get('type', None) == 'selector': 46 | for option in input.get('option_templates', []): 47 | properties = '' 48 | for p in option.get('property_blueprints', []): 49 | if p['type'] in PROPERTY_FIELDS: 50 | properties += p['name'] + ': { ' 51 | subproperties = [] 52 | for subproperty in PROPERTY_FIELDS[p['type']]: 53 | if type(subproperty) is tuple: 54 | subproperties.append( 55 | '{}: (( .properties.{}.{}.{}.{} ))'.format(subproperty[0], input['name'], 56 | option['name'], p['name'], subproperty[1])) 57 | else: 58 | subproperties.append( 59 | '{}: (( .properties.{}.{}.{}.{} ))'.format(subproperty, input['name'], option['name'], 60 | p['name'], subproperty)) 61 | properties += ', '.join(subproperties) + ' }\r\n' 62 | else: 63 | properties += p['name'] + ': (( .properties.' + input['name'] + '.' + option['name'] + '.' + p[ 64 | 'name'] + '.value ))\r\n' 65 | option['named_manifests'] = option.get("named_manifests", []) 66 | option['named_manifests'].append({ 67 | 'name': 'manifest_snippet', 68 | 'manifest': properties 69 | }) 70 | return input 71 | 72 | 73 | def render_yaml(input): 74 | # Inspired by https://stackoverflow.com/questions/50519454/python-yaml-dump-using-block-style-without-quotes 75 | def multiline_representer(dumper, data): 76 | return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|" if "\n" in data else None) 77 | 78 | yaml.SafeDumper.add_representer(str, multiline_representer) 79 | yaml.SafeDumper.ignore_aliases = lambda *args: True 80 | return yaml.safe_dump(input, default_flow_style=False, width=float("inf")) 81 | 82 | 83 | def render_yaml_literal(input): 84 | return yaml.safe_dump(input, default_flow_style=False, default_style='|', width=float("inf")) 85 | 86 | 87 | def render_shell_string(input): 88 | return '<%= Shellwords.escape ' + input + ' %>' 89 | 90 | 91 | def render_plans_json(input, escape=True, export=True): 92 | property_name = input['name'] 93 | variable_name = input.get('variable_name', property_name.upper()) 94 | value = '=<%= Shellwords.escape JSON.dump(plans) %>' if escape else '=<%= JSON.dump(plans) %>' 95 | export = 'export ' if export else '' 96 | return ('<%\n' 97 | ' plans = { }\n' 98 | ' p("' + property_name + '").each do |plan|\n' 99 | ' plan_name = plan[\'name\']\n' 100 | ' plans[plan_name] = plan\n' 101 | ' end\n' 102 | '%>\n' 103 | '' + export + variable_name + value) 104 | 105 | 106 | def render_selector_json(input, escape=True, export=True): 107 | value = '=<%= Shellwords.escape JSON.dump(hash) %>' if escape else '=<%= JSON.dump(hash) %>' 108 | export = 'export ' if export else '' 109 | return ('<%\n' 110 | ' hash = { }\n' 111 | ' hash["value"] = p("' + input + '")["value"]\n' 112 | ' hash["selected_option"] = { }\n' 113 | ' if p("' + input + '")["selected_option"]\n' 114 | ' p("' + input + '")["selected_option"].each_pair do |key, value|\n' 115 | ' hash["selected_option"][key] = value\n' 116 | ' end\n' 117 | ' end\n' 118 | '%>\n' 119 | '' + export + input.upper() + value) 120 | 121 | 122 | def render_collection_json(input, escape=True, export=True): 123 | value = '=<%= Shellwords.escape JSON.dump(array) %>' if escape else '=<%= JSON.dump(array) %>' 124 | export = 'export ' if export else '' 125 | return ('<%\n' 126 | ' array = [ ]\n' 127 | ' p("' + input + '").each do |m|\n' 128 | ' member = { }\n' 129 | ' m.each_pair do |key, value|\n' 130 | ' member[key] = value\n' 131 | ' end\n' 132 | ' array << member\n' 133 | ' end\n' 134 | '%>\n' 135 | '' + export + input.upper() + value) 136 | 137 | 138 | def render_property_json(input, escape=True, export=True): 139 | escape = ' Shellwords.escape' if escape else '' 140 | export = 'export ' if export else '' 141 | return (export + '{}=<%={} properties.{}.marshal_dump.to_json %>'.format(input.upper(), escape, input)) 142 | 143 | 144 | def render_property_value(input, escape=True, export=True): 145 | escape = ' Shellwords.escape' if escape else '' 146 | export = 'export ' if export else '' 147 | return (export + '{}=<%={} properties.{} %>'.format(input.upper(), escape, input)) 148 | 149 | 150 | def render_env_variable(property, escape=True, export=True): 151 | complex_types = ( 152 | 'simple_credentials', 153 | 'rsa_cert_credentials', 154 | 'rsa_pkey_credentials', 155 | 'salted_credentials', 156 | 'selector', 157 | ) 158 | if property['type'] in ['selector']: 159 | return render_selector_json(property['name'], escape, export) 160 | elif property['type'] in ['collection']: 161 | return render_collection_json(property['name'], escape, export) 162 | elif property['type'] in complex_types: 163 | return render_property_json(property['name'], escape, export) 164 | else: 165 | return render_property_value(property['name'], escape, export) 166 | 167 | 168 | def render_property(property): 169 | """Render a property for bosh manifest, according to its type.""" 170 | # This ain't the prettiest thing, but it should get the job done. 171 | # I don't think we have anything more elegant available at bosh-manifest-generation time. 172 | # See https://docs.pivotal.io/partners/product-template-reference.html for list. 173 | if 'type' in property and property['type'] in PROPERTY_FIELDS: 174 | fields = {} 175 | for field in PROPERTY_FIELDS[property['type']]: 176 | if type(field) is tuple: 177 | fields[field[0]] = '(( .properties.{}.{} ))'.format(property['name'], field[1]) 178 | else: 179 | fields[field] = '(( .properties.{}.{} ))'.format(property['name'], field) 180 | out = {property['name']: fields} 181 | else: 182 | if property.get('is_reference', False): 183 | out = {property['name']: property['default']} 184 | else: 185 | out = {property['name']: '(( .properties.{}.value ))'.format(property['name'])} 186 | return out 187 | 188 | 189 | def render_shell_variable_name(s): 190 | """Convert s to a shell variable identifier.""" 191 | return re.sub(r'[^a-zA-Z0-9]+', '_', s).upper() 192 | 193 | 194 | @pass_context 195 | def render(context, input): 196 | template = Template(input) 197 | return template.render(context) 198 | 199 | 200 | TEMPLATE_ENVIRONMENT = Environment(trim_blocks=True, lstrip_blocks=True, extensions=['jinja2.ext.do']) 201 | TEMPLATE_ENVIRONMENT.loader = FileSystemLoader(TEMPLATE_PATH) 202 | TEMPLATE_ENVIRONMENT.filters['hyphens'] = render_hyphens 203 | TEMPLATE_ENVIRONMENT.filters['expand_selector'] = expand_selector 204 | TEMPLATE_ENVIRONMENT.filters['yaml'] = render_yaml 205 | TEMPLATE_ENVIRONMENT.filters['yaml_literal'] = render_yaml_literal 206 | TEMPLATE_ENVIRONMENT.filters['shell_string'] = render_shell_string 207 | TEMPLATE_ENVIRONMENT.filters['shell_variable_name'] = render_shell_variable_name 208 | TEMPLATE_ENVIRONMENT.filters['plans_json'] = render_plans_json 209 | TEMPLATE_ENVIRONMENT.filters['property'] = render_property 210 | TEMPLATE_ENVIRONMENT.filters['env_variable'] = render_env_variable 211 | TEMPLATE_ENVIRONMENT.filters['render'] = render 212 | 213 | 214 | def render(target_path, template_file, config): 215 | target_dir = os.path.dirname(target_path) 216 | if target_dir != '': 217 | mkdir_p(target_dir) 218 | with open(target_path, 'wb') as target: 219 | target.write(bytes(TEMPLATE_ENVIRONMENT.get_template(template_file).render(config), 'utf-8')) 220 | 221 | 222 | def exists(template_file): 223 | return os.exists(path(template_file)) 224 | 225 | 226 | def path(template_file): 227 | return os.path.join(TEMPLATE_PATH, template_file) 228 | 229 | 230 | def mkdir_p(dir): 231 | try: 232 | os.makedirs(dir) 233 | except os.error as e: 234 | if e.errno != errno.EEXIST: 235 | raise 236 | --------------------------------------------------------------------------------