├── install ├── connectors │ ├── connector-otc.sh │ ├── connector-vcloud.sh │ ├── connector-exoscale.sh │ ├── connector-nuvlabox.sh │ ├── connector-openstack.sh │ ├── connector-cloudstack.sh │ ├── connector-opennebula.sh │ └── connector-ec2.sh ├── ss-ci-pipeline │ ├── ss-deployer_preinstall.sh │ ├── ss-tester_reports.sh │ ├── ss-builder_preinstall.sh │ ├── ss-tester_postinstall.sh │ ├── ss-tester.sh │ ├── ss-builder_postinstall.sh │ ├── ss-reports-collector.sh │ ├── ss-builder.sh │ ├── ss-deployer.sh │ └── ss-tester.py ├── ss-collectd.conf ├── slipstream-community.sh ├── slipstream-enterprise.sh ├── ss-install-riemann.sh ├── ss-install-backup.sh ├── slipstream-install.sh ├── ss-install-es-backup.sh ├── ss-install-connectors.sh ├── ss-install-nrpe.sh ├── ss-collectd-jmx.conf ├── ss-install-ref-conf.sh ├── ss-from-sources.sh └── slipstream.sh ├── .gitignore ├── src └── main │ └── resources │ └── versions.sh ├── Procfile ├── .circleci ├── config.yml └── settings.xml ├── git-pull.sh ├── licenses.sh ├── BUILD_STATUS.md ├── community └── pom.xml ├── README.md ├── enterprise └── pom.xml ├── tools └── updateSlipStreamRepos └── pom.xml /install/connectors/connector-otc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function deploy() { 4 | yum -y install slipstream-connector-otc 5 | } 6 | 7 | deploy 8 | -------------------------------------------------------------------------------- /install/connectors/connector-vcloud.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function deploy() { 4 | yum -y install slipstream-connector-vcloud 5 | } 6 | 7 | deploy 8 | -------------------------------------------------------------------------------- /install/connectors/connector-exoscale.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function deploy() { 4 | yum -y install slipstream-connector-exoscale 5 | } 6 | 7 | deploy 8 | -------------------------------------------------------------------------------- /install/connectors/connector-nuvlabox.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function deploy() { 4 | yum -y install slipstream-connector-nuvlabox 5 | } 6 | 7 | deploy 8 | -------------------------------------------------------------------------------- /install/connectors/connector-openstack.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function deploy() { 4 | yum -y install slipstream-connector-openstack 5 | } 6 | 7 | deploy 8 | -------------------------------------------------------------------------------- /install/connectors/connector-cloudstack.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function deploy() { 4 | yum -y install slipstream-connector-cloudstack 5 | } 6 | 7 | deploy 8 | -------------------------------------------------------------------------------- /install/connectors/connector-opennebula.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function deploy() { 4 | yum -y install slipstream-connector-opennebula 5 | } 6 | 7 | deploy 8 | -------------------------------------------------------------------------------- /install/ss-ci-pipeline/ss-deployer_preinstall.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | set -x 5 | 6 | yum install -y yum-utils epel-release 7 | yum-config-manager --enable epel 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .slipstream-build-all 2 | *~ 3 | .idea/ 4 | *.iml 5 | 6 | target 7 | 8 | pom.xml.next 9 | pom.xml.releaseBackup 10 | pom.xml.tag 11 | release.properties 12 | versions.sh 13 | -------------------------------------------------------------------------------- /src/main/resources/versions.sh: -------------------------------------------------------------------------------- 1 | TAG_VERSION=${parsedVersion.majorVersion}.${parsedVersion.minorVersion} 2 | NEXT_VERSION=${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}-SNAPSHOT 3 | -------------------------------------------------------------------------------- /install/ss-ci-pipeline/ss-tester_reports.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | test_results_dir=~/test-results 7 | 8 | [ -d $test_results_dir ] && cp -rp $test_results_dir $SLIPSTREAM_REPORT_DIR 9 | -------------------------------------------------------------------------------- /install/ss-ci-pipeline/ss-builder_preinstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # 7 | # upgrade system 8 | # 9 | echo statecustom "Upgrading system..." 10 | yum clean all 11 | yum install deltarpm -y 12 | yum upgrade -y 13 | -------------------------------------------------------------------------------- /install/ss-ci-pipeline/ss-tester_postinstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | curl -fsSL -o /usr/bin/lein \ 7 | https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein 8 | chmod 755 /usr/bin/lein 9 | 10 | export LEIN_ROOT=true 11 | lein -h 12 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | db: java -cp $HOME/.m2/repository/org/hsqldb/hsqldb/2.3.2/hsqldb-2.3.2.jar org.hsqldb.server.Server --database.0 file:slipstreamdb --dbname.0 slipstream 2 | web: mvn jetty:run-war -f ${PWD}/../SlipStreamServer/war/pom.xml -Dpersistence.unit=hsqldb-schema -Dstatic.content.location=file://${PWD}/../SlipStreamUI/src/slipstream/ui/views 3 | -------------------------------------------------------------------------------- /install/connectors/connector-ec2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function deploy() { 4 | # EPEL repo is required. slipstream-connector-ec2 depends on 5 | # slipstream-connector-ec2-python, which requires python-boto, 6 | # which in turn comes from EPEL. 7 | yum install -y --enablerepo=epel slipstream-connector-ec2 8 | } 9 | 10 | deploy 11 | -------------------------------------------------------------------------------- /install/ss-collectd.conf: -------------------------------------------------------------------------------- 1 | Hostname "SS_HOSTNAME" 2 | 3 | FQDNLookup false 4 | 5 | LoadPlugin logfile 6 | 7 | LogLevel "info" 8 | File "/var/log/collectd.log" 9 | Timestamp true 10 | PrintSeverity true 11 | 12 | 13 | # Collect JVM metrics over JMX from SS. 14 | LoadPlugin java 15 | 16 | # For sending metrics to Logstash. 17 | LoadPlugin network 18 | 19 | Server "SS_LOGSTASH_COLLECTD" "SS_LOGSTASH_COLLECTD_UDP" 20 | 21 | 22 | Include "/etc/collectd.d" 23 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Java Maven CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-java/ for more details 4 | # 5 | version: 2 6 | jobs: 7 | build: 8 | docker: 9 | # specify the version you desire here 10 | - image: sixsq/slipstream:build 11 | 12 | # Specify service dependencies here if necessary 13 | # CircleCI maintains a library of pre-built images 14 | # documented at https://circleci.com/docs/2.0/circleci-images/ 15 | # - image: circleci/postgres:9.4 16 | 17 | working_directory: ~/repo 18 | 19 | environment: 20 | # Customize the JVM maximum heap limit 21 | MAVEN_OPTS: -Xmx1600m 22 | 23 | steps: 24 | - checkout 25 | 26 | # Download and cache dependencies 27 | - restore_cache: 28 | keys: 29 | - v1-dependencies-{{ checksum "pom.xml" }} 30 | # fallback to using the latest cache if no exact match is found 31 | - v1-dependencies- 32 | 33 | - run: mvn -s .circleci/settings.xml --batch-mode dependency:go-offline -DskipModules 34 | 35 | - save_cache: 36 | paths: 37 | - ~/.m2 38 | key: v1-dependencies-{{ checksum "pom.xml" }} 39 | 40 | # build and deploy artifacts 41 | - run: mvn -s .circleci/settings.xml --batch-mode deploy -DskipModules 42 | -------------------------------------------------------------------------------- /install/slipstream-community.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | _SCRIPT_NAME=${0##*/} 6 | 7 | _YUM_REPO_KIND_DEFAULT=release 8 | 9 | function usage() { 10 | echo -e "usage:\n$_SCRIPT_NAME [repokind] 11 | - repokind: (default: ${_YUM_REPO_KIND_DEFAULT})" 12 | exit 1 13 | } 14 | 15 | while getopts : opt; do 16 | case $opt in 17 | \?) 18 | usage 19 | ;; 20 | esac 21 | done 22 | 23 | GH_BASE_URL=https://raw.githubusercontent.com/slipstream/SlipStream 24 | 25 | declare -A YUM_REPO_TO_GH_BRANCH 26 | YUM_REPO_TO_GH_BRANCH[local]=master 27 | YUM_REPO_TO_GH_BRANCH[snapshot]=master 28 | YUM_REPO_TO_GH_BRANCH[candidate]=candidate-latest 29 | YUM_REPO_TO_GH_TAG[${_YUM_REPO_KIND_DEFAULT}]=release-latest 30 | 31 | function _check_repo() { 32 | if ! test "${YUM_REPO_TO_GH_BRANCH[$1]+isset}"; then 33 | usage 34 | fi 35 | } 36 | 37 | function _download() { 38 | TO=$1 39 | shift 40 | FROM=$SCRIPT_BASE_URL/$TO 41 | echo -n "::: Downloading $@... " 42 | curl -sSf -k -o $TO $FROM || { echo "Failed downloading $FROM"; exit 1; } 43 | echo "done." 44 | chmod +x $TO 45 | } 46 | 47 | YUM_REPO_KIND=${1:-${_YUM_REPO_KIND_DEFAULT}} 48 | _check_repo $YUM_REPO_KIND 49 | SCRIPT_BASE_URL=$GH_BASE_URL/${YUM_REPO_TO_GH_BRANCH[$YUM_REPO_KIND]}/install 50 | 51 | SCRIPT=slipstream-install.sh 52 | _download $SCRIPT "SlipStream installation wrapper script" 53 | ./$SCRIPT community $YUM_REPO_KIND 54 | 55 | -------------------------------------------------------------------------------- /install/slipstream-enterprise.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | _SCRIPT_NAME=${0##*/} 6 | 7 | _YUM_REPO_KIND_DEFAULT=release 8 | 9 | function usage() { 10 | echo -e "usage:\n$_SCRIPT_NAME [repokind] 11 | - repokind: (default: ${_YUM_REPO_KIND_DEFAULT})" 12 | exit 1 13 | } 14 | 15 | while getopts : opt; do 16 | case $opt in 17 | \?) 18 | usage 19 | ;; 20 | esac 21 | done 22 | 23 | GH_BASE_URL=https://raw.githubusercontent.com/slipstream/SlipStream 24 | 25 | declare -A YUM_REPO_TO_GH_BRANCH 26 | YUM_REPO_TO_GH_BRANCH[local]=master 27 | YUM_REPO_TO_GH_BRANCH[snapshot]=master 28 | YUM_REPO_TO_GH_BRANCH[candidate]=candidate-latest 29 | YUM_REPO_TO_GH_TAG[${_YUM_REPO_KIND_DEFAULT}]=release-latest 30 | 31 | function _check_repo() { 32 | if ! test "${YUM_REPO_TO_GH_BRANCH[$1]+isset}"; then 33 | usage 34 | fi 35 | } 36 | 37 | function _download() { 38 | TO=$1 39 | shift 40 | FROM=$SCRIPT_BASE_URL/$TO 41 | echo -n "::: Downloading $@... " 42 | curl -sSf -k -o $TO $FROM || { echo "Failed downloading $FROM"; exit 1; } 43 | echo "done." 44 | chmod +x $TO 45 | } 46 | 47 | YUM_REPO_KIND=${1:-${_YUM_REPO_KIND_DEFAULT}} 48 | _check_repo $YUM_REPO_KIND 49 | SCRIPT_BASE_URL=$GH_BASE_URL/${YUM_REPO_TO_GH_BRANCH[$YUM_REPO_KIND]}/install 50 | 51 | SCRIPT=slipstream-install.sh 52 | _download $SCRIPT "SlipStream installation wrapper script" 53 | ./$SCRIPT enterprise $YUM_REPO_KIND 54 | 55 | -------------------------------------------------------------------------------- /git-pull.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | GITHUB_URLS=( 4 | "git@github.com:slipstream" 5 | "https://github.com/slipstream" 6 | ) 7 | 8 | REPOS=( 9 | "SlipStreamUI" 10 | "SlipStreamWebUI" 11 | "SlipStreamServer" 12 | "SlipStreamServerDeps" 13 | "SlipStreamClient" 14 | "SlipStreamClojureAPI" 15 | "SlipStreamConnectors" 16 | "SlipStreamPythonAPI" 17 | "SlipStreamJobEngine" 18 | "SlipStreamParent" 19 | "SlipStreamTests" 20 | 21 | # add enterprise-only repositories here -- do not remove this comment 22 | "SlipStreamConnectorsEnterprise" 23 | "SlipStreamServerEnterprise" 24 | ) 25 | 26 | # loop over possible repository locations for cloning 27 | clone_ss_repo(){ 28 | repo=${1} 29 | 30 | for i in "${GITHUB_URLS[@]}" 31 | do 32 | repo_url=${i}/${repo}.git 33 | echo "Cloning ${repo_url}" 34 | git clone ${repo_url} 35 | rc=$? 36 | if [ ${rc} -eq 0 ] 37 | then 38 | break; 39 | else 40 | echo "Cloning ${repo_url} failed!" 41 | fi 42 | done 43 | 44 | return ${rc} 45 | } 46 | 47 | # Update or clone the given repository. 48 | update_ss_repo(){ 49 | repo=${1} 50 | 51 | rc=0 52 | if [ -d ${repo} ] 53 | then 54 | echo "Updating ${repo}..." 55 | (cd ${repo}; git rev-parse --abbrev-ref HEAD; git pull) 56 | rc=$? 57 | else 58 | echo "Repo ${repo} is not yet cloned here." 59 | clone_ss_repo ${repo} 60 | rc=$? 61 | fi 62 | 63 | if [ ${rc} -ne 0 ] 64 | then 65 | echo "Clone of ${repo} failed." 66 | exit ${rc} 67 | fi 68 | } 69 | 70 | # Loop through all requested repositories and update or clone. 71 | for repo in ${REPOS[@]} 72 | do 73 | update_ss_repo ${repo} 74 | echo 75 | done 76 | -------------------------------------------------------------------------------- /install/ss-install-riemann.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Installs Riemann server on CentOS 7. 4 | # Requires EPEL repository. 5 | 6 | set -e 7 | set -x 8 | 9 | RIEMANN_VER=0.2.11-1 10 | ss_clj_client=/opt/slipstream/riemann/lib/SlipStreamRiemann.jar 11 | ss_riemann_conf=/etc/riemann/riemann-slipstream.config 12 | ss_riemann_streams=/opt/slipstream/riemann/streams 13 | 14 | function _inst() { 15 | yum install -y $@ 16 | } 17 | function srvc_start() { 18 | systemctl start $1 19 | } 20 | function srvc_enable() { 21 | systemctl enable $1 22 | } 23 | function _print() { 24 | echo -e "::: $@" 25 | } 26 | 27 | function _install_riemann() { 28 | yum localinstall -y https://aphyr.com/riemann/riemann-${RIEMANN_VER}.noarch.rpm 29 | srvc_enable riemann 30 | } 31 | 32 | function _add_ss_riemann_streams() { 33 | mkdir -p $ss_riemann_streams 34 | cat > $ss_riemann_conf<> /etc/sysconfig/riemann<&1 | tee /tmp/ss-execute-$CONNECTOR.log 51 | rc=$? 52 | if [ $rc -ne 0 ]; then 53 | msg="ERROR: Failed '$msg'" 54 | ss-display "$msg" 55 | echo $msg 56 | exit $rc 57 | fi 58 | done 59 | -------------------------------------------------------------------------------- /licenses.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # To use this script: 4 | # 1. Go to parent directory with all SlipStream modules: cd ../ 5 | # 2. Run script: ./SlipStream/licenses.sh > licenses.log 2>&1 6 | # 3. All license information in licenses-all.tar.gz 7 | # 4. Enjoy! 8 | 9 | ## Clean all license files. 10 | (cd SlipStream; mvn clean -P enterprise-build) 11 | 12 | ## Generates dependencies.html for all Java modules 13 | 14 | (cd SlipStream; mvn -P enterprise-build project-info-reports:dependencies) 15 | 16 | ## Clojure(Script) dependencies 17 | 18 | rm -f dirs.txt 19 | for i in `find . -name project.clj | grep -v module/ `; do (echo `dirname $i` >> dirs.txt); done 20 | for i in `cat dirs.txt`; do (cd $i; echo $i; mkdir -p target; lein licenses > target/licenses-clj.txt); done 21 | rm -f dirs.txt 22 | 23 | ## Node.js dependencies from WebUI 24 | 25 | (cd SlipStreamWebUI; mkdir -p target; npm install; ./node_modules/.bin/license-checker > target/licenses-npm.txt) 26 | 27 | ## Python dependencies 28 | 29 | rm -f dirs.txt 30 | for i in `find . -name requirements.txt`; do (echo `dirname $i` >> dirs.txt); done 31 | for i in `cat dirs.txt`; do (\ 32 | cd $i; \ 33 | echo $i; \ 34 | mkdir -p target ; \ 35 | virtualenv target/deps-env; \ 36 | source target/deps-env/bin/activate; \ 37 | pip install -r requirements.txt; \ 38 | pip install pip-licenses; \ 39 | pip-licenses > target/licenses-pypi.txt); \ 40 | done; 41 | 42 | rm -f dirs.txt 43 | 44 | ## Bundle all license information 45 | 46 | find . -path '*/site/*' -o -name licenses-\*.txt | tar zcf licenses-all.tar.gz -T - 47 | -------------------------------------------------------------------------------- /BUILD_STATUS.md: -------------------------------------------------------------------------------- 1 | - [![CircleCI](https://circleci.com/gh/slipstream/SlipStream/tree/master.svg?style=svg)](https://circleci.com/gh/slipstream/SlipStream/tree/master) SlipStream 2 | - [![CircleCI](https://circleci.com/gh/slipstream/SlipStreamServerDeps/tree/master.svg?style=svg)](https://circleci.com/gh/slipstream/SlipStreamServerDeps/tree/master) SlipStreamServerDeps 3 | - [![CircleCI](https://circleci.com/gh/slipstream/SlipStreamUI/tree/master.svg?style=svg)](https://circleci.com/gh/slipstream/SlipStreamUI/tree/master) SlipStreamUI 4 | - [![CircleCI](https://circleci.com/gh/slipstream/SlipStreamPythonAPI/tree/master.svg?style=svg)](https://circleci.com/gh/slipstream/SlipStreamPythonAPI/tree/master) SlipStreamPythonAPI 5 | - [![CircleCI](https://circleci.com/gh/slipstream/SlipStreamClient/tree/master.svg?style=svg)](https://circleci.com/gh/slipstream/SlipStreamClient/tree/master) SlipStreamClient 6 | - [![CircleCI](https://circleci.com/gh/slipstream/SlipStreamClojureAPI/tree/master.svg?style=svg)](https://circleci.com/gh/slipstream/SlipStreamClojureAPI/tree/master) SlipStreamClojureAPI 7 | - [![CircleCI](https://circleci.com/gh/slipstream/SlipStreamWebUI/tree/master.svg?style=svg)](https://circleci.com/gh/slipstream/SlipStreamWebUI/tree/master) SlipStreamWebUI 8 | - [![CircleCI](https://circleci.com/gh/slipstream/SlipStreamServer/tree/master.svg?style=svg)](https://circleci.com/gh/slipstream/SlipStreamServer/tree/master) SlipStreamServer 9 | - [![CircleCI](https://circleci.com/gh/slipstream/SlipStreamConnectors/tree/master.svg?style=svg)](https://circleci.com/gh/slipstream/SlipStreamConnectors/tree/master) SlipStreamConnectors 10 | - [![CircleCI](https://circleci.com/gh/slipstream/SlipStreamJobEngine/tree/master.svg?style=svg)](https://circleci.com/gh/slipstream/SlipStreamJobEngine/tree/master) SlipStreamJobEngine 11 | -------------------------------------------------------------------------------- /community/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | com.sixsq.slipstream 6 | SlipStreamCommunity 7 | pom 8 | 3.72-SNAPSHOT 9 | SlipStream/community 10 | http://sixsq.com/ 11 | 12 | 13 | com.sixsq.slipstream 14 | SlipStream 15 | 3.72-SNAPSHOT 16 | 17 | 18 | 19 | 20 | 21 | scm:git:https://github.com/slipstream 22 | scm:git:ssh://git@github.com/slipstream 23 | https://github.com/slipstream 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | true 32 | 33 | 34 | false 35 | 36 | slipstream.releases 37 | SlipStream Releases 38 | ${nexus}/releases-community${platform.os} 39 | 40 | 41 | 42 | 43 | false 44 | 45 | 46 | true 47 | 48 | slipstream.snapshots 49 | SlipStream Snapshots 50 | ${nexus}/snapshots-community${platform.os} 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SlipStream is now deprecated -> replaced by https://github.com/nuvla 2 | 3 | # SlipStream 4 | 5 | Developed by SixSq, SlipStream is a multi-cloud application management 6 | platform. See the [SlipStream product page][ss-product] for more 7 | detailed information. 8 | 9 | # Release Notes 10 | 11 | The [release notes][ss-release] are available from the main 12 | [SlipStream documentation][ss-docs] website. Stable releases are 13 | those that have been validated and are supported. Candidate releases 14 | are (usually) the result of each development iteration and may or may 15 | not be stable. These can be used but are not supported. 16 | 17 | # Getting started 18 | 19 | See the [Developer Guide][ss-dev] on the main [SlipStream 20 | documentation][ss-docs] website. This contains the full, supported 21 | build procedure for SlipStream, including the required build tools and 22 | dependencies. 23 | 24 | # License and copyright 25 | 26 | Copyright (C) 2017 SixSq Sarl (sixsq.com) 27 | 28 | The code in the public repositories is licensed under the Apache 29 | license. 30 | 31 | Licensed under the Apache License, Version 2.0 (the "License"); you 32 | may not use this file except in compliance with the License. You may 33 | obtain a copy of the License at 34 | 35 | http://www.apache.org/licenses/LICENSE-2.0 36 | 37 | Unless required by applicable law or agreed to in writing, software 38 | distributed under the License is distributed on an "AS IS" BASIS, 39 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 40 | implied. See the License for the specific language governing 41 | permissions and limitations under the License. 42 | 43 | [ss-release]: http://ssdocs.sixsq.com/en/latest/release_notes 44 | [ss-product]: http://sixsq.com/products/slipstream.html 45 | [ss-docs]: http://ssdocs.sixsq.com 46 | [ss-dev]: http://ssdocs.sixsq.com/en/latest/developer_guide 47 | -------------------------------------------------------------------------------- /.circleci/settings.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | nuvlabox.snapshots 8 | ${env.SIXSQ_NEXUS_USERNAME} 9 | ${env.SIXSQ_NEXUS_PASSWORD} 10 | 11 | 12 | nuvlabox.releases 13 | ${env.SIXSQ_NEXUS_USERNAME} 14 | ${env.SIXSQ_NEXUS_PASSWORD} 15 | 16 | 17 | sixsq.snapshots 18 | ${env.SIXSQ_NEXUS_USERNAME} 19 | ${env.SIXSQ_NEXUS_PASSWORD} 20 | 21 | 22 | sixsq.releases 23 | ${env.SIXSQ_NEXUS_USERNAME} 24 | ${env.SIXSQ_NEXUS_PASSWORD} 25 | 26 | 27 | sixsq.thirdparty 28 | ${env.SIXSQ_NEXUS_USERNAME} 29 | ${env.SIXSQ_NEXUS_PASSWORD} 30 | 31 | 32 | slipstream.snapshots 33 | ${env.SIXSQ_NEXUS_USERNAME} 34 | ${env.SIXSQ_NEXUS_PASSWORD} 35 | 36 | 37 | slipstream.releases 38 | ${env.SIXSQ_NEXUS_USERNAME} 39 | ${env.SIXSQ_NEXUS_PASSWORD} 40 | 41 | 42 | slipstream.snapshots.enterprise 43 | ${env.SIXSQ_NEXUS_USERNAME} 44 | ${env.SIXSQ_NEXUS_PASSWORD} 45 | 46 | 47 | slipstream.releases.enterprise 48 | ${env.SIXSQ_NEXUS_USERNAME} 49 | ${env.SIXSQ_NEXUS_PASSWORD} 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /install/ss-install-backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Install SlipStream backup to Amazon S3 3 | 4 | # Usage: 5 | 6 | set -e 7 | set -o pipefail 8 | set -o errtrace 9 | 10 | # E.g.: https://s3-eu-west-1.amazonaws.com/ 11 | S3_BUCKET=${1:?"Provide full URL to S3 bucket."} 12 | S3_ID=${2:?"Provide S3 ID"} 13 | S3_KEY=${3:?"Provide S3 Key"} 14 | INSTANCE_ID=${4:?"Provide instance ID (e.g., IP/hostname)."} 15 | 16 | LOG_FILE=/tmp/slipstream-backup-install.log 17 | exec 4>&2 3>&1 1>>${LOG_FILE} 2>&1 18 | 19 | SS_USER=slipstream 20 | 21 | function _print_on_trap() { 22 | _prints "\nERROR! Check log file ${LOG_FILE}\n... snippet ...\n$(tail -5 ${LOG_FILE})" 23 | } 24 | 25 | function _on_trap() { 26 | _print_on_trap 27 | } 28 | 29 | trap '_on_trap' ERR 30 | 31 | function _prints() { 32 | echo -e "$@" 1>&3 33 | } 34 | function _print() { 35 | _prints "::: $@" 36 | } 37 | function _printn() { 38 | echo -en "::: $@" 1>&3 39 | } 40 | 41 | function _install() { 42 | _printn " installing packages... " 43 | yum install -y slipstream-server-backup 44 | _prints "done." 45 | } 46 | 47 | function _configure() { 48 | _printn " configuring... " 49 | S3CURL_CONF=/opt/slipstream/server/.s3curl 50 | cp -f /opt/slipstream/backup/s3curl.cfg.tpl $S3CURL_CONF 51 | chmod 600 $S3CURL_CONF 52 | chown $SS_USER: $S3CURL_CONF 53 | sed -i -e "s|CHANGE_ME_ID|${S3_ID}|" $S3CURL_CONF 54 | sed -i -e "s|CHANGE_ME_KEY|${S3_KEY}|" $S3CURL_CONF 55 | 56 | sed -i -e "s|AMAZON_BUCKET=.*|AMAZON_BUCKET=${S3_BUCKET}|" \ 57 | -e "s|SS_HOSTNAME=.*|SS_HOSTNAME=${INSTANCE_ID}|" \ 58 | /etc/slipstream/slipstream-backup.conf \ 59 | /etc/slipstream/slipstream-es-backup.conf 60 | mkdir -p /var/log/slipstream/ 61 | chown slipstream. /var/log/slipstream/ 62 | _prints "done." 63 | } 64 | 65 | function install_backup_S3 () { 66 | _install 67 | _configure 68 | systemctl restart crond 69 | } 70 | 71 | _print "Installing SlipStream backup." 72 | install_backup_S3 73 | _print "SlipStream backup installed." 74 | -------------------------------------------------------------------------------- /enterprise/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | com.sixsq.slipstream 6 | SlipStreamEnterprise 7 | pom 8 | 3.72-SNAPSHOT 9 | SlipStream/enterprise 10 | http://sixsq.com/ 11 | 12 | 13 | com.sixsq.slipstream 14 | SlipStream 15 | 3.72-SNAPSHOT 16 | 17 | 18 | 19 | 20 | 21 | scm:git:https://github.com/SixSq 22 | scm:git:ssh://git@github.com/SixSq 23 | https://github.com/SixSq 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | true 32 | 33 | 34 | false 35 | 36 | slipstream.releases.enterprise 37 | SlipStream Releases 38 | ${nexus}/releases-enterprise${platform.os} 39 | 40 | 41 | 42 | 43 | false 44 | 45 | 46 | true 47 | 48 | slipstream.snapshots.enterprise 49 | SlipStream Snapshots 50 | ${nexus}/snapshots-enterprise${platform.os} 51 | 52 | 53 | 54 | 55 | 56 | 57 | sixsq.releases 58 | SixSq Releases 59 | ${nexus}/releases-enterprise${platform.os} 60 | 61 | 62 | 63 | sixsq.snapshots 64 | SixSq Snapshots 65 | ${nexus}/snapshots-enterprise${platform.os} 66 | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /install/slipstream-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | _SCRIPT_NAME=${0##*/} 6 | 7 | function usage() { 8 | echo -e "usage:\n$_SCRIPT_NAME [edition] [repokind] 9 | - edition: (default: community) 10 | - repokind: (default: release)" 11 | exit 1 12 | } 13 | 14 | while getopts : opt; do 15 | case $opt in 16 | \?) 17 | usage 18 | ;; 19 | esac 20 | done 21 | 22 | _GH_BASE_URL=https://raw.githubusercontent.com/slipstream/SlipStream 23 | 24 | declare -A YUM_REPO_TO_GH_BRANCH 25 | YUM_REPO_TO_GH_BRANCH[local]=master 26 | YUM_REPO_TO_GH_BRANCH[snapshot]=master 27 | YUM_REPO_TO_GH_BRANCH[candidate]=candidate-latest 28 | YUM_REPO_TO_GH_BRANCH[release]=release-latest 29 | 30 | function _check_yum_repo_kind_to_github_tag_map() { 31 | if ! test "${YUM_REPO_TO_GH_BRANCH[$1]+isset}"; then 32 | usage 33 | fi 34 | } 35 | 36 | YUM_REPO_EDITION=${1:-community} 37 | YUM_REPO_KIND=${2:-release} 38 | _check_yum_repo_kind_to_github_tag_map $YUM_REPO_KIND 39 | 40 | SCRIPT_BASE_URL=$_GH_BASE_URL/${YUM_REPO_TO_GH_BRANCH[$YUM_REPO_KIND]}/install 41 | 42 | function _download() { 43 | TO=$1 44 | shift 45 | FROM=$SCRIPT_BASE_URL/$TO 46 | echo -n "::: Downloading $@... " 47 | curl -sSf -k -o $TO $FROM || { echo "Failed downloading $FROM"; exit 1; } 48 | echo "done." 49 | chmod +x $TO 50 | } 51 | 52 | function install_slipstream_server() { 53 | echo -e ":::\n::: SlipStream Service.\n:::" 54 | SCRIPT=slipstream.sh 55 | _download $SCRIPT "SlipStream installation script" 56 | ./$SCRIPT -k $YUM_REPO_KIND -e $YUM_REPO_EDITION -x https://sixsq-build-artifacts.s3.amazonaws.com/slipstream/slipstream-3.71.repo 57 | } 58 | 59 | function install_slipstream_connectors() { 60 | echo -e ":::\n::: SlipStream Cloud Connectors.\n:::" 61 | SCRIPT=ss-install-connectors.sh 62 | _download $SCRIPT "SlipStream connectors installation script" 63 | CONNECTORS="cloudstack openstack opennebula" 64 | ./$SCRIPT -r $YUM_REPO_KIND $CONNECTORS 65 | systemctl restart slipstream 66 | echo -e "\n::: SlipStream connectors installed: $CONNECTORS" 67 | } 68 | 69 | install_slipstream_server 70 | if [ "$YUM_REPO_EDITION" == "community" ]; then 71 | install_slipstream_connectors 72 | fi 73 | -------------------------------------------------------------------------------- /install/ss-install-es-backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Install SlipStream ES backup to S3 3 | 4 | set -e 5 | set -o pipefail 6 | set -o errtrace 7 | 8 | 9 | S3_ID=${1:?"Provide S3 ID"} 10 | S3_KEY=${2:?"Provide S3 Key"} 11 | ES_HOST=${3:?"Provide ES host/IP."} 12 | ES_PORT=${4-9200} 13 | S3_ENDPOINT=${5-s3.amazonaws.com} 14 | S3_BUCKET=${6-slipstream-backup-es} 15 | S3_REGION=eu-west 16 | 17 | LOG_FILE=/tmp/slipstream-es-backup-install.log 18 | exec 4>&2 3>&1 1>>${LOG_FILE} 2>&1 19 | 20 | SS_USER=slipstream 21 | 22 | function _print_on_trap() { 23 | _prints "\nERROR! Check log file ${LOG_FILE}\n... snippet ...\n$(tail -5 ${LOG_FILE})" 24 | } 25 | 26 | function _on_trap() { 27 | _print_on_trap 28 | } 29 | 30 | trap '_on_trap' ERR 31 | 32 | function _prints() { 33 | echo -e "$@" 1>&3 34 | } 35 | function _print() { 36 | _prints "::: $@" 37 | } 38 | function _printn() { 39 | echo -en "::: $@" 1>&3 40 | } 41 | 42 | function _install() { 43 | _printn " installing packages... " 44 | yum install -y slipstream-server-backup 45 | sed -i \ 46 | -e "s|ES_HOST=.*|ES_HOST=${ES_HOST}|" \ 47 | -e "s|ES_PORT=.*|ES_PORT=${ES_PORT}|" \ 48 | /etc/slipstream/slipstream-es-backup.conf 49 | _prints "done." 50 | } 51 | 52 | function _create_backup_repo() { 53 | # Assumes repository-s3 plugin is installed on Elasticsearch. 54 | # We can't do this here as we might be running on another machine. 55 | # TODO: 56 | # - "server_side_encryption": true - not included; may not work on Exoscale. 57 | # - check client-side encryption in ES. See: 58 | # https://github.com/elastic/elasticsearch-cloud-aws/pull/118 59 | 60 | _printn " creating backup repo... " 61 | curl -XPUT \ 62 | "http://${ES_HOST}:${ES_PORT}/_snapshot/es_backup?verify=false&pretty=true" \ 63 | -d'{"type": "s3", 64 | "settings": { 65 | "endpoint": "'$S3_ENDPOINT'", 66 | "bucket": "'$S3_BUCKET'", 67 | "region": "eu-west", 68 | "access_key": "'$S3_ID'", 69 | "secret_key": "'$S3_KEY'", 70 | "compress": true, 71 | "server_side_encryption": true 72 | }}' 73 | _prints "done." 74 | } 75 | 76 | function install_es_backup_S3 () { 77 | _install 78 | _create_backup_repo 79 | } 80 | 81 | _print "Installing SlipStream ES backup." 82 | install_es_backup_S3 83 | _print "SlipStream backup ES installed." 84 | -------------------------------------------------------------------------------- /install/ss-ci-pipeline/ss-builder_postinstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # 7 | # install dependencies 8 | # 9 | echo statecustom "Installing build dependencies..." 10 | yum install -y epel-release yum-utils 11 | # epel may not be enabled though 12 | yum-config-manager --enable epel 13 | yum clean all 14 | yum install -y \ 15 | git \ 16 | java-1.8.0-openjdk-devel \ 17 | python \ 18 | python-devel \ 19 | python-pip \ 20 | python-mock \ 21 | python-nose \ 22 | python-coverage \ 23 | python-paramiko \ 24 | rpm-build \ 25 | createrepo \ 26 | npm \ 27 | gcc 28 | 29 | # Bug : https://bugzilla.redhat.com/show_bug.cgi?id=1479018 30 | # Extracted from yum 31 | pip install --upgrade pip 32 | pip install --upgrade --ignore-installed enum34 # fix bug because enum34 is already installed with yum. 33 | pip install pylint 34 | pip install tox 35 | 36 | # 37 | # install latest maven version 38 | # 39 | echo statecustom "Installing maven..." 40 | maven_version=3.3.9 41 | curl -o ~/apache-maven-${maven_version}-bin.tar.gz \ 42 | https://www-eu.apache.org/dist/maven/maven-3/${maven_version}/binaries/apache-maven-${maven_version}-bin.tar.gz 43 | tar zxf ~/apache-maven-${maven_version}-bin.tar.gz -C ~ 44 | export MAVEN_HOME=~/apache-maven-${maven_version} 45 | export MAVEN_OPTS=-Xmx2048M 46 | export PATH=$PATH:$MAVEN_HOME/bin 47 | cat >> ~/.bashrc << EOF 48 | export MAVEN_HOME=~/apache-maven-${maven_version} 49 | export MAVEN_OPTS=-Xmx2048M 50 | export PATH=$PATH:$MAVEN_HOME/bin 51 | EOF 52 | 53 | # 54 | # install leiningen 55 | # 56 | echo statecustom "Installing leiningen..." 57 | mkdir -p ~/bin 58 | curl -o ~/bin/lein https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein 59 | chmod a+x ~/bin/lein 60 | export PATH=$PATH:~/bin 61 | export LEIN_ROOT=true 62 | time lein 63 | cat >> ~/.bashrc << EOF 64 | export PATH=$PATH:~/bin 65 | export LEIN_ROOT=true 66 | EOF 67 | 68 | # 69 | # Install boot 70 | # 71 | echo statecustom "Installing boot..." 72 | boot=/usr/local/bin/boot 73 | curl -fsSLo $boot https://github.com/boot-clj/boot-bin/releases/download/latest/boot.sh 74 | chmod 755 $boot 75 | ln -sf $boot /usr/bin 76 | export BOOT_AS_ROOT=yes 77 | $boot 78 | cat >>~/.bashrc<> ~/.bashrc << EOF 105 | export HOME=~ 106 | EOF 107 | 108 | # 109 | # sanity! 110 | # 111 | echo statecustom "Installing sanity..." 112 | yum install -y emacs-nox vim bash-completion 113 | -------------------------------------------------------------------------------- /install/ss-install-connectors.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Install SlipStream connectors. 3 | 4 | set -e 5 | set -o pipefail 6 | 7 | _SCRIPT_NAME=${0##*/} 8 | 9 | _YUM_REPO_KIND_DEFAULT=release 10 | SLIPSTREAM_ETC=/etc/slipstream 11 | ES_COORDS=localhost:9300 12 | 13 | LOG_FILE=/tmp/slipstream-connectors-install.log 14 | exec 4>&2 3>&1 1>>${LOG_FILE} 2>&1 15 | 16 | function usage() { 17 | echo -e "usage:\n$_SCRIPT_NAME [-r repo] 18 | -a host:port of Elasticsearch (default: $ES_COORDS). 19 | -r repo: (default: ${_YUM_REPO_KIND_DEFAULT})" 1>&3 20 | exit 1 21 | } 22 | 23 | declare -A YUM_REPO_TO_GH_BRANCH 24 | YUM_REPO_TO_GH_BRANCH[local]=master 25 | YUM_REPO_TO_GH_BRANCH[snapshot]=master 26 | YUM_REPO_TO_GH_BRANCH[candidate]=candidate-latest 27 | YUM_REPO_TO_GH_BRANCH[${_YUM_REPO_KIND_DEFAULT}]=release-latest 28 | 29 | function _check_repo() { 30 | if ! test "${YUM_REPO_TO_GH_BRANCH[$1]+isset}"; then 31 | usage 32 | fi 33 | } 34 | 35 | SS_YUM_REPO_KIND=${_YUM_REPO_KIND_DEFAULT} 36 | 37 | while getopts :r:a: opt; do 38 | case $opt in 39 | r) 40 | SS_YUM_REPO_KIND=$OPTARG 41 | _check_repo $SS_YUM_REPO_KIND 42 | ;; 43 | a) 44 | ES_COORDS=$OPTARG 45 | ;; 46 | \?) 47 | usage 48 | ;; 49 | esac 50 | done 51 | 52 | shift $((OPTIND - 1)) 53 | 54 | export ES_HOST=${ES_COORDS%:*} 55 | export ES_PORT=${ES_COORDS##*:} 56 | 57 | GH_BASE_URL=https://raw.githubusercontent.com/slipstream/SlipStream 58 | 59 | function _prints() { 60 | echo -e "$@" 1>&3 61 | } 62 | function _print() { 63 | _prints "::: $@" 64 | } 65 | function _printn() { 66 | echo -en "::: $@" 1>&3 67 | } 68 | 69 | function _download() { 70 | TO=$1 71 | shift 72 | FROM=$SCRIPT_BASE_URL/$TO 73 | _printn "downloading $@... " 74 | curl -sSf -k -o $TO $FROM || { _print "ERROR: Failed downloading $FROM"; exit 1; } 75 | _prints "done." 76 | chmod +x $TO 77 | } 78 | 79 | function _push_connectors_configuration_to_db() { 80 | if [ -d $SLIPSTREAM_ETC/connectors ]; then 81 | SLIPSTREAM_CONNECTORS=$(find $SLIPSTREAM_ETC/connectors -name "*.edn") 82 | [ -z "$SLIPSTREAM_CONNECTORS" ] || \ 83 | { _print "Pushing connectors configuration to DB."; ss-config $SLIPSTREAM_CONNECTORS; } 84 | fi 85 | } 86 | 87 | SCRIPT_BASE_URL=$GH_BASE_URL/${YUM_REPO_TO_GH_BRANCH[$SS_YUM_REPO_KIND]}/install/connectors 88 | 89 | function _print_on_trap() { 90 | _prints "\nERROR! Check log file ${LOG_FILE}\n... snippet ...\n$(tail -5 ${LOG_FILE})" 91 | } 92 | 93 | function _on_trap() { 94 | _print_on_trap 95 | } 96 | 97 | trap '_on_trap' ERR 98 | 99 | CONNECTORS=${@} 100 | 101 | _print "Installing SlipStream connectors: $CONNECTORS" 102 | 103 | declare -A CONNECTORS_MAPPING 104 | CONNECTORS_MAPPING=( 105 | ['cloudstackadvancedzone']='cloudstack') 106 | 107 | for name in $CONNECTORS; do 108 | _print "---> ${name}" 109 | [ -n "${CONNECTORS_MAPPING[$name]}" ] && name=${CONNECTORS_MAPPING[$name]} 110 | script=connector-${name}.sh 111 | _download $script "$name connector installation script" 112 | _printn "installing... " 113 | ./$script 114 | _prints "done." 115 | done 116 | 117 | _push_connectors_configuration_to_db 118 | 119 | _print "SlipStream connectors installed: $CONNECTORS" 120 | 121 | -------------------------------------------------------------------------------- /install/ss-install-nrpe.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Install SlipStream NRPE monitoring 3 | 4 | # Usage: 5 | 6 | set -e 7 | set -o pipefail 8 | set -o errtrace 9 | 10 | # Nagios remote plugin executor 11 | NAGIOS_SERVER=${1:?"Provide Nagios server hostname/IP."} 12 | 13 | NRPE_DEPS="nrpe nagios-plugins-procs nagios-plugins-load nagios-plugins-disk" 14 | NRPE_D=/etc/nagios/nrpe.d 15 | LIB_DIR=/usr/lib64 16 | 17 | _NAGIOS_LIBEXEC=/usr/local/nagios/libexec 18 | mkdir -p ${_NAGIOS_LIBEXEC} 19 | CHECK_MEM_BIN=${_NAGIOS_LIBEXEC}/check_mem 20 | 21 | LOG_FILE=/tmp/slipstream-nrpe-install.log 22 | exec 4>&2 3>&1 1>>${LOG_FILE} 2>&1 23 | 24 | function _print_on_trap() { 25 | _prints "\nERROR! Check log file ${LOG_FILE}\n... snippet ...\n$(tail -5 ${LOG_FILE})" 26 | } 27 | 28 | function _on_trap() { 29 | _print_on_trap 30 | } 31 | 32 | trap '_on_trap' ERR 33 | 34 | function _prints() { 35 | echo -e "$@" 1>&3 36 | } 37 | function _print() { 38 | _prints "::: $@" 39 | } 40 | function _printn() { 41 | echo -en "::: $@" 1>&3 42 | } 43 | 44 | function _install_check_mem () { 45 | # check_mem 46 | # TODO: need to depend on a release version of check_mem.pl 47 | wget -O ${CHECK_MEM_BIN} \ 48 | https://raw.github.com/justintime/nagios-plugins/master/check_mem/check_mem.pl 49 | chmod +x ${CHECK_MEM_BIN} 50 | } 51 | 52 | function _iptables_get_accept_rule () { 53 | PORT=$1 54 | ACCEPT_RULE="-A INPUT -m state --state NEW -m tcp -p tcp --dport $PORT -j ACCEPT" 55 | echo $ACCEPT_RULE 56 | } 57 | 58 | function _install_iptables() { 59 | systemctl stop firewalld.service || true 60 | systemctl mask firewalld.service || true 61 | yum install -y iptables-services 62 | systemctl start iptables.service 63 | } 64 | 65 | function add_nrpe_firewall_rules () { 66 | _printn " adding NRPE firewall rules... " 67 | _install_iptables 68 | NRPE_PORT=5666 69 | NRPE_RULE="$(_iptables_get_accept_rule $NRPE_PORT)" 70 | 71 | grep -q "dport $NRPE_PORT.*ACCEPT" /etc/sysconfig/iptables && \ 72 | { _prints "(NRPE port already added) done."; return 0; } || true 73 | 74 | iptables-save > /etc/sysconfig/iptables 75 | sed -i.bak -e "/-A INPUT -j REJECT/i ${NRPE_RULE}" /etc/sysconfig/iptables 76 | systemctl restart iptables || \ 77 | { cp /etc/sysconfig/iptables.bak /etc/sysconfig/iptables; systemctl restart iptables; } 78 | _prints "done." 79 | } 80 | 81 | function install_nrpe_monitoring () { 82 | _printn " installing NRPE... " 83 | for pkg in $NRPE_DEPS; do 84 | yum -y --enablerepo=epel install $pkg 85 | done 86 | _prints "done." 87 | 88 | _printn " configuring SlipStream NRPE checks... " 89 | sed -i -e 's|^allowed_hosts=.*$|allowed_hosts='${NAGIOS_SERVER}'|' /etc/nagios/nrpe.cfg 90 | sed -i -e 's|^debug=.*$|debug=1|' /etc/nagios/nrpe.cfg 91 | sed -i -e 's|^include_dir=.*$|include_dir='${NRPE_D}'|' /etc/nagios/nrpe.cfg 92 | 93 | _install_check_mem 94 | 95 | mkdir -p $NRPE_D 96 | cat > $NRPE_D/slipstream.cfg << EOF 97 | command[check_hsqldb]=$LIB_DIR/nagios/plugins/check_procs -C java -a org.hsqldb.server.Server -c 1:1 98 | command[check_rootpart]=$LIB_DIR/nagios/plugins/check_disk -w 20% -c 10% -p / 99 | command[check_slipstream_backup]=/opt/slipstream/backup/nagios_check_slipstream_backup_timestamp 100 | command[check_slipstream_es_backup]=/opt/slipstream/backup/nagios_check_slipstream_es_backup_timestamp 101 | command[check_mem]=${CHECK_MEM_BIN} -w $ARG1$ -c $ARG2$ -f -C 102 | EOF 103 | _prints "done." 104 | 105 | _printn " starting NRPE service... " 106 | # enable arguments to NRPE commands 107 | sed -i -e 's/^dont_blame_nrpe=.*/dont_blame_nrpe=1/' /etc/nagios/nrpe.cfg 108 | 109 | systemctl enable nrpe 110 | 111 | systemctl restart nrpe 112 | _prints "done." 113 | } 114 | 115 | _print "Installing SlipStream NRPE monitoring." 116 | install_nrpe_monitoring 117 | add_nrpe_firewall_rules 118 | _print "SlipStream NRPE monitoring installed." 119 | 120 | -------------------------------------------------------------------------------- /install/ss-collectd-jmx.conf: -------------------------------------------------------------------------------- 1 | 2 | JVMArg "-verbose:jni" 3 | JVMArg "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/generic-jmx.jar" 4 | 5 | LoadPlugin "org.collectd.java.GenericJMX" 6 | 7 | 8 | ################ 9 | # MBean blocks # 10 | ################ 11 | # Number of classes being loaded. 12 | 13 | ObjectName "java.lang:type=ClassLoading" 14 | #InstancePrefix "" 15 | #InstanceFrom "" 16 | 17 | 18 | Type "gauge" 19 | InstancePrefix "loaded_classes" 20 | #InstanceFrom "" 21 | Table false 22 | Attribute "LoadedClassCount" 23 | 24 | 25 | 26 | # Time spent by the JVM compiling or optimizing. 27 | 28 | ObjectName "java.lang:type=Compilation" 29 | #InstancePrefix "" 30 | #InstanceFrom "" 31 | 32 | 33 | Type "total_time_in_ms" 34 | InstancePrefix "compilation_time" 35 | #InstanceFrom "" 36 | Table false 37 | Attribute "TotalCompilationTime" 38 | 39 | 40 | 41 | # Garbage collector information 42 | 43 | ObjectName "java.lang:type=GarbageCollector,*" 44 | InstancePrefix "gc-" 45 | InstanceFrom "name" 46 | 47 | Type "invocations" 48 | #InstancePrefix "" 49 | #InstanceFrom "" 50 | Table false 51 | Attribute "CollectionCount" 52 | 53 | 54 | 55 | Type "total_time_in_ms" 56 | InstancePrefix "collection_time" 57 | #InstanceFrom "" 58 | Table false 59 | Attribute "CollectionTime" 60 | 61 | 62 | # Not that useful, therefore commented out. 63 | 64 | Type "threads" 65 | #InstancePrefix "" 66 | #InstanceFrom "" 67 | Table false 68 | # Demonstration how to access composite types 69 | Attribute "LastGcInfo.GcThreadCount" 70 | 71 | 72 | 73 | ###################################### 74 | # Define the "jmx_memory" type as: # 75 | # jmx_memory value:GAUGE:0:U # 76 | # See types.db(5) for details. # 77 | ###################################### 78 | 79 | # Generic heap/nonheap memory usage. 80 | 81 | ObjectName "java.lang:type=Memory" 82 | #InstanceFrom "" 83 | InstancePrefix "memory" 84 | 85 | # Creates four values: committed, init, max, used 86 | 87 | Type "jmx_memory" 88 | #InstancePrefix "" 89 | #InstanceFrom "" 90 | Table true 91 | Attribute "HeapMemoryUsage" 92 | InstancePrefix "heap-" 93 | 94 | # Creates four values: committed, init, max, used 95 | 96 | Type "jmx_memory" 97 | #InstancePrefix "" 98 | #InstanceFrom "" 99 | Table true 100 | Attribute "NonHeapMemoryUsage" 101 | InstancePrefix "nonheap-" 102 | 103 | 104 | 105 | # Memory usage by memory pool. 106 | 107 | ObjectName "java.lang:type=MemoryPool,*" 108 | InstancePrefix "memory_pool-" 109 | InstanceFrom "name" 110 | 111 | 112 | Type "jmx_memory" 113 | #InstancePrefix "" 114 | #InstanceFrom "" 115 | Table true 116 | Attribute "Usage" 117 | 118 | 119 | 120 | ##################### 121 | # Connection blocks # 122 | ##################### 123 | 124 | ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:1099/jmxrmi" 125 | Host "slipstream-jmx" 126 | Collect "classes" 127 | Collect "compilation" 128 | Collect "garbage_collector" 129 | Collect "memory" 130 | Collect "memory_pool" 131 | 132 | 133 | 134 | 135 | -------------------------------------------------------------------------------- /tools/updateSlipStreamRepos: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script to maintain the SlipStream* repos up to date 4 | # 5 | # - Copy it out of this 'tools' folder to the root folder where you 6 | # want to clone the SlipStream* repos. 7 | # 8 | # - Running the script in a clean directory will git clone all 9 | # SliStream* repos within the current directory. 10 | # 11 | # - If the repo folders are already present, the script will update 12 | # them with git pull. Note that the branch will not be changed. 13 | # 14 | # - After the git clone or pull actions, all projects will be build 15 | # in the right order using mvn clean install. You can use --with-tests to 16 | # include the tests. Note that after a mvn clean some projects 17 | # require the tests to be build to work. 18 | # 19 | # - To see the full output of each command use -v. 20 | 21 | # Required for the pylint execution in project SlipStreamClient 22 | export LC_CTYPE="en_US.UTF-8" 23 | 24 | OK="$(tput setaf 2) OK$(tput sgr0)" 25 | ERROR="$(tput setaf 1) ERROR$(tput sgr0)" 26 | 27 | GIT_PULL_HAD_ERROR="false" 28 | MVN_INSTALL_HAD_ERROR="false" 29 | 30 | REPOS=() 31 | REPOS+=("SlipStream") 32 | REPOS+=("SlipStreamUI") 33 | REPOS+=("SlipStreamServer") 34 | REPOS+=("SlipStreamServerDeps") 35 | REPOS+=("SlipStreamClient") 36 | REPOS+=("SlipStreamClojureAPI") 37 | REPOS+=("SlipStreamConnectors") 38 | 39 | exec 3>&1 40 | exec 4>&2 41 | 42 | pecho() { 43 | echo -n "${1}" 1>&3 44 | } 45 | 46 | pecholn() { 47 | echo "${1}" 1>&3 48 | } 49 | 50 | SKIP_TESTS="true" 51 | SKIP_TESTS_FLAG="-DskipTests" 52 | if [ "${1}" == "--with-tests" ] 53 | then 54 | SKIP_TESTS="false" 55 | SKIP_TESTS_FLAG="" 56 | shift 57 | fi 58 | 59 | VERBOSE="false" 60 | if [ "${1}" == "-v" ] 61 | then 62 | VERBOSE="true" 63 | shift 64 | fi 65 | 66 | BUILD_SEPARATE_MODULES="false" 67 | if [ "${1}" == "--separate-modules" ]; then 68 | BUILD_SEPARATE_MODULES="true" 69 | shift 70 | fi 71 | 72 | if [ "${1}" != "" ]; then 73 | echo "$(basename $0): invalid option: ${1}." 74 | echo "$(basename $0): only --with-tests, -v and --separate-modules are valid in that order." 75 | exit 1 76 | fi 77 | 78 | if ! $VERBOSE 79 | then 80 | exec 1>/dev/null 81 | exec 2>/dev/null 82 | fi 83 | 84 | 85 | git_branch(){ 86 | pecho $(git rev-parse --abbrev-ref HEAD) 87 | } 88 | 89 | update_ss_repo(){ 90 | if [ -d ${1} ] 91 | then 92 | pecho " - Updating ${1}... " 93 | (cd ${1}; git_branch; if git pull && git gc --auto; then pecholn "$OK"; else pecholn "$ERROR"; exit 1; fi;) || GIT_PULL_HAD_ERROR="true"; 94 | else 95 | pecho " - Repo ${1} is not yet cloned here. Cloning ${1}... " 96 | git clone git@github.com:slipstream/${1}.git && pecholn "$OK" || pecholn "$ERROR" 97 | fi 98 | echo 99 | } 100 | 101 | install_project(){ 102 | if [ -d ${1} ] 103 | then 104 | pecho " - Installing ${1}... " 105 | (cd ${1}; git_branch; if mvn clean install --update-snapshots "${SKIP_TESTS_FLAG}" -DskipModules=true -Dmaven.test.skip=$SKIP_TESTS; then pecholn "$OK"; else pecholn "$ERROR"; exit 1; fi;) || MVN_INSTALL_HAD_ERROR="true"; 106 | else 107 | pecholn " - Project ${1} not found. Skipping." 108 | fi 109 | echo 110 | } 111 | 112 | pecholn "Updating repos:" 113 | 114 | for repo in "${REPOS[@]}" 115 | do 116 | update_ss_repo $repo 117 | done 118 | 119 | if $GIT_PULL_HAD_ERROR 120 | then 121 | pecholn 122 | pecholn "Some repos didn't update successfully. You might have uncommitted changes. Please fix it and re-run this command." 123 | exit 1 124 | fi 125 | 126 | pecholn 127 | pecholn "Installing projects:" 128 | 129 | if $BUILD_SEPARATE_MODULES; then 130 | for repo in "${REPOS[@]}" 131 | do 132 | install_project $repo 133 | done 134 | else 135 | (cd SlipStream && mvn clean install --update-snapshots --fail-at-end "${SKIP_TESTS_FLAG}" -Dmaven.test.skip=$SKIP_TESTS 1>&3 2>&4;) 136 | fi 137 | 138 | if $MVN_INSTALL_HAD_ERROR 139 | then 140 | pecholn 141 | pecholn "Verify following common mistakes:" 142 | pecholn " - All maven versions must be consistent (including the SlipStreamUI project)" 143 | pecholn " - Use 'export LC_CTYPE=\"en_US.UTF-8\"' for pylint in project 'SlipStreamClient' (automatically done with this script)" 144 | pecholn 145 | pecholn "Try building the project with:" 146 | pecholn " (cd SlipStream; touch .slipstream-build-all; mvn clean install; rm .slipstream-build-all)" 147 | pecholn "Source: https://github.com/slipstream/SlipStream/tree/master#build-everything" 148 | pecholn 149 | exit 1 150 | fi 151 | 152 | 153 | exit 154 | 155 | -------------------------------------------------------------------------------- /install/ss-ci-pipeline/ss-reports-collector.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # To be used on SS server to collect logs, configurations and data. 5 | 6 | LOGS= 7 | CONFS= 8 | DATA= 9 | ARCHIVE_TARGZ=/var/tmp/slipstream/ss-arch.tgz 10 | STARTSTOP= 11 | TO_ARCHIVE= 12 | 13 | SS_SERVICES=" 14 | kibana 15 | filebeat 16 | logstash 17 | graphite-api 18 | carbon-cache 19 | collectd 20 | nginx 21 | elasticsearch 22 | hsqldb 23 | ss-pricing 24 | cimi 25 | slipstream 26 | " 27 | 28 | USAGE="usage: -h -l -c -d -f -s\n 29 | \n 30 | -h print this help\n 31 | -l collect logs\n 32 | -c collect configurations\n 33 | -d collect data\n 34 | -s stop/start service before/after achriving\n 35 | -f full path to the tarball. Directories will be created. Default: $ARCHIVE_TARGZ.\n" 36 | 37 | _exit_usage() { 38 | echo -e $USAGE 39 | exit 1 40 | } 41 | 42 | while getopts f:lcdh opt; do 43 | case $opt in 44 | l) 45 | LOGS=true 46 | ;; 47 | c) 48 | CONFS=true 49 | ;; 50 | d) 51 | DATA=true 52 | ;; 53 | f) 54 | ARCHIVE_TARGZ=$OPTARG 55 | ;; 56 | s) 57 | STARTSTOP=true 58 | ;; 59 | *|h) 60 | _exit_usage 61 | ;; 62 | esac 63 | done 64 | 65 | [ -z "${LOGS}${CONFS}${DATA}" ] && \ 66 | { echo ":: ERROR: No resources to archive were requested."; exit 1; } 67 | 68 | mkdir -p `dirname $ARCHIVE_TARGZ` 69 | rm -rf $ARCHIVE_TARGZ 70 | 71 | _echo() { 72 | echo ":: $(date) $@" 73 | } 74 | 75 | _is_true() { 76 | if [ "x${1}" == "xtrue" ]; then 77 | return 0 78 | else 79 | return 1 80 | fi 81 | } 82 | 83 | _stop_services() { 84 | _is_true $STARTSTOP || return 0 85 | _echo "Stopping all SlipStream services." 86 | systemctl stop $SS_SERVICES || true 87 | _echo "Stopped all SlipStream services." 88 | } 89 | 90 | _start_services() { 91 | _is_true $STARTSTOP || return 0 92 | _echo "Starting all SlipStream services." 93 | systemctl start $SS_SERVICES || true 94 | _echo "Started all SlipStream services." 95 | } 96 | 97 | _add() { 98 | echo ".. Asked to add to archive: $@" 99 | actual=`ls -d $@ 2>/dev/null || true` 100 | [ -z "$actual" ] && \ 101 | { echo ".. WARNING: No existing resources to add to archive."; return 0; } 102 | 103 | echo ".. Existing resources to add to archive: $actual" 104 | for r in $actual; do 105 | echo "..... adding: $r" 106 | TO_ARCHIVE="$TO_ARCHIVE $r" 107 | done 108 | } 109 | 110 | _logs() { 111 | # Installation 112 | _add /tmp/slipstream*.log 113 | 114 | # SlipStream 115 | _add /var/log/slipstream 116 | 117 | # SS reports 118 | _add /var/tmp/slipstream/reports 119 | 120 | # HSQLSB 121 | _add /var/log/hsqldb.log 122 | 123 | # Elasticsearch 124 | _add /var/log/elasticsearch 125 | 126 | # Logstash 127 | _add /var/log/logstash 128 | 129 | # Filebeat 130 | _add /var/log/filebeat 131 | 132 | # Kibana 133 | _add /var/log/kibana 134 | 135 | # Collectd 136 | _add /var/log/collectd.log 137 | 138 | # Graphite 139 | _add /var/log/graphite-api.log 140 | 141 | # Carbon 142 | _add /var/log/carbon 143 | 144 | # System 145 | _add /var/log/messages 146 | } 147 | 148 | _configs() { 149 | # Global 150 | _add /etc/default 151 | 152 | # SlipStream 153 | _add /etc/slipstream \ 154 | /opt/slipstream/server/etc \ 155 | /opt/slipstream/server/start.ini 156 | 157 | # HSQLDB (TODO: enable service and sql logs) 158 | _add /etc/hsqldb.cfg 159 | 160 | # Elasticsearch 161 | _add /etc/elasticsearch 162 | 163 | # Logstash 164 | _add /etc/logstash 165 | 166 | # Filebeat 167 | _add /etc/filebeat 168 | 169 | # Kibana 170 | _add /etc/kibana 171 | 172 | # Collectd 173 | _add /etc/collectd.* 174 | 175 | # Graphite 176 | _add /etc/graphite-api.* 177 | 178 | # Carbon 179 | _add /etc/carbon 180 | } 181 | 182 | _data() { 183 | # HSQLDB (TODO: enable service and sql logs) 184 | _add /opt/slipstream/SlipStreamDB 185 | 186 | # Elasticsearch 187 | _add /var/lib/elasticsearch/nodes 188 | 189 | # Carbon 190 | _add /var/lib/carbon 191 | } 192 | 193 | _create_archive() { 194 | _echo "Collecting resources to be archived." 195 | if ( _is_true $LOGS ); then 196 | _logs 197 | fi 198 | if ( _is_true $CONFS ); then 199 | _configs 200 | fi 201 | if ( _is_true $DATA ); then 202 | _data 203 | fi 204 | if [ -z "$TO_ARCHIVE" ]; then 205 | echo ".. WARNINIG: No resources to archive were collected." 206 | exit 1 207 | fi 208 | _echo "Creating archive: $ARCHIVE_TARGZ" 209 | tar -zc $TO_ARCHIVE -f $ARCHIVE_TARGZ 210 | _echo "Created archive: $ARCHIVE_TARGZ" 211 | } 212 | 213 | _stop_services 214 | _create_archive 215 | _start_services 216 | 217 | -------------------------------------------------------------------------------- /install/ss-ci-pipeline/ss-builder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # input parameters 4 | # 5 | # maven_goal 6 | # nexus_creds 7 | # slipstream_version 8 | # slipstream_client_version 9 | # slipstream_connectors_version 10 | # slipstream_server_version 11 | # slipstream_server_deps_version 12 | # slipstream_ui_version 13 | # skip_tests 14 | # skip_containers 15 | # slipstream_edition (community|enterprise) 16 | 17 | # output parameters 18 | # 19 | # ss-repo-conf-url 20 | # ready 21 | 22 | set -x 23 | set -e 24 | 25 | 26 | # In case configuration file is not read. 27 | source ~/.bashrc 28 | 29 | export LC_ALL='en_US.UTF-8' 30 | 31 | maven_goal=`ss-get maven_goal` 32 | maven_options="`ss-get maven_options`" 33 | nexus_creds=`ss-get nexus_creds` 34 | slipstream_edition=`ss-get slipstream_edition` 35 | slipstream_version=`ss-get slipstream_version` 36 | slipstream_client_version=`ss-get slipstream_client_version` 37 | slipstream_connectors_version=`ss-get slipstream_connectors_version` 38 | slipstream_connectors_enterprise_version=`ss-get slipstream_connectors_enterprise_version` 39 | slipstream_server_version=`ss-get slipstream_server_version` 40 | slipstream_server_deps_version=`ss-get slipstream_server_deps_version` 41 | slipstream_ui_version=`ss-get slipstream_ui_version` 42 | slipstream_i18n_version=`ss-get slipstream_i18n_version` 43 | slipstream_pricing_version=`ss-get slipstream_pricing_version` 44 | skip_tests=`ss-get skip_tests` 45 | skip_containers=`ss-get skip_containers` 46 | _HOSTNAME=`ss-get hostname` 47 | 48 | function _install_git_creds() { 49 | [ "$nexus_creds" == "user:pass" ] && { echo "WARNING: Skipped intallation of git credentials."; return; } 50 | 51 | # Get and inflate git credentials. 52 | TARBALL=~/git-creds.tgz 53 | GIT_CREDS_URL=https://nexus.sixsq.com/service/local/repositories/releases-enterprise/content/com/sixsq/slipstream/sixsq-hudson-creds/1.0.0/sixsq-hudson-creds-1.0.0.tar.gz 54 | SSH_DIR=~/.ssh 55 | mkdir -p $SSH_DIR 56 | chmod 0700 $SSH_DIR 57 | _CREDS="-u $nexus_creds" 58 | curl -k -L -sSf $_CREDS -o $TARBALL $GIT_CREDS_URL 59 | tar -C $SSH_DIR -zxvf $TARBALL 60 | rm -f $TARBALL 61 | chown root:root ~/.ssh/* 62 | 63 | echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config 64 | } 65 | 66 | # 67 | # work from home directory 68 | # 69 | cd ~ 70 | 71 | # 72 | # get git creds for enterprise version 73 | # 74 | _install_git_creds 75 | 76 | # 77 | # create a settings.xml file (clobbering any existing file) 78 | # 79 | MAVEN_SETTINGS=$PWD/builder-settings.xml 80 | nexus_username=`echo -n ${nexus_creds} | cut -d ':' -f 1` 81 | nexus_password=`echo -n ${nexus_creds} | cut -d ':' -f 2` 82 | cat > $MAVEN_SETTINGS < 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | sixsq.snapshots 95 | ${nexus_username} 96 | ${nexus_password} 97 | 98 | 99 | sixsq.releases 100 | ${nexus_username} 101 | ${nexus_password} 102 | 103 | 104 | sixsq.thirdparty 105 | ${nexus_username} 106 | ${nexus_password} 107 | 108 | 109 | slipstream.snapshots 110 | ${nexus_username} 111 | ${nexus_password} 112 | 113 | 114 | slipstream.releases 115 | ${nexus_username} 116 | ${nexus_password} 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | EOF 127 | 128 | # 129 | # clone the SlipStream source code 130 | # 131 | YUM_REPO_NAME=SlipStream-FromSources-$slipstream_edition 132 | 133 | ss-set statecustom "Cloning SlipStream $slipstream_edition source code..." 134 | if [ "$slipstream_edition" == "enterprise" ]; then 135 | maven_profile="-P enterprise" 136 | else 137 | maven_profile="-P public" 138 | fi 139 | bootstrap_url=git@github.com:slipstream/SlipStreamBootstrap.git 140 | 141 | git clone $bootstrap_url 142 | 143 | cd SlipStreamBootstrap 144 | mvn ${maven_profile} \ 145 | --settings ${MAVEN_SETTINGS} \ 146 | ${maven_options} \ 147 | -B \ 148 | -Dslipstream.version=${slipstream_version} \ 149 | -Dslipstream.client.version=${slipstream_client_version} \ 150 | -Dslipstream.connectors.version=${slipstream_connectors_version} \ 151 | -Dslipstream.connectors.enterprise.version=${slipstream_connectors_enterprise_version} \ 152 | -Dslipstream.server.version=${slipstream_server_version} \ 153 | -Dslipstream.server.deps.version=${slipstream_server_deps_version} \ 154 | -Dslipstream.ui.version=${slipstream_ui_version} \ 155 | -Dslipstream.i18n.version=${slipstream_i18n_version} \ 156 | -Dslipstream.pricing.version=${slipstream_pricing_version} \ 157 | generate-sources 158 | 159 | # 160 | # build SlipStream 161 | # 162 | if [ "$slipstream_edition" == "enterprise" ]; then 163 | maven_options="${maven_options} -Denterprise" 164 | fi 165 | ss-set statecustom "Building SlipStream $slipstream_edition..." 166 | cd SlipStream 167 | mvn --settings ${MAVEN_SETTINGS} \ 168 | ${maven_options} \ 169 | -Dyum \ 170 | -B -DskipTests=${skip_tests} -DskipContainers=${skip_containers} clean ${maven_goal} 171 | 172 | # 173 | # make local yum repository 174 | # 175 | ss-set statecustom "Creating YUM repository $YUM_REPO_NAME..." 176 | mkdir -p /opt/slipstream 177 | cd /opt/slipstream 178 | tar zxf ~/SlipStreamBootstrap/SlipStream/yum/target/SlipStream*.tar.gz 179 | cd - 180 | 181 | # 182 | # expose the local repo via http 183 | # 184 | ss-set statecustom "Exporting YUM repository $YUM_REPO_NAME..." 185 | 186 | yum install -y httpd 187 | systemctl start httpd 188 | 189 | # Open TCP:80 in the local firewall. Insert before REJECT or append. 190 | 191 | systemctl stop firewalld.service || true 192 | systemctl mask firewalld.service || true 193 | yum -y install iptables-services 194 | systemctl start iptables.service 195 | 196 | RULE='-m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT' 197 | BEFORE_RULE_NUM=$(iptables -nL INPUT --line-numbers | grep REJECT | awk '{print $1}') 198 | if [ -n "$BEFORE_RULE_NUM" ]; then 199 | iptables -I INPUT $BEFORE_RULE_NUM $RULE 200 | else 201 | iptables -A INPUT $RULE 202 | fi 203 | rm -f /etc/sysconfig/iptables.bak 204 | cp /etc/sysconfig/iptables{,.bak} 205 | iptables-save > /etc/sysconfig/iptables 206 | 207 | ln -sf /opt/slipstream/yum /var/www/html/ 208 | cat > /var/www/html/slipstream.repo < -r -u -c -p 26 | -k -e -o '' 27 | -a Elasticsearch host:port. Default: $ES_HOST_PORT 28 | -r reference configuration URL as https://host/path/file.tgz. Mandatory parameter. 29 | -u credentials (user:pass) for URL with referece configuration. Optional parameter. 30 | -c YUM certificate tarball url as https://host/path/file.tgz Optional parameter. 31 | -p credentials (user:pass) for URL with YUM certificate tarball. Optional parameter. 32 | -k kind of the YUM repository to use: ${!YUM_REPO_TO_GH_BRANCH[@]}. Default: $_YUM_REPO_KIND_DEFAULT 33 | -e edition of the YUM repository to use: ${_YUM_REPO_EDITIONS[@]}. Default: $_YUM_REPO_EDITION_DEFAULT 34 | -o set of parameters to be passed to slipstream.sh installation script. 35 | -b force the GitHub branch to download the install scripts from. 36 | " 37 | exit 1 38 | } 39 | 40 | function _check_yum_repo_kind_to_github_tag_map() { 41 | if ! test "${YUM_REPO_TO_GH_BRANCH[$1]+isset}"; then 42 | usage_exit 43 | fi 44 | } 45 | 46 | function _check_repo_edition() { 47 | if [ "$1" != "community" ] && [ "$1" != "enterprise" ]; then 48 | usage_exit 49 | fi 50 | } 51 | 52 | while getopts a:r:u:c:p:k:e:o:b: opt; do 53 | case $opt in 54 | a) 55 | ES_HOST_PORT=$OPTARG 56 | ;; 57 | r) 58 | REFCONF_URL=$OPTARG 59 | ;; 60 | u) 61 | REFCONF_URL_USERPASS=$OPTARG 62 | ;; 63 | c) 64 | YUM_CREDS_URL=$OPTARG 65 | ;; 66 | p) 67 | YUM_CREDS_URL_USERPASS=$OPTARG 68 | ;; 69 | k) 70 | _check_yum_repo_kind_to_github_tag_map $OPTARG 71 | YUM_REPO_KIND=$OPTARG 72 | ;; 73 | e) 74 | _check_repo_edition $OPTARG 75 | YUM_REPO_EDITION=$OPTARG 76 | ;; 77 | o) 78 | SS_INSTALL_OPTIONS=$OPTARG 79 | ;; 80 | b) 81 | GH_BRANCH=$OPTARG 82 | ;; 83 | \?) 84 | usage_exit 85 | ;; 86 | esac 87 | done 88 | 89 | if [ -z "$REFCONF_URL" ]; then 90 | echo "ERROR: URL with reference configuration tarball was not provided." 91 | usage_exit 92 | fi 93 | if [ -z "$REFCONF_URL_USERPASS" ]; then 94 | echo "WARNING: Credentials for URL with reference configuration tarball were not provided." 95 | fi 96 | if [ -z "$YUM_CREDS_URL" ]; then 97 | echo "WARNING: URL with YUM certificates tarball was not provided." 98 | fi 99 | if [ -z "$YUM_CREDS_URL_USERPASS" ]; then 100 | echo "WARNING: Credentials for URL with YUM certificates tarball were not provided." 101 | fi 102 | 103 | if [ -n "$GH_BRANCH" ]; then 104 | branch=${GH_BRANCH} 105 | else 106 | branch=${YUM_REPO_TO_GH_BRANCH[${YUM_REPO_KIND}]} 107 | fi 108 | GH_BASE_URL=https://raw.githubusercontent.com/slipstream/SlipStream/${branch} 109 | 110 | SS_CONF_DIR=/etc/slipstream 111 | mkdir -p $SS_CONF_DIR 112 | 113 | function abort() { 114 | echo "!!! Aborting: $@" 115 | exit 1 116 | } 117 | 118 | function _now_sec() { 119 | date +%s 120 | } 121 | 122 | function _wait_listens() { 123 | # host port [timeout seconds] [sleep interval seconds] 124 | wait_time=${3:-60} 125 | sleep_interval=${4:-2} 126 | stop_time=$(($(_now_sec) + $wait_time)) 127 | while (( "$(_now_sec)" <= $stop_time )); do 128 | set +e 129 | res=$(ncat -v -4 $1 $2 < /dev/null 2>&1) 130 | if [ "$?" == "0" ]; then 131 | set -e 132 | return 0 133 | else 134 | if ( ! (echo $res | grep -q "Connection refused") ); then 135 | abort "Failed to check $1:$2 with:" $res 136 | fi 137 | fi 138 | set -e 139 | sleep $sleep_interval 140 | done 141 | abort "Timed out after ${wait_time} sec waiting for $1:$2" 142 | } 143 | 144 | function _install_yum_client_cert() { 145 | [ -z "$YUM_CREDS_URL" ] && { echo "WARNING: Skipped intallation of YUM credentials."; return; } 146 | # Get and inflate YUM certs. 147 | TARBALL=~/yum-certs.tgz 148 | _CREDS= 149 | if [ -n "$YUM_CREDS_URL_USERPASS" ]; then 150 | _CREDS="-u $YUM_CREDS_URL_USERPASS" 151 | fi 152 | curl -k -L -sSf $_CREDS -o $TARBALL $YUM_CREDS_URL 153 | tar -C $SS_CONF_DIR -zxvf $TARBALL 154 | chmod 400 $SS_CONF_DIR/yum-client.* 155 | rm -f $TARBALL 156 | } 157 | 158 | function _install_reference_configuration() { 159 | # Get and inflate the tarball with the server configuration. 160 | TARBALL=~/ss-ref-conf.tgz 161 | _CREDS= 162 | if [ -n "$REFCONF_URL_USERPASS" ]; then 163 | _CREDS="-u $REFCONF_URL_USERPASS" 164 | fi 165 | curl -k -L -sSf $_CREDS -o $TARBALL $REFCONF_URL 166 | tar -C $SS_CONF_DIR -zxvf $TARBALL 167 | rm -f $TARBALL 168 | 169 | # Discover connectors that have to be installed. 170 | CONNECTORS_TO_INSTALL=$(awk '/:cloudConnectorClass/,/("|,)$/' $SS_CONF_DIR/slipstream.edn | \ 171 | awk ' 172 | { 173 | gsub(/.*:cloudConnectorClass[ \t]*/, "", $0) 174 | # input: "foo:bar, baz" 175 | # ouput: bar baz 176 | split($0, cnames, ","); 177 | for (i in cnames) { 178 | split(cnames[i], cn, ":") 179 | if (length(cn) == 2) { 180 | cname = cn[2] 181 | } else { 182 | cname = cn[1] 183 | } 184 | gsub(/[ \t"\n]/, "", cname) 185 | print " " cname 186 | }; 187 | }' | sort -u) 188 | 189 | # Generate new passwords for the defined users. 190 | for passfile in /etc/slipstream/passwords/*; do 191 | echo -n $(uuidgen) | tail -c 12 > $passfile 192 | done 193 | } 194 | 195 | function _install_slipstream() { 196 | # Install SlipStream, but don't start it. 197 | curl -sSf -k -o slipstream.sh $GH_BASE_URL/install/slipstream.sh 198 | chmod +x slipstream.sh 199 | ./slipstream.sh -S -k $YUM_REPO_KIND -e $YUM_REPO_EDITION \ 200 | -a $ES_HOST_PORT $SS_INSTALL_OPTIONS 201 | } 202 | 203 | function _install_slipstream_connectors() { 204 | # Install required connectors. 205 | curl -sSf -k -o ss-install-connectors.sh \ 206 | $GH_BASE_URL/install/ss-install-connectors.sh 207 | chmod +x ss-install-connectors.sh 208 | ./ss-install-connectors.sh -a $ES_HOST_PORT -r $YUM_REPO_KIND \ 209 | $CONNECTORS_TO_INSTALL 210 | } 211 | 212 | function _start_slipstream() { 213 | # Start SlipStream. 214 | systemctl start cimi 215 | systemctl start slipstream 216 | } 217 | 218 | _install_yum_client_cert 219 | _install_reference_configuration 220 | _install_slipstream 221 | _install_slipstream_connectors 222 | _start_slipstream 223 | -------------------------------------------------------------------------------- /install/ss-from-sources.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | slipstream_version=`ss-get slipstream_version` 7 | slipstream_client_version=`ss-get slipstream_client_version` 8 | slipstream_connectors_version=`ss-get slipstream_connectors_version` 9 | slipstream_server_version=`ss-get slipstream_server_version` 10 | slipstream_server_deps_version=`ss-get slipstream_server_deps_version` 11 | slipstream_ui_version=`ss-get slipstream_ui_version` 12 | skip_tests=`ss-get skip_tests` 13 | install_examples=`ss-get install_examples` 14 | slipstream_backend=`ss-get slipstream_backend` 15 | with_refconf=`ss-get with_refconf` 16 | NEXUS_CREDS=`ss-get nexus_creds` 17 | REFCONF_NAME=`ss-get refconf_name` 18 | TEST_SERVICE=`ss-get test_service` 19 | 20 | echo "=== Parameters === 21 | slipstream_version = $slipstream_version 22 | slipstream_client_version = $slipstream_client_version 23 | slipstream_connectors_version = $slipstream_connectors_version 24 | slipstream_server_version = $slipstream_server_version 25 | slipstream_server_deps_version = $slipstream_server_deps_version 26 | slipstream_ui_version = $slipstream_ui_version 27 | skip_tests = $skip_tests 28 | install_examples = $install_examples 29 | slipstream_backend = $slipstream_backend 30 | with_refconf = $with_refconf 31 | NEXUS_CREDS = $NEXUS_CREDS 32 | REFCONF_NAME = $REFCONF_NAME 33 | TEST_SERVICE = $TEST_SERVICE 34 | === --- ===" 35 | 36 | 37 | function _is_true() { 38 | if [ "x${1}" == "xtrue" ]; then 39 | return 0 40 | else 41 | return 1 42 | fi 43 | } 44 | 45 | # 46 | # upgrade system 47 | # 48 | ss-set statecustom "Upgrading system..." 49 | yum clean all 50 | yum upgrade -y 51 | 52 | # 53 | # install dependencies 54 | # 55 | ss-set statecustom "Installing build dependencies..." 56 | yum install -y epel-release yum-utils 57 | # epel may not be enabled though 58 | yum-config-manager --enable epel 59 | yum clean all 60 | yum erase -y python-paramiko python-crypto 61 | yum install -y \ 62 | java-1.8.0-openjdk-devel \ 63 | python \ 64 | python-devel \ 65 | pylint \ 66 | python-pip \ 67 | python-mock \ 68 | gcc \ 69 | git \ 70 | rpm-build \ 71 | createrepo 72 | 73 | # 74 | # SlipStream python dependencies that require 75 | # versions that are more recent than packages. 76 | # 77 | ss-set statecustom "Installing python dependencies..." 78 | pip install nose coverage paramiko 79 | 80 | # 81 | # my sanity! 82 | # 83 | ss-set statecustom "Installing sanity..." 84 | yum install -y emacs-nox 85 | 86 | # 87 | # work from home directory 88 | # 89 | export HOME=/root 90 | cd ${HOME} 91 | 92 | # 93 | # install latest maven version 94 | # 95 | ss-set statecustom "Installing maven..." 96 | maven_version=3.3.3 97 | curl -o apache-maven-${maven_version}-bin.tar.gz \ 98 | http://mirror.switch.ch/mirror/apache/dist/maven/maven-3/${maven_version}/binaries/apache-maven-${maven_version}-bin.tar.gz 99 | tar zxf apache-maven-${maven_version}-bin.tar.gz 100 | 101 | export MAVEN_HOME=~/apache-maven-${maven_version} 102 | export MAVEN_OPTS=-Xmx2048M 103 | export PATH=$PATH:$MAVEN_HOME/bin:${HOME}/bin 104 | 105 | # 106 | # install leiningen 107 | # 108 | ss-set statecustom "Installing leiningen..." 109 | curl -o lein https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein 110 | mkdir ~/bin 111 | mv lein ~/bin 112 | chmod a+x ~/bin/lein 113 | export LEIN_ROOT=true 114 | lein 115 | 116 | # 117 | # clone the SlipStream source code 118 | # 119 | ss-set statecustom "Cloning SlipStream source code..." 120 | git clone https://github.com/slipstream/SlipStreamBootstrap 121 | 122 | cd SlipStreamBootstrap 123 | mvn -P public \ 124 | -B \ 125 | -Dslipstream.version=${slipstream_version} \ 126 | -Dslipstream.client.version=${slipstream_client_version} \ 127 | -Dslipstream.connectors.version=${slipstream_connectors_version} \ 128 | -Dslipstream.server.version=${slipstream_server_version} \ 129 | -Dslipstream.server.deps.version=${slipstream_server_deps_version} \ 130 | -Dslipstream.ui.version=${slipstream_ui_version} \ 131 | generate-sources 132 | 133 | # 134 | # build SlipStream 135 | # 136 | ss-set statecustom "Building SlipStream..." 137 | cd SlipStream 138 | mvn -B -DskipTests=${skip_tests} clean install 139 | 140 | # 141 | # make local yum repository 142 | # 143 | ss-set statecustom "Creating YUM repository..." 144 | mkdir -p /opt/slipstream 145 | cd /opt/slipstream 146 | tar zxf ~/SlipStreamBootstrap/SlipStream/yum/target/SlipStream*.tar.gz 147 | cd - 148 | 149 | # 150 | # installation from local repository 151 | # 152 | YUM_REPO_KIND=local 153 | YUM_REPO_EDITION=community 154 | 155 | declare -A YUM_REPO_TO_GH_BRANCH 156 | YUM_REPO_TO_GH_BRANCH[local]=master 157 | YUM_REPO_TO_GH_BRANCH[snapshot]=master 158 | YUM_REPO_TO_GH_BRANCH[candidate]=candidate-latest 159 | YUM_REPO_TO_GH_BRANCH[release]=release-latest 160 | 161 | _GH_PROJECT_URL=https://raw.githubusercontent.com/slipstream/SlipStream 162 | _GH_SCRIPTS_URL=$_GH_PROJECT_URL/${YUM_REPO_TO_GH_BRANCH[${YUM_REPO_KIND}]}/install 163 | _SS_PARAM_BACKEND="-d $slipstream_backend" 164 | if ( _is_true $with_refconf ); then 165 | ss-set statecustom "Installing SlipStream WITH reference configuration..." 166 | _NEXUS_URI=https://nexus.sixsq.com/service/local/artifact/maven/redirect 167 | curl -k -sSfL \ 168 | -o /tmp/ss-install-ref-conf.sh \ 169 | $_GH_SCRIPTS_URL/ss-install-ref-conf.sh 170 | chmod +x /tmp/ss-install-ref-conf.sh 171 | /tmp/ss-install-ref-conf.sh \ 172 | -r $_NEXUS_URI'?r=snapshots-enterprise-rhel7&g=com.sixsq.slipstream&a=SlipStreamReferenceConfiguration-'$REFCONF_NAME'-tar&p=tar.gz&c=bundle&v=LATEST' \ 173 | -u $NEXUS_CREDS \ 174 | -k $YUM_REPO_KIND \ 175 | -e $YUM_REPO_EDITION \ 176 | -o "$_SS_PARAM_BACKEND" 177 | 178 | # Get and publish configured users. 179 | _USER_PASS=$( 180 | for user in /etc/slipstream/passwords/*; do 181 | echo -n $(basename $user):$(cat $user),; 182 | done) 183 | _USER_PASS=${_USER_PASS%,} 184 | ss-set ss_users $_USER_PASS 185 | 186 | ### Get and publish connectors to test. 187 | declare -A CONNECTORS 188 | CONNECTORS=( 189 | ["nuv.la"]='exoscale-ch-gva ec2-eu-west nuvlabox-james-chadwick' 190 | ["connectors.community"]='ultimum-cz1') 191 | 192 | ss-set connectors_to_test "${CONNECTORS[$REFCONF_NAME]}" 193 | else 194 | ss-set statecustom "Installing SlipStream WITHOUT reference configuration..." 195 | export SLIPSTREAM_EXAMPLES=${install_examples} 196 | curl -k -sSfL \ 197 | -o /tmp/slipstream.sh \ 198 | $_GH_SCRIPTS_URL/slipstream.sh 199 | chmod +x /tmp/slipstream.sh 200 | /tmp/slipstream.sh $_SS_PARAM_BACKEND -e $YUM_REPO_EDITION -k $YUM_REPO_KIND 201 | fi 202 | 203 | # 204 | # restarting services (probably not necessary) 205 | systemctl restart slipstream 206 | systemctl restart cimi 207 | systemctl restart nginx 208 | 209 | # 210 | # set the service URL 211 | # 212 | ss-set statecustom "SlipStream Ready!" 213 | hostname=`ss-get hostname` 214 | ss_url="https://${hostname}" 215 | ss-set ss:url.service ${ss_url} 216 | 217 | # 218 | # validate that the installation worked 219 | # 220 | ss-set statecustom "Validating service..." 221 | 222 | SS_UNAME=super 223 | SS_UPASS=supeRsupeR 224 | if [ -f /etc/slipstream/passwords/$SS_UNAME ]; then 225 | SS_UPASS=$(cat /etc/slipstream/passwords/$SS_UNAME) 226 | fi 227 | profile_url="${ss_url}/user/$SS_UNAME" 228 | 229 | exit_code=0 230 | tries=0 231 | while [ $tries -lt 5 ]; do 232 | 233 | rc=`curl -k -s -u $SS_UNAME:$SS_UPASS -o /dev/null -w "%{http_code}" ${profile_url}` 234 | echo "Return code from $SS_UNAME profile page is " ${rc} 235 | if [ "${rc}" -ne "200" ]; then 236 | echo "Return code from $SS_UNAME profile page was not 200." 237 | exit_code=1 238 | else 239 | echo "Return code from $SS_UNAME profile page was 200." 240 | exit_code=0 241 | break 242 | fi 243 | 244 | sleep 10 245 | tries=$[$tries+1] 246 | 247 | done 248 | # the service failed the validation 249 | if [ "$exit_code" -ne "0" ]; then 250 | exit $exit_code 251 | fi 252 | 253 | # 254 | # test the service 255 | # 256 | 257 | if ( _is_true $TEST_SERVICE ); then 258 | CONNECTORS_TO_TEST=`ss-get connectors_to_test` 259 | msg="Running deployment tests for connectors: '${CONNECTORS_TO_TEST}'" 260 | ss-display "$msg" 261 | echo $msg 262 | USER=test 263 | for CONNECTOR in ${CONNECTORS_TO_TEST}; do 264 | msg="Running test on $CONNECTOR as $USER. $ss_url" 265 | ss-display "$msg" 266 | echo $msg 267 | ss-execute -v -u $USER -p $(cat /etc/slipstream/passwords/$USER) --endpoint=$ss_url \ 268 | -w 15 --kill-vms-on-error \ 269 | --parameters "testclient:cloudservice=$CONNECTOR,apache:cloudservice=$CONNECTOR" \ 270 | examples/tutorials/service-testing/system | tee /tmp/ss-execute-$CONNECTOR.log 271 | done 272 | else 273 | msg="Skipped running deployment tests. $ss_url" 274 | ss-display "$msg" 275 | echo $msg 276 | fi 277 | -------------------------------------------------------------------------------- /install/ss-ci-pipeline/ss-deployer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # input parameteres 4 | # 5 | # install_examples 6 | # with_refconf 7 | # nexus_creds 8 | # refconf_name 9 | # yum_repo_kind 10 | # yum_repo_edition 11 | # ss-repo-conf-url 12 | # builder.ready 13 | 14 | # output parameteres 15 | # 16 | # ss_service_url 17 | # connectors_to_test 18 | # ss_users 19 | # ready 20 | 21 | set -x 22 | set -e 23 | 24 | function _is_true() { 25 | if [ "x${1}" == "xtrue" ]; then 26 | return 0 27 | else 28 | return 1 29 | fi 30 | } 31 | 32 | function _is_none() { 33 | if [ "x${1}" == "xNone" ]; then 34 | return 0 35 | else 36 | return 1 37 | fi 38 | } 39 | 40 | function abort() { 41 | echo "!!! Aborting: $@" 42 | exit 1 43 | } 44 | 45 | function _now_sec() { 46 | date +%s 47 | } 48 | 49 | function _wait_listens() { 50 | # host port [timeout seconds] [sleep interval seconds] 51 | wait_time=${3:-60} 52 | sleep_interval=${4:-2} 53 | stop_time=$(($(_now_sec) + $wait_time)) 54 | while (( "$(_now_sec)" <= $stop_time )); do 55 | set +e 56 | res=$(ncat -v -4 $1 $2 < /dev/null 2>&1) 57 | if [ "$?" == "0" ]; then 58 | set -e 59 | return 0 60 | else 61 | if ( ! (echo $res | grep -q "Connection refused") ); then 62 | abort "Failed to check $1:$2 with:" $res 63 | fi 64 | fi 65 | set -e 66 | sleep $sleep_interval 67 | done 68 | abort "Timed out after ${wait_time} sec waiting for $1:$2" 69 | } 70 | 71 | yum clean all 72 | yum upgrade -y 73 | 74 | ss-get --timeout 3600 builder.ready 75 | 76 | install_examples=`ss-get install_examples` 77 | with_refconf=`ss-get with_refconf` 78 | NEXUS_CREDS=`ss-get nexus_creds` 79 | REFCONF_NAME=`ss-get refconf_name` 80 | YUM_REPO_KIND=`ss-get yum_repo_kind` 81 | YUM_REPO_EDITION=`ss-get yum_repo_edition` 82 | SS_REPO_CONF_URL=`ss-get ss-repo-conf-url` 83 | install_scripts_branch=`ss-get install_scripts_branch` 84 | ES_HOST_PORT=`ss-get es-host-port` 85 | 86 | CIMI_HOST=localhost 87 | CIMI_PORT=8201 88 | CIMI_ENDPOINT=http://$CIMI_HOST:$CIMI_PORT 89 | 90 | REPORTS_CLOUD_STORE=exoscale-ch-gva 91 | 92 | function _install_yum_client_cert() { 93 | SS_CONF_DIR=/etc/slipstream 94 | mkdir -p $SS_CONF_DIR 95 | 96 | TARBALL=~/yum-certs.tgz 97 | _CREDS= 98 | if [ -n "$NEXUS_CREDS" ]; then 99 | _CREDS="-u $NEXUS_CREDS" 100 | fi 101 | curl -k -L -sSf $_CREDS -o $TARBALL $1 102 | tar -C $SS_CONF_DIR -zxvf $TARBALL 103 | chmod 400 $SS_CONF_DIR/yum-client.* 104 | rm -f $TARBALL 105 | } 106 | 107 | function _configure_object_store_for_reports() { 108 | _wait_listens $CIMI_HOST $CIMI_PORT 600 109 | 110 | # search for a credential to use for saving reports 111 | curl -H'accept: application/json' -H'slipstream-authn-info: admin ADMIN' \ 112 | "$CIMI_ENDPOINT/api/credential?\$filter=type^='cloud-cred'%20and%20connector/href='connector/$REPORTS_CLOUD_STORE'" \ 113 | > credentials.json 114 | credential_id=`jq '.credentials[0] | .["id"]' credentials.json` 115 | rm -f credentials.json 116 | 117 | # copy the slipstream configuration and update with credential id 118 | curl -H'accept: application/json' -H'slipstream-authn-info: admin ADMIN' \ 119 | "$CIMI_ENDPOINT/api/configuration/slipstream" \ 120 | > ss-cfg.json 121 | sed -i -e 's|""|'$credential_id'|' ss-cfg.json 122 | 123 | # set the reports bucket to current seconds since the epoch 124 | # using /dev/urandom makes the script hang (presumably waiting for entropy) 125 | random_id=`date +%s` 126 | reports_bucket="slipstream-test-${random_id}" 127 | sed -i -e 's||'$reports_bucket'|' ss-cfg.json 128 | 129 | # push the modified configuration back into the server 130 | curl -H'content-type: application/json' -H'slipstream-authn-info: admin ADMIN' \ 131 | "$CIMI_ENDPOINT/api/configuration/slipstream" -XPUT --data @ss-cfg.json 132 | rm -f ss-cfg.json 133 | } 134 | 135 | # 136 | # for installation from local repository 137 | # 138 | 139 | declare -A YUM_REPO_TO_GH_BRANCH 140 | YUM_REPO_TO_GH_BRANCH[local]=master 141 | YUM_REPO_TO_GH_BRANCH[snapshot]=master 142 | YUM_REPO_TO_GH_BRANCH[candidate]=candidate-latest 143 | YUM_REPO_TO_GH_BRANCH[release]=release-latest 144 | 145 | _GH_PROJECT_URL=https://raw.githubusercontent.com/slipstream/SlipStream 146 | if [ "$install_scripts_branch" != "master" ]; then 147 | GH_BRANCH=$install_scripts_branch 148 | else 149 | GH_BRANCH=${YUM_REPO_TO_GH_BRANCH[${YUM_REPO_KIND}]} 150 | fi 151 | _GH_SCRIPTS_URL=$_GH_PROJECT_URL/$GH_BRANCH/install 152 | _NEXUS_URI=https://nexus.sixsq.com/service/local/artifact/maven/redirect 153 | if ( _is_true $with_refconf ); then 154 | ss-set statecustom "Installing SlipStream WITH reference configuration..." 155 | curl -k -sSfL \ 156 | -o /tmp/ss-install-ref-conf.sh \ 157 | $_GH_SCRIPTS_URL/ss-install-ref-conf.sh 158 | chmod +x /tmp/ss-install-ref-conf.sh 159 | REF_CONF_URL=$_NEXUS_URI'?r=snapshots-enterprise-rhel7&g=com.sixsq.slipstream&a=SlipStreamReferenceConfiguration-'$REFCONF_NAME'-tar&p=tar.gz&c=bundle&v=LATEST' 160 | ref_conf_params="-a $ES_HOST_PORT -r $REF_CONF_URL -u $NEXUS_CREDS -e $YUM_REPO_EDITION -b $GH_BRANCH" 161 | if ( _is_none ${SS_REPO_CONF_URL} ); then 162 | if [ "X$YUM_REPO_EDITION" == "Xenterprise" ]; then 163 | /tmp/ss-install-ref-conf.sh \ 164 | $ref_conf_params \ 165 | -k $YUM_REPO_KIND \ 166 | -c ${_NEXUS_URI}'?r=releases-enterprise&g=com.sixsq.slipstream&a=SlipStreamYUMCertsForSlipStreamInstaller&p=tgz&v=LATEST' \ 167 | -p $NEXUS_CREDS 168 | else 169 | /tmp/ss-install-ref-conf.sh \ 170 | $ref_conf_params 171 | fi 172 | else 173 | /tmp/ss-install-ref-conf.sh \ 174 | $ref_conf_params \ 175 | -o "-x $SS_REPO_CONF_URL" 176 | fi 177 | 178 | ### Get and publish connectors to test. 179 | declare -A CONNECTORS 180 | CONNECTORS=( 181 | ["nuv.la"]='exoscale-ch-gva ec2-eu-west' 182 | ["connectors.community"]='ultimum-cz1') 183 | 184 | ss-set connectors_to_test "${CONNECTORS[$REFCONF_NAME]}" 185 | else 186 | ss-set statecustom "Installing SlipStream WITHOUT reference configuration..." 187 | export SLIPSTREAM_EXAMPLES=${install_examples} 188 | curl -k -sSfL \ 189 | -o /tmp/slipstream.sh \ 190 | $_GH_SCRIPTS_URL/slipstream.sh 191 | chmod +x /tmp/slipstream.sh 192 | if [ "X$YUM_REPO_EDITION" == "Xenterprise" ]; then 193 | _install_yum_client_cert \ 194 | ${_NEXUS_URI}'?r=releases-enterprise&g=com.sixsq.slipstream&a=SlipStreamYUMCertsForSlipStreamInstaller&p=tgz&v=LATEST' 195 | fi 196 | if ( _is_none ${SS_REPO_CONF_URL} ); then 197 | /tmp/slipstream.sh -a $ES_HOST_PORT -e $YUM_REPO_EDITION -k $YUM_REPO_KIND 198 | else 199 | /tmp/slipstream.sh -a $ES_HOST_PORT -x $SS_REPO_CONF_URL 200 | fi 201 | fi 202 | 203 | # 204 | # Reduce the memory consumption of ElasticSearch 205 | # 206 | if [ "X$ES_HOST_PORT" == "Xlocalhost:9300" ]; then 207 | sed -i 's/^-Xms.*/-Xms256m/' /etc/elasticsearch/jvm.options 208 | sed -i 's/^-Xmx.*/-Xmx1g/' /etc/elasticsearch/jvm.options 209 | systemctl restart elasticsearch 210 | _wait_listens localhost 9300 211 | fi 212 | 213 | # 214 | # restarting cimi service (probably not necessary) 215 | systemctl restart cimi 216 | 217 | _wait_listens localhost 8201 600 218 | # ensure cimi is fully started and responds 219 | cep_url="http://localhost:8201/api/cloud-entry-point" 220 | tries=0 221 | while [ $tries -lt 10 ]; do 222 | 223 | rc=`curl -k -sS -o /dev/null -w '%{http_code}' ${cep_url}` 224 | echo "Return code from ${cep_url} is " ${rc} 225 | if [ "${rc}" -ne "200" ]; then 226 | echo "Return code from ${cep_url} was not 200." 227 | exit_code=1 228 | else 229 | echo "Return code from ${cep_url} was 200." 230 | exit_code=0 231 | break 232 | fi 233 | 234 | sleep 60 235 | tries=$[$tries+1] 236 | 237 | done 238 | 239 | # the service failed the validation with login 240 | if [ "$exit_code" -ne "0" ]; then 241 | ss-set statecustom "ERROR: Service failed to provide cloud-entry-point." 242 | exit $exit_code 243 | fi 244 | 245 | # 246 | # restart other services (probably not necessary) 247 | systemctl restart slipstream 248 | systemctl restart nginx 249 | systemctl restart slipstream-job-distributor@vms_collect 250 | systemctl restart slipstream-job-distributor@vms_cleanup 251 | systemctl restart slipstream-job-distributor@jobs_cleanup 252 | systemctl restart slipstream-job-executor 253 | 254 | # 255 | # set the service URL 256 | # 257 | ss-set statecustom "SlipStream Ready!" 258 | hostname=`ss-get hostname` 259 | ss_url="https://${hostname}" 260 | ss-set ss:url.service ${ss_url} 261 | ss-set ss_service_url ${ss_url} 262 | 263 | # 264 | # validate that the installation worked 265 | # 266 | ss-set statecustom "Validating service..." 267 | 268 | SS_UNAME=super 269 | SS_UPASS=supeRsupeR 270 | if [ -f /etc/slipstream/passwords/$SS_UNAME ]; then 271 | SS_UPASS=$(cat /etc/slipstream/passwords/$SS_UNAME) 272 | fi 273 | 274 | # Get and publish configured users. 275 | DEFAULT_USERPASS=$SS_UNAME:$SS_UPASS 276 | if [ -d /etc/slipstream/passwords/ ] 277 | then 278 | _USER_PASS=$( 279 | for user in /etc/slipstream/passwords/*; do 280 | echo -n $(basename $user):$(cat $user),; 281 | done) 282 | _USER_PASS=${_USER_PASS%,} 283 | fi 284 | _USER_PASS=${_USER_PASS:-$DEFAULT_USERPASS} 285 | ss-set ss_users $_USER_PASS 286 | 287 | exit_code=0 288 | 289 | # ensure slipstream (java) is fully started and responds 290 | # required so that HSQLDB is populated and available 291 | landing_page_url="${ss_url}/login" 292 | tries=0 293 | while [ $tries -lt 5 ]; do 294 | 295 | rc=`curl -k -sS -o /dev/null -w '%{http_code}' ${landing_page_url}` 296 | echo "Return code from ${landing_page_url} is " ${rc} 297 | if [ "${rc}" -ne "200" ]; then 298 | echo "Return code from ${landing_page_url} was not 200." 299 | exit_code=1 300 | else 301 | echo "Return code from ${landing_page_url} was 200." 302 | exit_code=0 303 | break 304 | fi 305 | 306 | sleep 10 307 | tries=$[$tries+1] 308 | 309 | done 310 | 311 | # the service failed the validation with login 312 | if [ "$exit_code" -ne "0" ]; then 313 | ss-set statecustom "ERROR: Service failed to provide landing page." 314 | exit $exit_code 315 | fi 316 | 317 | _configure_object_store_for_reports 318 | 319 | # authenticate with server using username and password 320 | authn_url="${ss_url}/api/session" 321 | tries=0 322 | while [ $tries -lt 5 ]; do 323 | 324 | rc=`curl -k --cookie-jar /root/cookies -b /root/cookies -sS -XPOST -d href='session-template/internal' -d username=${SS_UNAME} -d password=${SS_UPASS} -H content-type:application/x-www-form-urlencoded -o /dev/null -w '%{http_code}' ${authn_url}` 325 | echo "Return code from $SS_UNAME login is " ${rc} 326 | if [ "${rc}" -ne "201" ]; then 327 | echo "Return code from $SS_UNAME login was not 201." 328 | exit_code=1 329 | else 330 | echo "Return code from $SS_UNAME login was 201." 331 | exit_code=0 332 | break 333 | fi 334 | 335 | sleep 10 336 | tries=$[$tries+1] 337 | 338 | done 339 | 340 | # the service failed the validation with login 341 | if [ "$exit_code" -ne "0" ]; then 342 | ss-set statecustom "ERROR: Service failed login validation." 343 | exit $exit_code 344 | fi 345 | 346 | # check that the user's profile page is accessible 347 | profile_url="${ss_url}/user/$SS_UNAME" 348 | tries=0 349 | while [ $tries -lt 5 ]; do 350 | 351 | rc=`curl -k --cookie-jar ~/cookies -b ~/cookies -sS -o /dev/null -w "%{http_code}" ${profile_url}` 352 | echo "Return code from ${profile_url} is " ${rc} 353 | if [ "${rc}" -ne "200" ]; then 354 | echo "Return code from ${profile_url} was not 200." 355 | exit_code=1 356 | else 357 | echo "Return code from ${profile_url} was 200." 358 | exit_code=0 359 | break 360 | fi 361 | 362 | sleep 10 363 | tries=$[$tries+1] 364 | 365 | done 366 | 367 | # the service failed the user profile validation 368 | if [ "$exit_code" -ne "0" ]; then 369 | ss-set statecustom "ERROR: Service failed user profile validation." 370 | exit $exit_code 371 | fi 372 | 373 | ss-set statecustom "Service deployed and validated." 374 | ss-set ready true 375 | -------------------------------------------------------------------------------- /install/ss-ci-pipeline/ss-tester.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import json 5 | import tarfile 6 | import subprocess 7 | import shutil 8 | import collections 9 | import urllib2 10 | import base64 11 | 12 | from slipstream.ConfigHolder import ConfigHolder 13 | from slipstream.Client import Client 14 | from slipstream.util import (create_directory, filePutContent, fileAppendContent, execute, importETree) 15 | 16 | etree = importETree() 17 | 18 | NAGIOS_STATUS_URL = 'http://monitor.sixsq.com/nagios/statusJson.php' 19 | SS_SERVICES_IN_NAGIOS = ['nuv.la'] 20 | 21 | GIT_CREDS_URL = 'https://nexus.sixsq.com/service/local/repositories/releases-enterprise/content/' \ 22 | 'com/sixsq/slipstream/sixsq-hudson-creds/1.0.0/sixsq-hudson-creds-1.0.0.tar.gz' 23 | 24 | SSH_DIR = os.path.expanduser('~/.ssh') 25 | 26 | 27 | def download_file(src_url, dst_file, creds={}): 28 | """creds: {'cookie': '', 29 | 'username': '', 'password': ''} 30 | cookie is preferred over username and password. If none are provided, 31 | the download proceeds w/o authentication. 32 | """ 33 | request = urllib2.Request(src_url) 34 | if creds.get('cookie'): 35 | request.add_header('cookie', creds.get('cookie')) 36 | elif creds.get('username') and creds.get('password'): 37 | request.add_header('Authorization', 38 | (b'Basic ' + (creds.get('username') + b':' + creds.get('password')).encode('base64')).replace('\n', '')) 39 | src_fh = urllib2.urlopen(request) 40 | 41 | dst_fh = open(dst_file, 'wb') 42 | while True: 43 | data = src_fh.read() 44 | if not data: 45 | break 46 | dst_fh.write(data) 47 | src_fh.close() 48 | dst_fh.close() 49 | 50 | return dst_file 51 | 52 | 53 | def ss_get(param, ignore_abort=False, timeout=30, no_block=False): 54 | """Returns None if parameter is not set. 55 | Raises Exceptions.NotFoundError if parameter doesn't exist. 56 | """ 57 | ch = ConfigHolder(config={'foo': None}) 58 | ch.set('ignoreAbort', ignore_abort) 59 | ch.set('no_block', no_block) 60 | ch.set('timeout', timeout) 61 | client = Client(ch) 62 | return client.getRuntimeParameter(param) 63 | 64 | 65 | def ss_set(key, value, ignore_abort=False): 66 | ch = ConfigHolder(config={'foo': None}) 67 | ch.set('ignoreAbort', ignore_abort) 68 | client = Client(ch) 69 | client.setRuntimeParameter(key, value) 70 | 71 | 72 | def ss_display(msg, ignore_abort=False): 73 | ss_set('statecustom', msg, ignore_abort=ignore_abort) 74 | 75 | 76 | def _print(msg): 77 | ss_display(msg) 78 | print('::: %s' % msg) 79 | 80 | 81 | def _expanduser(path): 82 | return os.path.expanduser(path) 83 | 84 | 85 | def _cd_home(): 86 | os.chdir(_expanduser('~')) 87 | 88 | 89 | def _cd(path): 90 | os.chdir(path) 91 | 92 | 93 | def _mkdir(path, mode): 94 | path = os.path.expanduser(path) 95 | if not os.path.exists(path): 96 | os.makedirs(path) 97 | os.chmod(path, mode) 98 | 99 | 100 | def _rmdir(path, ignore_errors=True): 101 | shutil.rmtree(path, ignore_errors=ignore_errors) 102 | 103 | 104 | def _chown(path, uid, gid, recursive=False): 105 | if not recursive: 106 | os.chown(path, uid, gid) 107 | else: 108 | for root, dirs, files in os.walk(path): 109 | for d in dirs: 110 | os.chown(os.path.join(root, d), uid, gid) 111 | for f in files: 112 | os.chown(os.path.join(root, f), uid, gid) 113 | 114 | 115 | def _tar_extract(tarball, target_dir='.'): 116 | tarfile.open(tarball, 'r:gz').extractall(os.path.expanduser(target_dir)) 117 | 118 | 119 | def _check_call(cmd): 120 | subprocess.check_call(cmd, stdout=subprocess.PIPE) 121 | 122 | 123 | def _install_git_creds(nexus_user, nexus_pass): 124 | tarball = _expanduser('~/git-creds.tgz') 125 | download_file(GIT_CREDS_URL, tarball, creds={'username': nexus_user, 126 | 'password': nexus_pass}) 127 | _mkdir(SSH_DIR, 0700) 128 | _tar_extract(tarball, SSH_DIR) 129 | os.unlink(tarball) 130 | 131 | ssh_conf = _expanduser(os.path.join(SSH_DIR, 'config')) 132 | fileAppendContent(ssh_conf, "Host github.com\n\tStrictHostKeyChecking no\n") 133 | os.chmod(ssh_conf, 0644) 134 | 135 | _chown(SSH_DIR, os.getuid(), os.getgid(), recursive=True) 136 | 137 | 138 | def _install_ss_repo_creds_lein(nexus_user, nexus_pass): 139 | create_directory(_expanduser('~/.lein')) 140 | 141 | conf = """ 142 | { 143 | :auth {:repository-auth 144 | {#"https://nexus.sixsq.com/content/repositories/snapshots-community-rhel7/" 145 | {:username "%(user)s", :password "%(pass)s"} 146 | 147 | #"https://nexus.sixsq.com/content/repositories/releases-community-rhel7/" 148 | {:username "%(user)s", :password "%(pass)s"} 149 | 150 | #"https://nexus.sixsq.com/content/repositories/snapshots-enterprise-rhel7/" 151 | {:username "%(user)s", :password "%(pass)s"} 152 | 153 | #"https://nexus.sixsq.com/content/repositories/releases-enterprise-rhel7/" 154 | {:username "%(user)s", :password "%(pass)s"} 155 | 156 | #"https://nexus.sixsq.com/content/repositories/thirdparty/" 157 | {:username "%(user)s", :password "%(pass)s"} 158 | } 159 | } 160 | } 161 | """ % {'user': nexus_user, 'pass': nexus_pass} 162 | filePutContent(_expanduser('~/.lein/profiles.clj'), conf) 163 | 164 | 165 | def merge_dicts(x, y): 166 | z = x.copy() 167 | z.update(y) 168 | return z 169 | 170 | 171 | def _get_test_user_pass(): 172 | username = (ss_get('ss_test_user', no_block=True) or 'test').strip() 173 | users_passes = ss_get('ss_users') 174 | userpass = 'tesTtesT' 175 | if users_passes: 176 | # Comma separated list of colon separated user:pass pairs. 177 | userpass = dict(map(lambda x: x.split(':'), users_passes.split(','))).get(username, userpass) 178 | return username, userpass 179 | 180 | 181 | def _get_monitoring_status(): 182 | "Returns monitoring status as JSON." 183 | nagios_user, nagios_pass = ss_get('nagios_creds').split(':') 184 | 185 | request = urllib2.Request(NAGIOS_STATUS_URL) 186 | base64string = base64.b64encode('%s:%s' % (nagios_user, nagios_pass)) 187 | request.add_header("Authorization", "Basic %s" % base64string) 188 | request.add_header("Accept", "application/json") 189 | res = urllib2.urlopen(request) 190 | 191 | return json.loads(res.read()) 192 | 193 | 194 | def _check_enabled(check): 195 | return int(check.get('active_checks_enabled')) == 1 196 | 197 | 198 | def _check_error(check): 199 | return int(check.get('current_state', 10)) > 0 200 | 201 | 202 | def _enabled_and_error(check): 203 | return _check_enabled(check) and _check_error(check) 204 | 205 | 206 | def _failing_monitored_connectors(ss_servers): 207 | "ss_servers - list of SS server names as defined in monitoring app." 208 | 209 | status = _get_monitoring_status() 210 | 211 | ss_exec_checks_err = {} 212 | for s in ss_servers: 213 | for chn, ch in status.get("services", {}).get(s, {}).items(): 214 | if chn.startswith('ss-exec_') and _enabled_and_error(ch): 215 | if ss_exec_checks_err.has_key(chn): 216 | _ch = ss_exec_checks_err.get(chn) 217 | if int(_ch.get('last_check', 0)) < int(ch.get('last_check', 0)): 218 | ss_exec_checks_err[chn] = ch 219 | else: 220 | ss_exec_checks_err[chn] = ch 221 | return map(lambda x: x.replace('ss-exec_', ''), ss_exec_checks_err.keys()) 222 | 223 | 224 | def _get_connectors_to_test(monitored_ss): 225 | # Space separated list. 226 | requested = ss_get('connectors_to_test').split(' ') 227 | _print('Connectors requested to test: %s' % requested) 228 | monitored_failing = _failing_monitored_connectors(monitored_ss) 229 | _print('Connectors currently failing: %s on %s' % (monitored_failing, monitored_ss)) 230 | return list(set(requested) - set(monitored_failing)) 231 | 232 | 233 | def _tests_to_run(): 234 | return filter(None, 235 | (ss_get('tests_to_run', no_block=True) or '').strip().split(';')) 236 | 237 | 238 | class TestsRunner(object): 239 | """ 240 | Order in which tests get added with add_test() is preserved. 241 | """ 242 | 243 | def __init__(self, config_auth): 244 | self._tests = collections.OrderedDict() 245 | self._config_auth = config_auth 246 | self.failed_tests = [] 247 | 248 | def add_test(self, name, config={}, connectors=[], msg='', fail=False): 249 | self._tests[name] = {'config': merge_dicts(self._config_auth, config), 250 | 'connectors': connectors, 251 | 'msg': msg, 252 | 'fail': fail} 253 | 254 | def get_test_names(self): 255 | return self._tests.keys() 256 | 257 | def run(self, tests_to_run=[]): 258 | testnames = tests_to_run or self.get_test_names() 259 | 260 | for name in testnames: 261 | self._run_test(name, **self._tests[name]) 262 | 263 | def _run_test(self, tname, config={}, connectors=[], msg='', fail=False): 264 | if connectors: 265 | for connector in connectors: 266 | self._print_t(' '.join(filter(None, [msg, 'Connector: %s' % connector]))) 267 | config['connectors'] = connector 268 | self.__run_test(tname, config=config, fail=fail) 269 | else: 270 | if msg: 271 | self._print_t(msg) 272 | self.__run_test(tname, config=config, fail=fail) 273 | 274 | def __run_test(self, name, config={}, fail=False): 275 | cmd = ['make', name, "TESTOPTS=%s" % self._build_test_opts(config)] 276 | print('executing: %s ' % cmd) 277 | rc = execute(cmd) 278 | if rc != 0: 279 | self.failed_tests.append((name, config.get('connectors', ''))) 280 | if fail: 281 | raise Exception('Failed running test: %s' % name) 282 | 283 | def get_failed_tests(self): 284 | def _test_to_str(tpl): 285 | return '%s%s' % (tpl[0], (tpl[1] != '') and (' on ' + tpl[1]) or '') 286 | return map(_test_to_str, self.failed_tests) 287 | 288 | @staticmethod 289 | def _build_test_opts(config): 290 | opts = "" 291 | for k, v in config.items(): 292 | if k not in ['insecure?']: 293 | opts += ' --%s %s' % (k, v) 294 | if config.get('insecure?', False): 295 | opts += ' -i' 296 | return opts 297 | 298 | @staticmethod 299 | def _print(msg): 300 | ss_display(msg) 301 | print(msg) 302 | 303 | @staticmethod 304 | def _print_t(msg): 305 | _print(':t: %s' % msg) 306 | 307 | def info(self): 308 | _print('Tests to run: %s' % self.get_test_names()) 309 | 310 | 311 | ## 312 | ## Tests. 313 | ## 314 | 315 | test_repo_branch = ss_get('test_repo_branch') 316 | run_comp_uri = ss_get('run_comp_uri') 317 | scale_app_uri = ss_get('scale_app_uri') 318 | scale_comp_name = ss_get('scale_comp_name') 319 | tests_to_run = _tests_to_run() 320 | 321 | nexus_user, nexus_pass = ss_get('nexus_creds').split(':') 322 | 323 | _print('Installing git credentials.') 324 | _install_git_creds(nexus_user, nexus_pass) 325 | 326 | _print('Install SS repo credentials for lein.') 327 | _install_ss_repo_creds_lein(nexus_user, nexus_pass) 328 | 329 | _cd_home() 330 | 331 | _print('Cloning test repo.') 332 | test_repo_name = 'SlipStreamTests' 333 | _rmdir(_expanduser('~/%s' % test_repo_name), ignore_errors=True) 334 | _check_call(['git', 'clone', 'git@github.com:slipstream/%s.git' % test_repo_name]) 335 | 336 | # 337 | # Wait for deployer to deploy SlipStream. 338 | ss_get('deployer.ready', timeout=5400) 339 | 340 | test_username, test_userpass = _get_test_user_pass() 341 | 342 | endpoint = ss_get('ss_service_url') 343 | 344 | _print('Ready to run tests on %s as %s.' % (endpoint, test_username)) 345 | 346 | connectors_to_test = _get_connectors_to_test(SS_SERVICES_IN_NAGIOS) 347 | 348 | _cd(test_repo_name) 349 | _check_call(['git', 'checkout', test_repo_branch]) 350 | 351 | results_dir = _expanduser('~/test-results') 352 | _rmdir(results_dir) 353 | _mkdir(results_dir, 0755) 354 | 355 | config_auth = {'username': test_username, 356 | 'password': test_userpass, 357 | 'endpoint': endpoint, 358 | 'insecure?': True, 359 | 'results-dir': results_dir} 360 | 361 | tr = TestsRunner(config_auth) 362 | 363 | # smoke test 364 | tr.add_test('test-clojure-deps', 365 | msg='Check if local dependencies are available.', fail=True) 366 | 367 | tr.add_test('test-auth', 368 | msg='Authentication tests on %s as %s.' % (endpoint, test_username)) 369 | tr.add_test('test-run-comp', 370 | msg='Component deployment - %s on %s as %s.' % (run_comp_uri, endpoint, test_username), 371 | config={'comp-uri': run_comp_uri}, 372 | connectors=connectors_to_test) 373 | tr.add_test('test-run-app', 374 | msg='Application deployment - %s on %s as %s.' % (scale_app_uri, endpoint, test_username), 375 | config={'app-uri': scale_app_uri, 'comp-name': scale_comp_name}, 376 | connectors=connectors_to_test) 377 | tr.add_test('test-run-app-scale', 378 | msg='Scalable deployment - %s on %s as %s.' % (scale_app_uri, endpoint, test_username), 379 | config={'app-uri': scale_app_uri, 'comp-name': scale_comp_name}, 380 | connectors=connectors_to_test) 381 | 382 | tr.info() 383 | 384 | os.environ['LEIN_ROOT'] = 'true' 385 | tr.run(tests_to_run=tests_to_run) 386 | 387 | _print('All tests were run.') 388 | 389 | if tr.failed_tests: 390 | _print('Tests failed: %s' % ', '.join(tr.get_failed_tests())) 391 | exit(1) 392 | 393 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | 4.0.0 5 | 6 | com.sixsq.slipstream 7 | SlipStream 8 | pom 9 | 3.72-SNAPSHOT 10 | SlipStream 11 | http://sixsq.com/ 12 | 13 | 14 | community 15 | enterprise 16 | 17 | 18 | 19 | 20 | 21 | 22 | full-build 23 | 24 | 25 | !skipModules 26 | 27 | 28 | 29 | ../SlipStreamUI 30 | ../SlipStreamWebUI 31 | ../SlipStreamServer 32 | ../SlipStreamServerDeps 33 | ../SlipStreamClient 34 | ../SlipStreamClojureAPI 35 | ../SlipStreamConnectors 36 | ../SlipStreamPythonAPI 37 | ../SlipStreamJobEngine 38 | ../SlipStreamTests 39 | 40 | 41 | 42 | 43 | enterprise-build 44 | 45 | 46 | enterprise 47 | 48 | 49 | 50 | ../SlipStreamConnectorsEnterprise 51 | ../SlipStreamPricing 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | -rhel7 60 | 61 | UTF-8 62 | 63 | /opt/slipstream 64 | ${slipstream.base.dir}/server 65 | ${slipstream.base.dir}/downloads 66 | ${installation.dir}/webapps/slipstream.war/WEB-INF/lib 67 | 68 | 9.3.2.v20150730 69 | ${installation.dir}/lib/connectors 70 | ${installation.dir}/lib/ext 71 | 72 | el7 73 | 74 | 75 | 76 | scm:git:https://github.com/slipstream 77 | scm:git:ssh://git@github.com/slipstream 78 | https://github.com/slipstream 79 | 80 | 81 | 82 | https://nexus.sixsq.com/content/repositories 83 | https://nexus.sixsq.com/content/repositories 84 | 85 | 2.3.4 86 | 9.4.1209 87 | 88 | 0.18.0 89 | 2.46.1 90 | 91 | 3.1.0 92 | 93 | 94 | 95 | 96 | 97 | 0 98 | 99 | 100 | 101 | 102 | 103 | maven-restlet 104 | Public online Restlet repository 105 | https://maven.restlet.com 106 | 107 | 108 | 109 | sixsq.thirdparty 110 | ${nexus.thirdparty}/thirdparty 111 | 112 | 113 | 114 | clojars.org 115 | https://clojars.org/repo 116 | 117 | 118 | 119 | 120 | true 121 | 122 | 123 | false 124 | 125 | slipstream.releases 126 | SlipStream Releases 127 | ${nexus}/releases-community${platform.os} 128 | 129 | 130 | 131 | 132 | false 133 | 134 | 135 | true 136 | 137 | slipstream.snapshots 138 | SlipStream Snapshots 139 | ${nexus}/snapshots-community${platform.os} 140 | 141 | 142 | 143 | 144 | 145 | ${scm.read}/SlipStream.git 146 | ${scm.write}/SlipStream.git 147 | ${scm.public}/SlipStream 148 | HEAD 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | org.apache.maven.wagon 158 | wagon-file 159 | 2.5 160 | 161 | 162 | 163 | 164 | co.leantechniques 165 | maven-buildtime-extension 166 | 2.0.1 167 | 168 | 169 | 170 | 171 | 172 | 173 | eu.somatik.serviceloader-maven-plugin 174 | serviceloader-maven-plugin 175 | 1.0.7 176 | 177 | 178 | maven-release-plugin 179 | 2.5.3 180 | 181 | clean 182 | v@{project.version}-community 183 | 184 | 185 | 186 | org.apache.maven.plugins 187 | maven-compiler-plugin 188 | 3.5.1 189 | 190 | 1.8 191 | 1.8 192 | 193 | 194 | 195 | org.apache.maven.plugins 196 | maven-surefire-report-plugin 197 | 2.20.1 198 | 199 | 200 | org.apache.maven.plugins 201 | maven-deploy-plugin 202 | 2.8.2 203 | 204 | 205 | org.apache.maven.plugins 206 | maven-dependency-plugin 207 | 2.10 208 | 209 | 210 | org.apache.maven.plugins 211 | maven-install-plugin 212 | 2.5.2 213 | 214 | 215 | org.apache.maven.plugins 216 | maven-shade-plugin 217 | 2.4.3 218 | 219 | 220 | org.apache.maven.plugins 221 | maven-site-plugin 222 | 3.5.1 223 | 224 | 225 | org.apache.maven.plugins 226 | maven-assembly-plugin 227 | 2.6 228 | 229 | posix 230 | 231 | 232 | 233 | org.apache.maven.plugins 234 | maven-resources-plugin 235 | 3.0.1 236 | 237 | 238 | org.apache.maven.plugins 239 | maven-clean-plugin 240 | 3.0.0 241 | 242 | 243 | org.apache.maven.plugins 244 | maven-surefire-plugin 245 | 2.20.1 246 | 247 | 248 | org.apache.maven.plugins 249 | maven-jar-plugin 250 | 3.0.2 251 | 252 | 253 | org.apache.maven.plugins 254 | maven-antrun-plugin 255 | 1.8 256 | 257 | 258 | org.codehaus.mojo 259 | license-maven-plugin 260 | 1.9 261 | 262 | 263 | org.eclipse.jetty 264 | jetty-maven-plugin 265 | ${jetty.version} 266 | 267 | 268 | org.apache.maven.plugins 269 | maven-war-plugin 270 | 2.6 271 | 272 | 273 | org.codehaus.mojo 274 | findbugs-maven-plugin 275 | 3.0.4 276 | 277 | 278 | org.codehaus.mojo 279 | exec-maven-plugin 280 | 1.5.0 281 | 282 | 283 | 284 | org.codehaus.mojo 285 | buildnumber-maven-plugin 286 | 1.4 287 | 288 | 289 | 290 | org.codehaus.mojo 291 | rpm-maven-plugin 292 | 2.1.5 293 | 294 | 295 | 296 | org.kuali.maven.plugins 297 | graph-maven-plugin 298 | 1.2.3 299 | 300 | 301 | 302 | org.codehaus.mojo 303 | build-helper-maven-plugin 304 | 1.12 305 | 306 | 307 | 308 | net.ju-n.maven.plugins 309 | checksum-maven-plugin 310 | 1.3 311 | 312 | 313 | 314 | org.owasp 315 | dependency-check-maven 316 | 3.3.0 317 | 318 | 319 | 320 | check 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | org.codehaus.mojo 333 | build-helper-maven-plugin 334 | 335 | 336 | parse-version 337 | validate 338 | 339 | parse-version 340 | 341 | 342 | 343 | 344 | 345 | 346 | maven-resources-plugin 347 | 348 | 349 | copy-resources 350 | validate 351 | 352 | copy-resources 353 | 354 | 355 | ${basedir} 356 | 357 | 358 | src/main/resources 359 | true 360 | 361 | versions.sh 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | 377 | 378 | 379 | 380 | 381 | 382 | 383 | com.sixsq.slipstream 384 | SlipStreamDbBinding-jar-dep 385 | ${project.version} 386 | pom 387 | provided 388 | 389 | 390 | 391 | com.sixsq.slipstream 392 | SlipStreamDbBinding-jar 393 | ${project.version} 394 | 395 | 396 | 397 | com.sixsq.slipstream 398 | SlipStreamDbTesting-jar 399 | ${project.version} 400 | 401 | 402 | 403 | com.sixsq.slipstream 404 | SlipStreamDbSerializers-jar-dep 405 | ${project.version} 406 | pom 407 | provided 408 | 409 | 410 | 411 | com.sixsq.slipstream 412 | SlipStreamDbSerializers-jar 413 | ${project.version} 414 | 415 | 416 | 417 | com.sixsq.slipstream 418 | SlipStreamServerPRSlib-jar 419 | ${project.version} 420 | 421 | 422 | 423 | com.sixsq.slipstream 424 | Libcloud-SixSq-zip 425 | ${project.version} 426 | 427 | 428 | 429 | com.sixsq.slipstream 430 | SlipStreamAsync 431 | ${project.version} 432 | 433 | 434 | 435 | com.sixsq.slipstream 436 | SlipStreamCljResources-jar 437 | ${project.version} 438 | 439 | 440 | 441 | com.sixsq.slipstream 442 | SlipStreamCljResourcesTestServer-jar 443 | ${project.version} 444 | test 445 | 446 | 447 | 448 | com.sixsq.slipstream 449 | SlipStreamCredCache 450 | ${project.version} 451 | 452 | 453 | 454 | com.sixsq.slipstream 455 | SlipStreamService 456 | ${project.version} 457 | 458 | 459 | 460 | com.sixsq.slipstream 461 | SlipStreamService 462 | test-jar 463 | ${project.version} 464 | test 465 | 466 | 467 | 468 | com.sixsq.slipstream 469 | SlipStreamPersistence 470 | ${project.version} 471 | 472 | 473 | 474 | com.sixsq.slipstream 475 | SlipStreamPersistence 476 | test-jar 477 | ${project.version} 478 | test 479 | 480 | 481 | 482 | com.sixsq.slipstream 483 | SlipStreamConnector 484 | ${project.version} 485 | 486 | 487 | 488 | com.sixsq.slipstream 489 | SlipStreamConnector 490 | test-jar 491 | ${project.version} 492 | test 493 | 494 | 495 | 496 | com.sixsq.slipstream 497 | SlipStreamUI-dep 498 | ${project.version} 499 | pom 500 | provided 501 | 502 | 503 | 504 | com.sixsq.slipstream 505 | SlipStreamUI 506 | ${project.version} 507 | 508 | 509 | 510 | com.sixsq.slipstream 511 | SlipStreamServer-jar 512 | test-jar 513 | ${project.version} 514 | test 515 | 516 | 517 | 518 | com.sixsq.slipstream 519 | auth-dep 520 | ${project.version} 521 | pom 522 | provided 523 | 524 | 525 | 526 | com.sixsq.slipstream 527 | auth 528 | ${project.version} 529 | 530 | 531 | 532 | com.sixsq.slipstream 533 | token-dep 534 | ${project.version} 535 | pom 536 | provided 537 | 538 | 539 | 540 | com.sixsq.slipstream 541 | token-java 542 | ${project.version} 543 | 544 | 545 | 546 | com.sixsq.slipstream 547 | SlipStreamServer-ui-static-content 548 | ${project.version} 549 | zip 550 | 551 | 552 | 553 | com.sixsq.slipstream 554 | SlipStreamRiemann-jar 555 | ${project.version} 556 | 557 | 558 | 559 | com.sixsq.slipstream 560 | SlipStreamClojureAPI-cimi 561 | ${project.version} 562 | 563 | 564 | 565 | 566 | 567 | 568 | 569 | com.chemdrawcloud 570 | gwt-crypto 571 | 2.7.0 572 | 573 | 574 | 575 | 576 | commons-lang 577 | commons-lang 578 | 2.6 579 | 580 | 581 | 582 | 583 | commons-codec 584 | commons-codec 585 | 1.11 586 | 587 | 588 | 589 | 590 | com.jcraft 591 | jsch 592 | 0.1.55 593 | jar 594 | 595 | 596 | 597 | org.restlet.jee 598 | org.restlet 599 | 2.2.1 600 | 601 | 602 | 603 | org.restlet.jee 604 | org.restlet.ext.servlet 605 | 2.2.1 606 | 607 | 608 | 609 | org.restlet.jee 610 | org.restlet.ext.fileupload 611 | 2.2.1 612 | 613 | 614 | 615 | org.restlet.jee 616 | org.restlet.ext.xml 617 | 2.2.1 618 | 619 | 620 | 621 | org.restlet.jee 622 | org.restlet.ext.httpclient 623 | 2.2.1 624 | 625 | 626 | 627 | org.apache.httpcomponents 628 | httpclient 629 | 630 | 631 | org.apache.httpcomponents 632 | httpcore 633 | 634 | 635 | org.apache.httpcomponents 636 | httpmime 637 | 638 | 639 | 640 | 641 | 642 | org.simpleframework 643 | simple-xml 644 | 2.7.1 645 | 646 | 647 | 648 | com.google.code.typica 649 | typica 650 | 1.7.2 651 | 652 | 653 | 654 | com.google.code.gson 655 | gson 656 | 2.8.5 657 | 658 | 659 | 660 | org.json 661 | json 662 | 20160810 663 | 664 | 665 | 666 | 667 | javax.mail 668 | mail 669 | 1.4.7 670 | compile 671 | 672 | 673 | 674 | org.hibernate 675 | hibernate-entitymanager 676 | 4.3.9.Final 677 | 678 | 679 | 680 | org.hibernate 681 | hibernate-c3p0 682 | 4.3.9.Final 683 | 684 | 685 | 686 | com.mchange 687 | c3p0 688 | 0.9.5.3 689 | 690 | 691 | 692 | 693 | org.slf4j 694 | slf4j-jdk14 695 | 1.7.25 696 | 697 | 698 | 699 | org.hsqldb 700 | hsqldb 701 | ${hsqldb.version} 702 | 703 | 704 | 705 | org.postgresql 706 | postgresql 707 | ${postgresql.version} 708 | 709 | 710 | 711 | jline 712 | jline 713 | 1.0 714 | 715 | 716 | 717 | 718 | javax.servlet 719 | javax.servlet-api 720 | 4.0.1 721 | 722 | 723 | 724 | 725 | log4j 726 | log4j 727 | 1.2.17 728 | 729 | 730 | 731 | 732 | org.slf4j 733 | slf4j-log4j12 734 | 1.7.25 735 | 736 | 737 | 738 | 739 | org.apache.logging.log4j 740 | log4j-api 741 | 2.11.1 742 | 743 | 744 | 745 | 746 | org.apache.logging.log4j 747 | log4j-core 748 | 2.11.1 749 | 750 | 751 | 752 | io.dropwizard.metrics 753 | metrics-core 754 | ${metrics.version} 755 | 756 | 757 | 758 | io.dropwizard.metrics 759 | metrics-graphite 760 | ${metrics.version} 761 | 762 | 763 | 764 | io.dropwizard.metrics 765 | metrics-jvm 766 | ${metrics.version} 767 | 768 | 769 | 770 | 771 | 772 | 773 | 774 | 775 | org.clojure 776 | clojure 777 | 1.10.0 778 | 779 | 780 | 781 | 782 | org.clojure 783 | test.check 784 | 0.9.0 785 | 786 | 787 | 788 | 789 | 790 | 791 | 792 | junit 793 | junit 794 | 4.12 795 | test 796 | 797 | 798 | 799 | org.hamcrest 800 | hamcrest-all 801 | 1.3 802 | test 803 | 804 | 805 | 806 | expectations 807 | expectations 808 | 2.1.10 809 | test 810 | 811 | 812 | 813 | 814 | org.elasticsearch 815 | elasticsearch 816 | 6.2.4 817 | 818 | 819 | 820 | 821 | org.elasticsearch.client 822 | transport 823 | 6.2.4 824 | 825 | 826 | 827 | 828 | 829 | 830 | 831 | 832 | sixsq.releases 833 | SixSq Releases 834 | ${nexus}/releases-community${platform.os} 835 | 836 | 837 | 838 | sixsq.snapshots 839 | SixSq Snapshots 840 | ${nexus}/snapshots-community${platform.os} 841 | 842 | 843 | 844 | 845 | -------------------------------------------------------------------------------- /install/slipstream.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # SlipStream installation recipe. 4 | # The installation works on RedHat based distribution only. 5 | # The script does the following: 6 | # - installs SlipStream dependencies 7 | # - installs and starts RDBMS 8 | # - installs and starts SlipStream service (it's optionally possible not to 9 | # start the service). 10 | # NB! Installation of the SlipStream connectors is not done in this script. 11 | 12 | # Fail fast and fail hard. 13 | set -e 14 | set -x 15 | set -o pipefail 16 | 17 | # # # # # # # 18 | # DEFAULTS. 19 | # # # # # # # 20 | 21 | VERBOSE=false 22 | LOG_FILE=/tmp/slipstream-install.log 23 | 24 | # Defult YUM repository kind. 25 | _YUM_REPO_KIND_DEFAULT=release 26 | declare -A _YUM_REPO_KIND_MAP 27 | _YUM_REPO_KIND_MAP[local]=Local 28 | _YUM_REPO_KIND_MAP[snapshot]=Snapshots 29 | _YUM_REPO_KIND_MAP[candidate]=Candidates 30 | _YUM_REPO_KIND_MAP[${_YUM_REPO_KIND_DEFAULT}]=Releases 31 | SS_YUM_REPO_KIND=${_YUM_REPO_KIND_MAP[$_YUM_REPO_KIND_DEFAULT]} 32 | SS_YUM_REPO_DEF_URL= 33 | 34 | # Defult YUM repository edition. 35 | _YUM_REPO_EDITIONS=(enterprise community) 36 | SS_YUM_REPO_EDITION=community 37 | 38 | SS_THEME=default 39 | SS_LANG=en 40 | SS_START=true 41 | 42 | SS_SERVER_LOC=/opt/slipstream/server 43 | SS_SERVER_LOG_DIR=/var/log/slipstream/server 44 | 45 | # Elasticsearch via transport client on localhost. 46 | export ES_HOST=localhost 47 | export ES_PORT_BIN=9300 48 | export ES_PORT_HTTP=9200 49 | export ES_PORT=$ES_PORT_BIN 50 | ES_INSTALL=true 51 | 52 | # Logstash coordinates. 53 | SS_LOGSTASH_COLLECTD_UDP=25826 54 | export LOGSTASH_HOST=localhost 55 | export LOGSTASH_PORT=5043 56 | LOGSTASH_INSTALL=true 57 | 58 | ELK_XPACK=false 59 | 60 | # Zookeeper coordinates. 61 | export ZK_ENDPOINTS=localhost:2181 62 | 63 | USAGE="usage: -h -v -l -k -e -E -H -t -L \n 64 | -S -a -b -c\n 65 | \n 66 | -h print this help\n 67 | -v run in verbose mode\n 68 | -l log file (default: $LOG_FILE)\n 69 | -k kind of the repository to use: ${!_YUM_REPO_KIND_MAP[@]}. Default: $_YUM_REPO_KIND_DEFAULT\n 70 | -e edition of the repository to use: ${_YUM_REPO_EDITIONS[@]}. Default: $SS_YUM_REPO_EDITION\n 71 | -E don't load examples\n 72 | -H hostname or IP of the host. If not provided, an attempt to discover it is made.\n 73 | -t the theme for the service\n 74 | -L the language of the interface. Possilbe values: en, fr, de, jp. (default: en)\n 75 | -S don't start SlipStream service.\n 76 | -x URL with the YUM repo definition file.\n 77 | -a Elasticsearch coordinates. Default: $ES_HOST:$ES_PORT. If provided, and\n 78 | hostname/IP is localhost or 127.0.0.1, then Elasticsearch will be installed.\n 79 | -b Logstash coordinates. Default: $LOGSTASH_HOST:$LOGSTASH_PORT. If provided,\n 80 | and hostname/IP is localhost or 127.0.0.1, then Logstash will be installed.\n 81 | -c If provided, install X-Pack for ELK components.\n 82 | -z Zookeeper coordinates. Default: $ZK_ENDPOINTS. If provided,\n 83 | and hostname/IP is localhost or 127.0.0.1, then Zookeeper will be installed.\n" 84 | 85 | # Allow this to be set in the environment to avoid having to pass arguments 86 | # through all of the other installation scripts. 87 | SLIPSTREAM_EXAMPLES=${SLIPSTREAM_EXAMPLES:-true} 88 | 89 | function _exit_usage() { 90 | echo -e $USAGE 91 | exit 1 92 | } 93 | 94 | function _check_repo_edition() { 95 | if [ "$1" != "community" ] && [ "$1" != "enterprise" ]; then 96 | _exit_usage 97 | fi 98 | } 99 | 100 | function _check_repo_kind() { 101 | if ! test "${_YUM_REPO_KIND_MAP[$1]+isset}"; then 102 | _exit_usage 103 | fi 104 | } 105 | 106 | function _is_local_install() { 107 | if [[ $1 =~ (localhost|127.0.0.1) ]]; then 108 | echo true 109 | else 110 | echo false 111 | fi 112 | } 113 | 114 | while getopts a:b:l:H:t:L:k:e:d:x:vESch opt; do 115 | case $opt in 116 | v) 117 | VERBOSE=true 118 | ;; 119 | l) 120 | LOG_FILE=$OPTARG 121 | ;; 122 | k) 123 | _check_repo_kind $OPTARG 124 | SS_YUM_REPO_KIND=${_YUM_REPO_KIND_MAP[$OPTARG]} 125 | ;; 126 | e) 127 | _check_repo_edition $OPTARG 128 | SS_YUM_REPO_EDITION=$OPTARG 129 | ;; 130 | E) 131 | # Do not upload examples 132 | SLIPSTREAM_EXAMPLES=false 133 | ;; 134 | H) 135 | # hostname/ip 136 | SS_HOSTNAME=$OPTARG 137 | ;; 138 | t) 139 | # Theme name 140 | SS_THEME=$OPTARG 141 | ;; 142 | L) 143 | # Localization language 144 | SS_LANG=$OPTARG 145 | ;; 146 | S) 147 | # Don't start SlipStream service 148 | SS_START=false 149 | ;; 150 | x) 151 | SS_YUM_REPO_DEF_URL=$OPTARG 152 | ;; 153 | a) 154 | ES_HOST=${OPTARG%%:*} 155 | ES_PORT=${OPTARG#*:} 156 | ES_INSTALL=$(_is_local_install $ES_HOST) 157 | ;; 158 | b) 159 | LOGSTASH_HOST=${OPTARG%%:*} 160 | LOGSTASH_PORT=${OPTARG#*:} 161 | LOGSTASH_INSTALL=$(_is_local_install $LOGSTASH_HOST) 162 | ;; 163 | c) 164 | ELK_XPACK=true 165 | ;; 166 | *|h) 167 | _exit_usage 168 | ;; 169 | esac 170 | done 171 | 172 | SS_YUM_REPO=${SS_YUM_REPO_KIND}-${SS_YUM_REPO_EDITION} 173 | SS_YUM_REPO_COMMUNITY=${SS_YUM_REPO_KIND}-community 174 | 175 | shift $((OPTIND - 1)) 176 | 177 | VERBOSE=true 178 | if [ "$VERBOSE" = "true" ]; then 179 | exec 4>&2 3>&1 180 | else 181 | exec 4>&2 3>&1 1>>${LOG_FILE} 2>&1 182 | fi 183 | 184 | # # # # # # # 185 | # Utilities. 186 | # # # # # # # 187 | 188 | function abort() { 189 | echo "!!! Aborting: $@" 1>&4 190 | exit 1 191 | } 192 | 193 | function _print() { 194 | echo -e "::: $@" 1>&3 195 | } 196 | 197 | function _print_error() { 198 | _print "ERROR! $@" 199 | } 200 | 201 | function _print_on_trap() { 202 | if [ "$VERBOSE" != "true" ]; then 203 | _print "ERROR! Check log file ${LOG_FILE}\n... snippet ...\n$(tail -5 ${LOG_FILE})" 204 | fi 205 | } 206 | 207 | function _on_trap() { 208 | _print_on_trap 209 | } 210 | 211 | trap '_on_trap' ERR 212 | 213 | # Return first global IPv4 address. 214 | function _get_ip() { 215 | ip addr | awk '/inet .*global/ { split($2, x, "/"); print x[1] }' | head -1 216 | } 217 | 218 | # # # # # # # # # # # 219 | # Global parameters. 220 | # # # # # # # # # # # 221 | 222 | # First "global" IPv4 address 223 | HOST_IP=$(_get_ip) 224 | SS_HOSTNAME=${SS_HOSTNAME:-$HOST_IP} 225 | [ -z "${SS_HOSTNAME}" ] && \ 226 | abort "Could not determinee IP or hostname of the public interface 227 | for SlipStream to run on." 228 | 229 | # apache-libcloud 230 | CLOUD_CLIENT_LIBCLOUD_VERSION=0.18.0 231 | 232 | # Packages from PyPi for SlipStream Client 233 | PYPI_SCPCLIENT_VER=0.4 234 | 235 | # Riemann variables. 236 | RIEMANN_VER=0.2.11-1 237 | ss_clj_client=/opt/slipstream/riemann/lib/SlipStreamRiemann.jar 238 | ss_riemann_conf=/etc/riemann/riemann-slipstream.config 239 | ss_riemann_streams=/opt/slipstream/riemann/streams 240 | 241 | # # # # # # # # # # # # 242 | # Advanced parameters. 243 | # # # # # # # # # # # # 244 | 245 | CONFIGURE_FIREWALL=true 246 | 247 | SS_USERNAME=super 248 | # Deafult. Should be changed immediately after installation. 249 | # See SlipStream administrator manual. 250 | SS_PASSWORD=supeRsupeR 251 | 252 | if [ -f /etc/slipstream/passwords/$SS_USERNAME ]; then 253 | SS_PASSWORD=$(cat /etc/slipstream/passwords/$SS_USERNAME) 254 | fi 255 | 256 | # Default local coordinates of SlipStream. 257 | SS_LOCAL_PORT=8182 258 | SS_LOCAL_HOST=localhost 259 | SS_LOCAL_URL=http://$SS_LOCAL_HOST:$SS_LOCAL_PORT 260 | 261 | CIMI_LOCAL_HOST=localhost 262 | CIMI_LOCAL_PORT=8201 263 | CIMI_LOCAL_URL=http://$CIMI_LOCAL_HOST:$CIMI_LOCAL_PORT 264 | 265 | SLIPSTREAM_ETC=/etc/slipstream 266 | SLIPSTREAM_CONF=$SLIPSTREAM_ETC/slipstream.edn 267 | 268 | DEPS="unzip curl wget gnupg nc python-pip" 269 | CLEAN_PKG_CACHE="yum clean all" 270 | 271 | SS_JETTY_CONFIG=/etc/default/slipstream 272 | 273 | SS_USER=slipstream 274 | SS_GROUP=$SS_USER 275 | 276 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 277 | # Deployment. 278 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 279 | 280 | alias cp='cp' 281 | 282 | function _is_true() { 283 | if [ "x${1}" == "xtrue" ]; then 284 | return 0 285 | else 286 | return 1 287 | fi 288 | } 289 | 290 | function _inst() { 291 | yum install -y $@ 292 | } 293 | 294 | function srvc_start() { 295 | systemctl start $1 296 | } 297 | function srvc_stop() { 298 | systemctl stop $1 299 | } 300 | function srvc_restart() { 301 | systemctl restart $1 302 | } 303 | function srvc_enable() { 304 | systemctl enable $1 305 | } 306 | function srvc_mask() { 307 | systemctl mask $1 308 | } 309 | function srvc_() { 310 | systemctl $@ 311 | } 312 | 313 | function _now_sec() { 314 | date +%s 315 | } 316 | 317 | function _wait_listens() { 318 | # host port [timeout seconds] [sleep interval seconds] 319 | wait_time=${3:-60} 320 | sleep_interval=${4:-2} 321 | stop_time=$(($(_now_sec) + $wait_time)) 322 | while (( "$(_now_sec)" <= $stop_time )); do 323 | set +e 324 | res=$(ncat -v -4 $1 $2 < /dev/null 2>&1) 325 | if [ "$?" == "0" ]; then 326 | set -e 327 | return 0 328 | else 329 | if ( ! (echo $res | grep -q "Connection refused") ); then 330 | abort "Failed to check $1:$2 with:" $res 331 | fi 332 | fi 333 | set -e 334 | sleep $sleep_interval 335 | done 336 | abort "Timed out after ${wait_time} sec waiting for $1:$2" 337 | } 338 | 339 | 340 | function _configure_firewall () { 341 | _is_true $CONFIGURE_FIREWALL || return 0 342 | 343 | _print "- configuring firewall" 344 | 345 | # firewalld may not be installed 346 | srvc_stop firewalld || true 347 | srvc_mask firewalld || true 348 | 349 | _inst iptables-services 350 | srvc_enable iptables 351 | 352 | cat > /etc/sysconfig/iptables </dev/null 2>&1 441 | if [ $? -ne 0 ]; then 442 | getent hosts $HOST_IP 1>/dev/null 2>&1 443 | if [ $? -ne 0 ] ; then 444 | echo "$HOST_IP $(hostname)" >> /etc/hosts 445 | fi 446 | fi 447 | set -e 448 | } 449 | 450 | function prepare_node () { 451 | 452 | _print "Preparing node" 453 | 454 | _add_yum_repos 455 | _install_global_dependencies 456 | _configure_firewall 457 | _configure_selinux 458 | _install_time_sync_service 459 | _configure_hostname 460 | } 461 | 462 | function _deploy_haveged () { 463 | 464 | _print "- installing haveged (entropy generator)" 465 | 466 | srvc_stop haveged || true 467 | 468 | _inst haveged 469 | 470 | srvc_enable haveged 471 | 472 | srvc_start haveged 473 | } 474 | 475 | function _deploy_hsqldb () { 476 | 477 | _print "- installing HSQLDB" 478 | 479 | srvc_stop hsqldb || true 480 | kill -9 $(cat /var/run/hsqldb.pid) || true 481 | rm -f /var/run/hsqldb.pid 482 | 483 | _inst slipstream-hsqldb 484 | 485 | cat > ~/sqltool.rc <> /etc/default/cimi 588 | fi 589 | } 590 | 591 | function _set_theme() { 592 | # do not write this line if using the default theme for now 593 | if [ -n $SS_THEME -a "X$SS_THEME" != "Xdefault" ]; then 594 | _set_jetty_args slipstream.ui.util.theme.current-theme $SS_THEME 595 | fi 596 | } 597 | 598 | function _set_localization() { 599 | if [ -n $SS_LANG ]; then 600 | _set_jetty_args slipstream.ui.util.localization.lang-default $SS_LANG 601 | fi 602 | } 603 | 604 | function _stop_slipstream_service() { 605 | _print "- stopping SlipStream service" 606 | 607 | srvc_stop slipstream || true 608 | srvc_stop cimi || true 609 | } 610 | 611 | function _start_slipstream() { 612 | if ( _is_true $SS_START ); then 613 | _print "- starting SlipStream service" 614 | _start_slipstream_service 615 | else 616 | _print "- WARNING: requested not to start SlipStream service" 617 | fi 618 | } 619 | 620 | function _start_cimi_application() { 621 | # CIMI (increased to 600 because of slow startup) 622 | _wait_listens $CIMI_LOCAL_HOST $CIMI_LOCAL_PORT 600 623 | curl -m 60 -sfS -o /dev/null $CIMI_LOCAL_URL/api/cloud-entry-point 624 | } 625 | 626 | function _start_ss_application() { 627 | # SS 628 | _wait_listens $SS_LOCAL_HOST $SS_LOCAL_PORT 629 | curl -m 60 -sfS -o /dev/null $SS_LOCAL_URL 630 | } 631 | 632 | function _start_slipstream_service() { 633 | srvc_start cimi 634 | _start_cimi_application 635 | srvc_start slipstream 636 | _start_ss_application 637 | } 638 | 639 | function _enable_slipstream() { 640 | srvc_enable cimi 641 | srvc_enable slipstream 642 | } 643 | 644 | function _start_slipstream_application() { 645 | _start_cimi_application 646 | _start_ss_application 647 | } 648 | 649 | function _set_jetty_args() { 650 | prop_name=$1 651 | prop_value=${2:-""} 652 | if ( ! grep -q -- "-D$prop_name=" ${SS_JETTY_CONFIG} ); then 653 | cat >> ${SS_JETTY_CONFIG} </${SS_HOSTNAME}/" \ 666 | $@ 667 | } 668 | 669 | function _chown_slipstream_etc() { 670 | chown -R $SS_USER.$SS_GROUP $SLIPSTREAM_ETC 671 | } 672 | 673 | function _update_service_configuration() { 674 | 675 | # Configuration. 676 | [ -s $SLIPSTREAM_CONF ] || printf "{\n}\n" > $SLIPSTREAM_CONF 677 | 678 | chown $SS_USER.$SS_GROUP $SLIPSTREAM_CONF 679 | 680 | _update_hostname_in_conf_file $SLIPSTREAM_CONF 681 | 682 | # Update configuration file. 683 | ss-config \ 684 | -e id=configuration/slipstream \ 685 | -e clientURL=https://${SS_HOSTNAME}/downloads/slipstreamclient.tgz \ 686 | -e clientBootstrapURL=https://${SS_HOSTNAME}/downloads/slipstream.bootstrap \ 687 | -e connectorLibcloudURL=https://${SS_HOSTNAME}/downloads/libcloud.tgz \ 688 | -e serviceURL=https://${SS_HOSTNAME} \ 689 | -e connectorOrchPublicSSHKey=$SS_SERVER_LOC/.ssh/id_rsa.pub \ 690 | -e connectorOrchPrivateSSHKey=$SS_SERVER_LOC/.ssh/id_rsa \ 691 | $SLIPSTREAM_CONF 692 | 693 | # Push service configuration to DB. 694 | ss-config $SLIPSTREAM_CONF 695 | } 696 | 697 | function _update_connectors_configuration() { 698 | # Connectors. 699 | SLIPSTREAM_CONNECTORS= 700 | if [ -d $SLIPSTREAM_ETC/connectors ]; then 701 | SLIPSTREAM_CONNECTORS=$(find $SLIPSTREAM_ETC/connectors -name "*.edn") 702 | for cconf in $SLIPSTREAM_CONNECTORS; do 703 | _update_hostname_in_conf_file $cconf 704 | done 705 | fi 706 | 707 | # NB! Pushing connectors configuration to DB can only be done when connectors are installed. 708 | } 709 | 710 | function _update_slipstream_configuration() { 711 | _chown_slipstream_etc 712 | _update_service_configuration 713 | _update_connectors_configuration 714 | } 715 | 716 | function _install_kibana() { 717 | _print "- installing Kibana" 718 | 719 | _inst java-1.8.0-openjdk-headless 720 | _inst kibana 721 | 722 | _is_true $ELK_XPACK && \ 723 | /usr/share/kibana/bin/kibana-plugin install x-pack || true 724 | 725 | # SSL config. Steal certs from nginx. 726 | kibana_ssl=/etc/kibana/ssl 727 | if [ ! -d $kibana_ssl ]; then 728 | mkdir -p $kibana_ssl 729 | chmod 700 $kibana_ssl 730 | cp -p /etc/nginx/ssl/server.* $kibana_ssl 731 | chown -R kibana. $kibana_ssl 732 | chmod 400 $kibana_ssl/* 733 | fi 734 | 735 | log_dest=/var/log/kibana 736 | mkdir -p $log_dest 737 | chown -R kibana. $log_dest 738 | 739 | cat >> /etc/kibana/kibana.yml << EOF 740 | server.ssl.cert: $kibana_ssl/server.crt 741 | server.ssl.key: $kibana_ssl/server.key 742 | server.host: "0.0.0.0" 743 | elasticsearch.url: "http://$ES_HOST:$ES_PORT_HTTP" 744 | logging.dest: $log_dest/kibana.log 745 | EOF 746 | 747 | srvc_enable kibana.service 748 | srvc_start kibana 749 | } 750 | 751 | function _install_elasticsearch() { 752 | _print "- installing Elasticsearch" 753 | 754 | _inst java-1.8.0-openjdk-headless 755 | _inst elasticsearch 756 | 757 | # For authn with Kibana and more. 758 | _is_true $ELK_XPACK && \ 759 | /usr/share/elasticsearch/bin/elasticsearch-plugin install -b x-pack || true 760 | # To be added from elasticsearch v6.x 761 | 762 | mkdir -p /usr/share/elasticsearch/logs/ 763 | chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/logs 764 | 765 | mkdir -p /usr/share/elasticsearch/data/ 766 | chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/data 767 | 768 | # Configure elasticsearch 769 | # FIXME visible on localhost only 770 | elasticsearch_cfg=/etc/elasticsearch/elasticsearch.yml 771 | mv ${elasticsearch_cfg} ${elasticsearch_cfg}.orig 772 | cat > ${elasticsearch_cfg} < $LOGSTASH_PATTERNS_DIR/slipstream.grok< $LOGSTASH_PATTERNS_DIR/nginx.grok< $LOGSTASH_CONFD/ss-nginx.conf< "$LOGSTASH_PORT" 815 | } 816 | } 817 | filter { 818 | if [type] == "nginx-access" { 819 | grok { 820 | match => { "message" => "%{NGINXACCESS}" } 821 | patterns_dir => ["$LOGSTASH_PATTERNS_DIR"] 822 | } 823 | } 824 | if [type] == "ss-access" { 825 | grok { 826 | match => { "message" => "%{SS_ACCESS}" } 827 | patterns_dir => ["$LOGSTASH_PATTERNS_DIR"] 828 | } 829 | } 830 | } 831 | output { 832 | if [type] == "nginx-access" { 833 | elasticsearch { hosts => ["$ES_HOST:9200"] 834 | index => "$ES_INDEX_NGINX" 835 | } 836 | } 837 | if [type] == "ss-access" { 838 | elasticsearch { hosts => ["$ES_HOST:9200"] 839 | index => "$ES_INDEX_SS" 840 | } 841 | } 842 | } 843 | EOF 844 | 845 | # 846 | # Collectd listener. 847 | cat >$LOGSTASH_CONFD/collectd.conf< $SS_LOGSTASH_COLLECTD_UDP 851 | buffer_size => 1452 852 | codec => collectd { } 853 | type => "collectd" 854 | } 855 | } 856 | EOF 857 | 858 | echo config.reload.automatic: true >> /etc/logstash/logstash.yml 859 | 860 | srvc_enable logstash.service 861 | srvc_start logstash 862 | } 863 | 864 | function _install_logging_beats() { 865 | _print "- installing Logstash Beats and configurations" 866 | 867 | _inst filebeat 868 | 869 | cat > /etc/filebeat/filebeat.yml<$tmpl< /etc/default/slipstream-job-distributor< /etc/default/slipstream-job-executor< /etc/slipstream/san.json< $ss_riemann_conf<> /etc/sysconfig/riemann<$ss_type_db<>/etc/collectd.conf</etc/collectd.d/nginx.conf< 1107 | URL "http://localhost/nginx_status" 1108 | # User "user" 1109 | # Password "pass" 1110 | 1111 | EOF 1112 | 1113 | srvc_enable collectd 1114 | srvc_restart collectd 1115 | } 1116 | 1117 | function _install_monit_metricbeat() { 1118 | _inst metricbeat 1119 | sed -i -e "s/hosts: \[\"localhost:9200\"\]/hosts: [\"$ES_HOST:9200\"]/" \ 1120 | /etc/metricbeat/metricbeat.yml 1121 | srvc_enable metricbeat 1122 | srvc_start metricbeat 1123 | } 1124 | 1125 | function _install_monitoring() { 1126 | _enable_monit_jmx 1127 | _install_monit_collectd 1128 | _install_monit_metricbeat 1129 | _install_kibana 1130 | } 1131 | 1132 | function _install_logging() { 1133 | _is_true $LOGSTASH_INSTALL && _install_logstash || true 1134 | _install_logging_beats 1135 | } 1136 | 1137 | function deploy_logging_and_monitoring() { 1138 | _print "Installing logging" 1139 | _install_logging 1140 | 1141 | _print "Installing monitoring" 1142 | _install_monitoring 1143 | } 1144 | 1145 | function cleanup () { 1146 | $CLEAN_PKG_CACHE 1147 | } 1148 | 1149 | set -u 1150 | set -x 1151 | 1152 | _print $(date) 1153 | _print "Starting installation of SlipStream server (from ${SS_YUM_REPO})." 1154 | 1155 | prepare_node 1156 | deploy_slipstream_server_deps 1157 | deploy_slipstream_client 1158 | deploy_slipstream_server 1159 | deploy_slipstream_job_engine 1160 | deploy_prs_service 1161 | deploy_riemann 1162 | deploy_logging_and_monitoring 1163 | cleanup 1164 | 1165 | function _how_to_start_service() { 1166 | declare -f _start_slipstream_service | awk '/{/{x=1;next}/}/{x=0}x' 1167 | } 1168 | 1169 | if ( _is_true $SS_START ); then 1170 | _print "SlipStream server installed and accessible at https://$SS_HOSTNAME" 1171 | else 1172 | _print "SlipStream server installed, but wasn't started." 1173 | _print "To start the service run:\n$(_how_to_start_service)" 1174 | _print "SlipStream server will become accessible at https://$SS_HOSTNAME" 1175 | fi 1176 | _print "Please see Configuration section of the SlipStream administrator 1177 | manual for the next steps like changing the service default passwords, 1178 | adding cloud connectors and more." 1179 | _print "$(date)" 1180 | 1181 | exit 0 1182 | --------------------------------------------------------------------------------