├── setup.cfg ├── LICENSE ├── .travis.yml ├── README.md └── travis_after_all.py /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length=160 3 | exclude=build 4 | statistics=yes 5 | ignore = D100,D101,D102,D103,D104,D105,D200,D202,D204,D205,D207,D210,D211,D300,D301,D400,D401 6 | 7 | [pep8] 8 | exclude=build,lib,.tox,third,*.egg,docs,packages 9 | ;filename= 10 | ;select 11 | ignore=E501,E265,E402 12 | max-line-length=160 13 | count=1 14 | ;format 15 | ;quiet 16 | ;show-pep8 17 | ;show-source 18 | statistics=1 19 | ;verbose=1 20 | 21 | ;PEP8_OPTS="--filename=*.py --exclude=lib --ignore=E501 scripts" 22 | ;pep8 $PEP8_OPTS --show-source --repeat 23 | ;pep8 --statistics -qq $PEP8_OPTS 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Dmytro Makhno 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - '3.6' 4 | - '2.7' 5 | - '3.5' 6 | - '3.4' 7 | - '3.3' 8 | - '3.2' 9 | - 'pypy' 10 | - 'pypy3' 11 | env: 12 | # robot 13 | GITHUB_TOKEN_P1=7bea5d170ca6f7c162d7 14 | GITHUB_TOKEN_P2=58961507e413df080e9c 15 | sudo: false # Use container-based infrastructure 16 | install: "pip install flake8" 17 | script: 18 | - echo "Doing something..." 19 | - flake8 20 | - echo "Done" 21 | after_success: 22 | - export GITHUB_TOKEN=${GITHUB_TOKEN_P1}${GITHUB_TOKEN_P2} 23 | - python travis_after_all.py 24 | - export $(cat .to_export_back) > /dev/null 25 | - | 26 | if [ "$BUILD_LEADER" = "YES" ]; then 27 | if [ "$BUILD_AGGREGATE_STATUS" = "others_succeeded" ]; then 28 | echo "All jobs succeeded! PUBLISHING..." 29 | else 30 | echo "Some jobs failed" 31 | fi 32 | fi 33 | after_failure: 34 | - export GITHUB_TOKEN=${GITHUB_TOKEN_P1}${GITHUB_TOKEN_P2} 35 | - python travis_after_all.py 36 | - export $(cat .to_export_back) > /dev/null 37 | - | 38 | if [ "$BUILD_LEADER" = "YES" ]; then 39 | if [ "$BUILD_AGGREGATE_STATUS" = "others_failed" ]; then 40 | echo "All jobs failed" 41 | else 42 | echo "Some jobs failed" 43 | fi 44 | fi 45 | after_script: 46 | - echo leader=$BUILD_LEADER status=$BUILD_AGGREGATE_STATUS 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | travis_after_all 2 | ================ 3 | 4 | [![Build Status](https://travis-ci.org/dmakhno/travis_after_all.png?branch=master)](https://travis-ci.org/dmakhno/travis_after_all) 5 | 6 | This is a Travis CI helper to run particular work only once in matrix. 7 | 8 | This is a workaround for: https://github.com/travis-ci/travis-ci/issues/929 9 | 10 | 11 | Dependencies 12 | ------------- 13 | This script assumes an environment variable called `GITHUB_TOKEN` is always 14 | available for Travis CI builds. This token will be used for retrieving a temporary 15 | Travis token for the LEADER job to poll Travis about the state of the other 16 | jobs. 17 | 18 | Read more about creating a suitable GitHub token 19 | [here](https://docs.travis-ci.com/user/github-oauth-scopes/#Travis-CI-for-Private-Projects). 20 | 21 | Once you have a suitable token available, you can make sure it ends up 22 | encrypted in your .travis.yml file by doing: 23 | 24 | ```bash 25 | gem install travis 26 | travis encrypt GITHUB_TOKEN="github-token" --add 27 | ``` 28 | 29 | After this step you will find new lines in your Travis CI config: 30 | 31 | ```yaml 32 | env: 33 | global: 34 | secure: "encrypted-github-token" 35 | ``` 36 | 37 | Usage 38 | ------ 39 | The main goal of this script to have a single publish when a build has several jobs. Currently the first job is a leader, meaning a node that will do the publishing. 40 | 41 | An example .travis.yml shows how to ensure that `all_succeeded` or `all_failed`: 42 | 43 | ```yaml 44 | #... 45 | script: 46 | - curl -Lo travis_after_all.py https://raw.github.com/dmakhno/travis_after_all/master/travis_after_all.py 47 | after_success: 48 | - python travis_after_all.py https://api.travis-ci.com 49 | - export $(cat .to_export_back) 50 | - | 51 | if [ "$BUILD_LEADER" = "YES" ]; then 52 | if [ "$BUILD_AGGREGATE_STATUS" = "others_succeeded" ]; then 53 | echo "All jobs succeeded! PUBLISHING..." 54 | else 55 | echo "Some jobs failed" 56 | fi 57 | fi 58 | after_failure: 59 | - python travis_after_all.py https://api.travis-ci.com 60 | - export $(cat .to_export_back) 61 | - | 62 | if [ "$BUILD_LEADER" = "YES" ]; then 63 | if [ "$BUILD_AGGREGATE_STATUS" = "others_failed" ]; then 64 | echo "All jobs failed" 65 | else 66 | echo "Some jobs failed" 67 | fi 68 | fi 69 | after_script: 70 | - echo leader=$BUILD_LEADER status=$BUILD_AGGREGATE_STATUS 71 | ``` 72 | 73 | Limitations/Todo 74 | ---------------- 75 | 76 | - If other jobs will start late, can be global build timeout (think of passing leader role to others, for example who started last is the leader). 77 | - More flexible leader definition (in matrix not all jobs can publish). 78 | - Have several leaders, according to flavours (e.g. if in matrix slices responsible for platform). 79 | -------------------------------------------------------------------------------- /travis_after_all.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import json 4 | import time 5 | import logging 6 | 7 | try: 8 | from functools import reduce 9 | except ImportError: 10 | pass 11 | 12 | try: 13 | import urllib.request as urllib2 14 | except ImportError: 15 | import urllib2 16 | 17 | log = logging.getLogger("travis.leader") 18 | log.addHandler(logging.StreamHandler()) 19 | log.setLevel(logging.INFO) 20 | 21 | TRAVIS_JOB_NUMBER = 'TRAVIS_JOB_NUMBER' 22 | TRAVIS_BUILD_ID = 'TRAVIS_BUILD_ID' 23 | POLLING_INTERVAL = 'LEADER_POLLING_INTERVAL' 24 | GITHUB_TOKEN = 'GITHUB_TOKEN' 25 | 26 | 27 | # Travis API entry point, there are at least https://api.travis-ci.com and https://api.travis-ci.org 28 | travis_entry = sys.argv[1] if len(sys.argv) > 1 else 'https://api.travis-ci.org' 29 | 30 | build_id = os.getenv(TRAVIS_BUILD_ID) 31 | polling_interval = int(os.getenv(POLLING_INTERVAL, '5')) 32 | gh_token = os.getenv(GITHUB_TOKEN) 33 | 34 | # assume, first job is the leader 35 | 36 | 37 | def is_leader(job_number): 38 | return job_number.endswith('.1') 39 | 40 | 41 | job_number = os.getenv(TRAVIS_JOB_NUMBER) 42 | 43 | if not job_number: 44 | # seems even for builds with only one job, this won't get here 45 | log.fatal("Don't use defining leader for build without matrix") 46 | exit(1) 47 | elif is_leader(job_number): 48 | log.info("This is a leader") 49 | else: 50 | # since python is subprocess, env variables are exported back via file 51 | with open(".to_export_back", "w") as export_var: 52 | export_var.write("BUILD_MINION=YES") 53 | log.info("This is a minion") 54 | exit(0) 55 | 56 | 57 | class MatrixElement(object): 58 | 59 | def __init__(self, json_raw): 60 | self.is_finished = json_raw['finished_at'] is not None 61 | self.is_succeeded = json_raw['result'] == 0 62 | self.number = json_raw['number'] 63 | self.is_leader = is_leader(self.number) 64 | 65 | 66 | def matrix_snapshot(travis_token): 67 | """ 68 | :return: Matrix List 69 | """ 70 | headers = {'content-type': 'application/json', 'Authorization': 'token {}'.format(travis_token)} 71 | req = urllib2.Request("{0}/builds/{1}".format(travis_entry, build_id), headers=headers) 72 | response = urllib2.urlopen(req).read() 73 | raw_json = json.loads(response.decode('utf-8')) 74 | matrix_without_leader = [MatrixElement(job) for job in raw_json["matrix"] if not is_leader(job['number'])] 75 | return matrix_without_leader 76 | 77 | 78 | def wait_others_to_finish(travis_token): 79 | def others_finished(): 80 | """ 81 | Dumps others to finish 82 | Leader cannot finish, it is working now 83 | :return: tuple(True or False, List of not finished jobs) 84 | """ 85 | snapshot = matrix_snapshot(travis_token) 86 | finished = [job.is_finished for job in snapshot if not job.is_leader] 87 | return reduce(lambda a, b: a and b, finished), [job.number for job in snapshot if 88 | not job.is_leader and not job.is_finished] 89 | 90 | while True: 91 | finished, waiting_list = others_finished() 92 | if finished: 93 | break 94 | log.info("Leader waits for minions {0}...".format(waiting_list)) # just in case do not get "silence timeout" 95 | time.sleep(polling_interval) 96 | 97 | 98 | def get_token(): 99 | assert gh_token, 'GITHUB_TOKEN is not set' 100 | data = {"github_token": gh_token} 101 | headers = {'content-type': 'application/json', 'User-Agent': 'Travis/1.0'} 102 | 103 | req = urllib2.Request("{0}/auth/github".format(travis_entry), json.dumps(data).encode('utf-8'), headers) 104 | response = urllib2.urlopen(req).read() 105 | travis_token = json.loads(response.decode('utf-8')).get('access_token') 106 | 107 | return travis_token 108 | 109 | 110 | try: 111 | token = get_token() 112 | wait_others_to_finish(token) 113 | 114 | final_snapshot = matrix_snapshot(token) 115 | log.info("Final Results: {0}".format([(e.number, e.is_succeeded) for e in final_snapshot])) 116 | 117 | BUILD_AGGREGATE_STATUS = 'BUILD_AGGREGATE_STATUS' 118 | others_snapshot = [el for el in final_snapshot if not el.is_leader] 119 | if reduce(lambda a, b: a and b, [e.is_succeeded for e in others_snapshot]): 120 | os.environ[BUILD_AGGREGATE_STATUS] = "others_succeeded" 121 | elif reduce(lambda a, b: a and b, [not e.is_succeeded for e in others_snapshot]): 122 | log.error("Others Failed") 123 | os.environ[BUILD_AGGREGATE_STATUS] = "others_failed" 124 | else: 125 | log.warn("Others Unknown") 126 | os.environ[BUILD_AGGREGATE_STATUS] = "unknown" 127 | # since python is subprocess, env variables are exported back via file 128 | with open(".to_export_back", "w") as export_var: 129 | export_var.write("BUILD_LEADER=YES {0}={1}".format(BUILD_AGGREGATE_STATUS, os.environ[BUILD_AGGREGATE_STATUS])) 130 | 131 | except Exception as e: 132 | log.fatal(e) 133 | --------------------------------------------------------------------------------