├── add_sources.rb ├── common.sh ├── .gitignore ├── LICENSE ├── .travis.yml ├── README.md └── run_test.sh /add_sources.rb: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ruby 2 | 3 | require 'json' 4 | 5 | sources = JSON.parse(File.read(File.join(File.dirname($0), 'ubuntu.json'))) 6 | 7 | successes = sources.select do |src| 8 | puts "-------------------\nAdding #{src.inspect}\n" 9 | if src['key_url'] 10 | system("curl -sSL #{src['key_url'].untaint.inspect} | sudo -E env LANG=C.UTF-8 apt-key add -") && system("sudo -E env LANG=C.UTF-8 apt-add-repository -y #{src['sourceline'].untaint.inspect}") 11 | else 12 | system("sudo -E env LANG=C.UTF-8 apt-add-repository -y #{src['sourceline'].untaint.inspect}") 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /common.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | ANSI_RED="\033[31;1m" 4 | ANSI_GREEN="\033[32;1m" 5 | ANSI_RESET="\033[0m" 6 | ANSI_CLEAR="\033[0K" 7 | 8 | EXIT_SUCCESS=0 9 | EXIT_SOURCE_NOT_FOUND=1 10 | EXIT_SOURCE_HAS_SETUID=2 11 | EXIT_NOTHING_TO_COMMIT=3 12 | EXIT_DUPLICATE_EXISTS=4 13 | 14 | function notice() { 15 | msg=$1 16 | echo -e "\n${ANSI_GREEN}${msg}${ANSI_RESET}\n" 17 | } 18 | 19 | function warn() { 20 | msg=$1 21 | echo -e "\n${ANSI_RED}${msg}${ANSI_RESET}\n" 22 | } 23 | 24 | function fold_start() { 25 | echo -e "travis_fold:start:$1\033[33;1m$2\033[0m" 26 | } 27 | 28 | function fold_end() { 29 | echo -e "\ntravis_fold:end:$1\r" 30 | } 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | /.config 4 | /coverage/ 5 | /InstalledFiles 6 | /pkg/ 7 | /spec/reports/ 8 | /test/tmp/ 9 | /test/version_tmp/ 10 | /tmp/ 11 | 12 | ## Specific to RubyMotion: 13 | .dat* 14 | .repl_history 15 | build/ 16 | 17 | ## Documentation cache and generated files: 18 | /.yardoc/ 19 | /_yardoc/ 20 | /doc/ 21 | /rdoc/ 22 | 23 | ## Environment normalisation: 24 | /.bundle/ 25 | /vendor/bundle 26 | /lib/bundler/man/ 27 | 28 | # for a library or gem, you might want to ignore these files since the code is 29 | # intended to run in multiple environments; otherwise, check them in: 30 | # Gemfile.lock 31 | # .ruby-version 32 | # .ruby-gemset 33 | 34 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 35 | .rvmrc 36 | 37 | *.log 38 | *.log.[0-9] 39 | token 40 | 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Hiro Asari 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | 3 | sudo: required 4 | 5 | dist: trusty 6 | 7 | git: 8 | depth: 99999 9 | 10 | env: 11 | global: 12 | - DEBIAN_FRONTEND=noninteractive 13 | - PACKAGE=openscad 14 | - ISSUE_REPO=apt-package-safelist 15 | - ISSUE_NUMBER=4333 16 | - DIST=trusty 17 | 18 | before_install: 19 | - if [ -z $PACKAGE ]; then echo "\$PACKAGE not defined" && travis_terminate 2; fi 20 | - sudo apt-get update -y -qq 21 | - sudo apt-get install -yqq sshpass 22 | - if ! command -v docker ; then curl -sSL https://get.docker.io | bash; fi 23 | - sudo usermod -a -G docker travis 24 | - | 25 | if [[ $DIST = 'precise' ]]; then 26 | IMAGE=quay.io/travisci/travis-node-js:latest 27 | export DOCKER=true 28 | docker pull ${IMAGE} 29 | jq --version || sudo curl -sSL -o /usr/local/bin/jq http://stedolan.github.io/jq/download/linux64/jq && sudo chmod 0755 /usr/local/bin/jq 30 | docker images | grep -q -E '^travis\\s+\\bnode-js\\b\\s+' || docker tag ${IMAGE} travis:node-js 31 | docker run -v /var/tmp:/var/tmp -v /home/travis:/home/travis -d travis:node-js | tee docker_id 32 | docker inspect $(< docker_id) | jq -r .[0].NetworkSettings.IPAddress | tee docker_ip_address 33 | docker ps 34 | fi 35 | 36 | 37 | install: skip 38 | 39 | script: 40 | - ./run_test.sh 41 | 42 | notifications: 43 | email: false 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # apt-whitelist-checker 2 | 3 | This is a repository to automate testing [APT package whitelisting requests](https://github.com/travis-ci/apt-package-whitelist). 4 | 5 | ## Why? 6 | 7 | APT packages need to be checked before being added to the list, 8 | but that involves some tedious setup and manual labor. 9 | This repository aims to automate the tedious part, and 10 | human part more accessible. 11 | 12 | This is useful for only the maintainers of Travis CI repositories. 13 | 14 | ## How This Works 15 | 16 | First, you need to set up: 17 | 18 | `GITHUB_OAUTH_TOKEN`: You'd probably want to create one especially for use with this repo; https://github.com/settings/tokens 19 | 20 | Then, run: 21 | 22 | ```sh-session 23 | $ env REPO=travis-ci ruby run_tests.rb 24 | ``` 25 | 26 | This would go through the list of open GitHub issues in `REPO`: 27 | https://github.com/travis-ci/travis-ci/issues 28 | and print out what it intends to do. 29 | 30 | Specifically, it looks for tickets with the title of the form: `APT whitelist request for X` 31 | or `APT source whitelist request for Y`. 32 | For the latter, it will add a label and move on. 33 | For the former, it will make a commit to this repository, and make a push to this repository. 34 | 35 | The push, then, will trigger a build, as defined in `.travis.yml`. 36 | 37 | The build, in turn, adds the APT sources to a Docker container, 38 | downloads the source package in question and run: 39 | 40 | ``` 41 | grep -R -i -H -C5 -E --color 'set(uid|euid|gid)' --exclude install-sh . 42 | ``` 43 | 44 | (See [`build.sh`](build.sh).) 45 | 46 | If no such occurrence is found, a comment is posted, and a PR suggesting the addition 47 | is created on the APT package whitelist repo. 48 | 49 | If there are, a comment is posted on the original request issue, along with the list of 50 | packages the source package provides, and the build URL showing the problem. 51 | 52 | If no source package matching the given name is found, the label `apt-source-whitelist` is 53 | added to the issue, and a comment is posted. 54 | 55 | #### CI setup 56 | 57 | This repository's `run` branch is configured to automate the execution of the check 58 | described above. 59 | See https://travis-ci.org/travis-ci/apt-whitelist-checker/branches. 60 | 61 | ### REPO environment variable 62 | 63 | The environment variable `REPO` controls which repository to look for the issues. 64 | Historically, it was `travis-ci`, but the noise from issues became a problem, so we 65 | will be moving to `apt-package-whitelist`. 66 | For the time being, both will have to be monitored. 67 | -------------------------------------------------------------------------------- /run_test.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | source `dirname $0`/common.sh 4 | 5 | SSH_OPTS='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -q' 6 | 7 | BUILD_URL="https://travis-ci.org/${TRAVIS_REPO_SLUG}/builds/${TRAVIS_BUILD_ID}" 8 | ISSUE_REPO=${ISSUE_REPO:-"apt-package-whitelist"} # name of the repo that has issues, under "travis-ci" 9 | GITHUB_ISSUES_URL="https://api.github.com/repos/travis-ci/${ISSUE_REPO}/issues/${ISSUE_NUMBER}" 10 | 11 | if [[ -n $DOCKER ]]; then 12 | echo "Copying build.sh" 13 | cp build.sh common.sh add_sources.rb $HOME/build 14 | docker exec $(< docker_id) chown -R travis /home/travis 15 | docker exec -u travis $(< docker_id) mv $HOME/build/{build.sh,common.sh,add_sources.rb} $HOME 16 | fi 17 | 18 | echo "Running build.sh" 19 | if [[ -n $DOCKER ]]; then 20 | docker exec -u travis $(< docker_id) bash ${HOME}/build.sh ${PACKAGE} 21 | else 22 | bash ./build.sh ${PACKAGE} 23 | fi 24 | 25 | CHECK_RESULT=$? 26 | 27 | if [ $CHECK_RESULT -ne $EXIT_SOURCE_NOT_FOUND ]; then 28 | cp /var/tmp/deb-sources/packages . 29 | fi 30 | 31 | case $CHECK_RESULT in 32 | $EXIT_SUCCESS) 33 | notice "No suspicious bits found." 34 | notice "Setting up Git" 35 | git clone https://github.com/travis-ci/apt-package-whitelist.git 36 | cp packages apt-package-whitelist # so make_pr.sh can find it 37 | pushd apt-package-whitelist 38 | env GITHUB_OAUTH_TOKEN=${GITHUB_OAUTH_TOKEN} ./make_pr.sh -y ${ISSUE_REPO} ${ISSUE_NUMBER} ${DIST} 39 | if [ $? -eq $EXIT_NOTHING_TO_COMMIT ]; then 40 | COMMIT=$(git blame ubuntu-${DIST} | grep ${PACKAGE} | cut -f1 -d' ' | sort | uniq | head -1) 41 | curl -X POST -sS -H "Content-Type: application/json" -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ 42 | -d "{ \"body\": \"***This is an automated comment.***\r\n\r\nFailed to create a commit and a PR. This usually means that there has been a commit that resolved this request.\r\nInparticular, check https://github.com/travis-ci/apt-package-whitelist/commit/${COMMIT}\" }" \ 43 | ${GITHUB_ISSUES_URL}/comments 44 | curl -X POST -sS -H "Content-Type: application/json" -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ 45 | -d "[\"apt-whitelist-check-run\"]" \ 46 | ${GITHUB_ISSUES_URL}/labels 47 | elif [ $? -eq $EXIT_DUPLICATE_EXISTS ]; then 48 | warn "Duplicate exists" 49 | exit $EXIT_DUPLICATE_EXISTS 50 | fi 51 | popd 52 | ;; 53 | $EXIT_SOURCE_HAS_SETUID) 54 | warn "Found occurrences of setuid." 55 | echo -e "\n\n" 56 | echo -e "If these occurrences of \`setuid\`/\`seteuid\`/\`setgid\` are deemed harmless, add the following packages: $(< packages)\n" 57 | notice "Setting up Git" 58 | git clone https://github.com/travis-ci/apt-package-whitelist.git 59 | cp packages apt-package-whitelist # so make_pr.sh can find it 60 | pushd apt-package-whitelist 61 | env GITHUB_OAUTH_TOKEN=${GITHUB_OAUTH_TOKEN} ./make_pr.sh -s -y ${ISSUE_REPO} ${ISSUE_NUMBER} ${DIST} 62 | 63 | if [ $? -eq $EXIT_DUPLICATE_EXISTS ]; then 64 | warn "Duplicate exists" 65 | exit $EXIT_DUPLICATE_EXISTS 66 | fi 67 | 68 | cat <<-EOF > comment_payload 69 | { 70 | "body" : "***This is an automated comment.***\r\n\r\nRan tests and found setuid bits by purely textual search. Further analysis is required.\r\n\r\nIf these are found to be benign, examine http://github.com/travis-ci/apt-package-whitelist/compare/test-apt-package-whitelist-${ISSUE_NUMBER} and its PR.\r\n\r\nPackages found: $(< packages)\r\n\r\nSee ${BUILD_URL} for details." 71 | } 72 | EOF 73 | curl -X POST -sS -H "Content-Type: application/json" -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ 74 | -d @comment_payload \ 75 | ${GITHUB_ISSUES_URL}/comments 76 | curl -X POST -sS -H "Content-Type: application/json" -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ 77 | -d "[\"apt-whitelist-check-run\"]" \ 78 | ${GITHUB_ISSUES_URL}/labels 79 | pushd 80 | ;; 81 | $EXIT_SOURCE_NOT_FOUND) 82 | warn "Source not found." 83 | cat <<-EOF > comment_payload 84 | { 85 | "body" : "***This is an automated comment.***\r\n\r\nRan tests, but could not found source package. Either the source package for ${PACKAGE} does not exist, or the package needs an APT source. If you wish to add an APT source, please follow the directions on https://github.com/travis-ci/apt-source-whitelist#source-approval-process. Build results: ${BUILD_URL}." 86 | } 87 | EOF 88 | curl -X POST -sS -H "Content-Type: application/json" -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ 89 | -d @comment_payload \ 90 | ${GITHUB_ISSUES_URL}/comments 91 | curl -X POST -sS -H "Content-Type: application/json" -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ 92 | -d "[\"apt-source-whitelist\",\"apt-whitelist-check-run\"]" \ 93 | ${GITHUB_ISSUES_URL}/labels 94 | ;; 95 | *) 96 | warn "Something unexpected happened. Status: ${CHECK_RESULT}" 97 | ;; 98 | esac 99 | 100 | exit $CHECK_RESULT 101 | --------------------------------------------------------------------------------