├── requirements.txt ├── loop.sh ├── logrotate.conf ├── ssh-keys ├── loic.pub ├── tv.pub ├── cmccabe.pub ├── slang.pub ├── buck@bender.pub ├── dmick_angus.pub ├── nh.pub ├── samuelj.pub ├── yehuda.pub ├── joshd.pub ├── gregaf.pub ├── root@cephbooter └── sage.pub ├── autobuild-ceph.conf ├── rpm-autosign.exp ├── .gitignore ├── mark_all_as_pass.sh ├── reset-modules.sh ├── autobuild-ceph.init ├── use-mirror.sh ├── run ├── doc └── overview.dot ├── run-real ├── branches-local-samba ├── bootstrap ├── branches-local-hadoop ├── branches-local-apache-hadoop ├── get-libcephfs-java-jar.py ├── perf.patch ├── README.md ├── branches-local ├── lighttpd.conf └── fabfile.py /requirements.txt: -------------------------------------------------------------------------------- 1 | pycrypto 2 | fabric 3 | paramiko>=1.7.6 4 | -------------------------------------------------------------------------------- /loop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | while true ; do 4 | ./run 5 | sleep 5 6 | done 7 | -------------------------------------------------------------------------------- /logrotate.conf: -------------------------------------------------------------------------------- 1 | "logs/stdout.log" { 2 | rotate 30 3 | daily 4 | missingok 5 | compress 6 | } 7 | 8 | "logs/stderr.log" { 9 | rotate 30 10 | daily 11 | missingok 12 | compress 13 | } 14 | -------------------------------------------------------------------------------- /ssh-keys/loic.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEApKfP/I/WoSeX7MAFGpdntC56oHk0wJNn+qy0cud321yTaZRReJGQyqhoBk/sXsnqF4mOpXTf3+Ao1hQIiFazR5Fbmvk7DR0nKj8sGPV0iJB1F0+eyL+w7mgPyD8VDCw+ygCWG4ghMfeGKRAJQweO4v7rFx61UeYDxNk3roOgeuM= loic@inspiron.dachary.org 2 | -------------------------------------------------------------------------------- /autobuild-ceph.conf: -------------------------------------------------------------------------------- 1 | description "autobuild ceph" 2 | author "Tommi Virtanenh " 3 | 4 | start on runlevel [2345] 5 | stop on runlevel [!2345] 6 | 7 | respawn 8 | 9 | nice 10 10 | chdir /srv/autobuild-ceph 11 | 12 | exec ./run 13 | -------------------------------------------------------------------------------- /rpm-autosign.exp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/expect -f 2 | 3 | # First parameter should be the key such as: 4 | # --define "_gpg_name 03C3951A" 5 | 6 | set timeout -1 7 | spawn rpm --addsign {*}$argv 8 | expect -exact "Enter pass phrase: " 9 | send -- "\r" 10 | expect eof 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | .#* 3 | ## the next line needs to start with a backslash to avoid looking like 4 | ## a comment 5 | \#*# 6 | .*.swp 7 | 8 | *.pyc 9 | *.pyo 10 | *.egg-info 11 | /build 12 | /dist 13 | 14 | /virtualenv 15 | /bin 16 | 17 | /logs 18 | /gitbuilder.git 19 | rsync-key 20 | rsync-key.pub 21 | /gnupg 22 | -------------------------------------------------------------------------------- /mark_all_as_pass.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd /srv/autobuild-ceph/gitbuilder.git/build 4 | 5 | mkdir -p ../out/pass 6 | 7 | for b in `git branch -a | grep origin`; do 8 | touch ../out/pass/`git rev-parse $b` 9 | done 10 | 11 | for s in `git show-ref --tags -d | awk '{print $1}'`; do 12 | touch ../out/pass/$s 13 | done 14 | 15 | -------------------------------------------------------------------------------- /ssh-keys/tv.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDc7yOXy126/9f/5NLo3/6rGUsGZ/lAifBLzZfYLIum/ED+qj02qlhU07G1is7n8/FK7PgANazFVwSWAuFItoMCp1MhalNi1hansPKBuBotm7RqjFdtutAnKPcPJoiWp2F04Ja/YMq3SoDvDbMeptgBaHRj+KUjRXVPpDojtqM1Fy5EJ06wwh2ocHl3yTJmwZmWBV3iUW7Dpd9lTyXr/uOOhfcbqw1kXliaD6bsXeQAzRazNcr6RbvsLA1fAEt1QMP2/6kk8vhfCy7GZa0Mvh13cSXtU1UXBOJWcSHKuTuB19xBj0PVIDzc6lr1ugAOSAXEC8EQ4OTheWQZmTEvvfJR tv@dreamer 2 | -------------------------------------------------------------------------------- /ssh-keys/cmccabe.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAxT/EmlXe8YO4mJHpa8zMd4yibsO7ygg25n+8lIfkUeU2ugAn+Xt05IJKbofZ+6gok1dRO+sIUp4QMolCs2Sf9AuOJrvMbgZj398VQMmGyOc/3m9nUPiwzEXalrppn7TU5QLIHx0XccOuand2km/r3Bcoc3olc7VrIVBpJ8jBxOhaABoPtTYp6QiVYbeAYGpUqY+OyVpHVe23h5LFupMNr5EOgWDnA/8RViMHO/TO4Gw2Dkf7//o3r8BRY/HZHSQTRMA02Oq1D2kZK6Q1o3eQX528CaZkfVpd8RSSxIh9fiqVRJhXVZX/DkHoZbTOchFQtBpO9PnhjTkV84XTj3uH7Q== cmccabe@flab 2 | -------------------------------------------------------------------------------- /ssh-keys/slang.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC/UNzAKhZddfgwjUyytJjmSEu/UGBchhaWDhqGFqiZDRdN24I14LS49JAUykCibprLX746J3Mb7ecQu8tSFN0cAnKAP6EW7WsD/TMTno/N+hL1qx90bodFhQ+yVVjUff0rPmy5qloEzE7UoVzEat2HiCtp3kqvsRH2CpojEfOgTkA6u5uB6IJezRy/PJXDrg5dKxj4hcIpGGN+y5YYTDZgtuRG3QDPXtjfs9UhaoyLY+gPPQjxw800O1RjzQlQpaKAiNbvwtC1JZOZapxXL7a2QvU7xy/TGJBwjdLGfCAouzqP89u1RorPCkTwfQE82KVJPTvvztz8fwiE8kJ0nGBT slang@Scout 2 | -------------------------------------------------------------------------------- /ssh-keys/buck@bender.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDB5A134lPP6VjnOiezbe0V0A10/WI3nSyXLH4OSWno0Cb0BydhfAE3EyUE5QH6ERVHtpm89+IEUYOy0yeaX1pG5yUPjJb1gulgLFaOBcqXCNX5si8RMhOdHGrArjm+uwsIF2OSBdZHbE5mW+dBBsNWEjtR3BG2s1DTRuvMjbx9bIeYct7YoD7wcAugFROO8C/Od7TvOsX5Df4U0VIJwyBd8w/tRSh+Cnhf+jHgo/FcUv85x5C01m0EePhbTZjgUNizX1CqL5T8QLV34N+8BZ2TSW5H2e4pU6z35IPBimwscp4clkXcBsDGXDRcxmUz+Lwqgyh/4TVe+Ri2m6JBhtKf buck@bender 2 | -------------------------------------------------------------------------------- /ssh-keys/dmick_angus.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC305rmql/Qm8Si+DFvAs77hSUZakwiXhhAhSetS89bWq3AdPQTL1MUXJHFm3AMzwzb00tndvsl1AUmFseXLfAi87daNcKJdwEj/JvXgcHKzAXOpVRfArHh08JnOumQiP4EoLYvq7rL+3LjEFURKOWJLLnGxh4E+iiJAnxLTG3WuUcnz9kXVTwtmIs60+2yH+gZJkkB04mnivPYjX46sHigoKgtlE2JnQfOU1lBkeZaBeUhoZX2rIKwRyGELPPcUBwHk10izyk6tePLAw9bSWeS9uEaa/m+qa8neJDyR4/hWQNR9yRXp0YjKkiKoFog+qtsBt3xBNzTPsFLxZ5zugmp dmick@angus 2 | -------------------------------------------------------------------------------- /ssh-keys/nh.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHhYb/mGgHbEntAN2iPFTV1k8MYDs+jtsJDPE9MWtUZXksdeG8cEfXDBGhiBsprjELsW4+czJBqqESugFMevEysTVKIo5lc95bf1nApxfiRyMnAxb5wQ2OeLY62x26TWJhPu6HMajV+gb8CKjxDobf1Ix4ELbYlL77SOH1VVDtVZ6HiwtYngXAPMONCdI99cuwJlBpv5RjDoLwljOoxEXIh+1Fsvth+iymp0rIPuJx91DzYYgWZFfCLPHRgDy0H7YRdQWUzhrxTyzuT27vsNh5L8HgdowzOu9f2HwGzlTlxli94IJLMBY2359xsqjSswZzAv+BiPDAZMUwJpYvRvSt nhm@clusterfaq.org 2 | -------------------------------------------------------------------------------- /ssh-keys/samuelj.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCSSNGqNkIqGqeBuISkgVl2UsYEdA44FBCA9ZJA/3J0/jBYEXDpKizZg2f5n0bNUQED880DeAUEIEgn4fvrSF9PE4xKOTfRc8b3O8jej9uUWzMyZTXRCGxsOzXD7tAv0UANv1/d4J2I05iBC3H9hHFpsc/lsRbzI7d06JInTrxl00nI9CDbS9hRMiBJgsArLNCpb2uGBPOBaAiahwZGWpXhkLRhXJpXh9h/37p/n1IHFcZFnrfG7bev5n5FCyKwK1/7vO0a3gg4W84SRPqlMgGJrxWD/zfRijTz3UqrOlUL5R6Q18risw7apJntFfCEx6rl0fhhVSAA5KCj0eeXhfad samuelj@slider 2 | -------------------------------------------------------------------------------- /ssh-keys/yehuda.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAyRDKb+InDsnW4E/S12Pr6rK3wDfUG3j+iw/NsCWwN5xpJEA1/NhVlRzY8/jejc6ddmnNKWbm/UcYVrbo7yClRTGu45g0Qm9lmfvgDgi1k8mwUCy+edbQ/R8FB9cIHHCnUuWzRmE2XGaeDiSVQgiYqq3mOPY/hwe3G7jwFalYPsbmY1jNqpzODmKFMRd1bOwdHEOMTfh0owHWqhhXakU8em30lYraQVYH24FplH2NUBArxmTuctyfweHxx7yH3ZFcXFUpUMSLuVDWb1A7fMiL6eW6wiwh+dD+q5iFNvuioZITP2EjIsb9nS+eM9bOJZuZ+5pln2EsHee27LjunbwgWw== yehudasa@fatty 2 | -------------------------------------------------------------------------------- /ssh-keys/joshd.pub: -------------------------------------------------------------------------------- 1 | ssh-dss AAAAB3NzaC1kc3MAAACBAKdOnZE8IwiIy0aRMHKsY6PJ/k0xjULPST6Z4jbitIzEC/YjALKb75si6CVSu5axM5mm1LQASBziCfqAQuehhKf/P5aNbqicPSDAtHqicYQLw8v/RSyp4UlT9r1WXhISZdh183qf/1aG8eDEUHIBBjVbPUmb5jB91QTRwos0z+mVAAAAFQDnvTPm9e3ohOW4R59ByvnAK4lLlQAAAIAdn8thkcmkYHX1xcZsGvuX0Qj2rKHeMFgxzGOvLOTVBGNFxUptGlpqjC2384Xmm5bhbFwFC9nH2SkFwdScaSsf7Xh91PGjbWC+WUDr0BfFRSCVB52kB1dlGjVlm9rY1DXmHjxeqmJqw/Pg9CHyU2D3k7yQDiU2UarNmBv0ytKthwAAAIA/G+h4ynlVES7e7vAxaLi6oth4h1HFOq5tbtuqj23ccP0LwlcEz02o8/vynUp1lpYvuirbnXxJiyBofmD5A1TucbFxi79c+8p2cKXsicjSKenE/JK/fYlYGLVVOyXVccmjTPMkLXZ7R1oZHxFe+i363K26g5tDDafaZR73v5qXIA== josh.durgin 2 | -------------------------------------------------------------------------------- /ssh-keys/gregaf.pub: -------------------------------------------------------------------------------- 1 | ssh-dss AAAAB3NzaC1kc3MAAACBANaS5gZUnIp5UL/PM3xOrQiNVxfona7sD1XEXGYvNUyoojPWWAvdQRwdA/Ay8JsAjKeWUt0IwfWaPfbEoXU1TKAI+yyv5WzQUZAKmBFuYM0CUQnKrk72Dp6n8P2tnYen0ect22xEiW2+Y0JZ3wfV3IB3BQPoVCC9FQ+P3LHhYii/AAAAFQCgSceiZFeNItwJ2rJeQgVVTQKljwAAAIEA0Gd2VNCbbQ48vAF+m/i/CeQHJ9Dom9IOBN3v8XnSlOhtNzY1pJ0NRzrFQURCCeVQJaywr0et5FCXIDVqTuR4qfCu5XXU1KHpn/AvHstOd93Y0XYMwSXfNszOtpH7w9SEm7uBfKnLQNF843/MHZL4ZIdfxk73YjfMjrYFcvJwvM0AAACBAI9Wi2wG9Rrdbrgk8tqbnKMxxJCYt/Vakn5mQSc/WHqMUn14+Kb7oBtfTsfPFNK952FzU09ptNQvNWTPNMniI8l8lQi2B+Z28XUlFLL76uxJXWko99xFpf/d1kFBL0M7GRGmvvrMfeTCI+WIqh2jBVqbl4o8kjNx4p0VWyoiWeFJ gregf@pudgy 2 | -------------------------------------------------------------------------------- /ssh-keys/root@cephbooter: -------------------------------------------------------------------------------- 1 | ssh-dss AAAAB3NzaC1kc3MAAACBAJqV5n70j4YrwAgEaNozDuN5/gEHzi4EYmpcq8TZ1m0T3Fosv0Lfwv3UxBHHL7yK4DX3AiNpFYVyltQgUxOBEiN+z0XmVBaMRFSJFTnJLir9PaIZFUmSgxsNGf7Tniuey0gQfZPw4sm/wWbi6BpAy39IfETnog1neGNS5Ra2r5XdAAAAFQCnzbNBR2T+M5s0VqpPak9TDR9w/QAAAIBTku5kmcego1SF0hiP/NeuIQ6huPt7CnQROANn9K8l/8P3SajkjWUgmyh9hFkDdmSSiMn5Xzqj/a9kbpe+mipitTMwkcVQkGpTu0uixFH8gfENiiAic7nEHjHZfDNKClJsIcjasep/KhA3ujfrwap8VNuruWAgPKioLvYIQZE6vgAAAIAumZ6m3j5lblzQWQMube0JDsdvmZP2r6pyIIlLehO1+czZZcMaDD2/GNmGWJsPUHkPdDclWbY1usulgo1lFfaVNJQBiv6nBdUHbBH5NL+QLoNuyi8e2DiodpPDBG8zuEqlt6GR3Taly4B4vMQC42F8VssFwIZbuhGQLDNUpZCpZg== root@cephbooter 2 | -------------------------------------------------------------------------------- /reset-modules.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | git submodule foreach 'git clean -fdx && git reset --hard' 4 | rm -rf ceph-object-corpus 5 | rm -rf ceph-erasure-code-corpus 6 | rm -rf src/gmock 7 | rm -rf src/leveldb 8 | rm -rf src/libs3 9 | rm -rf src/mongoose 10 | rm -rf src/civetweb 11 | rm -rf src/rocksdb 12 | rm -rf src/erasure-code/jerasure/gf-complete 13 | rm -rf src/erasure-code/jerasure/jerasure 14 | rm -rf .git/modules/ 15 | git clean -fdx && git reset --hard 16 | /srv/git/bin/git submodule sync 17 | /srv/autobuild-ceph/use-mirror.sh 18 | /srv/git/bin/git submodule update --init 19 | git clean -fdx 20 | rm -fr /tmp/*virtualenv* 21 | -------------------------------------------------------------------------------- /autobuild-ceph.init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ### BEGIN INIT INFO 4 | # Provides: autobuild-ceph 5 | # Required-Start: $network $remote_fs $syslog 6 | # Required-Stop: $network $remote_fs $syslog 7 | # Default-Start: 2 3 4 5 8 | # Default-Stop: 9 | # Short-Description: Start ceph autobuilder 10 | ### END INIT INFO 11 | 12 | case $1 in 13 | start) 14 | pgrep loop.sh && exit 0 15 | cd /srv/autobuild-ceph 16 | setsid ./loop.sh & 17 | ;; 18 | stop) 19 | killall -g loop.sh 20 | ;; 21 | restart) 22 | $0 stop 23 | $0 start 24 | ;; 25 | *) 26 | esac 27 | -------------------------------------------------------------------------------- /use-mirror.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -x 2 | 3 | [ -e /srv/autobuild-ceph/no-mirror ] && exit 0 4 | 5 | # Change URL's of submodules. 6 | 7 | build=$(pwd) 8 | if [ -e ".gitmodule_mirrors" ] 9 | then 10 | # Chicken and the egg... Git submodule commands won't work if 11 | # they are not initialized. 12 | # Because of this we must use dirty hack to change submodule URLs 13 | for submodule in $(git config -l | grep submodule | cut -d'.' -f2) 14 | do 15 | if [ $(grep -ic ${submodule} ${build}/.gitmodule_mirrors) -gt 0 ] 16 | then 17 | giturl=$(grep ${submodule} ${build}/.gitmodule_mirrors | cut -d" " -f2) 18 | /srv/git/bin/git config --replace-all submodule.${submodule}.url ${giturl} 19 | fi 20 | done 21 | fi 22 | 23 | -------------------------------------------------------------------------------- /run: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # stdout/stderr are not yet redirected on /etc/init.d systems 5 | exec >>logs/stdout.log 2>>logs/stderr.log 6 | 7 | FILE_OWNER="$(stat --format='%U' gitbuilder.git/out)" 8 | CUR_USER="$(id -un)" 9 | if [ "$FILE_OWNER" = "root" ]; then 10 | echo "$0: root should never own the build tree, aborting." 1>&2 11 | exit 1 12 | fi 13 | 14 | if [ "$CUR_USER" = "root" ]; then 15 | # reexecute as the right user; 16 | # override /bin/false as shell in passwd 17 | echo "$0: I'm root, reexecuting as $FILE_OWNER" 18 | exec su -s /bin/sh -c "$0 $*" $FILE_OWNER 19 | fi 20 | 21 | if [ "$CUR_USER" != "$FILE_OWNER" ]; then 22 | echo "$0: not root and not file owner, aborting." 1>&2 23 | exit 1 24 | fi 25 | 26 | if command -v logrotate >/dev/null; then 27 | logrotate -v --state=logs/.logrotate.status logrotate.conf 28 | fi 29 | 30 | # if the log was rotated, move to the new files; if not 31 | # nothing changes 32 | exec >>logs/stdout.log 2>>logs/stderr.log 33 | 34 | 35 | cd gitbuilder.git 36 | exec ../run-real 37 | -------------------------------------------------------------------------------- /ssh-keys/sage.pub: -------------------------------------------------------------------------------- 1 | ssh-dss AAAAB3NzaC1kc3MAAACBAJhK1wkU1Drfn+UiBwK0WDL2IDGXpSiAQy9yr62gwSCBo+BRK1+wEsZ55bnl1wOXbYTPNpjYdMHL6kHyWyOsU2L+TbndZRqlLF/7Czq8qxirguZlL0rGI0cU/wAbmPlTReu79iCpiWrAgVCz9D7JfknZc+WPe9l7gvo0xZxadmnJAAAAFQDU1Sd06gjxy7rVDcNuGo5avW7pDQAAAIAAuLKZfwlFV0ng47jXUwApKPuipJWs8nLFdZstHFNfvk+mLan7gm8mGP/+c//6VaHIgH+4SsSIXydPHW+noR1qSuS4pmtJJzKYW3yocRRYXg1M0SbG7yfuhrcU6qBeC3KS2ErJY+Z2DnXg/ILPN8IZK1mYyGKe8/1g/biqgXCqnwAAAIEAlNbOGFpxjX2LabXLfNsf4ICN7AsJVoh9Doneh0GNCQoBmiQOE1S7uOm8knJLeXgtCEZt35tlMX9yZpbFxjkCeni5fuimujU1AK08vJFpYfILV6MP5PynwAfNBMCFp5n1UENnpSkn8drXa4F0F/riZk+GDeiS1R61rdC+5eZ0ieI= sage@ped 2 | ssh-dss AAAAB3NzaC1kc3MAAACBANFvOX/l/Q0sOxp9CoRl4XXYHKQGriGGbAgLEfqQz/NydGijoGrgDugjeG4Yzmx75LHb90v5mJejvtXM6QD1E6KVZwrSaWEHoSXN5LgG1BMZLUhWMJZI1dwaRlNGhIDz3pGxwmh9VXA8fNh2DYdmIuKMr76T3FMV7GeGKBiIGS3RAAAAFQDwCvlxKWXtM/4gdT+Ujxx4rojOZQAAAIEAmDBwmmdxsIftzOfbx/zTkg5FDr/jg0sfwRlSu+KFQaUMJNmYpJZMT44x9UB/OqbQitb1RhvGAkg/qE6aM0i6TYWP//5LPt5aLpVW7HUbwUot5j53KU/SyOmVbXVvi0FbdFtCuABLpMuxlY4XNaFrXnDVDPVWElVVA28U/FE6KzEAAACBAIBhOzyKHrT4erxG2av9PZ7JeGsfiPluiNHyEicvu7uitil86oZK7JjBagK0BGx+QQMdy1OjqpVHXx5KzN2UkknfgmgWW42UfgOsDpYnoGFuTtI+Pm9UX5g/K9UqoHNs2F+ulyVKfM4vuQkTZKG0mYoq7AGgEFCCDrUMBxfudCLH sage@autriche 3 | ssh-dss AAAAB3NzaC1kc3MAAACBAKfzZai/1ud7t3PS9WuR6CiSw0vpNShC3m8VoHXAgcQ0LLcC+b/+OpOvKdNhIqnR4uW1Hrm+BEdlUND0lcpuuJ8xOe0UK6AXXVAjtReKR6VzghpT5RDfL4pzWXxq/YzzE1+Yia++g+fJAmst6W9SYq63z80fKDLoRTi18ZN2GCFpAAAAFQDmAmlaOxukcQzTCrq2IcRi7KlMNQAAAIBxUTY4vWfJMgG9JqMoLQfNi98Ouyb31XJnuAAlT/KVSJebDNiMV/b5L79pOYGUjEVXvK1w6YNf4O9gcRsn93oys7NgcWaEsXizmp/7yS8A27Os+6bcWvZdbQd6NeCP17R/swiuUa3DjEjFmjMmt0MGZ/kL4MacEVJPWgaubg9aRwAAAIEApgppgTz98GP9rquAwksULxWy1zOHO2CL3lwKPD/fOOTLltIP0QxvNmRXJJUZQGxa93OwtU8aLvBv8VHYqw2BfZZvpdooTU3kxi56nevKXXda4w2c1TdVTKKP+aai+ijMrUoWOJQmuD6BSe+egLaAMc27YnuhuuLESBcm/V4dak4= sage@doctrine 4 | -------------------------------------------------------------------------------- /doc/overview.dot: -------------------------------------------------------------------------------- 1 | digraph "autobuild-overview" { 2 | // force edge ordering to match calling order 3 | { rank=same; autobuilder_sh; 4 | update_ref_symlinks; 5 | rsync_out; } 6 | autobuilder_sh -> update_ref_symlinks -> rsync_out [style=invis]; 7 | 8 | { rank=same; logrotate; run_real; } 9 | logrotate -> run_real [style=invis]; 10 | 11 | { rank=same; git_remote_update; 12 | branches_sh; 13 | autobuilder_sh_subshell; } 14 | git_remote_update -> branches_sh -> autobuilder_sh_subshell [style=invis]; 15 | 16 | { rank=same; next_rev_sh; 17 | run_build_sh; } 18 | next_rev_sh -> run_build_sh [style=invis]; 19 | 20 | // provided by base OS 21 | upstart [color="lightblue", style=filled]; 22 | 23 | // in autobuild-ceph.git 24 | autobuild_ceph_conf [label="autobuild-ceph.conf", color="#F05C56", style=filled]; 25 | autobuild_ceph_init [label="autobuild-ceph.init", color="#F05C56", style=filled]; 26 | run [color="#F05C56", style=filled]; 27 | logrotate [color="#F05C56", style=filled]; 28 | run_real [label="run-real", color="#F05C56", style=filled]; 29 | update_ref_symlinks [label="update ref\nsymlinks", color="#F05C56", style=filled]; 30 | rsync_out [label="rsync out", color="#F05C56", style=filled]; 31 | branches_local [label="../branches-local", color="#F05C56", style=filled]; 32 | build_sh [label="../build.sh\n(symlink based\non flavor)", color="#F05C56", style=filled]; 33 | 34 | // in gitbuilder.git 35 | autobuilder_sh [label="autobuilder.sh", color="lightgray", style=filled]; 36 | autobuilder_sh_loop [shape=point, color="lightgray"]; 37 | autobuilder_sh_subshell [label="(subshell)", color="lightgray", style=filled]; 38 | git_remote_update [label="git remote update\netc", color="lightgray", style=filled]; 39 | branches_sh [label="branches.sh", color="lightgray", style=filled]; 40 | branches_sh [label="branches.sh", color="lightgray", style=filled]; 41 | next_rev_sh [label="next-rev.sh", color="lightgray", style=filled]; 42 | run_build_sh [label="run-build.sh", color="lightgray", style=filled]; 43 | 44 | // who calls who 45 | upstart -> autobuild_ceph_conf; 46 | {autobuild_ceph_conf; autobuild_ceph_init} -> run; 47 | run -> { 48 | logrotate; 49 | run_real; 50 | }; 51 | run_real -> { 52 | autobuilder_sh; 53 | update_ref_symlinks; 54 | rsync_out; 55 | }; 56 | autobuilder_sh -> autobuilder_sh_loop [label="loop while did_something"]; 57 | autobuilder_sh_loop -> { 58 | git_remote_update; 59 | branches_sh; 60 | autobuilder_sh_subshell; 61 | }; 62 | branches_sh -> branches_local; 63 | branches_sh -> autobuilder_sh_subshell [style=dotted, weight=1000, label="pipe"]; 64 | autobuilder_sh_subshell -> { 65 | next_rev_sh; 66 | run_build_sh; 67 | }; 68 | run_build_sh -> build_sh; 69 | } 70 | -------------------------------------------------------------------------------- /run-real: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # use sun-java6 5 | export CPATH="$CPATH:/usr/lib/jvm/default-java/include:/usr/lib/jvm/default-java/include/linux" 6 | 7 | 8 | ./autobuilder.sh 9 | 10 | REF_OUTDIR="out/output/ref" 11 | SHA_OUTDIR="out/output/sha1" 12 | 13 | mkdir -p $REF_OUTDIR 14 | mkdir -p $SHA_OUTDIR 15 | 16 | # clean up temp files from aborted runs of below symlinking; all tmp 17 | # files are stale 18 | find "$REF_OUTDIR" -mindepth 1 -maxdepth 1 -name '*.tmp' -print0 \ 19 | | xargs -0 --no-run-if-empty rm -f -- 20 | 21 | # update symlinks that make branch point to the latest successful 22 | # output dir for that branch 23 | install -d -m0755 -- "$REF_OUTDIR" 24 | ( ./branches.sh -v ) \ 25 | | while read SHA REF; do 26 | # mangle unsafe characters in branch names, just in case (slashes 27 | # and leading periods); gitbuilder hanging on to "origin/" here 28 | # makes the typical result a bit ugly 29 | SAFE_REF="$(printf '%s' "$REF"|tr -c 'a-zA-Z0-9_.-' '_'|sed 's/^\./_/')" 30 | 31 | # strip out the leading origin_ prefix on branches; this might cause 32 | # collisions, but we're gonna ignore that for now 33 | SAFE_REF="${SAFE_REF#origin_}" 34 | 35 | # echo "ref $SAFE_REF $SHA" 36 | 37 | if [ -L "$REF_OUTDIR/$SAFE_REF" ]; then 38 | # keep symlink fresh 39 | touch --no-dereference "$REF_OUTDIR/$SAFE_REF" 40 | else 41 | if [ -e "out/pass/$SHA" ]; then 42 | # we did a successful build; make a new symlink. this only 43 | # happens if we aborted somewhere before, though; the pass.sh 44 | # really should have done this! 45 | ln -s -- "../sha1/$SHA" "$REF_OUTDIR/$SAFE_REF.tmp" 46 | mv --no-target-directory -- "$REF_OUTDIR/$SAFE_REF.tmp" "$REF_OUTDIR/$SAFE_REF" 47 | fi 48 | fi 49 | done 50 | 51 | target=`cat ../rsync-target` 52 | target_host=`cat ../rsync-target | sed 's/:.*//'` 53 | target_dir=`cat ../rsync-target | sed 's/.*://'` 54 | echo "rsync target host $target_host dir $target_dir" 55 | 56 | # remove refs symlinks that haven't been touched in 48 hours; as the 57 | # above will always touch everything, they are guaranteed fresh 58 | find "$REF_OUTDIR" -mindepth 1 -maxdepth 1 -mtime +1 \ 59 | | while read old ; do 60 | ref=`echo $old | sed 's/.*\///'` 61 | echo "removing old ref $ref" 62 | rm $old 63 | done 64 | 65 | # clean temp output dirs; since there cannot be a build running in 66 | # parallel with us, all tmp files are stale 67 | find "$SHA_OUTDIR" -mindepth 1 -maxdepth 1 -name '*.tmp' -print0 \ 68 | | xargs -0 --no-run-if-empty rm -rf -- 69 | 70 | # rsync --delete on the refs, so that stale refs on the target are 71 | # removed. remove remote sha1 dirs that are stale. 72 | if [ -n "$target" ]; then 73 | echo "removing old refs" 74 | rsync -auv --delete -e "ssh -i ../rsync-key -o StrictHostKeyChecking=no" \ 75 | "$REF_OUTDIR/" $target/ref 76 | 77 | # touch any active sha1 dirs, and remove any older than 4 days. 78 | echo "removing old builds" 79 | ssh -i ../rsync-key -o StrictHostKeyChecking=no $target_host \ 80 | touch $target_dir/ref/\*/. \; \ 81 | find $target_dir/sha1 -mindepth 1 -maxdepth 1 -mtime +7 \| \ 82 | xargs --no-run-if-empty -- rm -rfv -- 83 | fi 84 | -------------------------------------------------------------------------------- /branches-local-samba: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import optparse 3 | import subprocess 4 | import sys 5 | 6 | def get_refs(): 7 | p = subprocess.Popen( 8 | args=[ 9 | 'git', 10 | 'for-each-ref', 11 | '--format=%(objectname) %(*objectname) %(refname)', 12 | '--sort=-taggerdate', 13 | 'refs/remotes/origin/', 14 | 'refs/tags/', 15 | ], 16 | stdout=subprocess.PIPE, 17 | close_fds=True, 18 | ) 19 | (stdout, stderr) = p.communicate() 20 | # TODO check p.returncode 21 | assert not stderr 22 | for line in stdout.splitlines(): 23 | obj, tagobj, ref = line.split(' ', 2) 24 | if tagobj: 25 | obj = tagobj 26 | yield obj, ref 27 | 28 | def separate_tags_and_branches(refs): 29 | tags = [] 30 | branches = [] 31 | for obj, ref in refs: 32 | if ref.startswith('refs/tags/'): 33 | tags.append((obj, ref)) 34 | elif ref.startswith('refs/remotes/origin/'): 35 | branches.append((obj, ref)) 36 | else: 37 | raise RuntimeError('Bad refname: %r' % ref) 38 | return tags, branches 39 | 40 | PRIORITIZE = [ 41 | 'master', 42 | ] 43 | 44 | IGNORE = [ 45 | 'HEAD', 46 | ] 47 | 48 | IGNORE_PREFIX = [] 49 | 50 | IGNORE_TAGS = [] 51 | 52 | def separate_priority_branches(branches): 53 | prioritize = ['origin/{0}'.format(b) for b in PRIORITIZE] 54 | ignore = ['origin/{0}'.format(b) for b in IGNORE] 55 | ignore_prefix = ['origin/{0}'.format(b) for b in IGNORE_PREFIX] 56 | 57 | priority = [] 58 | normal = [] 59 | 60 | for obj, ref in branches: 61 | if ref in prioritize: 62 | priority.append((obj, ref)) 63 | continue 64 | 65 | if ref in ignore: 66 | continue 67 | 68 | if any(ref.startswith(prefix) for prefix in ignore_prefix): 69 | continue 70 | 71 | normal.append((obj, ref)) 72 | 73 | # priority lost its ordering, restore it 74 | priority.sort(key=lambda (obj,ref): prioritize.index(ref)) 75 | 76 | return priority, [] 77 | 78 | def strip_prefix(refs, prefix): 79 | for obj, ref in refs: 80 | assert ref.startswith(prefix) 81 | ref = ref[len(prefix):] 82 | yield obj, ref 83 | 84 | def filter_tags(ls): 85 | r = [] 86 | return r 87 | 88 | def doit(output, verbose): 89 | refs = get_refs() 90 | tags, branches = separate_tags_and_branches(refs) 91 | tags = strip_prefix(tags, 'refs/tags/') 92 | 93 | branches = strip_prefix(branches, 'refs/remotes/') 94 | priority, normal = separate_priority_branches(branches) 95 | 96 | filtered_tags = filter_tags(tags) 97 | 98 | if verbose: 99 | fmt = '{0} {1}' 100 | else: 101 | fmt = '{1}' 102 | 103 | for l in [priority, normal, filtered_tags]: 104 | for (obj, ref) in l: 105 | print >>output, fmt.format(obj, ref) 106 | 107 | def main(): 108 | parser = optparse.OptionParser( 109 | usage='%prog [-v]', 110 | description='Output interesting git tags and branches, for gitbuilder.', 111 | ) 112 | parser.add_option( 113 | '-v', '--verbose', 114 | action='store_true', 115 | ) 116 | opts, args = parser.parse_args() 117 | if args: 118 | parser.error('Unexpected arguments.') 119 | 120 | doit(sys.stdout, opts.verbose) 121 | 122 | if __name__ == '__main__': 123 | main() 124 | -------------------------------------------------------------------------------- /bootstrap: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | case "$(uname -s)" in 5 | Linux) 6 | case "$(lsb_release --id --short)" in 7 | Ubuntu|Debian) 8 | # Note that we install the system version of python-libvirt here 9 | # instead of relying on PyPI for that particular module. THe reason for 10 | # this is that Ubuntu Precise ships libvirt 0.9.8, and PyPI's 11 | # python-libvirt packages require libvirt >= 1.0.2. 12 | # Some options for resolving this situation would be choosing some or 13 | # all of the following: 14 | # A) Removing support for Precise, 15 | # B) Removing support for downburst, 16 | # C) Adding "Precise" conditionals somewhere, eg. conditionalizing 17 | # this bootstrap script to only use the python-libvirt package on 18 | # Ubuntu Precise. 19 | for package in python-dev python-pip python-virtualenv; do 20 | if [ "$(dpkg --status -- $package|sed -n 's/^Status: //p')" != "install ok installed" ]; then 21 | # add a space after old values 22 | missing="${missing:+$missing }$package" 23 | fi 24 | done 25 | if [ -n "$missing" ]; then 26 | echo "$0: missing required packages, please install them:" 1>&2 27 | echo "sudo apt-get install $missing" 28 | exit 1 29 | fi 30 | ;; 31 | Fedora) 32 | for package in python-pip python-virtualenv; do 33 | if [ "$(rpm -q $package)" == "package $package is not installed" ]; then 34 | missing="${missing:+$missing }$package" 35 | fi 36 | done 37 | if [ -n "$missing" ]; then 38 | echo "$0: missing required packages, please install them:" 1>&2 39 | echo "sudo yum install $missing" 40 | exit 1 41 | fi 42 | ;; 43 | "openSUSE project"|"SUSE LINUX") 44 | for package in python-pip python-virtualenv; do 45 | if [ "$(rpm -q $package)" == "package $package is not installed" ]; then 46 | missing="${missing:+$missing }$package" 47 | fi 48 | done 49 | if [ -n "$missing" ]; then 50 | echo "$0: missing required packages, please install them:" 1>&2 51 | echo "sudo zypper install $missing" 52 | exit 1 53 | fi 54 | ;; 55 | *) 56 | echo "This script does not support your Linux distribution yet. Patches encouraged!" 57 | exit 1 58 | ;; 59 | esac 60 | ;; 61 | Darwin) 62 | if ! which brew > /dev/null; then 63 | echo "You need Homebrew: http://brew.sh/" 64 | exit 1 65 | fi 66 | for keg in python; do 67 | if brew list $keg >/dev/null 2>&1; then 68 | echo "Found $keg" 69 | else 70 | brew install $keg 71 | fi 72 | done 73 | ;; 74 | *) 75 | echo "This script does not support your OS yet. Patches encouraged!" 76 | exit 1 77 | ;; 78 | esac 79 | 80 | if [ -z "$NO_CLOBBER" ] || [ ! -e ./virtualenv ]; then 81 | if ! which virtualenv > /dev/null; then 82 | pip install virtualenv 83 | fi 84 | 85 | virtualenv --setuptools virtualenv 86 | 87 | # avoid pip bugs 88 | ./virtualenv/bin/pip install --upgrade pip 89 | 90 | # work-around change in pip 1.5 91 | ./virtualenv/bin/pip install setuptools --no-use-wheel --upgrade 92 | fi 93 | 94 | ./virtualenv/bin/pip install -r requirements.txt 95 | 96 | #./virtualenv/bin/python setup.py develop 97 | 98 | ln -sf virtualenv/bin bin 99 | 100 | -------------------------------------------------------------------------------- /branches-local-hadoop: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import optparse 3 | import subprocess 4 | import sys 5 | 6 | def get_refs(): 7 | p = subprocess.Popen( 8 | args=[ 9 | 'git', 10 | 'for-each-ref', 11 | '--format=%(objectname) %(*objectname) %(refname)', 12 | '--sort=-taggerdate', 13 | 'refs/remotes/origin/', 14 | 'refs/tags/', 15 | ], 16 | stdout=subprocess.PIPE, 17 | close_fds=True, 18 | ) 19 | p2 = subprocess.Popen(['grep', '-i', 'cephfs'],stdin=p.stdout, stdout=subprocess.PIPE,close_fds=True,) 20 | (stdout, stderr) = p2.communicate() 21 | # TODO check p.returncode 22 | assert not stderr 23 | for line in stdout.splitlines(): 24 | obj, tagobj, ref = line.split(' ', 2) 25 | if tagobj: 26 | obj = tagobj 27 | yield obj, ref 28 | 29 | def separate_tags_and_branches(refs): 30 | tags = [] 31 | branches = [] 32 | for obj, ref in refs: 33 | if ref.startswith('refs/tags/'): 34 | tags.append((obj, ref)) 35 | elif ref.startswith('refs/remotes/origin/'): 36 | branches.append((obj, ref)) 37 | else: 38 | raise RuntimeError('Bad refname: %r' % ref) 39 | return tags, branches 40 | 41 | PRIORITIZE = [ 42 | 'master', 43 | 'next', 44 | 'stable', 45 | ] 46 | 47 | IGNORE = [ 48 | 'HEAD', 49 | ] 50 | 51 | IGNORE_PREFIX = [ 52 | 'historic/', 53 | ] 54 | 55 | def separate_priority_branches(branches): 56 | prioritize = ['origin/{0}'.format(b) for b in PRIORITIZE] 57 | ignore = ['origin/{0}'.format(b) for b in IGNORE] 58 | ignore_prefix = ['origin/{0}'.format(b) for b in IGNORE_PREFIX] 59 | 60 | priority = [] 61 | normal = [] 62 | 63 | for obj, ref in branches: 64 | if ref in prioritize: 65 | priority.append((obj, ref)) 66 | continue 67 | 68 | if ref in ignore: 69 | continue 70 | 71 | if any(ref.startswith(prefix) for prefix in ignore_prefix): 72 | continue 73 | 74 | normal.append((obj, ref)) 75 | 76 | # priority lost its ordering, restore it 77 | priority.sort(key=lambda (obj,ref): prioritize.index(ref)) 78 | 79 | return priority, normal 80 | 81 | def strip_prefix(refs, prefix): 82 | for obj, ref in refs: 83 | assert ref.startswith(prefix) 84 | ref = ref[len(prefix):] 85 | yield obj, ref 86 | 87 | NUM_OF_TAGS = 5 88 | 89 | def doit(output, verbose): 90 | refs = get_refs() 91 | tags, branches = separate_tags_and_branches(refs) 92 | del tags[NUM_OF_TAGS:] 93 | tags = strip_prefix(tags, 'refs/tags/') 94 | 95 | branches = strip_prefix(branches, 'refs/remotes/') 96 | priority, normal = separate_priority_branches(branches) 97 | 98 | if verbose: 99 | fmt = '{0} {1}' 100 | else: 101 | fmt = '{1}' 102 | 103 | for l in [priority, normal, tags]: 104 | for (obj, ref) in l: 105 | print >>output, fmt.format(obj, ref) 106 | 107 | def main(): 108 | parser = optparse.OptionParser( 109 | usage='%prog [-v]', 110 | description='Output interesting git tags and branches, for gitbuilder.', 111 | ) 112 | parser.add_option( 113 | '-v', '--verbose', 114 | action='store_true', 115 | ) 116 | opts, args = parser.parse_args() 117 | if args: 118 | parser.error('Unexpected arguments.') 119 | 120 | doit(sys.stdout, opts.verbose) 121 | 122 | if __name__ == '__main__': 123 | main() 124 | -------------------------------------------------------------------------------- /branches-local-apache-hadoop: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import optparse 3 | import subprocess 4 | import sys 5 | 6 | def get_refs(): 7 | p = subprocess.Popen( 8 | args=[ 9 | 'git', 10 | 'for-each-ref', 11 | '--format=%(objectname) %(*objectname) %(refname)', 12 | '--sort=-taggerdate', 13 | 'refs/remotes/origin/', 14 | 'refs/tags/', 15 | ], 16 | stdout=subprocess.PIPE, 17 | close_fds=True, 18 | ) 19 | p2 = subprocess.Popen(['grep', '-i', 'branch-1'],stdin=p.stdout, stdout=subprocess.PIPE,close_fds=True,) 20 | (stdout, stderr) = p2.communicate() 21 | # TODO check p.returncode 22 | assert not stderr 23 | for line in stdout.splitlines(): 24 | obj, tagobj, ref = line.split(' ', 2) 25 | if tagobj: 26 | obj = tagobj 27 | yield obj, ref 28 | 29 | def separate_tags_and_branches(refs): 30 | tags = [] 31 | branches = [] 32 | for obj, ref in refs: 33 | if ref.startswith('refs/tags/'): 34 | tags.append((obj, ref)) 35 | elif ref.startswith('refs/remotes/origin/'): 36 | branches.append((obj, ref)) 37 | else: 38 | raise RuntimeError('Bad refname: %r' % ref) 39 | return tags, branches 40 | 41 | PRIORITIZE = [ 42 | 'master', 43 | 'next', 44 | 'stable', 45 | ] 46 | 47 | IGNORE = [ 48 | 'HEAD', 49 | ] 50 | 51 | IGNORE_PREFIX = [ 52 | 'historic/', 53 | ] 54 | 55 | def separate_priority_branches(branches): 56 | prioritize = ['origin/{0}'.format(b) for b in PRIORITIZE] 57 | ignore = ['origin/{0}'.format(b) for b in IGNORE] 58 | ignore_prefix = ['origin/{0}'.format(b) for b in IGNORE_PREFIX] 59 | 60 | priority = [] 61 | normal = [] 62 | 63 | for obj, ref in branches: 64 | if ref in prioritize: 65 | priority.append((obj, ref)) 66 | continue 67 | 68 | if ref in ignore: 69 | continue 70 | 71 | if any(ref.startswith(prefix) for prefix in ignore_prefix): 72 | continue 73 | 74 | normal.append((obj, ref)) 75 | 76 | # priority lost its ordering, restore it 77 | priority.sort(key=lambda (obj,ref): prioritize.index(ref)) 78 | 79 | return priority, normal 80 | 81 | def strip_prefix(refs, prefix): 82 | for obj, ref in refs: 83 | assert ref.startswith(prefix) 84 | ref = ref[len(prefix):] 85 | yield obj, ref 86 | 87 | NUM_OF_TAGS = 5 88 | 89 | def doit(output, verbose): 90 | refs = get_refs() 91 | tags, branches = separate_tags_and_branches(refs) 92 | del tags[NUM_OF_TAGS:] 93 | tags = strip_prefix(tags, 'refs/tags/') 94 | 95 | branches = strip_prefix(branches, 'refs/remotes/') 96 | priority, normal = separate_priority_branches(branches) 97 | 98 | if verbose: 99 | fmt = '{0} {1}' 100 | else: 101 | fmt = '{1}' 102 | 103 | for l in [priority, normal, tags]: 104 | for (obj, ref) in l: 105 | print >>output, fmt.format(obj, ref) 106 | 107 | def main(): 108 | parser = optparse.OptionParser( 109 | usage='%prog [-v]', 110 | description='Output interesting git tags and branches, for gitbuilder.', 111 | ) 112 | parser.add_option( 113 | '-v', '--verbose', 114 | action='store_true', 115 | ) 116 | opts, args = parser.parse_args() 117 | if args: 118 | parser.error('Unexpected arguments.') 119 | 120 | doit(sys.stdout, opts.verbose) 121 | 122 | if __name__ == '__main__': 123 | main() 124 | -------------------------------------------------------------------------------- /get-libcephfs-java-jar.py: -------------------------------------------------------------------------------- 1 | 2 | import logging 3 | import urllib2 4 | import urlparse 5 | import os 6 | import sys 7 | from subprocess import Popen 8 | from subprocess import PIPE 9 | import glob 10 | 11 | 12 | logging.basicConfig() 13 | log = logging.getLogger(__name__) 14 | #log = logging.getLogger() 15 | log.setLevel(logging.INFO) 16 | 17 | def get_ceph_binary_url(package=None, 18 | branch=None, tag=None, sha1=None, dist=None, 19 | flavor=None, format=None, arch=None): 20 | BASE = 'http://gitbuilder.ceph.com/{package}-{format}-{dist}-{arch}-{flavor}/'.format( 21 | package=package, 22 | flavor=flavor, 23 | arch=arch, 24 | format=format, 25 | dist=dist 26 | ) 27 | 28 | log.info('BASE: %s' % (BASE)) 29 | if sha1 is not None: 30 | assert branch is None, "cannot set both sha1 and branch" 31 | assert tag is None, "cannot set both sha1 and tag" 32 | else: 33 | # gitbuilder uses remote-style ref names for branches, mangled to 34 | # have underscores instead of slashes; e.g. origin_master 35 | if tag is not None: 36 | ref = tag 37 | assert branch is None, "cannot set both branch and tag" 38 | else: 39 | if branch is None: 40 | branch = 'master' 41 | ref = branch 42 | 43 | sha1_url = urlparse.urljoin(BASE, 'ref/{ref}/sha1'.format(ref=ref)) 44 | log.info('sha1_url: %s' % (sha1_url)) 45 | log.info('Translating ref to sha1 using url %s', sha1_url) 46 | sha1_fp = urllib2.urlopen(sha1_url) 47 | sha1 = sha1_fp.read().rstrip('\n') 48 | sha1_fp.close() 49 | 50 | log.debug('Using %s %s sha1 %s', package, format, sha1) 51 | bindir_url = urlparse.urljoin(BASE, 'sha1/{sha1}/'.format(sha1=sha1)) 52 | log.info('sha1: %s bindir_url: %s' % (sha1, bindir_url)) 53 | return (sha1, bindir_url) 54 | 55 | def main(): 56 | package='ceph' 57 | format='tarball' 58 | dist='precise' 59 | arch='x86_64' 60 | flavor='basic' 61 | branch='master' 62 | 63 | sha1,bindir_url = get_ceph_binary_url(package, branch,None,None,dist,flavor,format,arch) 64 | log.info('sha1: %s bindir_url: %s' % (sha1, bindir_url)) 65 | 66 | p1 = Popen(args=[ 67 | 'install', '-d', '-m0755', '--', '/tmp/hadooptest/binary'], 68 | stdout=PIPE) 69 | p2 = Popen( args=[ 70 | 'uname', '-m',], stdin=p1.stdout, stdout=PIPE) 71 | p3 = Popen( args=[ 72 | 'sed', '-e', 's/^/ceph./; s/$/.tgz/',], stdin=p2.stdout, stdout=PIPE) 73 | p4 = Popen( args=[ 74 | 'wget', 75 | '-nv', 76 | '-O-', 77 | '--base={url}'.format(url=bindir_url), 78 | # need to use --input-file to make wget respect --base 79 | '--input-file=-',], stdin=p3.stdout, stdout=PIPE) 80 | p5 = Popen( args=[ 81 | 'tar', '-xzf', '-', '-C', '/tmp/hadooptest/binary',], 82 | stdin=p4.stdout,stdout=PIPE) 83 | p5.wait() 84 | 85 | log.info('copying libcephfs*.so to lib/') 86 | 87 | p1 = Popen(args=[ 88 | 'install', '-d', '-m0755', '--', 'lib'], 89 | stdout=PIPE) 90 | 91 | soFiles = glob.glob('/tmp/hadooptest/binary/usr/local/lib/libcephfs*.so') 92 | for libFile in soFiles: 93 | #log.info('soFile: %s' % libFile) 94 | p1 = Popen(args=[ 95 | 'cp', libFile, 'lib/']) 96 | p1.wait() 97 | 98 | log.info('copying jars to lib/') 99 | 100 | jarFiles = glob.glob('/tmp/hadooptest/binary/usr/local/share/java/*.jar') 101 | for jarFile in jarFiles: 102 | #log.info('soFile: %s' % libFile) 103 | p1 = Popen(args=[ 104 | 'cp', jarFile, 'lib/']) 105 | p1.wait() 106 | 107 | if __name__ == "__main__": 108 | main() 109 | 110 | -------------------------------------------------------------------------------- /perf.patch: -------------------------------------------------------------------------------- 1 | diff --git a/scripts/package/builddeb b/scripts/package/builddeb 2 | index acb8650..6ec4ec7 100644 3 | --- a/scripts/package/builddeb 4 | +++ b/scripts/package/builddeb 5 | @@ -78,17 +78,20 @@ tmpdir="$objtree/debian/tmp" 6 | fwdir="$objtree/debian/fwtmp" 7 | kernel_headers_dir="$objtree/debian/hdrtmp" 8 | libc_headers_dir="$objtree/debian/headertmp" 9 | +tools_dir="$objtree/debian/toolstmp" 10 | +tmp_build_dir="$objtree/debian/toolsbuild" 11 | packagename=linux-image-$version 12 | fwpackagename=linux-firmware-image 13 | kernel_headers_packagename=linux-headers-$version 14 | libc_headers_packagename=linux-libc-dev 15 | +tools_packagename=linux-tools-$version 16 | 17 | if [ "$ARCH" = "um" ] ; then 18 | packagename=user-mode-linux-$version 19 | fi 20 | 21 | # Setup the directory structure 22 | -rm -rf "$tmpdir" "$fwdir" "$kernel_headers_dir" "$libc_headers_dir" 23 | +rm -rf "$tmpdir" "$fwdir" "$kernel_headers_dir" "$libc_headers_dir" "$tools_dir" "$tmp_build_dir" 24 | mkdir -m 755 -p "$tmpdir/DEBIAN" 25 | mkdir -p "$tmpdir/lib" "$tmpdir/boot" "$tmpdir/usr/share/doc/$packagename" 26 | mkdir -m 755 -p "$fwdir/DEBIAN" 27 | @@ -98,6 +101,10 @@ mkdir -p "$libc_headers_dir/usr/share/doc/$libc_headers_packagename" 28 | mkdir -m 755 -p "$kernel_headers_dir/DEBIAN" 29 | mkdir -p "$kernel_headers_dir/usr/share/doc/$kernel_headers_packagename" 30 | mkdir -p "$kernel_headers_dir/lib/modules/$version/" 31 | +mkdir -m 755 -p "$tools_dir/DEBIAN" 32 | +mkdir -p "$tools_dir/usr/share/doc/$tools_packagename" 33 | +mkdir -p "$tools_dir/usr/bin/" 34 | +mkdir -p "$tmp_build_dir" 35 | if [ "$ARCH" = "um" ] ; then 36 | mkdir -p "$tmpdir/usr/lib/uml/modules/$version" "$tmpdir/usr/bin" 37 | fi 38 | @@ -120,6 +127,77 @@ else 39 | fi 40 | fi 41 | 42 | +# Build the tools 43 | +num_tools="0" 44 | +tools_flavour_abi=${version#*-} 45 | +tools_flavour=${tools_flavour_abi#*-} 46 | +tools_version=${version%-$tools_flavour} 47 | +debian_tools_version="${version%%-*}" 48 | +debian_tools_version="${version%.*}" 49 | +if grep -q '^CONFIG_PERF_EVENTS=y' .config ; then 50 | + # Build and install perf 51 | + ( 52 | + unset MAKEFLAGS 53 | + unset MFLAGS 54 | + unset MAKEOVERRIDES 55 | + 56 | + mkdir -p $objtree/tools/perf 57 | + cd "$srctree/tools/perf" 58 | + $MAKE DESTDIR="$tools_dir/usr/" LDFLAGS= O=$objtree/tools/perf/ install 59 | + mv "$tools_dir/usr/bin/perf" "$tools_dir/usr/bin/perf_$debian_tools_version" 60 | + ln -s "/usr/bin/perf_$debian_tools_version" "$tools_dir/usr/bin/perf_$tools_version" 61 | + mv "$tools_dir/usr/libexec/perf-core" "$tools_dir/usr/share/perf_$debian_tools_version-core" 62 | + rmdir "$tools_dir/usr/libexec" 63 | + 64 | + # Documentation can't be built out-of-tree so copy source 65 | + # over to objtree and build man pages there 66 | + mkdir -p $tmp_build_dir/perf 67 | + cp -a "$srctree/tools/perf/Documentation" "$tmp_build_dir/perf/" 68 | + cp -a "$srctree/tools/perf/config" "$tmp_build_dir/perf/" 69 | + cd "$tmp_build_dir/perf/Documentation" 70 | + $MAKE man 71 | + $MAKE DESTDIR="$tools_dir/usr/" install 72 | + for manpage in `find $tools_dir/usr/share/man/ -type f`; do 73 | + mv $manpage ${manpage/perf/perf_$debian_tools_version} 74 | + gzip -9 ${manpage/perf/perf_$debian_tools_version} 75 | + done 76 | + ) 77 | + num_tools=$(($num_tools+1)) 78 | +fi 79 | + 80 | +if [[ "$ARCH" = "i386" || "$ARCH" = "x86_64" ]]; then 81 | + # Build turbostat 82 | + ( 83 | + cp -a $srctree/tools/power/x86/turbostat "$tmp_build_dir" 84 | + ln -s $srctree/arch "$tmp_build_dir/../../../arch" 85 | + cd "$tmp_build_dir/turbostat/" 86 | + unset MAKEFLAGS 87 | + unset MFLAGS 88 | + unset MAKEOVERRIDES 89 | + $MAKE LDFLAGS= 90 | + cp turbostat "$tools_dir/usr/bin/turbostat_$debian_tools_version" 91 | + ln -s "/usr/bin/turbostat_$debian_tools_version" "$tools_dir/usr/bin/turbostat_$tools_version" 92 | + mkdir -p "$tools_dir/usr/share/man/man8" 93 | + cp turbostat.8 "$tools_dir/usr/share/man/man8/turbotstat_${debian_tools_version}.8" 94 | + gzip -9 "$tools_dir/usr/share/man/man8/turbotstat_${debian_tools_version}.8" 95 | + ) 96 | + 97 | + # Build x86_energy_perf_policy 98 | + ( 99 | + cp -a $srctree/tools/power/x86/x86_energy_perf_policy "$tmp_build_dir" 100 | + cd "$tmp_build_dir/x86_energy_perf_policy" 101 | + unset MAKEFLAGS 102 | + unset MFLAGS 103 | + unset MAKEOVERRIDES 104 | + $MAKE LDFLAGS= 105 | + cp x86_energy_perf_policy "$tools_dir/usr/bin/x86_energy_perf_policy_$debian_tools_version" 106 | + ln -s "/usr/bin/x86_energy_perf_policy_$debian_tools_version" "$tools_dir/usr/bin/x86_energy_perf_policy_$tools_version" 107 | + cp x86_energy_perf_policy.8 "$tools_dir/usr/share/man/man8/x86_energy_perf_policy_${debian_tools_version}.8" 108 | + gzip -9 "$tools_dir/usr/share/man/man8/x86_energy_perf_policy_${debian_tools_version}.8" 109 | + ) 110 | + num_tools=$(($num_tools+2)) 111 | +fi 112 | + 113 | if grep -q '^CONFIG_MODULES=y' .config ; then 114 | INSTALL_MOD_PATH="$tmpdir" $MAKE KBUILD_SRC= modules_install 115 | rm -f "$tmpdir/lib/modules/$version/build" 116 | @@ -292,9 +370,24 @@ Description: Linux support headers for userspace development 117 | are used by the installed headers for GNU glibc and other system libraries. 118 | EOF 119 | 120 | +cat <> debian/control 121 | + 122 | +Package: $tools_packagename 123 | +Section: devel 124 | +Architecture: $arch 125 | +Depends: \${shlibs:Depends} 126 | +Description: Linux kernel tools for version $version 127 | + This package provides the architecture dependant parts for kernel 128 | + version locked tools for version $version 129 | +EOF 130 | + 131 | if [ "$ARCH" != "um" ]; then 132 | create_package "$kernel_headers_packagename" "$kernel_headers_dir" 133 | create_package "$libc_headers_packagename" "$libc_headers_dir" 134 | + if [ "$num_tools" > "0" ]; then 135 | + dpkg-shlibdeps $tools_dir/usr/bin/* 136 | + create_package "$tools_packagename" "$tools_dir" 137 | + fi 138 | fi 139 | 140 | create_package "$packagename" "$tmpdir" 141 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Autobuilds for the Ceph Project 2 | 3 | This is a set of build scripts and a fabric file (fabfile.py) 4 | that allows remote deployment and setup of autobuilds for the 5 | ceph project. 6 | 7 | ## Quick Start 8 | 9 | To get started quickly, the following commands will allow you to setup 10 | and start a ceph autobuild on a given host: 11 | 12 | > git clone git@github.com:ceph/autobuild-ceph.git 13 | > cd autobuild-ceph 14 | > sudo apt-get install fabric 15 | > fab gitbuilder_ceph:host=@ 16 | 17 | That performs the appropriate setup on the host to run ceph builds 18 | continuously. There must be a password-less ssh access to the hostname 19 | and the must have a password-less sudo. 20 | 21 | An upstart service named autobuild-ceph gets created on the host that runs 22 | the autobuilder. 23 | 24 | If available, use ``sudo stop autobuild-ceph``, ``sudo start 25 | autobuild-ceph`` on the autobuilder host to manage the autobuilder. 26 | 27 | If stop/start do not work, ``/etc/init.d/autobuild-ceph stop``, 28 | ``/etc/init.d/autobuild-ceph start`` will start the daemon: it is a 29 | simple shell script. 30 | 31 | To get a list of other available commands, run ``fab -l``. Note 32 | that fabric expects to be able to ssh to the host you specify, so you 33 | should already have ssh keys setup for that host. If 34 | no host is specified, fabric will deploy to the set of hosts for the role associated with 35 | that command. Also note that the gitbuilder\_ceph command sets up the 36 | autobuilder to deploy the binary packages to the package server. This 37 | requires the rsync keys (rsync-key and rsync-key.pub) for the package 38 | server be located in your current directory (fabric copies them to the deployment 39 | host). You can get the keys from someone who already has access. 40 | 41 | ## Deploying autobuilders with fabric 42 | 43 | Fabric allows you to run commands to deploy a specific autobuilder 44 | build script on a node, setup ssh keys, and start the web server 45 | for displaying gitbuilder results. 46 | 47 | Fabric uses the fabfile.py file in your current working directory. 48 | The fabfile.py is essentially a set of roles and commands. The 49 | ``gitbuilder_ceph`` command runs the defined gitbuilder\_ceph 50 | function, sending remote commands to each of the hosts defined by 51 | the role(s) associated with that function. 52 | A role defines a list of hosts where a command will be run, for example, 53 | the ``gitbuilder_ceph`` role (happens to share the same name as the 54 | command) runs the gitbuilder\_ceph command on all all the VMs defined 55 | in that role. 56 | 57 | ## Implementing your own autobuild 58 | 59 | ### Create a build script 60 | 61 | For your project called __foo__, create a build script ``build-foo.sh`` 62 | in the top-level directory that executes the steps to build the foo project. 63 | The script should assume that the current working directory is the top-level 64 | checkout of the __foo__ repository. Gitbuilder controls cloning the foo repository 65 | and checking out to the desired branch. Gitbuilder checks the output of this 66 | script for lines that have "error:" or "warning:" messages, and reports those 67 | as such. If you need to ignore some warnings in the output of your build script, 68 | you can add the following echo statements around your build commands: 69 | 70 | echo --START-IGNORE-WARNINGS 71 | # build commands here... 72 | ./configure whatever 73 | echo --STOP-IGNORE-WARNINGS 74 | 75 | To limit which branches are built by gitbuilder, a branches-local script should 76 | be installed by the fabfile.py function/command for ``gitbuilder_foo`` that outputs 77 | only the branches that gitbuilder should build. See the branches-local script 78 | in this repo for an example that outputs the branches to build for the ceph autobuilder. 79 | 80 | ### Modify the fabfile.py 81 | 82 | First add a role definition called ``gitbuilder_foo`` to include a new function with a set of roles. 83 | The set of roles should include all the roles where you want to deploy your foo autobuilder. A 84 | basic gitbuilder function and role definition looks like this: 85 | 86 | @roles('gitbuilder_foo') 87 | def gitbuilder_foo(): 88 | _apt_install( 89 | 'make' 90 | 'libfoodep-dev', 91 | ) 92 | _gitbuilder( 93 | flavor='foo', 94 | git_repo='http://github.com/ceph/foo.git', 95 | extra_packages=[ 96 | 'fakeroot', 97 | 'reprepro', 98 | ], 99 | ) 100 | _sync_to_gitbuilder('foo', 'deb', 'basic') 101 | sudo('start autobuild-ceph || /etc/init.d/autobuild-ceph start') 102 | 103 | Note that the flavor you specify to the \_gitbuilder() function determines how your build script 104 | is chosen as the build script to run by the gitbuilder tool. The extra\_packages specify packages 105 | that need to be installed in order to create a deb repository for your autobuilt packages, and 106 | the \_sync\_to\_gitbuilder() function performs setup to rsync the binary packages created by the build 107 | to the repo hosting server. In order to perform the sync, rsync keys are required. You can get the 108 | keys from another user and place them in your checkout directory. 109 | 110 | As a final step, define a role that lists the hosts you want to deploy the foo autobuilder onto. By 111 | convention, the role shares the same name as the command, i.e. gitbuilder\_foo. See the other env.roledef 112 | lists at the top of the fabfile for examples. 113 | 114 | ### Deploying your autobuild 115 | 116 | Once you've created your build script and modified the fabfile.py to include your gitbuilder command 117 | and roles, you should be able to deploy your autobuild with: 118 | 119 | fab gitbuilder_foo 120 | 121 | ### Setting up the Autobuild web server 122 | 123 | A command to setup lighttpd and point it at the autobuild results exists in the fabfile.py. To start the web 124 | server, you can simply do: 125 | 126 | fab gitbuilder_serve:role=gitbuilder_foo 127 | 128 | ### How autobuilder works 129 | 130 | Running fabric with the autobuild-ceph fabfile.py does a clone of the autobuild-ceph repo into /srv/ on the host(s), installs 131 | other needed packages and creates a user to run autobuilder. It then sets up gitbuilder, checking out that 132 | repo into /srv/autobuild-ceph/gitbuilder.git, and 133 | creates a symlink from the build script you specified (build-foo.sh for example) to build.sh, and another symlink 134 | in gitbuilder.git/build.sh that points back to the build.sh in /srv/autobuild-ceph. It then clones the build repo (i.e. foo) into 135 | the build directory within gitbuilder.git/, and creates an upstart script in /etc/init/autobuild-ceph. The upstart script 136 | simply runs the ``run`` script in the /srv/autobuild-ceph directory, which in turn runs the gitbuilder autobuild.sh script. 137 | The script checks that new commits exist in the repository before attempting another build, and exits otherwise. The upstart 138 | script is configured to respawn once the previous process exits, so the script continuously checks for new commits to the repository. 139 | 140 | -------------------------------------------------------------------------------- /branches-local: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import optparse 3 | import subprocess 4 | import sys 5 | 6 | def get_refs(): 7 | p = subprocess.Popen( 8 | args=[ 9 | 'git', 10 | 'for-each-ref', 11 | '--format=%(objectname) %(*objectname) %(refname)', 12 | '--sort=-taggerdate', 13 | 'refs/remotes/origin/', 14 | 'refs/tags/', 15 | ], 16 | stdout=subprocess.PIPE, 17 | close_fds=True, 18 | ) 19 | (stdout, stderr) = p.communicate() 20 | # TODO check p.returncode 21 | assert not stderr 22 | for line in stdout.splitlines(): 23 | obj, tagobj, ref = line.split(' ', 2) 24 | if tagobj: 25 | obj = tagobj 26 | yield obj, ref 27 | 28 | def separate_tags_and_branches(refs): 29 | tags = [] 30 | branches = [] 31 | for obj, ref in refs: 32 | if ref.startswith('refs/tags/'): 33 | tags.append((obj, ref)) 34 | elif ref.startswith('refs/remotes/origin/'): 35 | branches.append((obj, ref)) 36 | else: 37 | raise RuntimeError('Bad refname: %r' % ref) 38 | return tags, branches 39 | 40 | PRIORITIZE = [ 41 | 'master', 42 | 'jewel', 43 | ] 44 | 45 | IGNORE = [ 46 | 'HEAD', 47 | ] 48 | 49 | IGNORE_PREFIX = [ 50 | 'historic/', 51 | ] 52 | 53 | IGNORE_TAGS = [ 54 | 'ceph-v0.20.1', 55 | 'v0.1', 56 | 'v0.2', 57 | 'v0.3', 58 | 'v0.4', 59 | 'v0.5', 60 | 'v0.6', 61 | 'v0.7', 62 | 'v0.7.1', 63 | 'v0.7.2', 64 | 'v0.7.3', 65 | 'v0.8', 66 | 'v0.9', 67 | 'v0.10', 68 | 'v0.11', 69 | 'v0.12', 70 | 'v0.13', 71 | 'v0.14', 72 | 'v0.15', 73 | 'v0.16', 74 | 'v0.16.1', 75 | 'v0.17', 76 | 'v0.18', 77 | 'v0.19', 78 | 'v0.19.1', 79 | 'v0.20', 80 | 'v0.20.1', 81 | 'v0.20.2', 82 | 'v0.21', 83 | 'v0.21.1', 84 | 'v0.21.2', 85 | 'v0.21.3', 86 | 'v0.22', 87 | 'v0.22.1', 88 | 'v0.22.2', 89 | 'v0.23', 90 | 'v0.23.1', 91 | 'v0.23.2', 92 | 'v0.24', 93 | 'v0.24.1', 94 | 'v0.24.2', 95 | 'v0.24.3', 96 | 'v0.25', 97 | 'v0.25.1', 98 | 'v0.25.2', 99 | 'v0.26', 100 | 'v0.27', 101 | 'v0.27.1', 102 | 'v0.28', 103 | 'v0.28.1', 104 | 'v0.28.2', 105 | 'v0.29', 106 | 'v0.29.1', 107 | 'v0.30', 108 | 'v0.31', 109 | 'v0.32', 110 | 'v0.33', 111 | 'v0.34', 112 | 'v0.35', 113 | 'v0.36', 114 | 'v0.37', 115 | 'v0.38', 116 | 'v0.39', 117 | 'v0.40', 118 | 'v0.41', 119 | 'v0.42', 120 | 'v0.42.1', 121 | 'v0.42.2', 122 | 'v0.43', 123 | 'v0.44', 124 | 'v0.44.1', 125 | 'v0.44.2', 126 | 'ceph-v0.10', 127 | 'ceph-v0.11', 128 | 'ceph-v0.12', 129 | 'ceph-v0.14', 130 | 'ceph-v0.15', 131 | 'ceph-v0.16', 132 | 'ceph-v0.16.1', 133 | 'ceph-v0.17', 134 | 'ceph-v0.18', 135 | 'ceph-v0.19', 136 | 'ceph-v0.19.1', 137 | 'ceph-v0.5', 138 | 'ceph-v0.8', 139 | 'foo-3.5', 140 | 'reconnect_gen_thing', 141 | 'v2.6.11', 142 | 'v2.6.11-tree', 143 | 'v2.6.12', 144 | 'v2.6.12-rc2', 145 | 'v2.6.12-rc3', 146 | 'v2.6.12-rc4', 147 | 'v2.6.12-rc5', 148 | 'v2.6.12-rc6', 149 | 'v2.6.13', 150 | 'v2.6.13-rc1', 151 | 'v2.6.13-rc2', 152 | 'v2.6.13-rc3', 153 | 'v2.6.13-rc4', 154 | 'v2.6.13-rc5', 155 | 'v2.6.13-rc6', 156 | 'v2.6.13-rc7', 157 | 'v2.6.14', 158 | 'v2.6.14-rc1', 159 | 'v2.6.14-rc2', 160 | 'v2.6.14-rc3', 161 | 'v2.6.14-rc4', 162 | 'v2.6.14-rc5', 163 | 'v2.6.15', 164 | 'v2.6.15-rc1', 165 | 'v2.6.15-rc2', 166 | 'v2.6.15-rc3', 167 | 'v2.6.15-rc4', 168 | 'v2.6.15-rc5', 169 | 'v2.6.15-rc6', 170 | 'v2.6.15-rc7', 171 | 'v2.6.16', 172 | 'v2.6.16-rc1', 173 | 'v2.6.16-rc2', 174 | 'v2.6.16-rc3', 175 | 'v2.6.16-rc4', 176 | 'v2.6.16-rc5', 177 | 'v2.6.16-rc6', 178 | 'v2.6.17', 179 | 'v2.6.17-rc1', 180 | 'v2.6.17-rc2', 181 | 'v2.6.17-rc3', 182 | 'v2.6.17-rc4', 183 | 'v2.6.17-rc5', 184 | 'v2.6.17-rc6', 185 | 'v2.6.18', 186 | 'v2.6.18-rc1', 187 | 'v2.6.18-rc2', 188 | 'v2.6.18-rc3', 189 | 'v2.6.18-rc4', 190 | 'v2.6.18-rc5', 191 | 'v2.6.18-rc6', 192 | 'v2.6.18-rc7', 193 | 'v2.6.19', 194 | 'v2.6.19-rc1', 195 | 'v2.6.19-rc2', 196 | 'v2.6.19-rc3', 197 | 'v2.6.19-rc4', 198 | 'v2.6.19-rc5', 199 | 'v2.6.19-rc6', 200 | 'v2.6.20', 201 | 'v2.6.20-rc1', 202 | 'v2.6.20-rc2', 203 | 'v2.6.20-rc3', 204 | 'v2.6.20-rc4', 205 | 'v2.6.20-rc5', 206 | 'v2.6.20-rc6', 207 | 'v2.6.20-rc7', 208 | 'v2.6.21', 209 | 'v2.6.21-rc1', 210 | 'v2.6.21-rc2', 211 | 'v2.6.21-rc3', 212 | 'v2.6.21-rc4', 213 | 'v2.6.21-rc5', 214 | 'v2.6.21-rc6', 215 | 'v2.6.21-rc7', 216 | 'v2.6.22', 217 | 'v2.6.22-rc1', 218 | 'v2.6.22-rc2', 219 | 'v2.6.22-rc3', 220 | 'v2.6.22-rc4', 221 | 'v2.6.22-rc5', 222 | 'v2.6.22-rc6', 223 | 'v2.6.22-rc7', 224 | 'v2.6.23', 225 | 'v2.6.23-rc1', 226 | 'v2.6.23-rc2', 227 | 'v2.6.23-rc3', 228 | 'v2.6.23-rc4', 229 | 'v2.6.23-rc5', 230 | 'v2.6.23-rc6', 231 | 'v2.6.23-rc7', 232 | 'v2.6.23-rc8', 233 | 'v2.6.23-rc9', 234 | 'v2.6.24', 235 | 'v2.6.24-rc1', 236 | 'v2.6.24-rc2', 237 | 'v2.6.24-rc3', 238 | 'v2.6.24-rc4', 239 | 'v2.6.24-rc5', 240 | 'v2.6.24-rc6', 241 | 'v2.6.24-rc7', 242 | 'v2.6.24-rc8', 243 | 'v2.6.25', 244 | 'v2.6.25-rc1', 245 | 'v2.6.25-rc2', 246 | 'v2.6.25-rc3', 247 | 'v2.6.25-rc4', 248 | 'v2.6.25-rc5', 249 | 'v2.6.25-rc6', 250 | 'v2.6.25-rc7', 251 | 'v2.6.25-rc8', 252 | 'v2.6.25-rc9', 253 | 'v2.6.26', 254 | 'v2.6.26-rc1', 255 | 'v2.6.26-rc2', 256 | 'v2.6.26-rc3', 257 | 'v2.6.26-rc4', 258 | 'v2.6.26-rc5', 259 | 'v2.6.26-rc6', 260 | 'v2.6.26-rc7', 261 | 'v2.6.26-rc8', 262 | 'v2.6.26-rc9', 263 | 'v2.6.27', 264 | 'v2.6.27-rc1', 265 | 'v2.6.27-rc2', 266 | 'v2.6.27-rc3', 267 | 'v2.6.27-rc4', 268 | 'v2.6.27-rc5', 269 | 'v2.6.27-rc6', 270 | 'v2.6.27-rc7', 271 | 'v2.6.27-rc8', 272 | 'v2.6.27-rc9', 273 | 'v2.6.28', 274 | 'v2.6.28-rc1', 275 | 'v2.6.28-rc2', 276 | 'v2.6.28-rc3', 277 | 'v2.6.28-rc4', 278 | 'v2.6.28-rc5', 279 | 'v2.6.28-rc6', 280 | 'v2.6.28-rc7', 281 | 'v2.6.28-rc8', 282 | 'v2.6.28-rc9', 283 | 'v2.6.29', 284 | 'v2.6.29-rc1', 285 | 'v2.6.29-rc2', 286 | 'v2.6.29-rc3', 287 | 'v2.6.29-rc4', 288 | 'v2.6.29-rc5', 289 | 'v2.6.29-rc6', 290 | 'v2.6.29-rc7', 291 | 'v2.6.29-rc8', 292 | 'v2.6.30', 293 | 'v2.6.30-rc1', 294 | 'v2.6.30-rc2', 295 | 'v2.6.30-rc3', 296 | 'v2.6.30-rc4', 297 | 'v2.6.30-rc5', 298 | 'v2.6.30-rc6', 299 | 'v2.6.30-rc7', 300 | 'v2.6.30-rc8', 301 | 'v2.6.31', 302 | 'v2.6.31-rc1', 303 | 'v2.6.31-rc2', 304 | 'v2.6.31-rc3', 305 | 'v2.6.31-rc4', 306 | 'v2.6.31-rc5', 307 | 'v2.6.31-rc6', 308 | 'v2.6.31-rc7', 309 | 'v2.6.31-rc8', 310 | 'v2.6.31-rc9', 311 | 'v2.6.32', 312 | 'v2.6.32-rc1', 313 | 'v2.6.32-rc2', 314 | 'v2.6.32-rc3', 315 | 'v2.6.32-rc4', 316 | 'v2.6.32-rc5', 317 | 'v2.6.32-rc6', 318 | 'v2.6.32-rc7', 319 | 'v2.6.32-rc8', 320 | 'v2.6.33', 321 | 'v2.6.33-rc1', 322 | 'v2.6.33-rc2', 323 | 'v2.6.33-rc3', 324 | 'v2.6.33-rc4', 325 | 'v2.6.33-rc5', 326 | 'v2.6.33-rc6', 327 | 'v2.6.33-rc7', 328 | 'v2.6.33-rc8', 329 | 'v2.6.34', 330 | 'v2.6.34-rc1', 331 | 'v2.6.34-rc2', 332 | 'v2.6.34-rc3', 333 | 'v2.6.34-rc4', 334 | 'v2.6.34-rc5', 335 | 'v2.6.34-rc6', 336 | 'v2.6.34-rc7', 337 | 'v2.6.35', 338 | 'v2.6.35-rc1', 339 | 'v2.6.35-rc2', 340 | 'v2.6.35-rc3', 341 | 'v2.6.35-rc4', 342 | 'v2.6.35-rc5', 343 | 'v2.6.35-rc6', 344 | 'v2.6.36', 345 | 'v2.6.36-rc1', 346 | 'v2.6.36-rc2', 347 | 'v2.6.36-rc3', 348 | 'v2.6.36-rc4', 349 | 'v2.6.36-rc5', 350 | 'v2.6.36-rc6', 351 | 'v2.6.36-rc7', 352 | 'v2.6.36-rc8', 353 | 'v2.6.37', 354 | 'v2.6.37-rc1', 355 | 'v2.6.37-rc2', 356 | 'v2.6.37-rc3', 357 | 'v2.6.37-rc4', 358 | 'v2.6.37-rc5', 359 | 'v2.6.37-rc6', 360 | 'v2.6.37-rc7', 361 | 'v2.6.37-rc8', 362 | 'v2.6.38', 363 | 'v2.6.38-rc1', 364 | 'v2.6.38-rc2', 365 | 'v2.6.38-rc3', 366 | 'v2.6.38-rc4', 367 | 'v2.6.38-rc5', 368 | 'v2.6.38-rc6', 369 | 'v2.6.38-rc7', 370 | 'v2.6.38-rc8', 371 | 'v2.6.39', 372 | 'v2.6.39-rc1', 373 | 'v2.6.39-rc2', 374 | 'v2.6.39-rc3', 375 | 'v2.6.39-rc4', 376 | 'v2.6.39-rc5', 377 | 'v2.6.39-rc6', 378 | 'v2.6.39-rc7', 379 | 'v3.0', 380 | 'v3.0-rc1', 381 | 'v3.0-rc2', 382 | 'v3.0-rc3', 383 | 'v3.0-rc4', 384 | 'v3.0-rc5', 385 | 'v3.0-rc6', 386 | 'v3.0-rc7', 387 | 'v3.1', 388 | 'v3.1-rc1', 389 | 'v3.1-rc10', 390 | 'v3.1-rc2', 391 | 'v3.1-rc3', 392 | 'v3.1-rc4', 393 | 'v3.1-rc5', 394 | 'v3.1-rc6', 395 | 'v3.1-rc7', 396 | 'v3.1-rc8', 397 | 'v3.1-rc9', 398 | 'v3.2', 399 | 'v3.2-rc1', 400 | 'v3.2-rc2', 401 | 'v3.2-rc3', 402 | 'v3.2-rc4', 403 | 'v3.2-rc5', 404 | 'v3.2-rc6', 405 | 'v3.2-rc7', 406 | 'v3.3', 407 | 'v3.3-rc1', 408 | 'v3.3-rc2', 409 | 'v3.3-rc3', 410 | 'v3.3-rc4', 411 | 'v3.3-rc5', 412 | 'v3.3-rc6', 413 | 'v3.3-rc7', 414 | 'v3.4', 415 | 'v3.4-rc1', 416 | 'v3.4-rc2', 417 | 'v3.4-rc3', 418 | 'v3.4-rc4', 419 | 'v3.4-rc5', 420 | 'v3.4-rc6', 421 | 'v3.4-rc7', 422 | 'v3.7', 423 | 'v3.7-rc1', 424 | 'v3.7-rc2', 425 | 'v3.7-rc3', 426 | 'v3.7-rc4', 427 | 'v3.7-rc5', 428 | 'v3.7-rc6', 429 | 'v3.7-rc7', 430 | 'v3.7-rc8', 431 | 'v3.8-rc1', 432 | ] 433 | 434 | def separate_priority_branches(branches): 435 | prioritize = ['origin/{0}'.format(b) for b in PRIORITIZE] 436 | ignore = ['origin/{0}'.format(b) for b in IGNORE] 437 | ignore_prefix = ['origin/{0}'.format(b) for b in IGNORE_PREFIX] 438 | 439 | require = [] 440 | try: 441 | f = open('/srv/autobuild-ceph/filter-branches') 442 | require = f.read().splitlines() 443 | except: 444 | pass 445 | 446 | priority = [] 447 | normal = [] 448 | 449 | for obj, ref in branches: 450 | if require and not any(str in ref for str in require): 451 | continue 452 | 453 | if ref in prioritize: 454 | priority.append((obj, ref)) 455 | continue 456 | 457 | if ref in ignore: 458 | continue 459 | 460 | if any(ref.startswith(prefix) for prefix in ignore_prefix): 461 | continue 462 | 463 | normal.append((obj, ref)) 464 | 465 | # priority lost its ordering, restore it 466 | priority.sort(key=lambda (obj,ref): prioritize.index(ref)) 467 | 468 | return priority, normal 469 | 470 | def strip_prefix(refs, prefix): 471 | for obj, ref in refs: 472 | assert ref.startswith(prefix) 473 | ref = ref[len(prefix):] 474 | yield obj, ref 475 | 476 | def filter_tags(ls): 477 | r = [] 478 | for x in ls: 479 | (sha1, tag) = x 480 | if tag not in IGNORE_TAGS: 481 | r.append(x) 482 | return r 483 | 484 | def doit(output, verbose): 485 | refs = get_refs() 486 | tags, branches = separate_tags_and_branches(refs) 487 | tags = strip_prefix(tags, 'refs/tags/') 488 | 489 | branches = strip_prefix(branches, 'refs/remotes/') 490 | priority, normal = separate_priority_branches(branches) 491 | 492 | filtered_tags = filter_tags(tags) 493 | 494 | if verbose: 495 | fmt = '{0} {1}' 496 | else: 497 | fmt = '{1}' 498 | 499 | for l in [priority, normal, filtered_tags]: 500 | for (obj, ref) in l: 501 | print >>output, fmt.format(obj, ref) 502 | 503 | def main(): 504 | parser = optparse.OptionParser( 505 | usage='%prog [-v]', 506 | description='Output interesting git tags and branches, for gitbuilder.', 507 | ) 508 | parser.add_option( 509 | '-v', '--verbose', 510 | action='store_true', 511 | ) 512 | opts, args = parser.parse_args() 513 | if args: 514 | parser.error('Unexpected arguments.') 515 | 516 | doit(sys.stdout, opts.verbose) 517 | 518 | if __name__ == '__main__': 519 | main() 520 | -------------------------------------------------------------------------------- /lighttpd.conf: -------------------------------------------------------------------------------- 1 | server.modules = ( "mod_cgi", "mod_accesslog" ) 2 | server.document-root = "/srv/autobuild-ceph/gitbuilder.git/out" 3 | 4 | # /etc/init.d script needs this 5 | server.pid-file = "/var/run/lighttpd.pid" 6 | 7 | server.username = "autobuild-ceph" 8 | server.groupname = "autobuild-ceph" 9 | 10 | # performance 11 | server.event-handler = "linux-sysepoll" 12 | server.network-backend = "linux-sendfile" 13 | server.stat-cache-engine = "simple" 14 | 15 | var.logdir = "/var/log/lighttpd" 16 | accesslog.filename = logdir + "/access.log" 17 | server.errorlog = logdir + "/error.log" 18 | 19 | # anything with .cgi is attempted, allows for shebangs 20 | cgi.assign = ( ".cgi" => "" ) 21 | index-file.names = ( "index.cgi", "index.html", "index.htm") 22 | 23 | mimetype.assign = ( 24 | ".ez" => "application/andrew-inset", 25 | ".anx" => "application/annodex", 26 | ".atom" => "application/atom+xml", 27 | ".atomcat" => "application/atomcat+xml", 28 | ".atomsrv" => "application/atomserv+xml", 29 | ".lin" => "application/bbolin", 30 | ".cap" => "application/cap", 31 | ".pcap" => "application/cap", 32 | ".cu" => "application/cu-seeme", 33 | ".davmount" => "application/davmount+xml", 34 | ".tsp" => "application/dsptype", 35 | ".es" => "application/ecmascript", 36 | ".spl" => "application/futuresplash", 37 | ".hta" => "application/hta", 38 | ".jar" => "application/java-archive", 39 | ".ser" => "application/java-serialized-object", 40 | ".class" => "application/java-vm", 41 | ".js" => "application/javascript", 42 | ".json" => "application/json", 43 | ".m3g" => "application/m3g", 44 | ".hqx" => "application/mac-binhex40", 45 | ".cpt" => "application/mac-compactpro", 46 | ".nb" => "application/mathematica", 47 | ".nbp" => "application/mathematica", 48 | ".mdb" => "application/msaccess", 49 | ".doc" => "application/msword", 50 | ".dot" => "application/msword", 51 | ".mxf" => "application/mxf", 52 | ".bin" => "application/octet-stream", 53 | ".oda" => "application/oda", 54 | ".ogx" => "application/ogg", 55 | ".one" => "application/onenote", 56 | ".onetoc2" => "application/onenote", 57 | ".onetmp" => "application/onenote", 58 | ".onepkg" => "application/onenote", 59 | ".pdf" => "application/pdf", 60 | ".key" => "application/pgp-keys", 61 | ".pgp" => "application/pgp-signature", 62 | ".prf" => "application/pics-rules", 63 | ".ps" => "application/postscript", 64 | ".ai" => "application/postscript", 65 | ".eps" => "application/postscript", 66 | ".epsi" => "application/postscript", 67 | ".epsf" => "application/postscript", 68 | ".eps2" => "application/postscript", 69 | ".eps3" => "application/postscript", 70 | ".rar" => "application/rar", 71 | ".rdf" => "application/rdf+xml", 72 | ".rss" => "application/rss+xml", 73 | ".rtf" => "application/rtf", 74 | ".stl" => "application/sla", 75 | ".smi" => "application/smil", 76 | ".smil" => "application/smil", 77 | ".xhtml" => "application/xhtml+xml", 78 | ".xht" => "application/xhtml+xml", 79 | ".xml" => "application/xml", 80 | ".xsl" => "application/xml", 81 | ".xsd" => "application/xml", 82 | ".xspf" => "application/xspf+xml", 83 | ".zip" => "application/zip", 84 | ".apk" => "application/vnd.android.package-archive", 85 | ".cdy" => "application/vnd.cinderella", 86 | ".kml" => "application/vnd.google-earth.kml+xml", 87 | ".kmz" => "application/vnd.google-earth.kmz", 88 | ".xul" => "application/vnd.mozilla.xul+xml", 89 | ".xls" => "application/vnd.ms-excel", 90 | ".xlb" => "application/vnd.ms-excel", 91 | ".xlt" => "application/vnd.ms-excel", 92 | ".thmx" => "application/vnd.ms-officetheme", 93 | ".cat" => "application/vnd.ms-pki.seccat", 94 | ".ppt" => "application/vnd.ms-powerpoint", 95 | ".pps" => "application/vnd.ms-powerpoint", 96 | ".odc" => "application/vnd.oasis.opendocument.chart", 97 | ".odb" => "application/vnd.oasis.opendocument.database", 98 | ".odf" => "application/vnd.oasis.opendocument.formula", 99 | ".odg" => "application/vnd.oasis.opendocument.graphics", 100 | ".otg" => "application/vnd.oasis.opendocument.graphics-template", 101 | ".odi" => "application/vnd.oasis.opendocument.image", 102 | ".odp" => "application/vnd.oasis.opendocument.presentation", 103 | ".otp" => "application/vnd.oasis.opendocument.presentation-template", 104 | ".ods" => "application/vnd.oasis.opendocument.spreadsheet", 105 | ".ots" => "application/vnd.oasis.opendocument.spreadsheet-template", 106 | ".odt" => "application/vnd.oasis.opendocument.text", 107 | ".odm" => "application/vnd.oasis.opendocument.text-master", 108 | ".ott" => "application/vnd.oasis.opendocument.text-template", 109 | ".oth" => "application/vnd.oasis.opendocument.text-web", 110 | ".pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation", 111 | ".sldx" => "application/vnd.openxmlformats-officedocument.presentationml.slide", 112 | ".ppsx" => "application/vnd.openxmlformats-officedocument.presentationml.slideshow", 113 | ".potx" => "application/vnd.openxmlformats-officedocument.presentationml.template", 114 | ".xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 115 | ".xltx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.template", 116 | ".docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document", 117 | ".dotx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.template", 118 | ".cod" => "application/vnd.rim.cod", 119 | ".mmf" => "application/vnd.smaf", 120 | ".sdc" => "application/vnd.stardivision.calc", 121 | ".sds" => "application/vnd.stardivision.chart", 122 | ".sda" => "application/vnd.stardivision.draw", 123 | ".sdd" => "application/vnd.stardivision.impress", 124 | ".sdf" => "application/vnd.stardivision.math", 125 | ".sdw" => "application/vnd.stardivision.writer", 126 | ".sgl" => "application/vnd.stardivision.writer-global", 127 | ".sxc" => "application/vnd.sun.xml.calc", 128 | ".stc" => "application/vnd.sun.xml.calc.template", 129 | ".sxd" => "application/vnd.sun.xml.draw", 130 | ".std" => "application/vnd.sun.xml.draw.template", 131 | ".sxi" => "application/vnd.sun.xml.impress", 132 | ".sti" => "application/vnd.sun.xml.impress.template", 133 | ".sxm" => "application/vnd.sun.xml.math", 134 | ".sxw" => "application/vnd.sun.xml.writer", 135 | ".sxg" => "application/vnd.sun.xml.writer.global", 136 | ".stw" => "application/vnd.sun.xml.writer.template", 137 | ".sis" => "application/vnd.symbian.install", 138 | ".vsd" => "application/vnd.visio", 139 | ".wbxml" => "application/vnd.wap.wbxml", 140 | ".wmlc" => "application/vnd.wap.wmlc", 141 | ".wmlsc" => "application/vnd.wap.wmlscriptc", 142 | ".wpd" => "application/vnd.wordperfect", 143 | ".wp5" => "application/vnd.wordperfect5.1", 144 | ".wk" => "application/x-123", 145 | ".7z" => "application/x-7z-compressed", 146 | ".abw" => "application/x-abiword", 147 | ".dmg" => "application/x-apple-diskimage", 148 | ".bcpio" => "application/x-bcpio", 149 | ".torrent" => "application/x-bittorrent", 150 | ".cab" => "application/x-cab", 151 | ".cbr" => "application/x-cbr", 152 | ".cbz" => "application/x-cbz", 153 | ".cdf" => "application/x-cdf", 154 | ".cda" => "application/x-cdf", 155 | ".vcd" => "application/x-cdlink", 156 | ".pgn" => "application/x-chess-pgn", 157 | ".mph" => "application/x-comsol", 158 | ".cpio" => "application/x-cpio", 159 | ".csh" => "application/x-csh", 160 | ".deb" => "application/x-debian-package", 161 | ".udeb" => "application/x-debian-package", 162 | ".dcr" => "application/x-director", 163 | ".dir" => "application/x-director", 164 | ".dxr" => "application/x-director", 165 | ".dms" => "application/x-dms", 166 | ".wad" => "application/x-doom", 167 | ".dvi" => "application/x-dvi", 168 | ".mm" => "application/x-freemind", 169 | ".gan" => "application/x-ganttproject", 170 | ".gnumeric" => "application/x-gnumeric", 171 | ".sgf" => "application/x-go-sgf", 172 | ".gcf" => "application/x-graphing-calculator", 173 | ".gtar" => "application/x-gtar", 174 | ".tgz" => "application/x-gtar-compressed", 175 | ".taz" => "application/x-gtar-compressed", 176 | ".hdf" => "application/x-hdf", 177 | ".rhtml" => "application/x-httpd-eruby", 178 | ".phtml" => "application/x-httpd-php", 179 | ".pht" => "application/x-httpd-php", 180 | ".php" => "application/x-httpd-php", 181 | ".phps" => "application/x-httpd-php-source", 182 | ".php3" => "application/x-httpd-php3", 183 | ".php3p" => "application/x-httpd-php3-preprocessed", 184 | ".php4" => "application/x-httpd-php4", 185 | ".php5" => "application/x-httpd-php5", 186 | ".ica" => "application/x-ica", 187 | ".info" => "application/x-info", 188 | ".ins" => "application/x-internet-signup", 189 | ".isp" => "application/x-internet-signup", 190 | ".iii" => "application/x-iphone", 191 | ".iso" => "application/x-iso9660-image", 192 | ".jam" => "application/x-jam", 193 | ".jnlp" => "application/x-java-jnlp-file", 194 | ".jmz" => "application/x-jmol", 195 | ".chrt" => "application/x-kchart", 196 | ".kil" => "application/x-killustrator", 197 | ".skp" => "application/x-koan", 198 | ".skd" => "application/x-koan", 199 | ".skt" => "application/x-koan", 200 | ".skm" => "application/x-koan", 201 | ".kpr" => "application/x-kpresenter", 202 | ".kpt" => "application/x-kpresenter", 203 | ".ksp" => "application/x-kspread", 204 | ".kwd" => "application/x-kword", 205 | ".kwt" => "application/x-kword", 206 | ".latex" => "application/x-latex", 207 | ".lha" => "application/x-lha", 208 | ".lyx" => "application/x-lyx", 209 | ".lzh" => "application/x-lzh", 210 | ".lzx" => "application/x-lzx", 211 | ".frm" => "application/x-maker", 212 | ".maker" => "application/x-maker", 213 | ".frame" => "application/x-maker", 214 | ".fm" => "application/x-maker", 215 | ".fb" => "application/x-maker", 216 | ".book" => "application/x-maker", 217 | ".fbdoc" => "application/x-maker", 218 | ".mif" => "application/x-mif", 219 | ".wmd" => "application/x-ms-wmd", 220 | ".wmz" => "application/x-ms-wmz", 221 | ".com" => "application/x-msdos-program", 222 | ".exe" => "application/x-msdos-program", 223 | ".bat" => "application/x-msdos-program", 224 | ".dll" => "application/x-msdos-program", 225 | ".msi" => "application/x-msi", 226 | ".nc" => "application/x-netcdf", 227 | ".pac" => "application/x-ns-proxy-autoconfig", 228 | ".dat" => "application/x-ns-proxy-autoconfig", 229 | ".nwc" => "application/x-nwc", 230 | ".o" => "application/x-object", 231 | ".oza" => "application/x-oz-application", 232 | ".p7r" => "application/x-pkcs7-certreqresp", 233 | ".crl" => "application/x-pkcs7-crl", 234 | ".pyc" => "application/x-python-code", 235 | ".pyo" => "application/x-python-code", 236 | ".qgs" => "application/x-qgis", 237 | ".shp" => "application/x-qgis", 238 | ".shx" => "application/x-qgis", 239 | ".qtl" => "application/x-quicktimeplayer", 240 | ".rdp" => "application/x-rdp", 241 | ".rpm" => "application/x-redhat-package-manager", 242 | ".rb" => "application/x-ruby", 243 | ".sci" => "application/x-scilab", 244 | ".sce" => "application/x-scilab", 245 | ".sh" => "application/x-sh", 246 | ".shar" => "application/x-shar", 247 | ".swf" => "application/x-shockwave-flash", 248 | ".swfl" => "application/x-shockwave-flash", 249 | ".scr" => "application/x-silverlight", 250 | ".sql" => "application/x-sql", 251 | ".sit" => "application/x-stuffit", 252 | ".sitx" => "application/x-stuffit", 253 | ".sv4cpio" => "application/x-sv4cpio", 254 | ".sv4crc" => "application/x-sv4crc", 255 | ".tar" => "application/x-tar", 256 | ".tcl" => "application/x-tcl", 257 | ".gf" => "application/x-tex-gf", 258 | ".pk" => "application/x-tex-pk", 259 | ".texinfo" => "application/x-texinfo", 260 | ".texi" => "application/x-texinfo", 261 | ".t" => "application/x-troff", 262 | ".tr" => "application/x-troff", 263 | ".roff" => "application/x-troff", 264 | ".man" => "application/x-troff-man", 265 | ".me" => "application/x-troff-me", 266 | ".ms" => "application/x-troff-ms", 267 | ".ustar" => "application/x-ustar", 268 | ".src" => "application/x-wais-source", 269 | ".wz" => "application/x-wingz", 270 | ".crt" => "application/x-x509-ca-cert", 271 | ".xcf" => "application/x-xcf", 272 | ".fig" => "application/x-xfig", 273 | ".xpi" => "application/x-xpinstall", 274 | ".amr" => "audio/amr", 275 | ".awb" => "audio/amr-wb", 276 | ".axa" => "audio/annodex", 277 | ".au" => "audio/basic", 278 | ".snd" => "audio/basic", 279 | ".csd" => "audio/csound", 280 | ".orc" => "audio/csound", 281 | ".sco" => "audio/csound", 282 | ".flac" => "audio/flac", 283 | ".mid" => "audio/midi", 284 | ".midi" => "audio/midi", 285 | ".kar" => "audio/midi", 286 | ".mpga" => "audio/mpeg", 287 | ".mpega" => "audio/mpeg", 288 | ".mp2" => "audio/mpeg", 289 | ".mp3" => "audio/mpeg", 290 | ".m4a" => "audio/mpeg", 291 | ".m3u" => "audio/mpegurl", 292 | ".oga" => "audio/ogg", 293 | ".ogg" => "audio/ogg", 294 | ".spx" => "audio/ogg", 295 | ".sid" => "audio/prs.sid", 296 | ".aif" => "audio/x-aiff", 297 | ".aiff" => "audio/x-aiff", 298 | ".aifc" => "audio/x-aiff", 299 | ".gsm" => "audio/x-gsm", 300 | ".wma" => "audio/x-ms-wma", 301 | ".wax" => "audio/x-ms-wax", 302 | ".ra" => "audio/x-pn-realaudio", 303 | ".rm" => "audio/x-pn-realaudio", 304 | ".ram" => "audio/x-pn-realaudio", 305 | ".pls" => "audio/x-scpls", 306 | ".sd2" => "audio/x-sd2", 307 | ".wav" => "audio/x-wav", 308 | ".alc" => "chemical/x-alchemy", 309 | ".cac" => "chemical/x-cache", 310 | ".cache" => "chemical/x-cache", 311 | ".csf" => "chemical/x-cache-csf", 312 | ".cbin" => "chemical/x-cactvs-binary", 313 | ".cascii" => "chemical/x-cactvs-binary", 314 | ".ctab" => "chemical/x-cactvs-binary", 315 | ".cdx" => "chemical/x-cdx", 316 | ".cer" => "chemical/x-cerius", 317 | ".c3d" => "chemical/x-chem3d", 318 | ".chm" => "chemical/x-chemdraw", 319 | ".cif" => "chemical/x-cif", 320 | ".cmdf" => "chemical/x-cmdf", 321 | ".cml" => "chemical/x-cml", 322 | ".cpa" => "chemical/x-compass", 323 | ".bsd" => "chemical/x-crossfire", 324 | ".csml" => "chemical/x-csml", 325 | ".csm" => "chemical/x-csml", 326 | ".ctx" => "chemical/x-ctx", 327 | ".cxf" => "chemical/x-cxf", 328 | ".cef" => "chemical/x-cxf", 329 | ".emb" => "chemical/x-embl-dl-nucleotide", 330 | ".embl" => "chemical/x-embl-dl-nucleotide", 331 | ".spc" => "chemical/x-galactic-spc", 332 | ".inp" => "chemical/x-gamess-input", 333 | ".gam" => "chemical/x-gamess-input", 334 | ".gamin" => "chemical/x-gamess-input", 335 | ".fch" => "chemical/x-gaussian-checkpoint", 336 | ".fchk" => "chemical/x-gaussian-checkpoint", 337 | ".cub" => "chemical/x-gaussian-cube", 338 | ".gau" => "chemical/x-gaussian-input", 339 | ".gjc" => "chemical/x-gaussian-input", 340 | ".gjf" => "chemical/x-gaussian-input", 341 | ".gal" => "chemical/x-gaussian-log", 342 | ".gcg" => "chemical/x-gcg8-sequence", 343 | ".gen" => "chemical/x-genbank", 344 | ".hin" => "chemical/x-hin", 345 | ".istr" => "chemical/x-isostar", 346 | ".ist" => "chemical/x-isostar", 347 | ".jdx" => "chemical/x-jcamp-dx", 348 | ".dx" => "chemical/x-jcamp-dx", 349 | ".kin" => "chemical/x-kinemage", 350 | ".mcm" => "chemical/x-macmolecule", 351 | ".mmd" => "chemical/x-macromodel-input", 352 | ".mmod" => "chemical/x-macromodel-input", 353 | ".mol" => "chemical/x-mdl-molfile", 354 | ".rd" => "chemical/x-mdl-rdfile", 355 | ".rxn" => "chemical/x-mdl-rxnfile", 356 | ".sd" => "chemical/x-mdl-sdfile", 357 | ".tgf" => "chemical/x-mdl-tgf", 358 | ".mcif" => "chemical/x-mmcif", 359 | ".mol2" => "chemical/x-mol2", 360 | ".gpt" => "chemical/x-mopac-graph", 361 | ".mop" => "chemical/x-mopac-input", 362 | ".mopcrt" => "chemical/x-mopac-input", 363 | ".mpc" => "chemical/x-mopac-input", 364 | ".zmt" => "chemical/x-mopac-input", 365 | ".moo" => "chemical/x-mopac-out", 366 | ".mvb" => "chemical/x-mopac-vib", 367 | ".asn" => "chemical/x-ncbi-asn1", 368 | ".prt" => "chemical/x-ncbi-asn1-ascii", 369 | ".ent" => "chemical/x-ncbi-asn1-ascii", 370 | ".val" => "chemical/x-ncbi-asn1-binary", 371 | ".aso" => "chemical/x-ncbi-asn1-binary", 372 | ".pdb" => "chemical/x-pdb", 373 | ".ros" => "chemical/x-rosdal", 374 | ".sw" => "chemical/x-swissprot", 375 | ".vms" => "chemical/x-vamas-iso14976", 376 | ".vmd" => "chemical/x-vmd", 377 | ".xtel" => "chemical/x-xtel", 378 | ".xyz" => "chemical/x-xyz", 379 | ".gif" => "image/gif", 380 | ".ief" => "image/ief", 381 | ".jpeg" => "image/jpeg", 382 | ".jpg" => "image/jpeg", 383 | ".jpe" => "image/jpeg", 384 | ".pcx" => "image/pcx", 385 | ".png" => "image/png", 386 | ".svg" => "image/svg+xml", 387 | ".svgz" => "image/svg+xml", 388 | ".tiff" => "image/tiff", 389 | ".tif" => "image/tiff", 390 | ".djvu" => "image/vnd.djvu", 391 | ".djv" => "image/vnd.djvu", 392 | ".wbmp" => "image/vnd.wap.wbmp", 393 | ".cr2" => "image/x-canon-cr2", 394 | ".crw" => "image/x-canon-crw", 395 | ".ras" => "image/x-cmu-raster", 396 | ".cdr" => "image/x-coreldraw", 397 | ".pat" => "image/x-coreldrawpattern", 398 | ".cdt" => "image/x-coreldrawtemplate", 399 | ".erf" => "image/x-epson-erf", 400 | ".ico" => "image/x-icon", 401 | ".art" => "image/x-jg", 402 | ".jng" => "image/x-jng", 403 | ".bmp" => "image/x-ms-bmp", 404 | ".nef" => "image/x-nikon-nef", 405 | ".orf" => "image/x-olympus-orf", 406 | ".psd" => "image/x-photoshop", 407 | ".pnm" => "image/x-portable-anymap", 408 | ".pbm" => "image/x-portable-bitmap", 409 | ".pgm" => "image/x-portable-graymap", 410 | ".ppm" => "image/x-portable-pixmap", 411 | ".rgb" => "image/x-rgb", 412 | ".xbm" => "image/x-xbitmap", 413 | ".xpm" => "image/x-xpixmap", 414 | ".xwd" => "image/x-xwindowdump", 415 | ".eml" => "message/rfc822", 416 | ".igs" => "model/iges", 417 | ".iges" => "model/iges", 418 | ".msh" => "model/mesh", 419 | ".mesh" => "model/mesh", 420 | ".silo" => "model/mesh", 421 | ".wrl" => "model/vrml", 422 | ".vrml" => "model/vrml", 423 | ".x3dv" => "model/x3d+vrml", 424 | ".x3d" => "model/x3d+xml", 425 | ".x3db" => "model/x3d+binary", 426 | ".manifest" => "text/cache-manifest", 427 | ".ics" => "text/calendar", 428 | ".icz" => "text/calendar", 429 | ".css" => "text/css", 430 | ".csv" => "text/csv", 431 | ".323" => "text/h323", 432 | ".html" => "text/html", 433 | ".htm" => "text/html", 434 | ".shtml" => "text/html", 435 | ".uls" => "text/iuls", 436 | ".mml" => "text/mathml", 437 | ".asc" => "text/plain", 438 | ".txt" => "text/plain", 439 | ".text" => "text/plain", 440 | ".pot" => "text/plain", 441 | ".brf" => "text/plain", 442 | ".rtx" => "text/richtext", 443 | ".sct" => "text/scriptlet", 444 | ".wsc" => "text/scriptlet", 445 | ".tm" => "text/texmacs", 446 | ".tsv" => "text/tab-separated-values", 447 | ".jad" => "text/vnd.sun.j2me.app-descriptor", 448 | ".wml" => "text/vnd.wap.wml", 449 | ".wmls" => "text/vnd.wap.wmlscript", 450 | ".bib" => "text/x-bibtex", 451 | ".boo" => "text/x-boo", 452 | ".h++" => "text/x-c++hdr", 453 | ".hpp" => "text/x-c++hdr", 454 | ".hxx" => "text/x-c++hdr", 455 | ".hh" => "text/x-c++hdr", 456 | ".c++" => "text/x-c++src", 457 | ".cpp" => "text/x-c++src", 458 | ".cxx" => "text/x-c++src", 459 | ".cc" => "text/x-c++src", 460 | ".h" => "text/x-chdr", 461 | ".htc" => "text/x-component", 462 | ".c" => "text/x-csrc", 463 | ".d" => "text/x-dsrc", 464 | ".diff" => "text/x-diff", 465 | ".patch" => "text/x-diff", 466 | ".hs" => "text/x-haskell", 467 | ".java" => "text/x-java", 468 | ".lhs" => "text/x-literate-haskell", 469 | ".moc" => "text/x-moc", 470 | ".p" => "text/x-pascal", 471 | ".pas" => "text/x-pascal", 472 | ".gcd" => "text/x-pcs-gcd", 473 | ".pl" => "text/x-perl", 474 | ".pm" => "text/x-perl", 475 | ".py" => "text/x-python", 476 | ".scala" => "text/x-scala", 477 | ".etx" => "text/x-setext", 478 | ".sfv" => "text/x-sfv", 479 | ".tk" => "text/x-tcl", 480 | ".tex" => "text/x-tex", 481 | ".ltx" => "text/x-tex", 482 | ".sty" => "text/x-tex", 483 | ".cls" => "text/x-tex", 484 | ".vcs" => "text/x-vcalendar", 485 | ".vcf" => "text/x-vcard", 486 | ".3gp" => "video/3gpp", 487 | ".axv" => "video/annodex", 488 | ".dl" => "video/dl", 489 | ".dif" => "video/dv", 490 | ".dv" => "video/dv", 491 | ".fli" => "video/fli", 492 | ".gl" => "video/gl", 493 | ".mpeg" => "video/mpeg", 494 | ".mpg" => "video/mpeg", 495 | ".mpe" => "video/mpeg", 496 | ".mp4" => "video/mp4", 497 | ".qt" => "video/quicktime", 498 | ".mov" => "video/quicktime", 499 | ".ogv" => "video/ogg", 500 | ".webm" => "video/webm", 501 | ".mxu" => "video/vnd.mpegurl", 502 | ".flv" => "video/x-flv", 503 | ".lsf" => "video/x-la-asf", 504 | ".lsx" => "video/x-la-asf", 505 | ".mng" => "video/x-mng", 506 | ".asf" => "video/x-ms-asf", 507 | ".asx" => "video/x-ms-asf", 508 | ".wm" => "video/x-ms-wm", 509 | ".wmv" => "video/x-ms-wmv", 510 | ".wmx" => "video/x-ms-wmx", 511 | ".wvx" => "video/x-ms-wvx", 512 | ".avi" => "video/x-msvideo", 513 | ".movie" => "video/x-sgi-movie", 514 | ".mpv" => "video/x-matroska", 515 | ".mkv" => "video/x-matroska", 516 | ".ice" => "x-conference/x-cooltalk", 517 | ".sisx" => "x-epoc/x-sisx-app", 518 | ".vrm" => "x-world/x-vrml", 519 | ) 520 | -------------------------------------------------------------------------------- /fabfile.py: -------------------------------------------------------------------------------- 1 | from fabric.context_managers import cd, hide, settings 2 | from fabric.api import * 3 | from fabric.contrib.files import exists, append, sed 4 | import os 5 | import sys 6 | 7 | env.roledefs['pre_cxx11'] = [ 8 | 'ubuntu@gitbuilder-ceph-deb-precise-amd64-basic.front.sepia.ceph.com', 9 | 'ubuntu@gitbuilder-ceph-deb-precise-amd64-notcmalloc.front.sepia.ceph.com', 10 | 'ubuntu@gitbuilder-ceph-deb-wheezy-amd64-basic.front.sepia.ceph.com', 11 | 'ubuntu@gitbuilder-ceph-rpm-centos6-5-amd64-basic.front.sepia.ceph.com', 12 | ] 13 | 14 | env.roledefs['gitbuilder_auto'] = [ 15 | 'ubuntu@gitbuilder-ceph-deb-jessie-amd64-basic.ovh.sepia.ceph.com', 16 | 'ubuntu@gitbuilder-ceph-deb-precise-amd64-basic.front.sepia.ceph.com', 17 | 'ubuntu@gitbuilder-ceph-deb-precise-amd64-notcmalloc.front.sepia.ceph.com', 18 | 'ubuntu@gitbuilder-ceph-deb-trusty-amd64-basic.front.sepia.ceph.com', 19 | 'ubuntu@gitbuilder-ceph-deb-trusty-amd64-notcmalloc.front.sepia.ceph.com', 20 | 'ubuntu@gitbuilder-ceph-deb-trusty-i386-basic.front.sepia.ceph.com', 21 | 'ubuntu@gitbuilder-ceph-deb-wheezy-amd64-basic.front.sepia.ceph.com', 22 | 'ubuntu@gitbuilder-ceph-tarball-trusty-amd64-basic.front.sepia.ceph.com', 23 | 'ubuntu@gitbuilder-ceph-tarball-trusty-amd64-cmake.ovh.sepia.ceph.com', 24 | 'ubuntu@gitbuilder-ceph-tarball-trusty-i386-basic.front.sepia.ceph.com', 25 | 'ubuntu@gitbuilder-ceph-deb-xenial-amd64-basic.front.sepia.ceph.com', 26 | ] 27 | 28 | env.roledefs['gitbuilder_ceph_rpm'] = [ 29 | 'ubuntu@gitbuilder-ceph-rpm-centos6-5-amd64-basic.front.sepia.ceph.com', 30 | 'ubuntu@gitbuilder-ceph-rpm-centos7-amd64-basic.ovh.sepia.ceph.com', 31 | 'ubuntu@gitbuilder-ceph-rpm-centos7-amd64-notcmalloc.ovh.sepia.ceph.com', 32 | 'ubuntu@gitbuilder-ceph-rpm-fedora22-amd64-basic.front.sepia.ceph.com', 33 | ] 34 | 35 | # kernels 36 | env.roledefs['gitbuilder_kernel_deb'] = [ 37 | 'ubuntu@gitbuilder-kernel-deb-precise-amd64-basic.front.sepia.ceph.com', 38 | 'ubuntu@gitbuilder-kernel-deb-precise-amd64-debug.front.sepia.ceph.com', 39 | ] 40 | env.roledefs['gitbuilder_kernel_rpm'] = [ 41 | 'ubuntu@gitbuilder-kernel-rpm-centos6-amd64-basic.front.sepia.ceph.com', 42 | 'ubuntu@gitbuilder-kernel-rpm-centos7-amd64-basic.front.sepia.ceph.com', 43 | 'ubuntu@gitbuilder-kernel-rpm-fedora20-amd64-basic.front.sepia.ceph.com', 44 | ] 45 | 46 | # special 47 | env.roledefs['gitbuilder_samba'] = [ 48 | 'ubuntu@gitbuilder-samba-deb-precise-amd64.front.sepia.ceph.com', 49 | ] 50 | 51 | env.roledefs['gitbuilder_hadoop'] = [ 52 | 'ubuntu@gitbuilder-precise-hadoop-amd64.front.sepia.ceph.com', 53 | ] 54 | 55 | env.roledefs['gitbuilder_apache_hadoop'] = [ 56 | 'ubuntu@gitbuilder-precise-apache-hadoop-amd64.front.sepia.ceph.com', 57 | ] 58 | 59 | 60 | ################# 61 | 62 | 63 | def _rpm_install(*packages): 64 | lsb = sudo("lsb_release -d") 65 | if '7.' in lsb: 66 | sudo("rpm -qa | grep epel-release || rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm") 67 | if '6.' in lsb: 68 | sudo("rpm -qa | grep epel-release || rpm -Uvh http://download.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm") 69 | # This will update to the newest release of the distro IE centos 6.3 to centos 6.5, etc... 70 | # sudo("yum --assumeyes --quiet update") 71 | sudo(' '.join( 72 | [ 73 | 'yum', 74 | '--quiet', 75 | '--assumeyes', 76 | 'install', 77 | ] 78 | + list(packages))) 79 | 80 | 81 | def _apt_add_testing_repo(branch): 82 | sudo('wget -q -O- https://raw.github.com/ceph/ceph/master/keys/autobuild.asc | sudo apt-key add -') 83 | sudo('echo deb http://gitbuilder.ceph.com/ceph-deb-$(lsb_release -sc)-x86_64-basic/ref/{branch} $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list'.format(branch=branch)) 84 | 85 | def _apt_install(*packages): 86 | sudo("apt-get update") 87 | sudo(' '.join( 88 | [ 89 | 'env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical', 90 | 'apt-get', 91 | '-q', 92 | '-y', 93 | '-o', 'Dpkg::Options::=--force-confnew', 94 | 'install', 95 | # '--no-install-recommends', 96 | # '--assume-yes', 97 | '--', 98 | ] 99 | + list(packages))) 100 | 101 | def _apt_reinstall_for_backports(*packages): 102 | if 'x86_64' not in env.host_string: 103 | return 104 | sudo("mkdir -p /srv/extras-backports") 105 | sudo("rm -f /srv/extras-backports/*") 106 | sudo("apt-get clean") 107 | sudo(' '.join( 108 | [ 109 | 'env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical', 110 | 'apt-get', 111 | '-q', 112 | '-y', 113 | '-o', 'Dpkg::Options::=--force-confnew', 114 | 'install', 115 | '--reinstall', 116 | # '--no-install-recommends', 117 | # '--assume-yes', 118 | '--', 119 | ] 120 | + list(packages))) 121 | debcache = [] 122 | for package in (list(packages)): 123 | debcache.append('/var/cache/apt/archives/{package}*'.format(package=package)) 124 | 125 | sudo(' '.join( 126 | [ 127 | 'cp', 128 | '-avf' 129 | ] 130 | + debcache + 131 | [ 132 | '/srv/extras-backports' 133 | ])) 134 | 135 | 136 | def _gem_install(*packages): 137 | sudo('gem install ' + ' '.join(list(packages))) 138 | 139 | def _rh_gitbuilder(flavor, git_repo, extra_remotes={}, extra_packages=[], ignore=[], branches_local_name='branches-local',branch_to_bundle='master'): 140 | """ 141 | extra_remotes will be fetch but not autobuilt. useful for tags. 142 | """ 143 | gitbuilder_commit='master' 144 | gitbuilder_origin='git://github.com/ceph/gitbuilder.git' 145 | 146 | sudo("initctl list|grep -q '^autobuild-ceph\s' && stop autobuild-ceph || /etc/init.d/autobuild-ceph stop || :") 147 | # 148 | # Install needed packages 149 | _rpm_install( 150 | 'ntp', 151 | 'ccache', 152 | 'git', 153 | 'logrotate', 154 | 'rsync', 155 | 'pkgconfig', 156 | 'tar', 157 | *extra_packages 158 | ) 159 | # 160 | # Create autobuild-ceph user 161 | with settings(warn_only=True): 162 | sudo( 163 | ' '.join([ 164 | 'adduser', 165 | '--system', 166 | '--home', '/home/autobuild-ceph', 167 | '--comment', '"Ceph autobuild"', 168 | '--user-group', 169 | #'--disabled-password', 170 | #'--disabled-login', 171 | 'autobuild-ceph', 172 | ]), 173 | ) 174 | sudo('mkdir -p /home/autobuild-ceph && chown autobuild-ceph /home/autobuild-ceph') 175 | with cd('/home/autobuild-ceph'): 176 | sudo('echo "%_find_debuginfo_dwz_opts %{nil}" > .rpmmacros') 177 | sudo('lsb_release -d -s | grep CentOS | grep -q release\ 7 && echo "%dist .el7" >> .rpmmacros') 178 | 179 | sudo('install -d -m0755 --owner=root --group=root /srv/autobuild-ceph') 180 | local('git bundle create bundle refs/heads/{branch_to_bundle}'.format(branch_to_bundle=branch_to_bundle)) 181 | put('bundle', 'bundle') 182 | local('rm -f bundle') 183 | with cd('/srv/autobuild-ceph'): 184 | sudo('git init') 185 | sudo('test -d /home/ubuntu || ln -sf /home/centos /home/ubuntu') 186 | sudo('git pull /home/ubuntu/bundle {branch_to_bundle}'.format(branch_to_bundle=branch_to_bundle)) 187 | sudo('ln -sf build-{flavor}.sh build.sh'.format(flavor=flavor)) 188 | brand_new = False 189 | if not exists('gitbuilder.git'): 190 | brand_new = True 191 | sudo('rm -rf gitbuilder.git.tmp') 192 | sudo('git clone %s gitbuilder.git.tmp' % gitbuilder_origin) 193 | with cd('gitbuilder.git.tmp'): 194 | sudo('git checkout %s' % gitbuilder_commit) 195 | sudo('ln -s ../build.sh ./') 196 | if branches_local_name != 'branches-local': 197 | sudo('mv ./branches-local ./branches-local-orig') 198 | sudo('ln -s ../{branches_local_name} ./branches-local'.format(branches_local_name=branches_local_name)) 199 | sudo('chown -R autobuild-ceph:autobuild-ceph out') 200 | sudo('mv gitbuilder.git.tmp gitbuilder.git') 201 | with cd('gitbuilder.git'): 202 | if not exists('build'): 203 | sudo('git clone {git_repo} build'.format(git_repo=git_repo)) 204 | sudo('chown -R autobuild-ceph:autobuild-ceph build') 205 | sudo('git remote set-url origin %s' % gitbuilder_origin) 206 | sudo('git fetch origin') 207 | sudo('git reset --hard %s' % gitbuilder_commit) 208 | 209 | with cd('gitbuilder.git/build'): 210 | sudo( 211 | 'git remote set-url origin {url}'.format( 212 | url=git_repo, 213 | ), 214 | user='autobuild-ceph', 215 | ) 216 | for name, url in extra_remotes.items(): 217 | sudo( 218 | 'git remote set-url {name} {url} || git remote add {name} {url}'.format( 219 | name=name, 220 | url=url, 221 | ), 222 | user='autobuild-ceph', 223 | ) 224 | sudo('git config remote.{name}.tagopt true'.format(name=name), 225 | user='autobuild-ceph') 226 | sudo('git config remote.origin.tagopt true', user='autobuild-ceph') 227 | if brand_new: 228 | sudo('/srv/autobuild-ceph/mark_all_as_pass.sh', 229 | user='autobuild-ceph') 230 | with cd('/srv'): 231 | if not exists('gnupg'): 232 | sudo('mkdir gnupg') 233 | sudo('chown autobuild-ceph:autobuild-ceph gnupg ; chmod 700 gnupg') 234 | with cd('gnupg'): 235 | if not exists('pubring.gpg'): 236 | # put doesn't honor cd() for some reason 237 | put('gnupg/pubring.gpg') 238 | put('gnupg/secring.gpg') 239 | put('trustdb.gpg') 240 | sudo("mv /home/ubuntu/*.gpg ./") 241 | sudo('chown autobuild-ceph:autobuild-ceph pubring.gpg secring.gpg trustdb.gpg') 242 | sudo('chmod 600 pubring.gpg secring.gpg trustdb.gpg') 243 | with cd('/srv/autobuild-ceph'): 244 | if ignore: 245 | sudo('install -d -m0755 --owner=autobuild-ceph --group=autobuild-ceph gitbuilder.git/out/ignore') 246 | for sha in ignore: 247 | sudo('touch gitbuilder.git/out/ignore/{sha}'.format(sha=sha)) 248 | sudo('install -d -m0755 --owner=autobuild-ceph --group=autobuild-ceph ccache') 249 | sudo('install -d -m0755 logs') 250 | 251 | sudo('install --owner=root --group=root -m0755 autobuild-ceph.init /etc/init.d/autobuild-ceph') 252 | run('rm bundle') 253 | sudo('chown -R autobuild-ceph:autobuild-ceph /srv/autobuild-ceph') 254 | install_git() 255 | 256 | def _gitbuilder(flavor, git_repo, extra_remotes={}, extra_packages=[], ignore=[], branches_local_name='branches-local', branch_to_bundle='master'): 257 | """ 258 | extra_remotes will be fetch but not autobuilt. useful for tags. 259 | """ 260 | gitbuilder_commit='master' 261 | gitbuilder_origin='git://github.com/ceph/gitbuilder.git' 262 | 263 | # shut down old instance, it exists 264 | sudo("initctl list|grep -q '^autobuild-ceph\s' && stop autobuild-ceph || /etc/init.d/autobuild-ceph stop ; systemctl stop autobuild-ceph || :") 265 | 266 | # sun-java6 is in partner repo. accept license. 267 | #sudo("echo 'deb http://archive.canonical.com/ubuntu maverick partner' > /etc/apt/sources.list.d/partner.list") 268 | #sudo("echo 'sun-java5-jdk shared/accepted-sun-dlj-v1-1 boolean true' | debconf-set-selections") 269 | 270 | _apt_install( 271 | 'ntp', 272 | 'build-essential', 273 | 'ccache', 274 | 'git', 275 | 'logrotate', 276 | # 'sun-java6-jdk', 277 | 'default-jdk', 278 | 'javahelper', 279 | 'rsync', 280 | 'pbuilder', 281 | *extra_packages 282 | ) 283 | 284 | # Reinstall for packport deps. 285 | _apt_reinstall_for_backports( 286 | 'libleveldb1', 287 | 'libcurl3-gnutls' 288 | ) 289 | 290 | sudo( 291 | ' '.join([ 292 | 'adduser', 293 | '--system', 294 | '--home', '/nonexistent', 295 | '--gecos', '"Ceph autobuild"', 296 | '--group', 297 | '--disabled-password', 298 | '--disabled-login', 299 | 'autobuild-ceph', 300 | ]), 301 | ) 302 | sudo('install -d -m0755 --owner=root --group=root /srv/autobuild-ceph') 303 | local('git bundle create bundle refs/heads/{branch_to_bundle}'.format(branch_to_bundle=branch_to_bundle)) 304 | put('bundle', 'bundle') 305 | local('rm -f bundle') 306 | with cd('/srv/autobuild-ceph'): 307 | sudo('git init') 308 | # blarg 309 | sudo('test -d /home/ubuntu || ln -sf /home/debian /home/ubuntu') 310 | sudo('git pull /home/ubuntu/bundle {branch_to_bundle}'.format(branch_to_bundle=branch_to_bundle)) 311 | sudo('ln -sf build-{flavor}.sh build.sh'.format(flavor=flavor)) 312 | brand_new = False 313 | if not exists('gitbuilder.git'): 314 | brand_new = True 315 | sudo('rm -rf gitbuilder.git.tmp') 316 | sudo('git clone %s gitbuilder.git.tmp' % gitbuilder_origin) 317 | with cd('gitbuilder.git.tmp'): 318 | sudo('git checkout %s' % gitbuilder_commit) 319 | sudo('ln -s ../build.sh ./') 320 | if branches_local_name != 'branches-local': 321 | sudo('mv ../branches-local ../branches-local-orig') 322 | sudo('ln -s ../branches-local {branches_local_name}'.format(branches_local_name=branches_local_name)) 323 | sudo('chown -R autobuild-ceph:autobuild-ceph out') 324 | sudo('mv gitbuilder.git.tmp gitbuilder.git') 325 | with cd('gitbuilder.git'): 326 | if not exists('build'): 327 | sudo('git clone {git_repo} build'.format(git_repo=git_repo)) 328 | sudo('chown -R autobuild-ceph:autobuild-ceph build') 329 | sudo('git remote set-url origin %s' % gitbuilder_origin) 330 | sudo('git fetch origin') 331 | sudo('git reset --hard %s' % gitbuilder_commit) 332 | with cd('gitbuilder.git/build'): 333 | sudo( 334 | 'git remote set-url origin {url}'.format( 335 | url=git_repo, 336 | ), 337 | user='autobuild-ceph', 338 | ) 339 | for name, url in extra_remotes.items(): 340 | sudo( 341 | 'git remote set-url {name} {url} || git remote add {name} {url}'.format( 342 | name=name, 343 | url=url, 344 | ), 345 | user='autobuild-ceph', 346 | ) 347 | sudo('git config remote.{name}.tagopt true'.format(name=name), 348 | user='autobuild-ceph') 349 | sudo('git config remote.origin.tagopt true', user='autobuild-ceph') 350 | if brand_new: 351 | sudo('/srv/autobuild-ceph/mark_all_as_pass.sh', 352 | user='autobuild-ceph') 353 | if ignore: 354 | sudo('install -d -m0755 --owner=autobuild-ceph --group=autobuild-ceph gitbuilder.git/out/ignore') 355 | for sha in ignore: 356 | sudo('touch gitbuilder.git/out/ignore/{sha}'.format(sha=sha)) 357 | sudo('install -d -m0755 --owner=autobuild-ceph --group=autobuild-ceph ccache') 358 | sudo('install -d -m0755 logs') 359 | 360 | sudo('install --owner=root --group=root -m0644 autobuild-ceph.conf /etc/init/autobuild-ceph.conf ; install --owner=root --group=root -m0755 autobuild-ceph.init /etc/init.d/autobuild-ceph ; exit 0') 361 | run('rm bundle') 362 | sudo('chown -R autobuild-ceph:autobuild-ceph /srv/autobuild-ceph') 363 | install_git() 364 | 365 | def _deb_install_extras(): 366 | with cd('/srv'): 367 | if not exists('gnupg'): 368 | sudo('mkdir gnupg') 369 | if not exists('aptcache'): 370 | sudo('mkdir aptcache ; chown autobuild-ceph:autobuild-ceph aptcache') 371 | 372 | sudo('chown autobuild-ceph:autobuild-ceph gnupg ; chmod 700 gnupg') 373 | with cd('gnupg'): 374 | if not exists('pubring.gpg', use_sudo=True): 375 | # put doesn't honor cd() for some reason 376 | put('gnupg/pubring.gpg', use_sudo=True) 377 | put('gnupg/secring.gpg', use_sudo=True) 378 | sudo("mv /home/ubuntu/*.gpg ./") 379 | sudo('chown autobuild-ceph:autobuild-ceph pubring.gpg secring.gpg') 380 | sudo('chmod 600 pubring.gpg secring.gpg') 381 | if not exists('ceph-build'): 382 | sudo('git clone https://github.com/ceph/ceph-build.git') 383 | with cd('ceph-build'): 384 | sudo('git pull') 385 | sudo('grep -q autobuild-ceph /etc/sudoers || echo "autobuild-ceph ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers') 386 | 387 | 388 | def _kernel_deps(): 389 | _apt_install( 390 | # kernel tools 391 | 'bison', 392 | 'flex', 393 | 'asciidoc', 394 | 'libdw-dev', 395 | 'libnewt-dev', 396 | 'xmlto', 397 | 'libgtk2.0-dev', 398 | 'libunwind-setjmp0-dev', 399 | 'libunwind7-dev', 400 | 'libaudit-dev', 401 | 'binutils-dev', 402 | 'python-dev', 403 | ) 404 | 405 | def _kernel_rpm_deps(): 406 | _rpm_install( 407 | 'pkgconfig', 408 | 'automake', 409 | 'autoconf', 410 | 'make', 411 | 'libtool', 412 | 'libaio', 413 | 'libaio-devel', 414 | 'libedit', 415 | 'libedit-devel', 416 | 'libuuid', 417 | 'libuuid-devel', 418 | 'libblkid', 419 | 'libblkid-devel', 420 | 'gcc-c++', 421 | 'expat', 422 | 'expat-devel', 423 | 'sharutils', 424 | 'gnupg', 425 | 'expect', 426 | 'yasm', 427 | 'rpm-sign', 428 | 'createrepo', 429 | 'rpmdevtools', 430 | 'yum-utils', 431 | 'bc', 432 | 'zlib-devel' 433 | ) 434 | 435 | @roles('gitbuilder_kernel_deb') 436 | def gitbuilder_kernel_deb(): 437 | _kernel_deps() 438 | _gitbuilder( 439 | flavor='auto', 440 | git_repo='https://github.com/ceph/ceph-client.git', 441 | extra_remotes=dict( 442 | korg='git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git', 443 | ), 444 | extra_packages=[ 445 | 'fakeroot', 446 | 'reprepro', 447 | ], 448 | ignore=[ 449 | 'fbeb94b65cf784ed8bf852131e28c9fb5c4c760f', 450 | ], 451 | ) 452 | _sync_to_gitbuilder_from_hostname() 453 | sudo('start autobuild-ceph || /etc/init.d/autobuild-ceph start ; systemctl enable autobuild-ceph || true ; systemctl start autobuild-ceph || true') 454 | 455 | 456 | 457 | @roles('gitbuilder_kernel_rpm') 458 | def gitbuilder_kernel_rpm(): 459 | _kernel_rpm_deps() 460 | _rh_gitbuilder( 461 | flavor='kernel-rpm', 462 | git_repo='https://github.com/ceph/ceph-client.git', 463 | extra_remotes=dict( 464 | korg='git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git', 465 | ), 466 | extra_packages=[ 467 | 'fakeroot', 468 | 'reprepro', 469 | ], 470 | ignore=[ 471 | 'fbeb94b65cf784ed8bf852131e28c9fb5c4c760f', 472 | ], 473 | ) 474 | _sync_to_gitbuilder('kernel','rpm','basic') 475 | sudo('start autobuild-ceph || /etc/init.d/autobuild-ceph start ; systemctl enable autobuild-ceph || true ; systemctl start autobuild-ceph || true') 476 | 477 | 478 | def _hadoop_deps(): 479 | #_apt_add_testing_repo('master') 480 | _apt_install( 481 | 'openjdk-6-jdk', 482 | 'ant', 483 | 'automake', 484 | 'libtool', 485 | ) 486 | 487 | def _samba_deps(): 488 | _apt_add_testing_repo('master') 489 | _apt_install( 490 | 'build-essential', 491 | 'libacl1-dev', 492 | 'libattr1-dev', 493 | 'libblkid-dev', 494 | 'libgnutls-dev', 495 | 'libreadline-dev', 496 | 'python-dev', 497 | 'python-dnspython', 498 | 'gdb', 499 | 'pkg-config', 500 | 'libpopt-dev', 501 | 'libldap2-dev', 502 | 'dnsutils', 503 | 'libbsd-dev', 504 | 'attr', 505 | 'krb5-user', 506 | 'ruby1.8-dev', 507 | 'rubygems', 508 | 'libcephfs-dev', 509 | 'libncurses-dev', 510 | 'dpkg-sig', 511 | ) 512 | 513 | _gem_install('fpm') 514 | 515 | @roles('gitbuilder_samba') 516 | def gitbuilder_samba(): 517 | _samba_deps() 518 | _gitbuilder( 519 | flavor='samba-deb', 520 | git_repo='git://apt-mirror.front.sepia.ceph.com/samba.git', 521 | extra_packages=[ 522 | 'fakeroot', 523 | 'reprepro', 524 | ], 525 | branches_local_name='branches-local-samba', 526 | ) 527 | _deb_install_extras() 528 | sudo('start autobuild-ceph || /etc/init.d/autobuild-ceph start ; systemctl enable autobuild-ceph || true ; systemctl start autobuild-ceph || true') 529 | _sync_to_gitbuilder('samba', 'deb', 'basic') 530 | 531 | @roles('gitbuilder_hadoop') 532 | def gitbuilder_hadoop(): 533 | _hadoop_deps() 534 | _gitbuilder( 535 | flavor='hadoop', 536 | git_repo='https://github.com/ceph/hadoop-common.git', 537 | extra_packages=[ 538 | 'fakeroot', 539 | 'reprepro', 540 | ], 541 | branches_local_name='branches-local-hadoop', 542 | ) 543 | _sync_to_gitbuilder('hadoop', 'jar', 'basic') 544 | sudo('start autobuild-ceph || /etc/init.d/autobuild-ceph start ; systemctl enable autobuild-ceph || true ; systemctl start autobuild-ceph || true') 545 | 546 | @roles('gitbuilder_apache_hadoop') 547 | def gitbuilder_apache_hadoop(): 548 | _hadoop_deps() 549 | _gitbuilder( 550 | flavor='apache-hadoop', 551 | git_repo='git://git.apache.org/hadoop-common.git', 552 | extra_packages=[ 553 | 'fakeroot', 554 | 'reprepro', 555 | ], 556 | branches_local_name='branches-local-apache-hadoop', 557 | ) 558 | _sync_to_gitbuilder('apache-hadoop', 'jar', 'basic') 559 | sudo('start autobuild-ceph || /etc/init.d/autobuild-ceph start ; systemctl enable autobuild-ceph || true ; systemctl start autobuild-ceph || true') 560 | 561 | @roles('gitbuilder_ceph') 562 | def gitbuilder_ceph(): 563 | _gitbuilder_ceph('ceph') 564 | _sync_to_gitbuilder('ceph', 'tarball', 'basic') 565 | 566 | def _gitbuilder_ceph(flavor): 567 | _gitbuilder( 568 | flavor=flavor, 569 | git_repo='https://github.com/ceph/ceph.git', 570 | extra_remotes=dict( 571 | ci='https://github.com/ceph/ceph-ci.git', 572 | ), 573 | extra_packages=[ 574 | 'automake', 575 | 'libtool', 576 | 'pkg-config', 577 | 'libboost-dev', 578 | 'libboost-thread-dev', 579 | 'libedit-dev', 580 | 'libssl-dev', 581 | 'libcrypto++-dev', 582 | 'libgtkmm-2.4-dev', 583 | 'xfslibs-dev', 584 | 'libfuse-dev', 585 | 'libexpat1-dev', 586 | 'libfcgi-dev', 587 | 'libcurl4-gnutls-dev', 588 | 'libatomic-ops-dev', 589 | # 'libgoogle-perftools-dev', 590 | 'libkeyutils-dev', 591 | 'uuid-dev', 592 | 'libblkid-dev', 593 | 'libbz2-dev', 594 | 'libudev-dev', 595 | 'python-pip', 596 | 'python-requests', 597 | 'python-virtualenv', 598 | 'python-argparse', 599 | 'libaio-dev', 600 | 'libxml2-dev', 601 | 'libnss3-dev', 602 | 'junit4', 603 | 'xmlstarlet', 604 | 'yasm', 605 | 'python-nose', 606 | 'cryptsetup-bin', 607 | # for kernel build, perf etc 608 | 'flex', 609 | 'bison', 610 | 'libdw-dev', 611 | 'binutils-dev', 612 | 'libnewt-dev', 613 | 'libsnappy-dev', 614 | ], 615 | ) 616 | sudo('start autobuild-ceph || /etc/init.d/autobuild-ceph start ; systemctl enable autobuild-ceph || true ; systemctl start autobuild-ceph || true') 617 | 618 | def _deb_builder(git_url, flavor, extra_remotes={}): 619 | _gitbuilder( 620 | flavor=flavor, 621 | git_repo=git_url, 622 | extra_remotes=extra_remotes, 623 | extra_packages=[ 624 | 'automake', 625 | 'libtool', 626 | 'pkg-config', 627 | 'libboost-dev', 628 | 'libboost-thread-dev', 629 | 'libboost-program-options-dev', 630 | 'libedit-dev', 631 | 'libssl-dev', 632 | 'libcrypto++-dev', 633 | 'libgtkmm-2.4-dev', 634 | 'xfslibs-dev', 635 | 'libfuse-dev', 636 | 'libexpat1-dev', 637 | 'libfcgi-dev', 638 | 'libcurl4-gnutls-dev', 639 | 'libatomic-ops-dev', 640 | # 'libgoogle-perftools-dev', 641 | 'libkeyutils-dev', 642 | 'uuid-dev', 643 | 'uuid-runtime', 644 | 'libblkid-dev', 645 | 'libbz2-dev', 646 | 'libudev-dev', 647 | 'libaio-dev', 648 | 'libxml2-dev', 649 | 'libnss3-dev', 650 | 'xmlstarlet', 651 | 'python-pip', 652 | 'python-requests', 653 | 'python-virtualenv', 654 | 'python-argparse', 655 | 'python-sphinx', 656 | 'pbuilder', 657 | 'gnupg', 658 | 'devscripts', 659 | 'lintian', 660 | 'flex', 'byacc', # collectd 661 | 'debhelper', 662 | 'reprepro', 663 | 'fakeroot', 664 | 'junit4', 665 | 'sharutils', 666 | 'libdistro-info-perl', # needed by raring 667 | 'libboost-system-dev', 668 | 'libleveldb-dev', 669 | 'yasm', 670 | 'python-nose', 671 | 'libsnappy-dev', 672 | ], 673 | ) 674 | _deb_install_extras() 675 | 676 | @roles('gitbuilder_auto') 677 | def gitbuilder_auto(): 678 | _deb_builder('https://github.com/ceph/ceph.git', 'auto', 679 | extra_remotes=dict( 680 | ci='https://github.com/ceph/ceph-ci.git' 681 | )) 682 | sudo('start autobuild-ceph || /etc/init.d/autobuild-ceph start ; systemctl enable autobuild-ceph || true ; systemctl start autobuild-ceph || true') 683 | _sync_to_gitbuilder_from_hostname() 684 | 685 | @roles('gitbuilder_ceph_rpm') 686 | def gitbuilder_ceph_rpm(): 687 | _gitbuilder_ceph_rpm('https://github.com/ceph/ceph.git', 'auto', 688 | extra_remotes=dict( 689 | ci='https://github.com/ceph/ceph-ci.git' 690 | )) 691 | hostname = run('hostname -s') 692 | flavor = hostname.split('-')[-1] 693 | _sync_to_gitbuilder('ceph', 'rpm', flavor) 694 | 695 | def _gitbuilder_ceph_rpm(url, flavor, extra_remotes={}): 696 | if '6-' in run('hostname -s'): 697 | sphinx = 'python-sphinx10' 698 | else: 699 | sphinx = 'python-sphinx' 700 | _rh_gitbuilder( 701 | flavor=flavor, 702 | git_repo=url, 703 | extra_remotes=extra_remotes, 704 | extra_packages=[ 705 | 'pkgconfig', 706 | 'automake', 707 | 'autoconf', 708 | 'make', 709 | 'libtool', 710 | 'libaio', 711 | 'libaio-devel', 712 | 'libedit', 713 | 'libedit-devel', 714 | 'libuuid', 715 | 'libuuid-devel', 716 | 'libblkid', 717 | 'libblkid-devel', 718 | 'bzip2-devel', 719 | 'fcgi', 720 | 'fcgi-devel', 721 | 'xfsprogs', 722 | 'xfsprogs-devel', 723 | 'fuse', 724 | 'fuse-libs', 725 | 'fuse-devel', 726 | 'gperftools-devel', 727 | 'mod_fcgid', 728 | 'keyutils-libs-devel', 729 | 'cryptopp-devel', 730 | 'gcc-c++', 731 | 'expat', 732 | 'expat-devel', 733 | 'libatomic_ops-devel', 734 | 'boost', 735 | 'boost-devel', 736 | 'boost-program-options', 737 | 'libcurl', 738 | 'libcurl-devel', 739 | 'rpm-build', 740 | 'libxml2-devel', 741 | 'xmlstarlet', 742 | 'nss-devel', 743 | 'gtkmm24', 744 | 'gtkmm24-devel', 745 | # 'junit4', 746 | 'sharutils', 747 | 'gnupg', 748 | 'expect', 749 | 'yasm', 750 | 'python-nose', 751 | 'hdparm', 752 | 'rpm-sign', 753 | 'createrepo', 754 | 'leveldb-devel', 755 | 'snappy-devel', 756 | 'wget', 757 | 'lighttpd', 758 | 'zlib-devel', 759 | 'python-requests', 760 | 'python-virtualenv', 761 | 'python-argparse', 762 | sphinx, 763 | 'lttng-ust-devel', 764 | 'libbabeltrace-devel', 765 | 'cryptsetup', 766 | ] 767 | ) 768 | with cd('/srv/autobuild-ceph'): 769 | sudo('echo centos6 > dists') 770 | sudo('start autobuild-ceph || /etc/init.d/autobuild-ceph start ; systemctl enable autobuild-ceph || true ; systemctl start autobuild-ceph || true') 771 | 772 | 773 | def _sync_to_gitbuilder(package, format, flavor): 774 | dist_or_codename = '`lsb_release -s -c`' 775 | if format == 'rpm': 776 | dist_or_codename = '`lsb_release -s -i | tr A-Z a-z``lsb_release -s -r | sed -s "s;\..*;;g"`' 777 | with cd('/srv/autobuild-ceph'): 778 | # fugliness 779 | sudo("echo gitbuilder@gitbuilder.ceph.com:gitbuilder.ceph.com/{package}-{format}-{dist_or_codename}-`uname -m`-{flavor} > rsync-target".format( 780 | package=package, 781 | format=format, 782 | dist_or_codename=dist_or_codename, 783 | flavor=flavor)) 784 | sudo('sed -i "s;redhatenterpriseserver;rhel;g" rsync-target') 785 | _sync_rsync_keys() 786 | 787 | def _sync_rsync_keys(): 788 | if not exists('rsync-key'): 789 | if not os.path.exists('rsync-key'): 790 | print >> sys.stderr, 'Required rsync keys to gitbuilder.ceph.com missing!' 791 | sys.exit(1) 792 | # for whatever reason, put doesn't seem to honor cd and use_sudo fails 793 | put("rsync-key") 794 | put("rsync-key.pub") 795 | sudo("mv /home/ubuntu/rsync-key* ./") 796 | sudo("chmod 600 rsync-key* ; chown autobuild-ceph.autobuild-ceph rsync-key*") 797 | 798 | def _sync_to_gitbuilder_from_hostname(): 799 | with cd('/srv/autobuild-ceph'): 800 | # fugliness 801 | sudo("echo gitbuilder@gitbuilder.ceph.com:gitbuilder.ceph.com/`hostname | cut --delimiter=- -f 2`-`hostname | cut --delimiter=- -f 3`-`lsb_release -s -c`-`uname -m`-`hostname | cut --delimiter=- -f 6` > rsync-target") 802 | _sync_rsync_keys() 803 | 804 | @roles('gitbuilder_modfastcgi_deb_precise') 805 | def gitbuilder_modfastcgi_deb_precise(): 806 | _deb_builder('git://ceph.newdream.net/git/libapache-mod-fastcgi-2.4.7-0910052141.git', 'deb') 807 | with cd('/srv/autobuild-ceph'): 808 | sudo('echo precise > dists') 809 | sudo('echo libapache-mod-fastcgi > pkgname') 810 | _sync_to_gitbuilder('libapache-mod-fastcgi','deb','basic') 811 | 812 | @roles('gitbuilder_kernel_deb', 813 | 'gitbuilder_kernel_rpm', 814 | 'gitbuilder_samba', 815 | 'gitbuilder_hadoop' 816 | ) 817 | def gitbuilder_serve(): 818 | # kill any remaining thttpd's in favor of lighttpd. Do this before 819 | # installing lighttpd so that lighttpd can start without errors 820 | # (albeit with the default config) 821 | 822 | sudo('/etc/init.d/thttpd stop || true') 823 | 824 | _apt_install( 825 | 'lighttpd', 826 | ) 827 | 828 | put('lighttpd.conf', '/tmp/lighttpd.conf') 829 | 830 | with settings(hide('warnings'), warn_only = True): 831 | same = sudo('diff -q /etc/lighttpd/lighttpd.conf /tmp/lighttpd.conf') 832 | if same.succeeded == False: 833 | sudo('/etc/init.d/lighttpd stop') 834 | sudo('mv /etc/lighttpd/lighttpd.conf /etc/lighttpd.orig') 835 | sudo('mv /tmp/lighttpd.conf /etc/lighttpd/lighttpd.conf') 836 | sudo('chown -R autobuild-ceph:autobuild-ceph /var/log/lighttpd') 837 | sudo('/etc/init.d/lighttpd start') 838 | else: 839 | sudo('rm /tmp/lighttpd.conf') 840 | sudo('/etc/init.d/lighttpd start') 841 | 842 | @roles('gitbuilder_ceph_rpm', 843 | 'gitbuilder_kernel_rpm' 844 | ) 845 | def gitbuilder_serve_rpm(): 846 | # kill any remaining thttpd's in favor of lighttpd. Do this before 847 | # installing lighttpd so that lighttpd can start without errors 848 | # (albeit with the default config) 849 | 850 | _rpm_install( 851 | 'lighttpd', 852 | ) 853 | 854 | put('lighttpd.conf', '/tmp/lighttpd.conf') 855 | 856 | with settings(hide('warnings'), warn_only = True): 857 | same = sudo('diff -q /etc/lighttpd/lighttpd.conf /tmp/lighttpd.conf') 858 | if same.succeeded == False: 859 | sudo('/etc/init.d/lighttpd stop') 860 | sudo('systemctl stop lighttpd') 861 | sudo('mv /etc/lighttpd/lighttpd.conf /etc/lighttpd.orig') 862 | sudo('mv /tmp/lighttpd.conf /etc/lighttpd/lighttpd.conf') 863 | sudo('chown -R autobuild-ceph:autobuild-ceph /var/log/lighttpd') 864 | sudo('/etc/init.d/lighttpd start') 865 | sudo('systemctl start lighttpd') 866 | sudo('systemctl enable lighttpd') 867 | sudo('chkconfig --add lighttpd') 868 | else: 869 | sudo('chown -R autobuild-ceph:autobuild-ceph /var/log/lighttpd') 870 | sudo('rm /tmp/lighttpd.conf') 871 | sudo('/etc/init.d/lighttpd start') 872 | sudo('systemctl start lighttpd') 873 | 874 | 875 | @roles('gitbuilder_ceph', 876 | 'gitbuilder_ceph_notcmalloc', 877 | 'gitbuilder_kernel_deb', 878 | 'gitbuilder_kernel_rpm', 879 | 'gitbuilder_ceph_rpm', 880 | 'gitbuilder_samba', 881 | 'gitbuilder_hadoop', 882 | ) 883 | def authorize_ssh_keys(): 884 | keyfile = '.ssh/authorized_keys' 885 | keydir = os.path.join( 886 | os.path.dirname(__file__), 887 | 'ssh-keys', 888 | ) 889 | keys = [] 890 | for filename in os.listdir(keydir): 891 | if filename.startswith('.'): 892 | continue 893 | if not filename.endswith('.pub'): 894 | continue 895 | keys.extend(line.rstrip('\n') for line in file(os.path.join(keydir, filename))) 896 | with hide('running'): 897 | for key in keys: 898 | run('grep -q "%s" %s || echo "%s" >> %s' % (key, keyfile, key, keyfile)) 899 | 900 | @roles('pre_cxx11') 901 | def install_filter_branches(): 902 | """install a filter-branches file so old builders will only look at branches before Infernalis 903 | """ 904 | filter_branches_path = '/srv/autobuild-ceph/filter-branches' 905 | # only keep the branches with following keywords in it. 906 | pre_cxx11_branches = ['hammer', 907 | 'giant', 908 | 'firefly', 909 | 'emperor', 910 | 'dumpling'] 911 | sudo('touch {filter}'.format(filter=filter_branches_path)) 912 | append(filter_branches_path, 913 | '\n'.join(pre_cxx11_branches), 914 | use_sudo=True) 915 | 916 | 917 | def install_git(): 918 | # Install newer git from source 919 | # for bug fixes. 920 | 921 | git_version = '1.8.5.3' 922 | if not exists('/srv/git/bin'): 923 | sudo ('mkdir -p /srv/git/src') 924 | with cd('/srv/git/src'): 925 | sudo('wget -O /srv/git/src/git-{version}.tar.gz http://ceph.com/qa/git-{version}.tar.gz'.format(version=git_version)) 926 | sudo('tar xzf /srv/git/src/git-{version}.tar.gz'.format(version=git_version)) 927 | sudo('rm -f /srv/git/src/git-{version}.tar.gz'.format(version=git_version)) 928 | with cd('/srv/git/src/git-{version}'.format(version=git_version)): 929 | sudo('./configure --prefix=/srv/git/') 930 | sudo('make -j8') 931 | sudo('make install') 932 | sudo('rm -Rf /srv/src/git-{version}'.format(version=git_version)) 933 | --------------------------------------------------------------------------------