├── .dockerignore ├── .gitignore ├── Dockerfile.builder ├── Dockerfile.cacher ├── README.md ├── benchmarks.py ├── docker-compose.yml ├── nginx.conf ├── plot_results.py └── requirements.txt /.dockerignore: -------------------------------------------------------------------------------- 1 | cacher_root -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | cacher_root 3 | timings_*.json 4 | /*.png -------------------------------------------------------------------------------- /Dockerfile.builder: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | 3 | ENV BAZEL_VERSION 0.5.2 4 | 5 | RUN apt-get update \ 6 | && apt-get install -y --no-install-recommends \ 7 | build-essential \ 8 | ca-certificates \ 9 | curl \ 10 | wget \ 11 | git \ 12 | && rm -rf /var/lib/apt/lists/* 13 | 14 | # JDK for Bazel 15 | RUN apt-get update \ 16 | && apt-get install -y software-properties-common \ 17 | && apt-add-repository ppa:webupd8team/java --yes \ 18 | && apt-get update \ 19 | && echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections \ 20 | && apt-get install -y --no-install-recommends \ 21 | oracle-java8-installer \ 22 | unzip \ 23 | && rm -rf /var/lib/apt/lists/* 24 | 25 | # Bazel 26 | RUN cd /tmp \ 27 | && curl -L -o install-bazel.sh https://github.com/bazelbuild/bazel/releases/download/0.5.2/bazel-$BAZEL_VERSION-without-jdk-installer-linux-x86_64.sh \ 28 | && bash install-bazel.sh \ 29 | && /usr/local/bin/bazel --batch version \ 30 | && rm -rf install-bazel.sh 31 | 32 | # drake 33 | RUN apt-get update \ 34 | && apt-get install -y --no-install-recommends \ 35 | software-properties-common \ 36 | lsb-core \ 37 | && wget -q -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add - \ 38 | && add-apt-repository -y "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-3.9 main" \ 39 | && add-apt-repository ppa:george-edison55/cmake-3.x \ 40 | && apt-get update 41 | 42 | 43 | RUN apt-get install -y --no-install-recommends \ 44 | clang-3.9 gfortran cmake \ 45 | autoconf automake bison doxygen freeglut3-dev git graphviz \ 46 | libboost-dev libboost-system-dev libgtk2.0-dev libhtml-form-perl \ 47 | libjpeg-dev libmpfr-dev libpng-dev libterm-readkey-perl libtinyxml-dev \ 48 | libtool libvtk5-dev libwww-perl make ninja-build \ 49 | patchutils perl pkg-config \ 50 | python-bs4 python-dev python-gtk2 python-html5lib python-numpy \ 51 | python-pip python-sphinx python-yaml unzip valgrind zip \ 52 | && rm -rf /var/lib/apt/lists/* 53 | 54 | RUN curl -L -o /usr/local/bin/cloc \ 55 | https://github.com/AlDanial/cloc/releases/download/v1.72/cloc-1.72.pl \ 56 | && chmod +x /usr/local/bin/cloc 57 | 58 | WORKDIR /src 59 | 60 | # Python dependencies for test scripts 61 | COPY requirements.txt /src/ 62 | RUN pip install --upgrade pip \ 63 | && pip install setuptools \ 64 | && pip install -r requirements.txt 65 | 66 | ENV BENCHMARK_GIT_REPO_PATH /code 67 | ENV BENCHMARK_GIT_REPO_URL https://github.com/RobotLocomotion/drake.git 68 | # Two random revisions of the code, ~ 1 week apart 69 | ENV BENCHMARK_GIT_REV_OLD 290724e 70 | ENV BENCHMARK_GIT_REV_NEW 60b5ed9 71 | 72 | # ENV BENCHMARK_BUILD_TARGET "//..." # ~30m 73 | ENV BENCHMARK_BUILD_TARGET "//drake/examples:simple_continuous_time_system" # ~30s 74 | # ENV BENCHMARK_BUILD_TARGET "//drake/examples/QPInverseDynamicsForHumanoids/system:valkyrie_controller" # ~5m 75 | 76 | CMD bash 77 | -------------------------------------------------------------------------------- /Dockerfile.cacher: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y --no-install-recommends \ 5 | nginx \ 6 | nginx-extras \ 7 | && rm -rf /var/lib/apt/lists/* 8 | 9 | COPY nginx.conf /etc/nginx/nginx.conf 10 | 11 | CMD nginx -g "daemon off;" -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Bazel build benchmarks 2 | ====================== 3 | 4 | [Bazel](http://bazel.build) is Google's open source build system. This repo is 5 | a companion to my [blog post]() and contains the configuration and scripts that 6 | I used for benchmarks. 7 | 8 | The `cacher` _docker compose` service is a simple instance of nginx + WebDAV to 9 | serve as a remote cache. The `builder` does the actual C++ builds with Bazel and 10 | clang 3.9. 11 | 12 | Set-up 13 | ------ 14 | 15 | You'll need _docker_ and _docker-compose_ installed. Then: 16 | 17 | docker-compose build && docker-compose up 18 | 19 | will build the images and start the cache server. Then, get a builder shell: 20 | 21 | docker-compose run builder bash 22 | 23 | Interactive demo 24 | ---------------- 25 | 26 | git clone $BENCHMARK_GIT_REPO_URL $BENCHMARK_GIT_REPO_PATH 27 | ./benchmarks.py configure_bazel --cache 28 | 29 | cd $BENCHMARK_GIT_REPO_PATH 30 | cloc . 31 | 32 | # Let's build some code now 33 | git checkout $BENCHMARK_GIT_REV_OLD 34 | 35 | # The cache is empty 36 | rm -rf /src/cacher_root/* 37 | 38 | bazel build //drake/examples:simple_continuous_time_system 39 | # This took x seconds 40 | # A no-op build is very fast: 41 | bazel build //drake/examples:simple_continuous_time_system 42 | 43 | # We filled up the cache: 44 | du -sh /src/cacher_root 45 | 46 | # Let's checkout a different revision 47 | git checkout $BENCHMARK_GIT_REV_NEW 48 | # There's quite a difference between the two 49 | git diff $BENCHMARK_GIT_REV_OLD $BENCHMARK_GIT_REV_NEW | wc -l 50 | 51 | # Build this new revision 52 | bazel build //drake/examples:simple_continuous_time_system 53 | 54 | # Let's check the cache again 55 | du -sh /src/cacher_root 56 | 57 | # Go back to the first 58 | git checkout $BENCHMARK_GIT_REV_OLD 59 | 60 | # Build again 61 | bazel build //drake/examples:simple_continuous_time_system 62 | 63 | # Let's spin up a new instance and make sure it can use the cache 64 | exit 65 | 66 | docker-compose run builder bash 67 | 68 | git clone $BENCHMARK_GIT_REPO_URL $BENCHMARK_GIT_REPO_PATH 69 | ./benchmarks.py configure_bazel --cache 70 | 71 | git checkout $BENCHMARK_GIT_REV_OLD 72 | 73 | # The cache is still there: 74 | du -sh /src/cacher_root 75 | 76 | bazel build //drake/examples:simple_continuous_time_system 77 | -------------------------------------------------------------------------------- /benchmarks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import print_function 4 | 5 | import json 6 | import os 7 | from subprocess import check_call, check_output 8 | import sys 9 | 10 | import click 11 | import stopwatch 12 | 13 | 14 | GIT_REPO_PATH = os.environ['BENCHMARK_GIT_REPO_PATH'] 15 | GIT_REPO_URL = os.environ['BENCHMARK_GIT_REPO_URL'] 16 | # Two random revisions of the code, ~ 1 week apart 17 | GIT_REV_OLD = os.environ['BENCHMARK_GIT_REV_OLD'] 18 | GIT_REV_NEW = os.environ['BENCHMARK_GIT_REV_NEW'] 19 | BUILD_TARGET = os.environ['BENCHMARK_BUILD_TARGET'] 20 | 21 | 22 | def _get_timings(sw): 23 | " Output stopwatch timings in JSON format. """ 24 | report = sw.get_last_aggregated_report() 25 | values = report.aggregated_values 26 | 27 | # fetch all values only for main stopwatch, ignore all the tags 28 | log_names = sorted( 29 | log_name for log_name in values if "+" not in log_name 30 | ) 31 | if not log_names: 32 | return 33 | 34 | data = {} 35 | 36 | for log_name in log_names[1:]: 37 | delta_ms, count, bucket = values[log_name] 38 | short_name = log_name[log_name.rfind("#") + 1:] 39 | data[short_name] = delta_ms 40 | 41 | return data 42 | 43 | 44 | def _sh(cmd, *args, **kwargs): 45 | return check_call(cmd, *args, shell=True, **kwargs) 46 | 47 | 48 | # 49 | 50 | def _checkout_code(revision): 51 | if not os.path.exists(GIT_REPO_PATH): 52 | _sh('git clone {} {}'.format(GIT_REPO_URL, GIT_REPO_PATH)) 53 | _sh('git checkout {}'.format(revision), cwd=GIT_REPO_PATH) 54 | 55 | 56 | def _build(): 57 | _sh('bazel build {} --verbose_failures'.format(BENCHMARK_BUILD_TARGET), 58 | cwd=GIT_REPO_PATH) 59 | 60 | 61 | def _clean(): 62 | if os.path.exists(GIT_REPO_PATH): 63 | _sh('bazel clean --expunge', cwd=GIT_REPO_PATH) 64 | 65 | 66 | def _configure_bazel(enable_cache): 67 | """ Write .bazelrc to enable remote caching. """ 68 | bazelrc_path = os.path.join(GIT_REPO_PATH, '.bazelrc') 69 | 70 | DEFAULT_BAZELRC = """ 71 | startup --host_jvm_args=-Dbazel.DigestFunction=SHA1 72 | build --compiler=clang-3.9 73 | """ 74 | 75 | with open(bazelrc_path, 'w') as f: 76 | f.write(DEFAULT_BAZELRC) 77 | 78 | if enable_cache: 79 | f.write(""" 80 | build --spawn_strategy=remote 81 | build --remote_rest_cache=http://cacher:7070/cache 82 | """) 83 | 84 | 85 | @click.group() 86 | def cli(): 87 | pass 88 | 89 | 90 | @cli.command() 91 | @click.option('--cache/--no-cache') 92 | def configure_bazel(cache): 93 | _configure_bazel(cache) 94 | 95 | 96 | @cli.command() 97 | @click.option('--enable-cache', is_flag=True) 98 | def between_commits(enable_cache): 99 | """ Benchmark builds going back and forth between two 100 | commits. """ 101 | 102 | _clean() 103 | _checkout_code(GIT_REV_OLD) 104 | 105 | if enable_cache: 106 | _enable_cache() 107 | 108 | # Count the number of lines in the diff 109 | num_diff_lines = check_output( 110 | 'git diff {} {} | wc -l'.format(GIT_REV_OLD, GIT_REV_NEW), 111 | shell=True, 112 | cwd=GIT_REPO_PATH).strip() 113 | 114 | sw = stopwatch.StopWatch() 115 | 116 | with sw.timer('between_commits'): 117 | with sw.timer('1_clean_build_old'): 118 | _build() 119 | with sw.timer('2_no_op_build'): 120 | _build() 121 | 122 | # Jump to the newer commit and build again 123 | _checkout_code(GIT_REV_NEW) 124 | with sw.timer('3_build_new_commit'): 125 | _build() 126 | with sw.timer('4_no_op_build'): 127 | _build() 128 | 129 | # Revert to the old one and build again 130 | _checkout_code(GIT_REV_OLD) 131 | with sw.timer('5_build_old_again'): 132 | _build() 133 | 134 | # Save and print results 135 | timings_data = _get_timings(sw) 136 | output_fname = 'timings_cache_{}.json'.format(enable_cache) 137 | with open(output_fname, 'w') as f: 138 | json.dump(timings_data, f, indent=2) 139 | print(json.dumps(timings_data, indent=2)) 140 | 141 | print('{} diff lines between commits {} and {}'.format( 142 | num_diff_lines, GIT_REV_OLD, GIT_REV_NEW)) 143 | 144 | 145 | if __name__ == '__main__': 146 | cli() 147 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | cacher: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile.cacher 7 | volumes: 8 | - ./cacher_root:/var/www/cache 9 | ports: 10 | - "7070:7070" 11 | builder: 12 | build: 13 | context: . 14 | dockerfile: Dockerfile.builder 15 | volumes: 16 | - .:/src 17 | -------------------------------------------------------------------------------- /nginx.conf: -------------------------------------------------------------------------------- 1 | user root; 2 | worker_processes auto; 3 | pid /run/nginx.pid; 4 | 5 | events { 6 | worker_connections 768; 7 | # multi_accept on; 8 | } 9 | 10 | http { 11 | sendfile on; 12 | tcp_nopush on; 13 | tcp_nodelay on; 14 | keepalive_timeout 65; 15 | types_hash_max_size 2048; 16 | # server_tokens off; 17 | 18 | include /etc/nginx/mime.types; 19 | default_type application/octet-stream; 20 | 21 | access_log /dev/stdout; 22 | error_log /dev/stderr; 23 | 24 | gzip on; 25 | gzip_disable "msie6"; 26 | 27 | server { 28 | listen 7070 default_server; 29 | 30 | root /var/www; 31 | 32 | location /cache/ { 33 | dav_methods PUT; 34 | autoindex on; 35 | allow all; 36 | client_max_body_size 512M; 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /plot_results.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | 5 | import click 6 | from IPython import embed 7 | from matplotlib import pyplot as plt 8 | import pandas as pd 9 | import seaborn as sns 10 | 11 | 12 | def df_from_file(json_path, label): 13 | """ Load timings json data into a pd.DataFrame. """ 14 | with open(json_path) as json_f: 15 | data = json.load(json_f) 16 | 17 | df = pd.DataFrame.from_dict(data, orient='index') 18 | df.columns = [label] 19 | df[label] /= 1000 20 | df = df.sort_index() 21 | 22 | return df 23 | 24 | 25 | @click.command() 26 | @click.option('--with-cache', is_flag=True) 27 | def main(with_cache): 28 | df = df_from_file('timings_cache_False.json', 'no_cache') 29 | 30 | if with_cache: 31 | # Load and merge data from the no-cache case 32 | df2 = df_from_file('timings_cache_True.json', 'with_cache') 33 | df = df.merge(df2, left_index=True, right_index=True) 34 | 35 | ax = df.plot(kind='bar', 36 | legend=with_cache, 37 | figsize=(6,4), 38 | rot=45,) 39 | ax.figure.subplots_adjust(bottom=0.35) 40 | ax.set_ylabel('Time [s]') 41 | plt.tight_layout() 42 | 43 | output_file_name = 'results_with_cache.png' \ 44 | if with_cache \ 45 | else 'results_without_cache.png' 46 | ax.figure.savefig(output_file_name) 47 | 48 | 49 | if __name__ == '__main__': 50 | main() 51 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | click==6.7 2 | dbx-stopwatch==1.5 3 | --------------------------------------------------------------------------------