├── data └── .keep ├── work └── .keep ├── docker ├── build.sh ├── install_rbenv.sh ├── benchmark_software.json ├── test_benchmark_run.rb ├── Dockerfile ├── benchmark_discourse_setup.rb ├── app.yml └── build_benchmark_software.rb ├── packer ├── 90-tune-kernel.conf ├── discourse_install_rvm.sh ├── discourse_install_node.sh ├── nginx_install.sh ├── nginx.default.conf ├── discourse_install_postgres.sh ├── rc.local ├── benchmark_software.json ├── misc_install.sh ├── discourse_install.sh ├── passenger-enterprise-install.sh ├── setup_discourse_gems.rb ├── ami.json ├── ami-with-passenger-enterprise.json ├── setup_discourse.rb ├── README.md └── setup.rb ├── TODO ├── .gitignore ├── bin └── setup ├── INSTALL.md ├── in_each_ruby.rb ├── Gemfile ├── update_discourse.sh ├── scripts ├── foreach_ip.rb ├── multithread_each_ip.rb ├── launch_instance.rb ├── test_instances.rb └── session_mgmt.sh ├── graph └── README.md ├── example_runners ├── 3_0_final_runner.rb ├── 2_7_jit_runner.rb ├── 3_0_runner.rb └── modified_2_7_jit_runner.rb ├── runner.rb ├── seed_db_data.rb ├── process.rb ├── alice.txt ├── user_simulator.rb ├── README.md ├── Gemfile.lock ├── COPYING └── start.rb /data/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /work/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docker/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker build . --no-cache --tag noahgibbs/rails_ruby_bench/base:build --squash 3 | -------------------------------------------------------------------------------- /packer/90-tune-kernel.conf: -------------------------------------------------------------------------------- 1 | net.ipv4.tcp_tw_recycle = 1 2 | net.ipv4.tcp_tw_reuse = 1 3 | net.ipv4.ip_conntrack_max = 20464 4 | 5 | net.ipv6.conf.all.disable_ipv6 = 1 6 | -------------------------------------------------------------------------------- /packer/discourse_install_rvm.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | rvm install 2.4.1 4 | rvm --default use 2.4.1 # If this error out check https://rvm.io/integration/gnome-terminal 5 | gem install bundler -v1.17.3 6 | gem install mailcatcher 7 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | 2 | Explicit JSON-check step at start of Packer build to quickly catch JSON errors in benchmark_software.json 3 | 4 | Turn off automatic Ubuntu updates in Packer image: https://libre-software.net/ubuntu-automatic-updates/ 5 | -------------------------------------------------------------------------------- /packer/discourse_install_node.sh: -------------------------------------------------------------------------------- 1 | # Load nvm 2 | export NVM_DIR="/home/ubuntu/.nvm" 3 | . "$NVM_DIR/nvm.sh" 4 | 5 | set -e 6 | 7 | nvm install 6.2.0 8 | nvm alias default 6.2.0 9 | npm install -g svgo phantomjs-prebuilt 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | work/discourse 2 | work/ruby 3 | packer/ec2_amazon-ebs.pem 4 | 5 | process_output.json 6 | tmp/cache 7 | packer/benchmark_software_*.json 8 | packer/passenger-enterprise-license 9 | packer/passenger-download-token 10 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | cd work 4 | git clone https://github.com/discourse/discourse.git 5 | cd discourse 6 | bundle 7 | RAILS_ENV=profile rake db:create db:migrate # If necessary, db:drop first 8 | cd .. 9 | 10 | bundle 11 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | # UPDATE FOR: https://github.com/discourse/discourse/blob/master/docs/DEVELOPMENT-OSX-NATIVE.md 2 | 3 | # WHEN LINUXIFYING: https://github.com/discourse/discourse/blob/master/docs/DEVELOPER-ADVANCED.md 4 | 5 | brew services start postgresql 6 | brew services start redis 7 | -------------------------------------------------------------------------------- /in_each_ruby.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # Run start.rb with the specified arguments in all the marked-as-installed Rubies in the Rails Ruby Bench installation. 4 | 5 | RUBIES = File.read("/home/ubuntu/benchmark_ruby_versions.txt").split("\n") 6 | 7 | RUBIES.each do |ruby| 8 | system("bash -l -c \"rvm use #{ruby} && #{ARGV.join(" ")}\"") 9 | end 10 | -------------------------------------------------------------------------------- /packer/nginx_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # To be run as "ubuntu" user w/ sudo access 6 | 7 | sudo apt-get -yqq install nginx 8 | 9 | # Set up directory for benchmark to write into, served by NGinX 10 | sudo mkdir -p /var/www/html/benchmark-results 11 | sudo chown -R ubuntu /var/www/html/benchmark-results 12 | sudo chmod -R ugo+r /var/www/html/benchmark-results 13 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | gem 'gabbler' 2 | gem 'get_process_mem' 3 | 4 | gem 'rexml', '=3.2.4' # Seems to be required in Ruby 3.0.0-preview1 and higher 5 | gem 'bigdecimal', '=1.3.5' # Compatible before Ruby 2.6.0? 6 | gem 'thwait', '=0.1.0' # Undeclared requirement for discourse_image_optim circa 2017 7 | gem 'e2mmap', '=0.1.0' # Undeclared requirement of thwait (prev gem) 8 | 9 | eval_gemfile 'work/discourse/Gemfile' 10 | -------------------------------------------------------------------------------- /update_discourse.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | #set -x 5 | 6 | cd work/discourse 7 | git pull 8 | bundle 9 | 10 | RAILS_ENV=profile rake db:drop db:create db:migrate 11 | RAILS_ENV=profile rake assets:precompile 12 | 13 | mkdir public/uploads || echo "Fine that public/uploads already exists." 14 | 15 | cd ../.. 16 | 17 | # Needed after the drop-and-recreate in Discourse 18 | RAILS_ENV=profile ruby seed_db_data.rb 19 | -------------------------------------------------------------------------------- /scripts/foreach_ip.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | raise "Provide exactly one argument!" if ARGV.size != 1 4 | 5 | ips = File.read("#{ENV["HOME"]}/multi_inst_ips.txt").split("\n").map(&:strip) 6 | puts "IPs: #{ips.inspect}, arg: #{ARGV[0]}" 7 | 8 | arg = ARGV[0] 9 | 10 | if arg["0.0.0.0"] 11 | ips.each do |ip| 12 | real_arg = arg.gsub("0.0.0.0", ip) 13 | system real_arg 14 | end 15 | else 16 | ips.each do |ip| 17 | system "#{arg} #{ip}" 18 | end 19 | end 20 | 21 | -------------------------------------------------------------------------------- /packer/nginx.default.conf: -------------------------------------------------------------------------------- 1 | # Default server configuration 2 | # 3 | server { 4 | listen 80 default_server; 5 | listen [::]:80 default_server; 6 | 7 | root /var/www/html; 8 | 9 | # Add index.php to the list if you are using PHP 10 | index index.html index.htm index.nginx-debian.html; 11 | 12 | server_name _; 13 | 14 | location / { 15 | # First attempt to serve request as file, then 16 | # as directory, then fall back to displaying a 404. 17 | try_files $uri $uri/ =404; 18 | autoindex on; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /scripts/multithread_each_ip.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | raise "Provide exactly one argument!" if ARGV.size != 1 4 | 5 | ips = File.read("#{ENV["HOME"]}/multi_inst_ips.txt").split("\n").map(&:strip) 6 | 7 | arg = ARGV[0] 8 | threads = [] 9 | 10 | ips.each do |ip| 11 | t = Thread.new do 12 | if arg["0.0.0.0"] 13 | real_arg = arg.gsub("0.0.0.0", ip) 14 | system real_arg 15 | else 16 | system "#{arg} #{ip}" 17 | end 18 | end 19 | threads << t 20 | end 21 | 22 | threads.each { |t| t.join } 23 | -------------------------------------------------------------------------------- /graph/README.md: -------------------------------------------------------------------------------- 1 | For awhile I tried to keep a reasonable graphing solution in 2 | here. Unfortunately, "graphing time data" very quickly turns into a 3 | general graphing package, which I do not want to write or maintain. 4 | 5 | You can see my specific individual graphing attempts in the repository 6 | "https://github.com/noahgibbs/rrb_datavis", which contains both my 7 | collected data and my data visualization for RRB-based talks and blog 8 | posts. It includes a number of examples of graphing the results, 9 | especially using the D3 JavaScript visualization library. 10 | -------------------------------------------------------------------------------- /packer/discourse_install_postgres.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | # To be run as user "postgres" 4 | createuser --createdb --superuser -Upostgres $(cat /tmp/username) 5 | psql -c "ALTER USER $(cat /tmp/username) WITH PASSWORD 'password';" 6 | psql -c "create database discourse_development owner $(cat /tmp/username) encoding 'UTF8' TEMPLATE template0;" 7 | psql -c "create database discourse_test owner $(cat /tmp/username) encoding 'UTF8' TEMPLATE template0;" 8 | psql -d discourse_development -c "CREATE EXTENSION hstore;" 9 | psql -d discourse_development -c "CREATE EXTENSION pg_trgm;" 10 | exit 11 | -------------------------------------------------------------------------------- /packer/rc.local: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | # 3 | # rc.local 4 | # 5 | # This script is executed at the end of each multiuser runlevel. 6 | # Make sure that the script will "exit 0" on success or any other 7 | # value on error. 8 | # 9 | # In order to enable or disable this script just change the execution 10 | # bits. 11 | # 12 | # By default this script does nothing. 13 | 14 | #sudo su ubuntu /bin/bash -l -c "cd /home/ubuntu/rails_ruby_bench && ruby ./start.rb --out-dir /var/www/html/benchmark-results 2>/tmp/benchmark_boot_stderr >> /tmp/benchmark_boot_console" 15 | 16 | sudo /etc/init.d/redis-server start 17 | 18 | exit 0 19 | -------------------------------------------------------------------------------- /docker/install_rbenv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | 6 | git clone https://github.com/rbenv/rbenv.git ~/.rbenv 7 | cd ~/.rbenv && src/configure && make -C src # This is technically optional 8 | 9 | echo 'export PATH="$HOME/.rbenv/shims:$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile 10 | echo 'eval "$(rbenv init -)"' >> ~/.bash_profile 11 | . ~/.bash_profile 12 | 13 | # rbenv-doctor script: Remove once this is working 14 | #curl -fsSL https://github.com/rbenv/rbenv-installer/raw/master/bin/rbenv-doctor | bash 15 | 16 | # Install ruby-build 17 | mkdir -p "$(rbenv root)"/plugins 18 | git clone https://github.com/rbenv/ruby-build.git "$(rbenv root)"/plugins/ruby-build 19 | -------------------------------------------------------------------------------- /packer/benchmark_software.json: -------------------------------------------------------------------------------- 1 | { 2 | "rails_ruby_bench": { 3 | "git_url": "https://github.com/noahgibbs/rails_ruby_bench.git", 4 | "git_tag": "" 5 | }, 6 | "discourse": { 7 | "git_url": "https://github.com/discourse/discourse.git", 8 | "git_tag": "v1.8.0.beta13" 9 | }, 10 | "compare_rubies": [ 11 | { 12 | "rvm_name": "2.0.0-p0", 13 | "discourse": false 14 | }, 15 | { 16 | "rvm_name": "2.1.10", 17 | "discourse": false 18 | }, 19 | { 20 | "rvm_name": "2.2.10", 21 | "discourse": false 22 | }, 23 | { 24 | "rvm_name": "2.3.8", 25 | "discourse": false 26 | }, 27 | { 28 | "rvm_name": "2.4.5" 29 | }, 30 | { 31 | "rvm_name": "2.5.3" 32 | }, 33 | { 34 | "rvm_name": "2.6.0" 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /packer/misc_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # To be run as "ubuntu" user w/ sudo access 6 | 7 | sudo apt-get -yqq install apache2-utils libjemalloc-dev libtcmalloc-minimal4 openjdk-8-jdk 8 | 9 | # Passenger install for comparing app servers 10 | sudo apt-get install -y dirmngr gnupg 11 | sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 561F9B9CAC40B2F7 12 | sudo apt-get install -y apt-transport-https ca-certificates 13 | 14 | sudo sh -c 'echo deb https://oss-binaries.phusionpassenger.com/apt/passenger xenial main > /etc/apt/sources.list.d/passenger.list' 15 | sudo apt-get update 16 | 17 | sudo apt-get install -y passenger 18 | 19 | # Wrk 20 | cd /home/ubuntu 21 | git clone https://github.com/wg/wrk.git 22 | cd wrk 23 | make 24 | # Install wrk binary into /usr/local/bin 25 | sudo cp wrk /usr/bin/ 26 | 27 | # You know what sucks? Having a huge, benchmark-busting cron job start at 28 | # a random-ish time, screwing up all your results. 29 | sudo apt-get remove -y unattended-upgrades 30 | -------------------------------------------------------------------------------- /docker/benchmark_software.json: -------------------------------------------------------------------------------- 1 | { 2 | "rails_ruby_bench": { 3 | "git_url": "https://github.com/noahgibbs/rails_ruby_bench.git", 4 | "git_tag": "" 5 | }, 6 | "discourse": { 7 | "git_url": "https://github.com/discourse/discourse.git", 8 | "git_tag": "v1.8.0.beta13" 9 | }, 10 | "compare_rubies": [ 11 | { 12 | "name": "mri-2.5.0", 13 | "git_url": "git://github.com/ruby/ruby.git", 14 | "git_tag": "v2_5_0" 15 | }, 16 | { 17 | "name": "mri-2.6.0-preview1", 18 | "git_url": "git://github.com/ruby/ruby.git", 19 | "git_tag": "v2_6_0_preview1" 20 | }, 21 | { 22 | "name": "mri-2.6.0-preview2", 23 | "git_url": "git://github.com/ruby/ruby.git", 24 | "git_tag": "v2_6_0_preview2" 25 | }, 26 | { 27 | "name": "mri-head-aug-17", 28 | "git_url": "git://github.com/ruby/ruby.git", 29 | "git_tag": "753219768ffe7dce6d974f42b275077d1b02ee17" 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /docker/test_benchmark_run.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | RAILS_BENCH_DIR = "/var/rails_ruby_bench" 4 | 5 | class DockerBuildError < RuntimeError; end 6 | 7 | # Checked system - error if the command fails 8 | def csystem(cmd, err, opts = {}) 9 | cmd = "bash -l -c \"#{cmd}\"" if opts[:bash] 10 | print "Running command: #{cmd.inspect}\n" if opts[:debug] || opts["debug"] 11 | system(cmd, out: $stdout, err: :out) 12 | unless $?.success? || opts[:fail_ok] || opts["fail_ok"] 13 | puts "Error running command:\n#{cmd.inspect}" 14 | raise DockerBuildError.new(err) 15 | end 16 | end 17 | 18 | # And check to make sure the benchmark actually runs... But just do a few iterations. 19 | Dir.chdir(RAILS_BENCH_DIR) do 20 | begin 21 | csystem "./start.rb -s 1 -n 1 -i 10 -w 0 -o /tmp/ -c 1", "Couldn't successfully run the benchmark!", :bash => true 22 | rescue DockerBuildError 23 | # Before dying, let's look at that Rails logfile... Redirect stdout to stderr. 24 | print "Error running test iterations of the benchmark, printing Rails log to console!\n==========\n" 25 | print `tail -60 work/discourse/log/profile.log` # If we echo too many lines they just get cut off by Packer 26 | print "=============\n" 27 | raise # Re-raise the error, we still want to die. 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /packer/discourse_install.sh: -------------------------------------------------------------------------------- 1 | # To be run as "ubuntu" user with sudo access. 2 | 3 | set -e 4 | 5 | #sleep 30 # Time for OS to boot properly during AMI build 6 | 7 | # No ~/.bash_profile? Make one that sources ~/.bashrc. Otherwise you won't like what happens 8 | # if rvm creates it for you. 9 | if [ ! -f ~/.bash_profile ]; then 10 | cat >~/.bash_profile < /tmp/username 21 | sudo add-apt-repository ppa:chris-lea/redis-server 22 | sudo apt-get -yqq update 23 | sudo apt-get -yqq --allow-unauthenticated install python-software-properties vim curl expect debconf-utils git-core build-essential zlib1g-dev libssl-dev openssl libcurl4-openssl-dev libreadline6-dev libpcre3 libpcre3-dev imagemagick postgresql postgresql-contrib-9.5 libpq-dev postgresql-server-dev-9.5 redis-server advancecomp gifsicle jhead jpegoptim libjpeg-turbo-progs optipng pngcrush pngquant gnupg2 24 | sudo /etc/init.d/redis-server start 25 | 26 | # Ruby 27 | curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - 28 | curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import - 29 | curl -sSL https://get.rvm.io | bash -s stable 30 | echo 'gem: --no-document' >> ~/.gemrc 31 | 32 | # Node 33 | curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.31.1/install.sh | bash 34 | -------------------------------------------------------------------------------- /example_runners/3_0_final_runner.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # Simple example runner script, editable for different uses 4 | 5 | RUBIES = [ 6 | #"2.5.3", 7 | "2.6.6", 8 | "2.7.1", 9 | "ext-new_fake_2.8", 10 | ] 11 | 12 | TESTS = [ 13 | "gem install bundler -v 1.17.3 && bundle _1.17.3_ && bundle _1.17.3_ exec ./start.rb -i 15000 -w 1000 -s 0 --no-warm-start -o data/", 14 | ] 15 | 16 | TIMES = 30 17 | 18 | # Checked system - error if the command fails 19 | def csystem(cmd, err, opts = {}) 20 | cmd = "bash -l -c \"#{cmd}\"" if opts[:bash] || opts["bash"] 21 | print "Running command: #{cmd.inspect}\n" if opts[:to_console] || opts["to_console"] || opts[:debug] || opts["debug"] 22 | if opts[:to_console] || opts["to_console"] 23 | system(cmd, out: $stdout, err: :out) 24 | else 25 | out = `#{cmd}` 26 | end 27 | unless $?.success? || opts[:fail_ok] || opts["fail_ok"] 28 | puts "Error running command:\n#{cmd.inspect}" 29 | puts "Output:\n#{out}\n=====" if out 30 | raise err 31 | end 32 | end 33 | 34 | commands = [] 35 | RUBIES.each do |ruby| 36 | TESTS.each_with_index do |test, test_index| 37 | invocation = "rvm use #{ruby} && export RUBY_RUNNER_TEST_INDEX=#{test_index} && #{test}" 38 | commands.concat([invocation] * TIMES) 39 | end 40 | end 41 | 42 | rand_commands = commands.shuffle 43 | 44 | rand_commands.each do |command| 45 | csystem(command, "Error running test!", bash: true, to_console: true) 46 | end 47 | 48 | csystem("touch #{ENV["HOME"]}/run_finished.txt", "Error creating run-finished file!", bash: false, to_console: true) 49 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # NAME: rails_ruby_bench/discourse 2 | # VERSION: release 3 | FROM discourse/base:2.0.20180907 4 | 5 | #LABEL maintainer="Noah Gibbs" 6 | 7 | ENV RRB_GIT_URL https://github.com/noahgibbs/rails_ruby_bench.git 8 | 9 | RUN echo 2.0.`date +%Y%m%d` > /RRB_VERSION 10 | 11 | RUN chown discourse:discourse /var 12 | 13 | # Additional packages to compile Rubies 14 | RUN apt-get install -y bison autoconf build-essential libssl-dev libyaml-dev libreadline6-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm-dev 15 | 16 | # Additional SQLite3 package, mostly for Mailcatcher gem 17 | RUN apt-get install -y libsqlite3-dev 18 | 19 | # Clone Rails Ruby Bench 20 | RUN sudo -H -u discourse git clone ${RRB_GIT_URL} /var/rails_ruby_bench 21 | 22 | # Install RRB gems into system Ruby 23 | RUN cd /var/rails_ruby_bench && bundle 24 | 25 | ADD install_rbenv.sh /tmp/install_rbenv.sh 26 | RUN chmod +x /tmp/install_rbenv.sh && sudo -H -u discourse /tmp/install_rbenv.sh 27 | 28 | # Copy in Ruby settings, which may be modified from checked-in version 29 | ADD benchmark_software.json /tmp/benchmark_software.json 30 | 31 | ADD build_benchmark_software.rb /tmp/build_benchmark_software.rb 32 | RUN chmod +x /tmp/build_benchmark_software.rb && sudo -H -u discourse /tmp/build_benchmark_software.rb 33 | 34 | ADD benchmark_discourse_setup.rb /tmp/benchmark_discourse_setup.rb 35 | RUN chmod +x /tmp/benchmark_discourse_setup.rb && sudo -H -u discourse /tmp/benchmark_discourse_setup.rb 36 | 37 | ADD test_benchmark_run.rb /tmp/test_benchmark_run.rb 38 | RUN chmod +x /tmp/test_benchmark_run.rb && sudo -H -u discourse /tmp/test_benchmark_run.rb 39 | -------------------------------------------------------------------------------- /scripts/launch_instance.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "json" 4 | 5 | # While you might get some useful ideas here, this script is *not* general-purpose 6 | # and will *not* do exactly what you wish it would. It's pretty specific to my workflow. 7 | 8 | latest_ami = 'ami-0e7a9d0f34bbb44e9' 9 | inst_name = ENV['INSTANCE_NAME'] || 'RailsRubyBenchTestInstance' 10 | inst_type = ENV['INSTANCE_TYPE'] || 'm4.2xlarge' 11 | placement = ENV['PLACEMENT'] ? "--placement #{ENV['PLACEMENT']}" : "" # --placement Tenancy=dedicated 12 | json_out = `aws ec2 run-instances --count 1 --instance-type #{inst_type} --key-name rrb-1 #{placement} --image-id #{latest_ami} --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=#{inst_name}}]'` 13 | 14 | ec2_info = JSON.parse(json_out) 15 | 16 | # This is really not a sensible way to handle multiple instances (which I don't use currently) 17 | ec2_info["Instances"].each do |instance| 18 | id = instance["InstanceId"] 19 | ec2_type = instance["InstanceType"] 20 | 21 | inst_ip = `aws ec2 describe-instances --instance-ids #{id} --query 'Reservations[*].Instances[*].PublicIpAddress' --output text`.strip 22 | 23 | puts "Launched EC2 instance: #{ec2_type.inspect} / #{id.inspect}" 24 | cmd_lines = <> /etc/apt/auth.conf' 17 | sudo sh -c 'echo deb https://www.phusionpassenger.com/enterprise_apt xenial main > /etc/apt/sources.list.d/passenger.list' 18 | sudo chown root: /etc/apt/sources.list.d/passenger.list 19 | sudo chmod 644 /etc/apt/sources.list.d/passenger.list 20 | sudo chown root: /etc/apt/auth.conf 21 | sudo chmod 600 /etc/apt/auth.conf 22 | sudo apt-get update 23 | 24 | # Install Passenger Enterprise + Nginx module 25 | #sudo apt-get install -y libnginx-mod-http-passenger-enterprise 26 | sudo apt-get install -y nginx-extras passenger-enterprise 27 | 28 | #if [ ! -f /etc/nginx/modules-enabled/50-mod-http-passenger.conf ] 29 | #then sudo ln -s /usr/share/nginx/modules-available/mod-http-passenger.load /etc/nginx/modules-enabled/50-mod-http-passenger.conf 30 | #fi 31 | #sudo ls /etc/nginx/conf.d/mod-http-passenger.conf 32 | 33 | sudo service nginx restart 34 | 35 | # sudo /usr/bin/passenger-config validate-install 36 | -------------------------------------------------------------------------------- /example_runners/2_7_jit_runner.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # Simple example runner script, editable for different uses 4 | 5 | RUBIES = [ 6 | "2.6.0", 7 | "2.7.0-rc1", 8 | ] 9 | 10 | TESTS = [ 11 | "gem install bundler -v 1.17.3 && bundle _1.17.3_ && bundle _1.17.3_ exec ./start.rb -i 10000 -w 1000 -s 0 --no-warm-start -o data/", 12 | ] 13 | 14 | TIMES = 30 15 | 16 | # Checked system - error if the command fails 17 | def csystem(cmd, err, opts = {}) 18 | cmd = "bash -l -c \"#{cmd}\"" if opts[:bash] || opts["bash"] 19 | print "Running command: #{cmd.inspect}\n" if opts[:to_console] || opts["to_console"] || opts[:debug] || opts["debug"] 20 | if opts[:to_console] || opts["to_console"] 21 | system(cmd, out: $stdout, err: :out) 22 | else 23 | out = `#{cmd}` 24 | end 25 | unless $?.success? || opts[:fail_ok] || opts["fail_ok"] 26 | puts "Error running command:\n#{cmd.inspect}" 27 | puts "Output:\n#{out}\n=====" if out 28 | raise err 29 | end 30 | end 31 | 32 | commands = [] 33 | RUBIES.each do |ruby| 34 | TESTS.each_with_index do |test, test_index| 35 | # For JIT, set RUBYOPT to turn JIT on. For either JIT or non-JIT, set a RRB_WITH_JIT variable that gets picked up in 'environment' because it has RUBY in the name. 36 | invocation_jit = "rvm use #{ruby} && export RRB_WITH_JIT=YES && export RUBYOPT='--jit' && export RRB_RUNNER_TEST_INDEX=#{test_index} && #{test}" 37 | invocation_no_jit = "rvm use #{ruby} && export RRB_WITH_JIT=NO && export RRB_RUNNER_TEST_INDEX=#{test_index} && #{test}" 38 | commands.concat([invocation_no_jit,invocation_jit] * TIMES) 39 | end 40 | end 41 | 42 | rand_commands = commands.shuffle 43 | 44 | rand_commands.each do |command| 45 | csystem(command, "Error running test!", bash: true, to_console: true) 46 | end 47 | -------------------------------------------------------------------------------- /runner.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # Simple example runner script, editable for different uses 4 | 5 | RUBIES = [ 6 | "2.6.0", 7 | "2.6.5", 8 | "ext-mri-head", 9 | ] 10 | 11 | TESTS = [ 12 | "gem install bundler -v 1.17.3 && bundle _1.17.3_ && bundle _1.17.3_ exec ./start.rb -i 10000 -w 1000 -s 0 --no-warm-start -o data/", 13 | ] 14 | 15 | TIMES = 30 16 | 17 | # Some potentially useful snippets 18 | WITH_COMPACT="export RUBY_COMPACT=YES && echo GC.compact > ~/rails_ruby_bench/work/discourse/config/initializers/900-gc-compact.rb" 19 | NO_COMPACT="export RUBY_COMPACT=NO && rm -f ~/rails_ruby_bench/work/discourse/config/initializers/900-gc-compact.rb" 20 | 21 | # Checked system - error if the command fails 22 | def csystem(cmd, err, opts = {}) 23 | cmd = "bash -l -c \"#{cmd}\"" if opts[:bash] || opts["bash"] 24 | print "Running command: #{cmd.inspect}\n" if opts[:to_console] || opts["to_console"] || opts[:debug] || opts["debug"] 25 | if opts[:to_console] || opts["to_console"] 26 | system(cmd, out: $stdout, err: :out) 27 | else 28 | out = `#{cmd}` 29 | end 30 | unless $?.success? || opts[:fail_ok] || opts["fail_ok"] 31 | puts "Error running command:\n#{cmd.inspect}" 32 | puts "Output:\n#{out}\n=====" if out 33 | raise err 34 | end 35 | end 36 | 37 | commands = [] 38 | RUBIES.each do |ruby| 39 | TESTS.each_with_index do |test, test_index| 40 | invocation = "rvm use #{ruby} && export RUBY_RUNNER_TEST_INDEX=#{test_index} && #{test}" 41 | commands.concat([invocation] * TIMES) 42 | end 43 | end 44 | 45 | rand_commands = commands.shuffle 46 | 47 | rand_commands.each do |command| 48 | csystem(command, "Error running test!", bash: true, to_console: true) 49 | end 50 | 51 | csystem("touch #{ENV["HOME"]}/run_finished.txt", "Error creating run-finished file!", bash: false, to_console: true) 52 | -------------------------------------------------------------------------------- /scripts/test_instances.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "json" 4 | 5 | launch_ami = 'ami-0e7a9d0f34bbb44e9' 6 | inst_name = ENV['INSTANCE_NAME'] || 'RRBMultiTestInstance' 7 | inst_type = ENV['INSTANCE_TYPE'] || 'm4.2xlarge' 8 | inst_count = ENV['INSTANCE_COUNT'] || 1 9 | placement = ENV['PLACEMENT'] ? "--placement #{ENV['PLACEMENT']}" : "" # --placement Tenancy=dedicated 10 | json_out = `aws ec2 run-instances --count #{inst_count} --instance-type #{inst_type} --key-name rrb-1 #{placement} --image-id #{launch_ami} --tag-specifications 'ResourceType=instance,Tags=[{Key=InstTypeComment,Value=#{inst_name}}]'` 11 | 12 | ec2_info = JSON.parse(json_out) 13 | # File.open("#{ENV["HOME"]}/multi_inst_json.txt", "w") { |f| f.write JSON.pretty_generate(ec2_info) } 14 | 15 | ids = ec2_info["Instances"].map { |i| i["InstanceId"] }.join(" ") 16 | cmd_lines = < 800 do 43 | sentence << @gabbler.sentence 44 | sentence << "\n" 45 | end 46 | sentence 47 | end 48 | 49 | def create_admin(seq) 50 | User.new.tap { |admin| 51 | admin.email = "admin@fake#{seq}.appfolio.com" 52 | admin.username = "admin#{seq}" 53 | admin.password = "longpassword" 54 | admin.save! 55 | admin.grant_admin! 56 | admin.change_trust_level!(TrustLevel[4]) 57 | admin.email_tokens.update_all(confirmed: true) 58 | admin.activate # Added after activation seemed not to work 59 | admin.approved = true 60 | admin.active = true 61 | admin.save! 62 | } 63 | end 64 | 65 | require File.expand_path(File.join(File.dirname(__FILE__), "work/discourse/config/environment")) 66 | 67 | unless Rails.env == "profile" 68 | puts "This script should only be used in the profile environment" 69 | exit 70 | end 71 | 72 | if do_drop 73 | system "cd work/discourse && RAILS_ENV=profile rake db:drop db:create db:migrate" 74 | end 75 | 76 | SiteSetting.queue_jobs = false 77 | 78 | # by default, Discourse has a "system" account 79 | if User.count > 1 80 | puts "Only run this script against an empty DB (and in RAILS_ENV=profile)" 81 | exit 82 | end 83 | 84 | puts "Creating 100 users" 85 | users = 100.times.map do |i| 86 | putc "." 87 | create_admin(i) 88 | end 89 | 90 | puts 91 | puts "Creating 10 categories" 92 | categories = 10.times.map do |i| 93 | putc "." 94 | Category.create(name: "category#{i}", text_color: "ffffff", color: "000000", user: users.first) 95 | end 96 | 97 | puts 98 | puts "Creating 100 topics" 99 | 100 | topic_ids = 100.times.map do 101 | post = PostCreator.create(users.sample, raw: sentence, title: sentence[0..50].strip, category: categories.sample.name, skip_validations: true) 102 | 103 | putc "." 104 | post.topic_id 105 | end 106 | 107 | puts 108 | puts "creating 200 replies" # PostCreator is just crazy slow. Why? 109 | 200.times do 110 | putc "." 111 | PostCreator.create(users.sample, raw: sentence, topic_id: topic_ids.sample, skip_validations: true) 112 | end 113 | 114 | # no sidekiq so update some stuff 115 | Category.update_stats 116 | Jobs::PeriodicalUpdates.new.execute(nil) 117 | 118 | -------------------------------------------------------------------------------- /packer/setup_discourse_gems.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "fileutils" 4 | require "json" 5 | 6 | # Pass --local to run the setup on a local machine, or set RRB_LOCAL 7 | LOCAL = (ARGV.delete '--local') || ENV["RRB_LOCAL"] 8 | # Whether to build rubies with rvm 9 | BUILD_RUBY = !LOCAL 10 | USE_BASH = BUILD_RUBY 11 | # Print all commands and show their full output 12 | #VERBOSE = LOCAL 13 | VERBOSE = true 14 | 15 | base = LOCAL ? File.expand_path('..', __FILE__) : "/home/ubuntu" 16 | benchmark_software = JSON.load(File.read("#{base}/benchmark_software.json")) 17 | 18 | class SystemPackerBuildError < RuntimeError; end 19 | 20 | print < true 74 | end 75 | end 76 | end 77 | end 78 | 79 | if !first_ruby 80 | raise "Couldn't find any Discourse-capable Ruby to run the benchmark..." 81 | end 82 | 83 | # And check to make sure the benchmark actually runs... But just do a few iterations. 84 | Dir.chdir(RAILS_BENCH_DIR) do 85 | begin 86 | csystem "rvm use #{first_ruby} && bundle exec ./start.rb -s 1 -n 1 -i 10 -w 0 -o /tmp/ -c 1", "Couldn't successfully run the benchmark!", :bash => true 87 | rescue SystemPackerBuildError 88 | # Before dying, let's look at that Rails logfile... Redirect stdout to stderr. 89 | print "Error running test iterations of the benchmark, printing Rails log to console!\n==========\n" 90 | print `tail -60 work/discourse/log/profile.log` # If we echo too many lines they just get cut off by Packer 91 | print "=============\n" 92 | raise # Re-raise the error, we still want to die. 93 | end 94 | end 95 | 96 | FileUtils.touch "/tmp/setup_discourse_gems_ran_correctly" 97 | -------------------------------------------------------------------------------- /docker/app.yml: -------------------------------------------------------------------------------- 1 | ## this is the all-in-one, standalone Discourse Docker container template 2 | ## 3 | ## After making changes to this file, you MUST rebuild 4 | ## /var/discourse/launcher rebuild app 5 | ## 6 | ## BE *VERY* CAREFUL WHEN EDITING! 7 | ## YAML FILES ARE SUPER SUPER SENSITIVE TO MISTAKES IN WHITESPACE OR ALIGNMENT! 8 | ## visit http://www.yamllint.com/ to validate this file as needed 9 | 10 | templates: 11 | - "templates/postgres.template.yml" 12 | - "templates/redis.template.yml" 13 | - "templates/web.template.yml" 14 | - "templates/web.ratelimited.template.yml" 15 | ## Uncomment these two lines if you wish to add Lets Encrypt (https) 16 | #- "templates/web.ssl.template.yml" 17 | #- "templates/web.letsencrypt.ssl.template.yml" 18 | 19 | ## which TCP/IP ports should this container expose? 20 | ## If you want Discourse to share a port with another webserver like Apache or nginx, 21 | ## see https://meta.discourse.org/t/17247 for details 22 | expose: 23 | - "80:80" # http 24 | - "443:443" # https 25 | 26 | params: 27 | db_default_text_search_config: "pg_catalog.english" 28 | 29 | ## Set db_shared_buffers to a max of 25% of the total memory. 30 | ## will be set automatically by bootstrap based on detected RAM, or you can override 31 | db_shared_buffers: "1024MB" 32 | 33 | ## can improve sorting performance, but adds memory usage per-connection 34 | #db_work_mem: "40MB" 35 | 36 | ## Which Git revision should this container use? (default: tests-passed) 37 | #version: tests-passed 38 | 39 | env: 40 | LANG: en_US.UTF-8 41 | # DISCOURSE_DEFAULT_LOCALE: en 42 | 43 | ## How many concurrent web requests are supported? Depends on memory and CPU cores. 44 | ## will be set automatically by bootstrap based on detected CPUs, or you can override 45 | UNICORN_WORKERS: 4 46 | 47 | ## TODO: The domain name this Discourse instance will respond to 48 | ## Required. Discourse will not work with a bare IP number. 49 | DISCOURSE_HOSTNAME: discourse.example.com 50 | 51 | ## Uncomment if you want the container to be started with the same 52 | ## hostname (-h option) as specified above (default "$hostname-$config") 53 | #DOCKER_USE_HOSTNAME: true 54 | 55 | ## TODO: List of comma delimited emails that will be made admin and developer 56 | ## on initial signup example 'user1@example.com,user2@example.com' 57 | DISCOURSE_DEVELOPER_EMAILS: 'me@example.com,you@example.com' 58 | 59 | ## TODO: The SMTP mail server used to validate new accounts and send notifications 60 | # SMTP ADDRESS, username, and password are required 61 | # WARNING the char '#' in SMTP password can cause problems! 62 | DISCOURSE_SMTP_ADDRESS: smtp.example.com 63 | DISCOURSE_SMTP_PORT: 587 64 | DISCOURSE_SMTP_USER_NAME: user@example.com 65 | DISCOURSE_SMTP_PASSWORD: "pa$$word" 66 | #DISCOURSE_SMTP_ENABLE_START_TLS: true # (optional, default true) 67 | 68 | ## If you added the Lets Encrypt template, uncomment below to get a free SSL certificate 69 | #LETSENCRYPT_ACCOUNT_EMAIL: me@example.com 70 | 71 | ## The CDN address for this Discourse instance (configured to pull) 72 | ## see https://meta.discourse.org/t/14857 for details 73 | #DISCOURSE_CDN_URL: //discourse-cdn.example.com 74 | 75 | ## The Docker container is stateless; all data is stored in /shared 76 | volumes: 77 | - volume: 78 | host: /var/discourse/shared/standalone 79 | guest: /shared 80 | - volume: 81 | host: /var/discourse/shared/standalone/log/var-log 82 | guest: /var/log 83 | 84 | ## Plugins go here 85 | ## see https://meta.discourse.org/t/19157 for details 86 | hooks: 87 | after_code: 88 | - exec: 89 | cd: $home/plugins 90 | cmd: 91 | - git clone https://github.com/discourse/docker_manager.git 92 | 93 | ## Any custom commands to run after building 94 | run: 95 | - exec: echo "Beginning of custom commands" 96 | ## If you want to set the 'From' email address for your first registration, uncomment and change: 97 | ## After getting the first signup email, re-comment the line. It only needs to run once. 98 | #- exec: rails r "SiteSetting.notification_email='info@unconfigured.discourse.org'" 99 | - exec: echo "End of custom commands" 100 | -------------------------------------------------------------------------------- /packer/ami.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "aws_access_key": "", 4 | "aws_secret_key": "", 5 | "builder_suffix": "" 6 | }, 7 | "builders": [{ 8 | "type": "amazon-ebs", 9 | "access_key": "{{user `aws_access_key`}}", 10 | "secret_key": "{{user `aws_secret_key`}}", 11 | "region": "us-east-1", 12 | "source_ami": "ami-263d0b5c", 13 | "instance_type": "t2.small", 14 | "ssh_username": "ubuntu", 15 | "ami_name": "rails-ruby-benchmark {{timestamp}}", 16 | "run_tags": { 17 | "Name": "Packer RailsRubyBench Builder{{user `builder_suffix`}}" 18 | }, 19 | "launch_block_device_mappings": [ 20 | { 21 | "device_name": "/dev/sda1", 22 | "volume_size": 40, 23 | "volume_type": "gp2", 24 | "delete_on_termination": true 25 | } 26 | ] 27 | }], 28 | "provisioners": [{ 29 | "type": "shell", 30 | "inline": [ 31 | "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done" 32 | ] 33 | }, { 34 | "type": "file", 35 | "source": "ami.json", 36 | "destination": "/tmp/ami.json" 37 | }, { 38 | "type": "shell", 39 | "inline": "sudo mv /tmp/ami.json /home/ubuntu/ami.json" 40 | }, { 41 | "type": "shell", 42 | "script": "discourse_install.sh" 43 | }, { 44 | "type": "shell", 45 | "execute_command": "{{ .Vars }} /bin/bash -e -l {{ .Path }}", 46 | "script": "discourse_install_rvm.sh" 47 | }, { 48 | "type": "shell", 49 | "execute_command": "{{ .Vars }} /bin/bash -e -l {{ .Path }}", 50 | "script": "discourse_install_node.sh" 51 | }, { 52 | "type": "shell", 53 | "execute_command": "chmod +x {{ .Path }}; sudo su postgres -c '{{ .Vars }} {{ .Path }}'", 54 | "script": "discourse_install_postgres.sh" 55 | }, { 56 | "type": "shell", 57 | "inline": [ 58 | "cd /home/ubuntu && git clone https://github.com/noahgibbs/rails_ruby_bench.git && git clone https://github.com/noahgibbs/rsb.git" 59 | ] 60 | }, { 61 | "type": "file", 62 | "source": "benchmark_software.json", 63 | "destination": "/home/ubuntu/benchmark_software.json" 64 | }, { 65 | "type": "shell", 66 | "execute_command": "cd /home/ubuntu; . .rvm/scripts/rvm; {{ .Vars }} ruby {{ .Path }}", 67 | "script": "setup_discourse.rb" 68 | }, { 69 | "type": "shell", 70 | "inline": "[ -f /tmp/setup_discourse_ran_correctly ] || (echo 'Setup_discourse.rb failed to run to completion...'; exit 1) # Make sure setup_discourse.rb ran correctly" 71 | }, { 72 | "type": "shell", 73 | "script": "misc_install.sh" 74 | }, { 75 | "type": "shell", 76 | "environment_vars": [ 77 | "RAILS_RUBY_BENCH_URL=https://github.com/noahgibbs/rails_ruby_bench.git", 78 | "RAILS_RUBY_BENCH_TAG=" 79 | ], 80 | "execute_command": "cd /home/ubuntu; . .rvm/scripts/rvm; {{ .Vars }} ruby {{ .Path }}", 81 | "script": "setup.rb" 82 | }, { 83 | "type": "shell", 84 | "inline": "[ -f /tmp/setup_ran_correctly ] || (echo 'Setup.rb failed to run to completion...'; exit 1) # Make sure setup.rb ran correctly" 85 | }, { 86 | "type": "shell", 87 | "execute_command": "cd /home/ubuntu; . .rvm/scripts/rvm; {{ .Vars }} ruby {{ .Path }}", 88 | "script": "setup_discourse_gems.rb" 89 | }, { 90 | "type": "shell", 91 | "inline": "[ -f /tmp/setup_discourse_gems_ran_correctly ] || (echo 'Setup_discourse.rb failed to run to completion...'; exit 1) # Make sure setup_discourse_gems.rb ran correctly" 92 | }, { 93 | "type": "shell", 94 | "script": "nginx_install.sh" 95 | }, { 96 | "type": "file", 97 | "source": "rc.local", 98 | "destination": "/tmp/rc.local" 99 | }, { 100 | "type": "shell", 101 | "inline": "sudo mv /tmp/rc.local /etc/rc.local" 102 | }, { 103 | "type": "shell", 104 | "inline": "sudo chmod ugo+x /etc/rc.local" 105 | }, { 106 | "type": "file", 107 | "source": "90-tune-kernel.conf", 108 | "destination": "/tmp/90-tune-kernel.conf" 109 | }, { 110 | "type": "shell", 111 | "inline": "sudo mv /tmp/90-tune-kernel.conf /etc/sysctl.d/" 112 | }, { 113 | "type": "file", 114 | "source": "nginx.default.conf", 115 | "destination": "/tmp/nginx.default.conf" 116 | }, { 117 | "type": "shell", 118 | "inline": "sudo mv /tmp/nginx.default.conf /etc/nginx/sites-available/default" 119 | } 120 | ] 121 | } 122 | -------------------------------------------------------------------------------- /packer/ami-with-passenger-enterprise.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "aws_access_key": "", 4 | "aws_secret_key": "", 5 | "builder_suffix": "" 6 | }, 7 | "builders": [{ 8 | "type": "amazon-ebs", 9 | "access_key": "{{user `aws_access_key`}}", 10 | "secret_key": "{{user `aws_secret_key`}}", 11 | "region": "us-east-1", 12 | "source_ami": "ami-263d0b5c", 13 | "instance_type": "t2.small", 14 | "ssh_username": "ubuntu", 15 | "ami_name": "rails-ruby-benchmark {{timestamp}}", 16 | "run_tags": { 17 | "Name": "Packer RailsRubyBench Builder{{user `builder_suffix`}}" 18 | }, 19 | "launch_block_device_mappings": [ 20 | { 21 | "device_name": "/dev/sda1", 22 | "volume_size": 40, 23 | "volume_type": "gp2", 24 | "delete_on_termination": true 25 | } 26 | ] 27 | }], 28 | "provisioners": [{ 29 | "type": "shell", 30 | "inline": [ 31 | "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done" 32 | ] 33 | }, { 34 | "type": "file", 35 | "source": "ami.json", 36 | "destination": "/tmp/ami.json" 37 | }, { 38 | "type": "shell", 39 | "inline": "sudo mv /tmp/ami.json /home/ubuntu/ami.json" 40 | }, { 41 | "type": "shell", 42 | "script": "discourse_install.sh" 43 | }, { 44 | "type": "shell", 45 | "execute_command": "{{ .Vars }} /bin/bash -e -l {{ .Path }}", 46 | "script": "discourse_install_rvm.sh" 47 | }, { 48 | "type": "shell", 49 | "execute_command": "{{ .Vars }} /bin/bash -e -l {{ .Path }}", 50 | "script": "discourse_install_node.sh" 51 | }, { 52 | "type": "shell", 53 | "execute_command": "chmod +x {{ .Path }}; sudo su postgres -c '{{ .Vars }} {{ .Path }}'", 54 | "script": "discourse_install_postgres.sh" 55 | }, { 56 | "type": "shell", 57 | "inline": [ 58 | "cd /home/ubuntu && git clone https://github.com/noahgibbs/rails_ruby_bench.git && git clone https://github.com/noahgibbs/rsb.git" 59 | ] 60 | }, { 61 | "type": "file", 62 | "source": "benchmark_software.json", 63 | "destination": "/home/ubuntu/benchmark_software.json" 64 | }, { 65 | "type": "shell", 66 | "execute_command": "cd /home/ubuntu; . .rvm/scripts/rvm; {{ .Vars }} ruby {{ .Path }}", 67 | "script": "setup_discourse.rb" 68 | }, { 69 | "type": "shell", 70 | "inline": "[ -f /tmp/setup_discourse_ran_correctly ] || (echo 'Setup_discourse.rb failed to run to completion...'; exit 1) # Make sure setup_discourse.rb ran correctly" 71 | }, { 72 | "type": "shell", 73 | "script": "misc_install.sh" 74 | }, { 75 | "type": "shell", 76 | "environment_vars": [ 77 | "RAILS_RUBY_BENCH_URL=https://github.com/noahgibbs/rails_ruby_bench.git", 78 | "RAILS_RUBY_BENCH_TAG=" 79 | ], 80 | "execute_command": "cd /home/ubuntu; . .rvm/scripts/rvm; {{ .Vars }} ruby {{ .Path }}", 81 | "script": "setup.rb" 82 | }, { 83 | "type": "shell", 84 | "inline": "[ -f /tmp/setup_ran_correctly ] || (echo 'Setup.rb failed to run to completion...'; exit 1) # Make sure setup.rb ran correctly" 85 | }, { 86 | "type": "shell", 87 | "execute_command": "cd /home/ubuntu; . .rvm/scripts/rvm; {{ .Vars }} ruby {{ .Path }}", 88 | "script": "setup_discourse_gems.rb" 89 | }, { 90 | "type": "shell", 91 | "inline": "[ -f /tmp/setup_discourse_gems_ran_correctly ] || (echo 'Setup_discourse.rb failed to run to completion...'; exit 1) # Make sure setup_discourse_gems.rb ran correctly" 92 | }, { 93 | "type": "shell", 94 | "script": "nginx_install.sh" 95 | }, { 96 | "type": "file", 97 | "source": "rc.local", 98 | "destination": "/tmp/rc.local" 99 | }, { 100 | "type": "shell", 101 | "inline": "sudo mv /tmp/rc.local /etc/rc.local" 102 | }, { 103 | "type": "shell", 104 | "inline": "sudo chmod ugo+x /etc/rc.local" 105 | }, { 106 | "type": "file", 107 | "source": "90-tune-kernel.conf", 108 | "destination": "/tmp/90-tune-kernel.conf" 109 | }, { 110 | "type": "shell", 111 | "inline": "sudo mv /tmp/90-tune-kernel.conf /etc/sysctl.d/" 112 | }, { 113 | "type": "file", 114 | "source": "nginx.default.conf", 115 | "destination": "/tmp/nginx.default.conf" 116 | }, { 117 | "type": "shell", 118 | "inline": "sudo mv /tmp/nginx.default.conf /etc/nginx/sites-available/default" 119 | }, 120 | { 121 | "type": "file", 122 | "source": "passenger-enterprise-license", 123 | "destination": "/tmp/passenger-enterprise-license" 124 | }, 125 | { 126 | "type": "file", 127 | "source": "passenger-download-token", 128 | "destination": "/tmp/passenger-download-token" 129 | }, 130 | { 131 | "type": "file", 132 | "source": "passenger-enterprise-install.sh", 133 | "destination": "/tmp/passenger-enterprise-install.sh" 134 | }, 135 | { 136 | "type": "shell", 137 | "script": "passenger-enterprise-install.sh" 138 | } 139 | ] 140 | } 141 | -------------------------------------------------------------------------------- /docker/build_benchmark_software.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "fileutils" 4 | require "json" 5 | 6 | # Print all commands and show their full output 7 | VERBOSE = true 8 | 9 | Dir.mkdir "/var/rubies" 10 | benchmark_software = JSON.load(File.read("/tmp/benchmark_software.json")) 11 | 12 | DISCOURSE_DIR = "/var/www/discourse" 13 | RAILS_BENCH_DIR = "/var/rails_ruby_bench" 14 | 15 | class SystemPackerBuildError < RuntimeError; end 16 | 17 | # Checked system - error if the command fails 18 | def csystem(cmd, err, opts = {}) 19 | cmd = "bash -l -c \"#{cmd}\"" if opts[:bash] 20 | print "Running command: #{cmd.inspect}\n" if VERBOSE || opts[:debug] || opts["debug"] 21 | if VERBOSE 22 | system(cmd, out: $stdout, err: :out) 23 | else 24 | out = `#{cmd}` 25 | end 26 | unless $?.success? || opts[:fail_ok] || opts["fail_ok"] 27 | puts "Error running command:\n#{cmd.inspect}" 28 | puts "Output:\n#{out}\n=====" if out 29 | raise SystemPackerBuildError.new(err) 30 | end 31 | end 32 | 33 | def clone_or_update_repo(repo_url, tag, work_dir) 34 | unless Dir.exist?(work_dir) 35 | csystem "git clone #{repo_url} #{work_dir}", "Couldn't 'git clone' into #{work_dir}!", :debug => true 36 | end 37 | 38 | Dir.chdir(work_dir) do 39 | csystem "git fetch", "Couldn't 'git fetch' in #{work_dir}!", :debug => true 40 | 41 | if tag && tag.strip != "" 42 | tag = tag.strip 43 | csystem "git checkout #{tag}", "Couldn't 'git checkout #{tag}' in #{work_dir}!", :debug => true 44 | else 45 | csystem "git pull", "Couldn't 'git pull' in #{work_dir}!", :debug => true 46 | end 47 | end 48 | end 49 | 50 | def clone_or_update_by_json(h, work_dir) 51 | clone_or_update_repo(h["git_url"], h["git_tag"], h["checkout_dir"] || work_dir) 52 | end 53 | 54 | def build_and_mount_ruby(source_dir, prefix_dir, mount_name, options = {}) 55 | puts "Build and mount Ruby: Source dir: #{source_dir.inspect} Prefix dir: #{prefix_dir.inspect} Mount name: #{mount_name.inspect}" 56 | Dir.chdir(source_dir) do 57 | unless File.exists?("configure") 58 | csystem "autoconf", "Couldn't run autoconf in #{source_dir}!" 59 | end 60 | unless File.exists?("Makefile") 61 | configure_options = options["configure_options"] || "" 62 | csystem "./configure --prefix #{prefix_dir} #{configure_options}", "Couldn't run configure in #{source_dir}!" 63 | end 64 | csystem "make", "Make failed in #{source_dir}!" 65 | # This should install to the benchmark ruby dir 66 | csystem "make install", "Installing Ruby failed in #{source_dir}!" 67 | end 68 | csystem "ln -s #{source_dir} ~/.rbenv/versions/#{mount_name}", "Couldn't mount #{source_dir.inspect} as #{mount_name}!", :bash => true 69 | end 70 | 71 | def autogen_name 72 | @autogen_number ||= 1 73 | name = "autogen-name-#{@autogen_number}" 74 | @autogen_number += 1 75 | name 76 | end 77 | 78 | def clone_or_update_ruby_by_json(h, work_dir) 79 | clone_or_update_by_json(h, work_dir) 80 | mount_name = h["name"] || autogen_name 81 | prefix_dir = h["prefix_dir"] || "/home/discourse/.rbenv/versions/#{mount_name.gsub("/", "_")}" 82 | 83 | build_and_mount_ruby(h["checkout_dir"], prefix_dir, mount_name, { "configure_options" => h["configure_options"] || "" } ) 84 | h["mount_name"] = mount_name 85 | end 86 | 87 | # First, clone all the Rubies to get the network errors together up-front 88 | benchmark_software["compare_rubies"].each do |ruby_hash| 89 | puts "Installing Ruby: #{ruby_hash.inspect}" 90 | # Clone the Ruby, then build and mount if necessary 91 | if ruby_hash["git_url"] 92 | work_dir = File.join("/var/rubies", ruby_hash["name"]) 93 | ruby_hash["checkout_dir"] = work_dir 94 | clone_or_update_by_json(ruby_hash, work_dir) 95 | end 96 | end 97 | 98 | # Build and mount each Ruby and install all gems 99 | benchmark_software["compare_rubies"].each do |ruby_hash| 100 | puts "Installing Ruby: #{ruby_hash.inspect}" 101 | # Clone the Ruby, then build and mount if necessary 102 | if ruby_hash["git_url"] 103 | clone_or_update_ruby_by_json(ruby_hash, ruby_hash["checkout_dir"]) 104 | end 105 | 106 | puts "Mounted the built Ruby: #{ruby_hash.inspect}" 107 | 108 | mount_ruby_name = ruby_hash["mount_name"] || ruby_hash["name"] 109 | csystem "rbenv shell #{mount_ruby_name} && gem install bundler mailcatcher", "Couldn't install bundler and/or mailcatcher for Ruby #{mount_ruby_name.inspect}!", :bash => true 110 | 111 | Dir.chdir(RAILS_BENCH_DIR) do 112 | csystem "rbenv shell #{mount_ruby_name} && bundle", "Couldn't install RRB gems in #{RAILS_BENCH_DIR} for Ruby #{mount_ruby_name.inspect}!", :bash => true 113 | end 114 | 115 | Dir.chdir(DISCOURSE_DIR) do 116 | csystem "rbenv shell #{mount_ruby_name} && bundle", "Couldn't install Discourse gems in #{DISCOURSE_DIR} for Ruby #{mount_ruby_name.inspect}!", :bash => true 117 | end 118 | end 119 | 120 | puts "Create benchmark_ruby_versions.txt" 121 | File.open("/var/benchmark_ruby_versions.txt", "w") do |f| 122 | rubies = benchmark_software["compare_rubies"].map { |h| h["mount_name"] || h["name"] } 123 | f.print rubies.join("\n") 124 | end 125 | 126 | #Dir.chdir(DISCOURSE_DIR) do 127 | # csystem("RAILS_ENV=profile bundle exec rake db:create db:migrate", "Couldn't create Discourse database!", :bash => true) 128 | # unless File.exists?("public/assets") 129 | # csystem("RAILS_ENV=profile bundle exec rake assets:precompile", "Failed to precompile Discourse assets!", :bash => true) 130 | # end 131 | #end 132 | -------------------------------------------------------------------------------- /packer/setup_discourse.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "fileutils" 4 | require "json" 5 | 6 | # Pass --local to run the setup on a local machine, or set RRB_LOCAL 7 | LOCAL = (ARGV.delete '--local') || ENV["RRB_LOCAL"] 8 | # Whether to build rubies with rvm 9 | BUILD_RUBY = !LOCAL 10 | USE_BASH = BUILD_RUBY 11 | # Print all commands and show their full output 12 | VERBOSE = LOCAL 13 | 14 | base = LOCAL ? File.expand_path('..', __FILE__) : "/home/ubuntu" 15 | benchmark_software = JSON.load(File.read("#{base}/benchmark_software.json")) 16 | 17 | DISCOURSE_GIT_URL = benchmark_software["discourse"]["git_url"] 18 | DISCOURSE_TAG = benchmark_software["discourse"]["git_tag"] 19 | 20 | class SystemPackerBuildError < RuntimeError; end 21 | 22 | print < true 47 | end 48 | 49 | Dir.chdir(work_dir) do 50 | csystem "git fetch", "Couldn't 'git fetch' in #{work_dir}!", :debug => true 51 | 52 | if tag && tag.strip != "" 53 | tag = tag.strip 54 | csystem "git checkout #{tag}", "Couldn't 'git checkout #{tag}' in #{work_dir}!", :debug => true 55 | else 56 | csystem "git pull", "Couldn't 'git pull' in #{work_dir}!", :debug => true 57 | end 58 | end 59 | end 60 | 61 | if LOCAL 62 | RAILS_BENCH_DIR = File.expand_path("../..", __FILE__) 63 | else 64 | RAILS_BENCH_DIR = File.join(Dir.pwd, "rails_ruby_bench") 65 | end 66 | DISCOURSE_DIR = File.join(RAILS_BENCH_DIR, "work", "discourse") 67 | 68 | clone_or_update_repo DISCOURSE_GIT_URL, DISCOURSE_TAG, DISCOURSE_DIR 69 | 70 | # Install Discourse gems into RVM-standard Ruby installed for Discourse 71 | Dir.chdir(DISCOURSE_DIR) do 72 | csystem "gem install bundler -v1.17.3", "Couldn't install bundler for #{DISCOURSE_DIR} for Discourse's system Ruby!", :bash => true 73 | csystem "bundle _1.17.3_", "Couldn't install Discourse gems for #{DISCOURSE_DIR} for Discourse's system Ruby!", :bash => true 74 | end 75 | 76 | if LOCAL 77 | puts "\nIf not done already, you should setup the dependencies for Discourse: redis, postgres and node" 78 | puts "https://github.com/discourse/discourse/blob/v1.8.0.beta13/docs/DEVELOPER-ADVANCED.md#preparing-a-fresh-ubuntu-install" 79 | puts 80 | end 81 | 82 | Dir.chdir(DISCOURSE_DIR) do 83 | csystem "RAILS_ENV=profile bundle _1.17.3_ exec rake db:create", "Couldn't create Rails database!", :bash => true 84 | csystem "RAILS_ENV=profile bundle _1.17.3_ exec rake db:migrate", "Failed running 'rake db:migrate' in #{DISCOURSE_DIR}!", :bash => true 85 | 86 | # TODO: use a better check for whether to rebuild precompiled assets 87 | unless File.exists? "public/assets" 88 | csystem "RAILS_ENV=profile bundle _1.17.3_ exec rake assets:precompile", "Failed running 'rake assets:precompile' in #{DISCOURSE_DIR}!", :bash => true 89 | end 90 | unless File.exists? "public/uploads" 91 | FileUtils.mkdir "public/uploads" 92 | end 93 | conf_db = File.read "config/database.yml" 94 | new_contents = conf_db.gsub("pool: 5", "pool: 30") # Increase database.yml thread pool, including for profile environment 95 | if new_contents != conf_db 96 | File.open("config/database.yml", "w") do |f| 97 | f.print new_contents 98 | end 99 | end 100 | end 101 | 102 | puts "Add assets.rb initializer for Discourse" 103 | # Minor bugfix for this version of Discourse. Can remove when I only use 1.8.0+ Discourse? 104 | ASSETS_INIT = File.join(DISCOURSE_DIR, "config/initializers/assets.rb") 105 | unless File.exists?(ASSETS_INIT) 106 | File.open(ASSETS_INIT, "w") do |f| 107 | f.write <<-INITIALIZER 108 | Rails.application.config.assets.precompile += %w( jquery_include.js ) 109 | INITIALIZER 110 | end 111 | end 112 | 113 | puts "Hack to disable CSRF protection during benchmark..." 114 | # Turn off CSRF protection for Discourse in the benchmark. I have no idea why 115 | # user_simulator's CSRF handling stopped working between Discourse 1.7.X and 116 | # 1.8.0.beta10, but it clearly did. This is a horrible workaround and should 117 | # be fixed when I figure out the problem. 118 | APP_CONTROLLER = File.join(DISCOURSE_DIR, "app/controllers/application_controller.rb") 119 | contents = File.read(APP_CONTROLLER) 120 | original_line = "protect_from_forgery" 121 | patched_line = "#protect_from_forgery" 122 | unless contents[patched_line] 123 | File.open(APP_CONTROLLER, "w") do |f| 124 | f.print contents.gsub(original_line, patched_line) 125 | end 126 | end 127 | 128 | # Oh and hey, looks like Bootsnap breaks something in the load order when 129 | # including config/environment into start.rb. Joy. 130 | # TODO: merge this patching into a little table of patches? 131 | path = File.join(DISCOURSE_DIR, "config/boot.rb") 132 | contents = File.read(path) 133 | original_line = "if ENV['RAILS_ENV'] != 'production'" 134 | patched_line = "if ENV['RAILS_ENV'] != 'production' && ENV['RAILS_ENV'] != 'profile'" 135 | unless contents[patched_line] 136 | File.open(path, "w") do |f| 137 | f.print contents.gsub(original_line, patched_line) 138 | end 139 | end 140 | 141 | FileUtils.touch "/tmp/setup_discourse_ran_correctly" 142 | -------------------------------------------------------------------------------- /packer/README.md: -------------------------------------------------------------------------------- 1 | # Using Packer to Build an AMI 2 | 3 | Instead of using a pre-built AMI, you can build your own. 4 | 5 | This only builds a combined AMI, with the load-tester, database and Rails server on the same instance. You're welcome to adapt it to other configurations, though, if you'd like to benchmark in that configuration. I'd love a Pull Request! 6 | 7 | To build your own AMI, the usual invocation is "packer build ami.json" from this directory. 8 | 9 | # Passenger Enterprise 10 | 11 | Passenger Enterprise is *not* included in the default ami.json. There 12 | is a file called ami-with-passenger-enterprise.json. You'll also need 13 | two more files called passenger-enterprise-license and 14 | passenger-download-token, with the license and download codes you 15 | received when you purchased Passenger Enterprise. 16 | 17 | If you run this without those codes, you can't build an AMI with 18 | Passenger Enterprise on it. You shouldn't need a production 19 | (user-facing) license for this. But given that you're benchmarking, 20 | you usually won't want to spend money on a Passenger Enterprise 21 | license. It's not in the default image, and I can't hand out Passenger 22 | license codes in any case. 23 | 24 | NOTE FOR LATER: Passenger has changed their method of installation 25 | between Xenial (current) and Bionic (not yet RRB's default.) The 26 | Passenger Enterprise Install script will need to be updated when we 27 | move to Bionic. 28 | 29 | Xenial: https://www.phusionpassenger.com/docs/advanced_guides/install_and_upgrade/nginx/install/enterprise/xenial.html 30 | 31 | Bionic: https://www.phusionpassenger.com/docs/advanced_guides/install_and_upgrade/nginx/install/enterprise/bionic.html 32 | 33 | # Installing Packer 34 | 35 | Packer is annoyingly unfriendly to most packaging systems. On Linux, they just want you to download and manually install the Packer binary from their site. 36 | 37 | On Mac, Homebrew has a package for it which just does the same thing. But at least Homebrew will do it for you: 38 | 39 | brew install packer 40 | 41 | # Getting Your AWS Account Set Up 42 | 43 | See Packer's documentation on building AMIs: https://www.packer.io/intro/getting-started/build-image.html 44 | 45 | The short answer is that you can install the AWS credentials in the 46 | standard ways and Packer will use them. For Mac OS, you can use homebrew: 47 | 48 | brew install aws-cli 49 | aws configure 50 | 51 | ## Canonical Benchmark Timing 52 | 53 | The region shouldn't matter much, but I recommend us-east-1 -- it's the cheapest, and this benchmark shouldn't need locality to any specific world location. You need to build an AMI in the specific region where you'll be testing. 54 | 55 | However, you do *not* need to build an AMI with the exact same instance type where you'll be benchmarking. So you can save a few pennies when building your instance, if you want. 56 | 57 | If you build an AMI and keep it in your account, that may also cost around $0.01 USD/month (plus the initial $0.02 when you build it.) So if you don't need to preserve it, delete it. I don't know if that's still true for a free AWS account, so YMMV. 58 | 59 | If you want more realistic performance numbers... Well, a benchmark may not be your best bet. But you *can* configure these processes to run on multiple instances, which will exchange some sources of error for other ones. However, I don't supply AMI build scripts for separate instances for database, load-tester, etc, nor do I supply a way to coordinate them. You'll need to set that up for yourself. It's not hard if you're used to setting up multi-instance Rails apps - Discourse is pretty standard in how it gets set up. 60 | 61 | ## Configuring Your AMI 62 | 63 | When choosing a source AMI, you'll need to match the region you're building in. Building in us-east-1? Use a us-east-1 AMI. I recommend Packer's EBS builder, but then you'll need to use EBS instead of instance store. And if you build on a cheap instance like t2.micro, you can probably only use an HVM AMI (fully virtualized.) To build a ParaVirtualized (PV) AMI, I think you'll need to build your AMI from a ParaVirtualized source AMI -- feel free to correct me if I'm wrong in the form of a Pull Request or Issue :-) 64 | 65 | You don't need to fully match the final instance size for the benchmark when you build your AMI. Though if you match it 100%, it'll certainly be compatible. The only reason I don't match them up perfectly is that it's generally cheaper to build your AMI on a smaller instance when you can. 66 | 67 | ## IAM Roles 68 | 69 | See Packer's documentation on setting up AWS and IAM roles: https://www.packer.io/docs/builders/amazon.html 70 | 71 | Also: 72 | 73 | { 74 | "Version": "2012-10-17", 75 | "Statement": [{ 76 | "Effect": "Allow", 77 | "Action" : [ 78 | "ec2:AttachVolume", 79 | "ec2:AuthorizeSecurityGroupIngress", 80 | "ec2:CopyImage", 81 | "ec2:CreateImage", 82 | "ec2:CreateKeypair", 83 | "ec2:CreateSecurityGroup", 84 | "ec2:CreateSnapshot", 85 | "ec2:CreateTags", 86 | "ec2:CreateVolume", 87 | "ec2:DeleteKeypair", 88 | "ec2:DeleteSecurityGroup", 89 | "ec2:DeleteSnapshot", 90 | "ec2:DeleteVolume", 91 | "ec2:DeregisterImage", 92 | "ec2:DescribeImageAttribute", 93 | "ec2:DescribeImages", 94 | "ec2:DescribeInstances", 95 | "ec2:DescribeRegions", 96 | "ec2:DescribeSecurityGroups", 97 | "ec2:DescribeSnapshots", 98 | "ec2:DescribeSubnets", 99 | "ec2:DescribeTags", 100 | "ec2:DescribeVolumes", 101 | "ec2:DetachVolume", 102 | "ec2:GetPasswordData", 103 | "ec2:ModifyImageAttribute", 104 | "ec2:ModifyInstanceAttribute", 105 | "ec2:ModifySnapshotAttribute", 106 | "ec2:RegisterImage", 107 | "ec2:RunInstances", 108 | "ec2:StopInstances", 109 | "ec2:TerminateInstances" 110 | ], 111 | "Resource" : "*" 112 | }] 113 | } 114 | 115 | -------------------------------------------------------------------------------- /process.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "json" 4 | require "optparse" 5 | 6 | cohorts_by = "RUBY_VERSION,warmup_iterations,discourse_revision,random_seed" 7 | input_glob = "rails_ruby_bench_*.json" 8 | 9 | OptionParser.new do |opts| 10 | opts.banner = "Usage: ruby process.rb [options]" 11 | opts.on("-c", "--cohorts-by COHORTS", "Variables to partition data by, incl. RUBY_VERSION,warmup_iterations,etc.") do |c| 12 | cohorts_by = c #.to_i 13 | end 14 | opts.on("-i", "--input-glob GLOB", "File pattern to match on (default *.json)") do |s| 15 | input_glob = s 16 | end 17 | end.parse! 18 | 19 | OUTPUT_FILE = "process_output.json" 20 | 21 | cohort_indices = cohorts_by.strip.split(",") 22 | 23 | req_time_by_cohort = {} 24 | run_by_cohort = {} 25 | throughput_by_cohort = {} 26 | startup_by_cohort = {} 27 | 28 | INPUT_FILES = Dir[input_glob] 29 | 30 | process_output = { 31 | cohort_indices: cohort_indices, 32 | input_files: INPUT_FILES, 33 | req_time_by_cohort: req_time_by_cohort, 34 | run_by_cohort: run_by_cohort, 35 | throughput_by_cohort: throughput_by_cohort, 36 | startup_by_cohort: startup_by_cohort, 37 | processed: { 38 | :cohort => {}, 39 | }, 40 | } 41 | 42 | INPUT_FILES.each do |f| 43 | begin 44 | d = JSON.load File.read(f) 45 | rescue JSON::ParserError 46 | raise "Error parsing JSON in file: #{f.inspect}" 47 | end 48 | 49 | # Assign a cohort to these samples 50 | cohort_parts = cohort_indices.map do |cohort_elt| 51 | raise "Unexpected file format for file #{f.inspect}!" unless d && d["settings"] && d["environment"] 52 | item = nil 53 | if d["settings"].has_key?(cohort_elt) 54 | item = d["settings"][cohort_elt] 55 | elsif d["environment"].has_key?(cohort_elt) 56 | item = d["environment"][cohort_elt] 57 | else 58 | raise "Can't find setting or environment object #{cohort_elt}!" 59 | end 60 | item 61 | end 62 | cohort = cohort_parts.join(",") 63 | 64 | # Update data format to latest version 65 | if d["version"].nil? 66 | times = d["requests"]["times"].flat_map do |items| 67 | out_items = [] 68 | cur_time = 0.0 69 | items.each do |i| 70 | out_items.push(i - cur_time) 71 | cur_time = i 72 | end 73 | out_items 74 | end 75 | runs = d["requests"]["times"].map { |thread_times| thread_times[-1] } 76 | raise "Error with request times! #{d["requests"]["times"].inspect}" if runs.nil? || runs.any?(:nil?) 77 | elsif [2,3].include?(d["version"]) 78 | times = d["requests"]["times"].flatten(1) 79 | runs = d["requests"]["times"].map { |thread_times| thread_times.inject(0.0, &:+) } 80 | else 81 | raise "Unrecognized data version #{d["version"].inspect} in JSON file #{f.inspect}!" 82 | end 83 | 84 | startup_by_cohort[cohort] ||= [] 85 | startup_by_cohort[cohort].concat d["startup"]["times"] 86 | 87 | req_time_by_cohort[cohort] ||= [] 88 | req_time_by_cohort[cohort].concat times 89 | 90 | run_by_cohort[cohort] ||= [] 91 | run_by_cohort[cohort].push runs 92 | 93 | throughput_by_cohort[cohort] ||= [] 94 | throughput_by_cohort[cohort].push (d["requests"]["times"].flatten.size / runs.max) unless runs.empty? 95 | end 96 | 97 | def percentile(list, pct) 98 | len = list.length 99 | how_far = pct * 0.01 * (len - 1) 100 | prev_item = how_far.to_i 101 | return list[prev_item] if prev_item >= len - 1 102 | return list[0] if prev_item < 0 103 | 104 | linear_combination = how_far - prev_item 105 | list[prev_item] + (list[prev_item + 1] - list[prev_item]) * linear_combination 106 | end 107 | 108 | def array_mean(arr) 109 | return nil if arr.empty? 110 | arr.inject(0.0, &:+) / arr.size 111 | end 112 | 113 | # Calculate variance based on the Wikipedia article of algorithms for variance. 114 | # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance 115 | # Includes Bessel's correction. 116 | def array_variance(arr) 117 | n = arr.size 118 | return nil if arr.empty? || n < 2 119 | 120 | ex = ex2 = 0 121 | arr.each do |x| 122 | diff = x - arr[0] 123 | ex += diff 124 | ex2 += diff * diff 125 | end 126 | 127 | (ex2 - (ex * ex) / arr.size) / (arr.size - 1) 128 | end 129 | 130 | req_time_by_cohort.keys.sort.each do |cohort| 131 | data = req_time_by_cohort[cohort] 132 | data.sort! # Sort request times lowest-to-highest for use with percentile() 133 | runs = run_by_cohort[cohort] 134 | flat_runs = runs.flatten.sort 135 | run_longest = runs.map { |worker_times| worker_times.max } 136 | throughputs = throughput_by_cohort[cohort].sort 137 | startup_times = startup_by_cohort[cohort].sort 138 | 139 | cohort_printable = cohort_indices.zip(cohort.split(",")).map { |a, b| "#{a}: #{b}" }.join(", ") 140 | print "=====\nCohort: #{cohort_printable}, # of data points: #{data.size} http / #{startup_times.size} startup, full runs: #{runs.size}\n" 141 | process_output[:processed][:cohort][cohort] = { 142 | data_points: data.size, 143 | full_runs: runs.size, 144 | request_percentiles: {}, 145 | run_percentiles: {}, 146 | throughputs: throughputs, 147 | } 148 | [0, 1, 5, 10, 50, 90, 95, 99, 100].each do |p| 149 | process_output[:processed][:cohort][cohort][:request_percentiles][p.to_s] = percentile(data, p) 150 | print " #{"%2d" % p}%ile: #{percentile(data, p)}\n" 151 | end 152 | 153 | print "--\n Overall thread completion times:\n" 154 | [0, 10, 50, 90, 100].each do |p| 155 | process_output[:processed][:cohort][cohort][:run_percentiles][p.to_s] = percentile(flat_runs, p) 156 | print " #{"%2d" % p}%ile: #{percentile(flat_runs, p)}\n" 157 | end 158 | 159 | print "--\n Throughput in reqs/sec for each full run:\n" 160 | print " Mean: #{array_mean(throughputs).inspect} Median: #{percentile(throughputs, 50).inspect} Variance: #{array_variance(throughputs).inspect}\n" 161 | process_output[:processed][:cohort][cohort][:throughput_mean] = array_mean(throughputs) 162 | process_output[:processed][:cohort][cohort][:throughput_median] = percentile(throughputs, 50) 163 | process_output[:processed][:cohort][cohort][:throughput_variance] = array_variance(throughputs) 164 | print " #{throughputs.inspect}\n\n" 165 | 166 | process_output[:processed][:cohort][cohort][:startup_mean] = array_mean(startup_times) 167 | process_output[:processed][:cohort][cohort][:startup_median] = percentile(startup_times, 50) 168 | process_output[:processed][:cohort][cohort][:startup_variance] = array_variance(startup_times) 169 | print "--\n Startup times for this cohort:\n" 170 | print " Mean: #{array_mean(startup_times).inspect} Median: #{percentile(startup_times, 50).inspect} Variance: #{array_variance(startup_times).inspect}\n" 171 | end 172 | 173 | print "******************\n" 174 | 175 | File.open(OUTPUT_FILE, "w") do |f| 176 | f.print JSON.pretty_generate(process_output) 177 | end 178 | -------------------------------------------------------------------------------- /alice.txt: -------------------------------------------------------------------------------- 1 | Alice was beginning to get very tired of sitting by her sister on the 2 | bank, and of having nothing to do: once or twice she had peeped into the 3 | book her sister was reading, but it had no pictures or conversations in 4 | it, 'and what is the use of a book,' thought Alice 'without pictures or 5 | conversation?' 6 | 7 | So she was considering in her own mind (as well as she could, for the 8 | hot day made her feel very sleepy and stupid), whether the pleasure 9 | of making a daisy-chain would be worth the trouble of getting up and 10 | picking the daisies, when suddenly a White Rabbit with pink eyes ran 11 | close by her. 12 | 13 | There was nothing so VERY remarkable in that; nor did Alice think it so 14 | VERY much out of the way to hear the Rabbit say to itself, 'Oh dear! 15 | Oh dear! I shall be late!' (when she thought it over afterwards, it 16 | occurred to her that she ought to have wondered at this, but at the time 17 | it all seemed quite natural); but when the Rabbit actually TOOK A WATCH 18 | OUT OF ITS WAISTCOAT-POCKET, and looked at it, and then hurried on, 19 | Alice started to her feet, for it flashed across her mind that she had 20 | never before seen a rabbit with either a waistcoat-pocket, or a watch 21 | to take out of it, and burning with curiosity, she ran across the field 22 | after it, and fortunately was just in time to see it pop down a large 23 | rabbit-hole under the hedge. 24 | 25 | In another moment down went Alice after it, never once considering how 26 | in the world she was to get out again. 27 | 28 | The rabbit-hole went straight on like a tunnel for some way, and then 29 | dipped suddenly down, so suddenly that Alice had not a moment to think 30 | about stopping herself before she found herself falling down a very deep 31 | well. 32 | 33 | Either the well was very deep, or she fell very slowly, for she had 34 | plenty of time as she went down to look about her and to wonder what was 35 | going to happen next. First, she tried to look down and make out what 36 | she was coming to, but it was too dark to see anything; then she 37 | looked at the sides of the well, and noticed that they were filled with 38 | cupboards and book-shelves; here and there she saw maps and pictures 39 | hung upon pegs. She took down a jar from one of the shelves as 40 | she passed; it was labelled 'ORANGE MARMALADE', but to her great 41 | disappointment it was empty: she did not like to drop the jar for fear 42 | of killing somebody, so managed to put it into one of the cupboards as 43 | she fell past it. 44 | 45 | 'Well!' thought Alice to herself, 'after such a fall as this, I shall 46 | think nothing of tumbling down stairs! How brave they'll all think me at 47 | home! Why, I wouldn't say anything about it, even if I fell off the top 48 | of the house!' (Which was very likely true.) 49 | 50 | Down, down, down. Would the fall NEVER come to an end! 'I wonder how 51 | many miles I've fallen by this time?' she said aloud. 'I must be getting 52 | somewhere near the centre of the earth. Let me see: that would be four 53 | thousand miles down, I think--' (for, you see, Alice had learnt several 54 | things of this sort in her lessons in the schoolroom, and though this 55 | was not a VERY good opportunity for showing off her knowledge, as there 56 | was no one to listen to her, still it was good practice to say it over) 57 | '--yes, that's about the right distance--but then I wonder what Latitude 58 | or Longitude I've got to?' (Alice had no idea what Latitude was, or 59 | Longitude either, but thought they were nice grand words to say.) 60 | 61 | Presently she began again. 'I wonder if I shall fall right THROUGH the 62 | earth! How funny it'll seem to come out among the people that walk with 63 | their heads downward! The Antipathies, I think--' (she was rather glad 64 | there WAS no one listening, this time, as it didn't sound at all the 65 | right word) '--but I shall have to ask them what the name of the country 66 | is, you know. Please, Ma'am, is this New Zealand or Australia?' (and 67 | she tried to curtsey as she spoke--fancy CURTSEYING as you're falling 68 | through the air! Do you think you could manage it?) 'And what an 69 | ignorant little girl she'll think me for asking! No, it'll never do to 70 | ask: perhaps I shall see it written up somewhere.' 71 | 72 | Down, down, down. There was nothing else to do, so Alice soon began 73 | talking again. 'Dinah'll miss me very much to-night, I should think!' 74 | (Dinah was the cat.) 'I hope they'll remember her saucer of milk at 75 | tea-time. Dinah my dear! I wish you were down here with me! There are no 76 | mice in the air, I'm afraid, but you might catch a bat, and that's very 77 | like a mouse, you know. But do cats eat bats, I wonder?' And here Alice 78 | began to get rather sleepy, and went on saying to herself, in a dreamy 79 | sort of way, 'Do cats eat bats? Do cats eat bats?' and sometimes, 'Do 80 | bats eat cats?' for, you see, as she couldn't answer either question, 81 | it didn't much matter which way she put it. She felt that she was dozing 82 | off, and had just begun to dream that she was walking hand in hand with 83 | Dinah, and saying to her very earnestly, 'Now, Dinah, tell me the truth: 84 | did you ever eat a bat?' when suddenly, thump! thump! down she came upon 85 | a heap of sticks and dry leaves, and the fall was over. 86 | 87 | Alice was not a bit hurt, and she jumped up on to her feet in a moment: 88 | she looked up, but it was all dark overhead; before her was another 89 | long passage, and the White Rabbit was still in sight, hurrying down it. 90 | There was not a moment to be lost: away went Alice like the wind, and 91 | was just in time to hear it say, as it turned a corner, 'Oh my ears 92 | and whiskers, how late it's getting!' She was close behind it when she 93 | turned the corner, but the Rabbit was no longer to be seen: she found 94 | herself in a long, low hall, which was lit up by a row of lamps hanging 95 | from the roof. 96 | 97 | There were doors all round the hall, but they were all locked; and when 98 | Alice had been all the way down one side and up the other, trying every 99 | door, she walked sadly down the middle, wondering how she was ever to 100 | get out again. 101 | 102 | Suddenly she came upon a little three-legged table, all made of solid 103 | glass; there was nothing on it except a tiny golden key, and Alice's 104 | first thought was that it might belong to one of the doors of the hall; 105 | but, alas! either the locks were too large, or the key was too small, 106 | but at any rate it would not open any of them. However, on the second 107 | time round, she came upon a low curtain she had not noticed before, and 108 | behind it was a little door about fifteen inches high: she tried the 109 | little golden key in the lock, and to her great delight it fitted! 110 | 111 | Alice opened the door and found that it led into a small passage, not 112 | much larger than a rat-hole: she knelt down and looked along the passage 113 | into the loveliest garden you ever saw. How she longed to get out of 114 | that dark hall, and wander about among those beds of bright flowers and 115 | those cool fountains, but she could not even get her head through the 116 | doorway; 'and even if my head would go through,' thought poor Alice, 'it 117 | would be of very little use without my shoulders. Oh, how I wish I could 118 | shut up like a telescope! I think I could, if I only know how to begin.' 119 | For, you see, so many out-of-the-way things had happened lately, 120 | that Alice had begun to think that very few things indeed were really 121 | impossible. 122 | 123 | -------------------------------------------------------------------------------- /user_simulator.rb: -------------------------------------------------------------------------------- 1 | # Initially based on Discourse's user_simulator script 2 | 3 | #require 'gabbler' 4 | 5 | # Example options array passed into these functions 6 | #options = { 7 | # :user_offset => 0, 8 | # :random_seed => 1234567890, 9 | # :delay => nil, 10 | # :iterations => 100, 11 | # :warmup_iterations => 0, 12 | # :port_num => 4567, 13 | # :worker_threads => 5, 14 | # :out_dir => "/tmp", 15 | #} 16 | 17 | def sentence 18 | @gabbler ||= Gabbler.new.tap do |gabbler| 19 | story = File.read(File.dirname(__FILE__) + "/alice.txt") 20 | gabbler.learn(story) 21 | end 22 | 23 | sentence = "" 24 | until sentence.length > 800 do 25 | sentence << @gabbler.sentence 26 | sentence << "\n" 27 | end 28 | sentence 29 | end 30 | 31 | ACTIONS = [:read_topic, :post_reply, :post_topic, :get_latest] # Not active: :save_draft, :delete_reply. See below. 32 | 33 | class DiscourseClient 34 | def initialize(options) 35 | @cookies = nil 36 | @csrf = nil 37 | @prefix = "http://localhost:#{options[:port_num]}" 38 | 39 | @last_topics = Topic.order('id desc').limit(10).pluck(:id) 40 | @last_posts = Post.order('id desc').limit(10).pluck(:id) 41 | end 42 | 43 | def get_csrf_token 44 | resp = RestClient.get "#{@prefix}/session/csrf.json" 45 | @cookies = resp.cookies 46 | @csrf = JSON.parse(resp.body)["csrf"] 47 | end 48 | 49 | def request(method, url, payload = nil) 50 | args = { :method => method, :url => "#{@prefix}#{url}", :cookies => @cookies, :headers => { "X-CSRF-Token" => @csrf } } 51 | args[:payload] = payload if payload 52 | begin 53 | resp = RestClient::Request.execute args 54 | rescue RestClient::Found => e # 302 redirect 55 | resp = e.response 56 | rescue RestClient::Exception # Any other RestClient failure 57 | STDERR.puts "Got exception when #{method.to_s.upcase}ing #{url.inspect}..." 58 | raise 59 | end 60 | @cookies = resp.cookies # Maintain continuity of cookies 61 | resp 62 | end 63 | 64 | # Given the randomized parameters for an action, take that action. 65 | # See below for randomized parameter generation from the random 66 | # seed. 67 | def action_from_args(action_type, text, fp) 68 | case action_type 69 | when :read_topic 70 | # Read Topic 71 | topic_id = @last_topics[-1] 72 | request(:get, "/t/#{topic_id}.json?track_visit=true&forceLoad=true") 73 | when :save_draft 74 | # Save draft - currently not active, need to fix 403. Wrong topic ID? 75 | topic_id = @last_topics[-1] 76 | post_id = @last_posts[-1] # Not fully correct 77 | draft_hash = { "reply" => text * 5, "action" => "edit", "title" => "Title of draft reply", "categoryId" => 11, "postId" => post_id, "archetypeId" => "regular", "metaData" => nil, "sequence" => 0 } 78 | request(:post, "/draft.json", "draft_key" => "topic_#{topic_id}", "data" => draft_hash.to_json) 79 | when :post_reply 80 | # Post reply 81 | request(:post, "/posts", "raw" => text * 5, "unlist_topic" => "false", "category" => "9", "topic_id" => topic_id, "is_warning" => "false", "archetype" => "regular", "typing_during_msecs" => "2900", "composer_open_duration_msecs" => "12114", "featured_link" => "", "nested_post" => "true") 82 | # TODO: request(:delete, "/draft.json", "draft_key" => "topic_XX", "sequence" => "0") 83 | # TODO: update @last_posts 84 | when :post_topic 85 | # Post new topic 86 | request(:post, "/posts", "raw" => "", "title" => text, "unlist_topic" => "false", "category" => "", "is_warning" => "false", "archetype" => "regular", "typing_duration_msecs" => "6300", "composer_open_duration_msecs" => "31885", "nested_post" => "true") 87 | # TODO: request(:delete, "/draft.json", "topic_id" => "topic_XX") 88 | # TODO: request(:get, "/t/#{topic_id}.json?track_visit=true&forceLoad=true") 89 | # TODO: update @last_topics 90 | =begin 91 | Started GET "/composer_messages?composer_action=createTopic&_=1483481672874" for ::1 at 2017-01-03 14:39:19 -0800 92 | lProcessing by ComposerMessagesController#index as JSON 93 | Parameters: {"composer_action"=>"createTopic", "_"=>"1483481672874"} 94 | Completed 200 OK in 27ms (Views: 0.1ms | ActiveRecord: 1.6ms) 95 | Started GET "/similar_topics?title=This%20is%20a%20new%20topic.%20Totally.&raw=And%20this%20is%20the%20body.%20Yup!%20It%27s%20awesome.%0A&_=1483481672875" for ::1 at 2017-01-03 14:39:32 -0800 96 | Processing by SimilarTopicsController#index as JSON 97 | Parameters: {"title"=>"This is a new topic. Totally.", "raw"=>"And this is the body. Yup! It's awesome.\n", "_"=>"1483481672875"} 98 | Completed 200 OK in 35ms (Views: 0.1ms | ActiveRecord: 16.0ms) 99 | Started POST "/draft.json" for ::1 at 2017-01-03 14:39:34 -0800 100 | Processing by DraftController#update as JSON 101 | Parameters: {"draft_key"=>"new_topic", "data"=>"{\"reply\":\"And this is the body. Yup! It's awesome.\\n\",\"action\":\"createTopic\",\"title\":\"This is a new topic. Totally.\",\"categoryId\":null,\"postId\":null,\"archetypeId\":\"regular\",\"metaData\":null,\"composerTime\":14745,\"typingTime\":5000}", "sequence"=>"2"} 102 | Completed 200 OK in 14ms (Views: 0.3ms | ActiveRecord: 5.1ms) 103 | Started GET "/similar_topics?title=This%20is%20a%20new%20topic.%20Totally.&raw=And%20this%20is%20the%20body.%20Yup!%20It%27s%20awesome.%20Totally%20awesome.%0A&_=1483481672876" for ::1 at 2017-01-03 14:39:42 -0800 104 | Processing by SimilarTopicsController#index as JSON 105 | Parameters: {"title"=>"This is a new topic. Totally.", "raw"=>"And this is the body. Yup! It's awesome. Totally awesome.\n", "_"=>"1483481672876"} 106 | Completed 200 OK in 23ms (Views: 0.1ms | ActiveRecord: 8.9ms) 107 | Started POST "/draft.json" for ::1 at 2017-01-03 14:39:42 -0800 108 | Processing by DraftController#update as JSON 109 | Parameters: {"draft_key"=>"new_topic", "data"=>"{\"reply\":\"And this is the body. Yup! It's awesome. Totally awesome.\\n\",\"action\":\"createTopic\",\"title\":\"This is a new topic. Totally.\",\"categoryId\":null,\"postId\":null,\"archetypeId\":\"regular\",\"metaData\":null,\"composerTime\":23385,\"typingTime\":6300}", "sequence"=>"2"} 110 | Completed 200 OK in 8ms (Views: 0.2ms | ActiveRecord: 1.4ms) 111 | =end 112 | when :delete_reply 113 | # Delete reply, currently not active, need to get correct Post ID 114 | request(:delete, "/posts/#{post_num}") 115 | request(:get, "/posts/#{post_num - 1}") 116 | # TODO: update @last_posts 117 | when :get_latest 118 | # Get latest 119 | request(:get, "/latest.json?order=default") 120 | else 121 | raise "Something is wrong! Illegal value: #{action_type}" 122 | end 123 | end 124 | end 125 | 126 | def log(s) 127 | print "[#{Process.pid}]: #{s}\n" 128 | end 129 | 130 | def time_actions(actions, user_offset, port_num) 131 | user = User.offset(user_offset).first 132 | unless user 133 | print "No user at offset #{user_offset.inspect}! Exiting.\n" 134 | exit -1 135 | end 136 | 137 | log "Simulating activity for user id #{user.id}: #{user.name}" 138 | 139 | log "Getting Rails CSRF token..." 140 | client = DiscourseClient.new(port_num: port_num) 141 | client.get_csrf_token 142 | 143 | log "Logging in as #{user.username.inspect}... (not part of benchmark request time(s))" 144 | client.request :post, "/session", { "login" => user.username, "password" => "longpassword" } 145 | client.request :post, "/login", { "login" => user.username, "password" => "longpassword", "redirect" => "http://localhost:#{port_num}/" } 146 | 147 | times = [] 148 | t_last = Time.now 149 | actions.each do |action| 150 | client.action_from_args *action 151 | current = Time.now 152 | times.push (current - t_last) 153 | t_last = current 154 | end 155 | times 156 | end 157 | 158 | def actions_for_iterations(num_iterations) 159 | (1..num_iterations).map { |i| [ ACTIONS.sample, sentence, rand() ] } 160 | end 161 | 162 | def multithreaded_actions(actions, worker_threads, port_num) 163 | output_mutex = Mutex.new 164 | output_times = [] 165 | 166 | #actions = (1..iterations).map { |i| [ ACTIONS.sample, sentence, rand() ] } 167 | actions_per_thread = (actions.size + worker_threads - 1) / worker_threads # Round up 168 | 169 | threads = (0..(worker_threads-1)).map do |offset| 170 | Thread.new do 171 | begin 172 | # Grab just this one thread's worth of actions 173 | my_actions = actions[ (actions_per_thread * offset) .. (actions_per_thread * (offset + 1) - 1) ] 174 | 175 | # Only a few warmup iterations with lots of load threads? In that case, rounding error can result 176 | # in a few "empty" threads at high offsets. In that case, you have "too many" threads - just 177 | # have the ones with no work assigned do nothing. 178 | unless my_actions == nil || my_actions.size == 0 179 | thread_times = time_actions(my_actions, offset, port_num) 180 | output_mutex.synchronize do 181 | output_times << thread_times 182 | end 183 | end 184 | rescue Exception => e 185 | STDERR.print "Exception in worker thread: #{e.message}\n#{e.backtrace.join("\n")}\n" 186 | raise e # Re-raise the exception 187 | end 188 | end 189 | end 190 | 191 | threads.each(&:join) 192 | output_times 193 | end 194 | -------------------------------------------------------------------------------- /packer/setup.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "fileutils" 4 | require "json" 5 | 6 | # Pass --local to run the setup on a local machine, or set RRB_LOCAL 7 | LOCAL = (ARGV.delete '--local') || ENV["RRB_LOCAL"] 8 | # Whether to build rubies with rvm 9 | BUILD_RUBY = !LOCAL 10 | USE_BASH = BUILD_RUBY 11 | # Print all commands and show their full output 12 | VERBOSE = LOCAL 13 | 14 | base = LOCAL ? File.expand_path('..', __FILE__) : "/home/ubuntu" 15 | benchmark_software = JSON.load(File.read("#{base}/benchmark_software.json")) 16 | 17 | RAILS_RUBY_BENCH_URL = ENV["RAILS_RUBY_BENCH_URL"] # Cloned in ami.json 18 | RAILS_RUBY_BENCH_TAG = ENV["RAILS_RUBY_BENCH_TAG"] 19 | 20 | DISCOURSE_DIR = ENV["DISCOURSE_DIR"] || File.join(__dir__, "work", "discourse") 21 | DISCOURSE_URL = ENV["DISCOURSE_URL"] || benchmark_software["discourse"]["git_url"] 22 | DISCOURSE_TAG = ENV["DISCOURSE_TAG"] || benchmark_software["discourse"]["git_tag"] 23 | 24 | class SystemPackerBuildError < RuntimeError; end 25 | 26 | print < true 56 | end 57 | 58 | Dir.chdir(work_dir) do 59 | csystem "git fetch", "Couldn't 'git fetch' in #{work_dir}!", :debug => true 60 | 61 | if tag && tag.strip != "" 62 | tag = tag.strip 63 | csystem "git checkout #{tag}", "Couldn't 'git checkout #{tag}' in #{work_dir}!", :debug => true 64 | else 65 | csystem "git pull", "Couldn't 'git pull' in #{work_dir}!", :debug => true 66 | end 67 | end 68 | end 69 | 70 | def clone_or_update_by_json(h, work_dir) 71 | clone_or_update_repo(h["git_url"], h["git_tag"], h["checkout_dir"] || work_dir) 72 | end 73 | 74 | def build_and_mount_ruby(source_dir, prefix_dir, mount_name, options = {}) 75 | puts "Build and mount Ruby: Source dir: #{source_dir.inspect} Prefix dir: #{prefix_dir.inspect} Mount name: #{mount_name.inspect}" 76 | Dir.chdir(source_dir) do 77 | unless File.exists?("configure") 78 | csystem "autoconf", "Couldn't run autoconf in #{source_dir}!" 79 | end 80 | unless File.exists?("Makefile") 81 | configure_options = options["configure_options"] || "" 82 | csystem "./configure --prefix #{prefix_dir} #{configure_options}", "Couldn't run configure in #{source_dir}!" 83 | end 84 | csystem "make", "Make failed in #{source_dir}!" 85 | # This should install to the benchmark ruby dir 86 | csystem "make install", "Installing Ruby failed in #{source_dir}!" 87 | end 88 | csystem "rvm mount #{prefix_dir} -n #{mount_name}", "Couldn't mount #{source_dir.inspect} as #{mount_name}!", :bash => true 89 | csystem "rvm use --default ext-#{mount_name}", "Couldn't set ext-#{mount_name} to rvm default!", :bash => true 90 | end 91 | 92 | def autogen_name 93 | @autogen_number ||= 1 94 | name = "autogen-name-#{@autogen_number}" 95 | @autogen_number += 1 96 | name 97 | end 98 | 99 | def clone_or_update_ruby_by_json(h, work_dir) 100 | clone_or_update_by_json(h, work_dir) 101 | mount_name = h["name"] || autogen_name 102 | prefix_dir = h["prefix_dir"] || File.join(RAILS_BENCH_DIR, "work", "prefix", mount_name.gsub("/", "_")) 103 | 104 | build_and_mount_ruby(h["checkout_dir"], prefix_dir, mount_name, { "configure_options" => h["configure_options"] || "" } ) 105 | h["mount_name"] = "ext-" + mount_name 106 | end 107 | 108 | # When you run with "rvm use", you wind up with a bunch of extra 109 | # output that you usually don't want. You need to cut out just the 110 | # last line, remove extraneous newlines, make sure .bash_profile has 111 | # been sourced... 112 | def last_line_with_ruby(cmd, ruby) 113 | output = `bash -l -c \"rvm use #{ruby} && #{cmd}\"` 114 | unless $?.success? 115 | puts "Something went wrong running command, returning nil... #{$?.inspect} / #{cmd.inspect}" 116 | return nil 117 | end 118 | output.split("\n").compact[-1] 119 | end 120 | 121 | if LOCAL 122 | RAILS_BENCH_DIR = File.expand_path("../..", __FILE__) 123 | else 124 | RAILS_BENCH_DIR = File.join(Dir.pwd, "rails_ruby_bench") 125 | end 126 | 127 | # Cloned in ami.json, but go ahead and update anyway. This shouldn't normally do anything. 128 | if RAILS_RUBY_BENCH_URL && RAILS_RUBY_BENCH_URL.strip != "" 129 | Dir.chdir(RAILS_BENCH_DIR) do 130 | csystem "git remote add benchmark-url #{RAILS_RUBY_BENCH_URL} && git fetch benchmark-url", "error fetching commits from Rails Ruby Bench at #{RAILS_RUBY_BENCH_URL.inspect}" 131 | if RAILS_RUBY_BENCH_TAG.strip != "" 132 | csystem "git checkout benchmark-url/#{RAILS_RUBY_BENCH_TAG}", "Error checking out Rails Ruby Bench tag #{RAILS_RUBY_BENCH_TAG.inspect}" 133 | end 134 | end 135 | end 136 | 137 | # Install Rails Ruby Bench gems into system Ruby 138 | Dir.chdir(RAILS_BENCH_DIR) do 139 | csystem "gem install bundler -v1.17.3", "Couldn't install bundler for #{RAILS_BENCH_DIR} for system Ruby!", :bash => true 140 | csystem "bundle _1.17.3_", "Couldn't install RRB gems for #{RAILS_BENCH_DIR} for system Ruby!", :bash => true 141 | end 142 | 143 | if BUILD_RUBY 144 | benchmark_software["compare_rubies"].each do |ruby_hash| 145 | puts "Installing Ruby: #{ruby_hash.inspect}" 146 | # Clone the Ruby, then build and mount if necessary 147 | if ruby_hash["git_url"] 148 | work_dir = File.join(RAILS_BENCH_DIR, "work", ruby_hash["name"]) 149 | ruby_hash["checkout_dir"] = work_dir 150 | clone_or_update_ruby_by_json(ruby_hash, work_dir) 151 | 152 | #csystem "rvm list #2", "Error running rvm list [2] on Ruby #{ruby_hash.inspect}!", :debug => true 153 | puts "Mount the built Ruby: #{ruby_hash.inspect}" 154 | 155 | rvm_ruby_name = ruby_hash["mount_name"] || ruby_hash["name"] 156 | Dir.chdir(RAILS_BENCH_DIR) do 157 | # In Ruby 2.6.0preview3 and later, Bundler is installed as part of Ruby. Check if that's present. 158 | bundle_path = last_line_with_ruby("which bundle", rvm_ruby_name) 159 | 160 | puts "Checking bundler path: #{bundle_path.inspect}" 161 | if !bundle_path || bundle_path == '' 162 | # Okay, so no Bundler is in the path yet. Install the gem. 163 | puts "No builtin or installed Bundler, installing the gem" 164 | csystem "rvm use #{rvm_ruby_name} && gem install bundler -v1.17.3", "Couldn't install Bundler in #{RAILS_BENCH_DIR} for Ruby #{rvm_ruby_name.inspect}!", :bash => true 165 | end 166 | 167 | if !ruby_hash.has_key?("discourse") || ruby_hash["discourse"] 168 | which_bundle = last_line_with_ruby("which bundle", rvm_ruby_name) 169 | puts "Fell through, trying to run bundle. Executable: #{which_bundle.inspect}" 170 | csystem "rvm use #{rvm_ruby_name} && bundle _1.17.3_", "Couldn't install RRB gems in #{RAILS_BENCH_DIR} for Ruby #{rvm_ruby_name.inspect}!", :bash => true 171 | end 172 | end 173 | 174 | elsif ruby_hash["rvm_name"] 175 | csystem "rvm install #{ruby_hash["rvm_name"]}", "Couldn't use RVM to install Ruby named #{ruby_hash["rvm_name"]}!" 176 | if ruby_hash["discourse"] 177 | csystem "rvm use #{ruby_hash["rvm_name"]} && cd #{RAILS_BENCH_DIR} && bundle _1.17.3_", "Couldn't install RRB gems in #{RAILS_BENCH_DIR} for RVM-installed Ruby #{ruby_hash["rvm_name"]}!", :bash => true 178 | end 179 | csystem "rvm use #{ruby_hash["rvm_name"]} && gem install bundler -v1.17.3", "Couldn't install Bundler in #{RAILS_BENCH_DIR} for Ruby #{ruby_hash["rvm_name"].inspect}!", :bash => true 180 | end 181 | 182 | end 183 | 184 | puts "Create benchmark_ruby_versions.txt" 185 | File.open("/home/ubuntu/benchmark_ruby_versions.txt", "w") do |f| 186 | rubies = benchmark_software["compare_rubies"].map { |h| h["mount_name"] || h["name"] || h["rvm_name"] || h["name"] } 187 | f.print rubies.join("\n") 188 | end 189 | end 190 | 191 | clone_or_update_repo(DISCOURSE_URL, DISCOURSE_TAG, DISCOURSE_DIR) 192 | 193 | if LOCAL 194 | Dir.chdir(DISCOURSE_DIR) { csystem "bundle _1.17.3_", "Couldn't install Discourse gems into system Ruby!", :bash => true } 195 | end 196 | 197 | Dir.chdir(RAILS_BENCH_DIR) do 198 | # If there are already users added, this should exit without error and not change the database 199 | puts "Adding seed data..." 200 | csystem "RAILS_ENV=profile ruby seed_db_data.rb", "Couldn't seed the database with profiling sample data!", :bash => true 201 | end 202 | 203 | FileUtils.touch "/tmp/setup_ran_correctly" 204 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rails Ruby Bench 2 | 3 | Rails Ruby Bench (aka RRB) is a Discourse-based benchmark to measure 4 | the speed of the Ruby language. It can incidentally be used to measure 5 | the speed of a number of other things. 6 | 7 | RRB is a "Real World" benchmark, in the sense of running a large Rails 8 | app in a concurrent configuration with a lot of complexity and 9 | variation in what it does. That makes it wonderful for measuring 10 | end-to-end effects of significant changes, and terrible for optimizing 11 | operations that don't take a lot of runtime. 12 | 13 | This Discourse-based benchmark steals some code from Discourse 14 | (e.g. user\_simulator.rb, seed\_db\_data.rb), so it's licensed 15 | GPLv2. It also *uses* Discourse. 16 | 17 | I normally run this benchmark by building an AWS image using Packer 18 | and running it on a dedicated EC2 instance. For a variety of reasons, 19 | that gives very consistent benchmark results. It's also annoying for 20 | some use cases. If you can easily use AWS, I recommend it. 21 | 22 | This benchmark was written and, for the first few years, maintained 23 | via AppFolio's sponsorship (https://engineering.appfolio.com). Thank 24 | you AppFolio! 25 | 26 | ## Command-Line Options 27 | 28 | Start.rb supports a number of options: 29 | 30 | -r NUMBER Set the random seed 31 | -i NUMBER Number of total iterations (default: 1500) 32 | -n NUMBER Number of load threads in the user simulator 33 | -s NUMBER Number of start/stop iterations, measuring time to first successful request 34 | -w NUMBER Number of warmup HTTP requests before timing 35 | -p NUMBER Port number for Puma server (default: 4567) 36 | -o DIR Directory for JSON output 37 | -t NUMBER Threads per Puma server 38 | -c NUMBER Number of cluster processes for Puma 39 | 40 | ## Running the Benchmark Locally (Incomplete Version) 41 | 42 | If you think your computer is basically set up for Discourse already, 43 | here's the short version of the configuration. Are you worried that 44 | you may need more software? Scroll down to the complete version of the 45 | setup below. 46 | 47 | Make sure to run: 48 | 49 | $ ./bin/setup 50 | 51 | That script will install dependencies and Discourse, and then it will try to setup the database for Discourse. 52 | 53 | Note that Discourse (at least version 1.8) does not support Postgresql 54 | 10, version 9 is needed. 55 | 56 | Then, run the database seeding script: 57 | 58 | $ cd ../.. # Back to root directory rather than work/discourse 59 | $ RAILS_ENV=profile ruby seed_db_data.rb # And wait awhile - it's slow 60 | 61 | Now you can run the benchmark: 62 | 63 | $ ./start.rb 64 | 65 | ## Running the Benchmark Locally (Complete Version) 66 | 67 | For the AWS build, RRB uses Packer to create an image configured for 68 | Discourse, 100% from scratch. That means the Packer directory includes 69 | all the necessary configuration scripts to make that happen. 70 | 71 | For very good reasons, RRB's Packer build separates these scripts into 72 | a lot of small steps. So the configuration is annoying :-( 73 | 74 | If you want to be 100% sure you have all the latest configuration 75 | steps, read packer/ami.json - that's the configuration file that 76 | Packer actually uses to build the image, so it's basically guaranteed 77 | to work. It runs a lot of other small scripts which are stored in the 78 | packer directory. If you have trouble with these instructions, you may 79 | want to refer to the runnable script. 80 | 81 | RRB tries to keep an up-to-date local install script, but it *does* 82 | sometimes get out of date: 83 | 84 | ```bash 85 | Switch to a 2.3.4 or 2.4.1 Ruby 86 | $ ruby packer/setup.rb --local 87 | ``` 88 | 89 | You'll also need Discourse's dependencies - the benchmark uses 90 | Discourse, so you'll need to manage a local copy of Discourse. 91 | There's a script in the Discourse directory you can look at on OS X 92 | under work/discourse/script/osx_dev. Refer to 93 | [this page](https://github.com/discourse/discourse/blob/v1.8.0.beta13/docs/DEVELOPER-ADVANCED.md#preparing-a-fresh-ubuntu-install) 94 | for Linux. 95 | 96 | Rerun the setup until it succeeds. 97 | 98 | Then run start.rb to run the server and the benchmark. 99 | 100 | ## Customizing the Benchmark 101 | 102 | The benchmark uses the Ruby and Discourse versions found in 103 | setup.json. You can customize them there, then re-run setup.rb. 104 | You may need to clear the cloned versions in the work directory 105 | first. 106 | 107 | ## Definitive Benchmark Numbers and AWS 108 | 109 | The definitive version of the benchmark uses an AWS m4.2xlarge 110 | instance and an AMI. This has 8 vCPUs (4 virtual cores) as discussed 111 | in the design documentation, and doesn't have an excessive amount of 112 | memory or I/O priority. It's a realistic hosting choice at roughly 113 | $270/month if running continuously. Your benchmark should run in well 114 | under an hour and cost about $0.40 (40 cents) in USD. 115 | 116 | To create your own AMI, see packer/README.md in this Git repo. 117 | 118 | With your AMI (or using a public AMI), you can launch an instance as 119 | normal for AWS. Here's an example command line: 120 | 121 | aws ec2 run-instances --image-id ami-f678218d --count 1 --instance-type m4.2xlarge --key-name MyKeyPair --placement Tenancy=dedicated 122 | 123 | Replace "MyKeyPair" with the name of your own AWS keypair. You can, of course, replace the AMI ID with the current latest public AMI, or one you built. 124 | 125 | The current publicly available AMIs are: 126 | 127 | ami-554a4543 for Discourse v1.5 and Ruby 2.0.0 through 2.3.4 128 | 129 | ami-f678218d for Discourse v1.8 and Ruby 2.3.4 and 2.4.1 130 | 131 | ## Debugging with the AMI 132 | 133 | Example command lines: 134 | 135 | aws ec2 run-instances --count 1 --instance-type m4.2xlarge --key-name my-ssh-key-name --placement Tenancy=dedicated --image-id ami-f678218d 136 | ssh -i ~/.ssh/my-ssh-key-name.pem ubuntu@ec2-34-228-227-234.compute-1.amazonaws.com 137 | cd rails_ruby_bench 138 | ./in_each_ruby.rb "for i in {1..20}; do ./start.rb -i 3000 -w 100 -s 0; done" 139 | 140 | You'll need to find the public DNS name of the VM you created 141 | somehow. I normally use the EC2 dashboard. Similarly, you should use 142 | an SSH key name that exists in your AWS account. The parameters above 143 | are for an m4.2xlarge dedicated instance. That's a bit expensive, but 144 | will also give you reproducible results that you can compare directly 145 | with mine. If you use a much smaller instance, you'll want to reduce 146 | the number of load-testing threads and Puma processes and threads. 147 | 148 | I normally copy the JSON files back to my own machine, something like: 149 | 150 | > scp -i ~/.ssh/my-ssh-key-name.pem ubuntu@ec2-34-228-227-234.compute-1.amazonaws.com:~/rails_ruby_bench/*.json ./my_local_directory 151 | 152 | ## Debugging and AWS 153 | 154 | By default, the Rails benchmark is git-cloned under 155 | ~ubuntu/rails\_ruby\_bench, and Discourse is cloned under the work 156 | subdirectory of that repository. The built Rubies are mounted using 157 | rvm for the benchmark. You can see them with "rvm list". If you want 158 | to change anything when starting your own instance, those are great 159 | places to begin. 160 | 161 | By default, the image won't update or rebuild Ruby or Discourse, nor 162 | update the Rails Ruby benchmark on boot. It will use the versions of 163 | all of those things that were current when the AMI was built. But you 164 | can modify your own image later however you like. 165 | 166 | If you'd like to change the behavior of the AWS image, you can use AWS 167 | user data to run a script on boot. See 168 | "http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html". 169 | 170 | You can also just log in and make whatever changes you want, of course. 171 | 172 | ## Decisions and Intent 173 | 174 | * Configurable Ruby, Rails and Discourse versions. This makes it easy 175 | to test a particular optimization or fix to any of Ruby, Rails or 176 | Discourse. To test other fixes to most software, the gem version in 177 | Rails can be updated at a different URL. This doesn't make it as 178 | easy to test fixes to system libraries or programs (e.g. libxml, 179 | postgres) which are less likely to be patched just because of Ruby 180 | performance. 181 | 182 | * Use Discourse code because it's a real Rails application, used in 183 | production, with a reasonably stable REST API. This tests many code 184 | paths in realistic proportions. 185 | 186 | * Use random seed for client because it provides a reasonable balance 187 | between unpredictability and stability. Note that no multithreaded 188 | or multiprocess benchmark using a host (real or virtual) is going to 189 | be fully reproducible or stable. But the random seed for client 190 | requests provides a baseline of reproducibility, within the limits 191 | of a benchmark that isn't extremely artificial. 192 | 193 | * Test with multiple requests at once because better Ruby concurrency 194 | is an explicit goal of Ruby 3x3. The first question most people ask 195 | of any Ruby optimization is "how much will it improve my Rails 196 | performance?" Concurrency is key to answering this question. 197 | 198 | * Use Puma because we want multithreaded operation. Multithreaded 199 | means Puma or Thin or commercial Passenger. Puma is more commonly 200 | used as a high-performance multithreaded Ruby application server at 201 | this point. 202 | 203 | * Use AWS because it's common, it's an industry standard and it's easy 204 | to test. Also, nearly all other cloud offerings are measured against 205 | AWS. AWS numbers will be treated as meaningful on their face. 206 | 207 | * Use postgres on same machine: on same machine to avoid AWS time in 208 | benchmarking, require Postgres because Discourse does and can't be 209 | easily changed. (https://meta.discourse.org/t/why-not-support-mysql/2568/2) 210 | 211 | * Use Sidekiq on same machine: on same machine to avoid AWS time in 212 | benchmarking, use Sidekiq because it's fast, simple, in Ruby and not 213 | trivial to change. 214 | 215 | * Don't use Discourse's existing benchmark script because it simply 216 | runs a (very) small set of URLs, and only tests one URL at once. 217 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | actionmailer (4.2.8) 5 | actionpack (= 4.2.8) 6 | actionview (= 4.2.8) 7 | activejob (= 4.2.8) 8 | mail (~> 2.5, >= 2.5.4) 9 | rails-dom-testing (~> 1.0, >= 1.0.5) 10 | actionpack (4.2.8) 11 | actionview (= 4.2.8) 12 | activesupport (= 4.2.8) 13 | rack (~> 1.6) 14 | rack-test (~> 0.6.2) 15 | rails-dom-testing (~> 1.0, >= 1.0.5) 16 | rails-html-sanitizer (~> 1.0, >= 1.0.2) 17 | actionview (4.2.8) 18 | activesupport (= 4.2.8) 19 | builder (~> 3.1) 20 | erubis (~> 2.7.0) 21 | rails-dom-testing (~> 1.0, >= 1.0.5) 22 | rails-html-sanitizer (~> 1.0, >= 1.0.3) 23 | active_model_serializers (0.8.3) 24 | activemodel (>= 3.0) 25 | activejob (4.2.8) 26 | activesupport (= 4.2.8) 27 | globalid (>= 0.3.0) 28 | activemodel (4.2.8) 29 | activesupport (= 4.2.8) 30 | builder (~> 3.1) 31 | activerecord (4.2.8) 32 | activemodel (= 4.2.8) 33 | activesupport (= 4.2.8) 34 | arel (~> 6.0) 35 | activesupport (4.2.8) 36 | i18n (~> 0.7) 37 | minitest (~> 5.1) 38 | thread_safe (~> 0.3, >= 0.3.4) 39 | tzinfo (~> 1.1) 40 | addressable (2.5.1) 41 | public_suffix (~> 2.0, >= 2.0.2) 42 | annotate (2.7.1) 43 | activerecord (>= 3.2, < 6.0) 44 | rake (>= 10.4, < 12.0) 45 | arel (6.0.4) 46 | aws-sdk (2.5.3) 47 | aws-sdk-resources (= 2.5.3) 48 | aws-sdk-core (2.5.3) 49 | jmespath (~> 1.0) 50 | aws-sdk-resources (2.5.3) 51 | aws-sdk-core (= 2.5.3) 52 | babel-source (5.8.34) 53 | babel-transpiler (0.7.0) 54 | babel-source (>= 4.0, < 6) 55 | execjs (~> 2.0) 56 | barber (0.11.2) 57 | ember-source (>= 1.0, < 3) 58 | execjs (>= 1.2, < 3) 59 | better_errors (2.1.1) 60 | coderay (>= 1.0.0) 61 | erubis (>= 2.6.6) 62 | rack (>= 0.9.0) 63 | binding_of_caller (0.7.2) 64 | debug_inspector (>= 0.0.1) 65 | bootsnap (0.2.14) 66 | msgpack (~> 1.0) 67 | builder (3.2.3) 68 | bullet (5.4.2) 69 | activesupport (>= 3.0.0) 70 | uniform_notifier (~> 1.10.0) 71 | byebug (9.0.6) 72 | certified (1.0.0) 73 | coderay (1.1.1) 74 | concurrent-ruby (1.0.5) 75 | connection_pool (2.2.0) 76 | crack (0.4.3) 77 | safe_yaml (~> 1.0.0) 78 | crass (1.0.2) 79 | debug_inspector (0.0.2) 80 | diff-lcs (1.3) 81 | discourse-qunit-rails (0.0.9) 82 | railties 83 | discourse_image_optim (0.24.5) 84 | exifr (~> 1.2, >= 1.2.2) 85 | fspath (~> 3.0) 86 | image_size (~> 1.5) 87 | in_threads (~> 1.3) 88 | progress (~> 3.0, >= 3.0.1) 89 | domain_name (0.5.25) 90 | unf (>= 0.0.5, < 1.0.0) 91 | email_reply_trimmer (0.1.6) 92 | ember-data-source (2.2.1) 93 | ember-source (>= 1.8, < 3.0) 94 | ember-handlebars-template (0.7.5) 95 | barber (>= 0.11.0) 96 | sprockets (>= 3.3, < 4) 97 | ember-rails (0.18.5) 98 | active_model_serializers 99 | ember-data-source (>= 1.0.0.beta.5) 100 | ember-handlebars-template (>= 0.1.1, < 1.0) 101 | ember-source (>= 1.1.0) 102 | jquery-rails (>= 1.0.17) 103 | railties (>= 3.1) 104 | ember-source (2.10.0) 105 | erubis (2.7.0) 106 | excon (0.53.0) 107 | execjs (2.7.0) 108 | exifr (1.2.5) 109 | fabrication (2.9.8) 110 | fakeweb (1.3.0) 111 | faraday (0.11.0) 112 | multipart-post (>= 1.2, < 3) 113 | fast_blank (1.0.0) 114 | fast_xor (1.1.3) 115 | rake 116 | rake-compiler 117 | fast_xs (0.8.0) 118 | fastimage (2.1.0) 119 | ffi (1.9.18) 120 | flamegraph (0.9.5) 121 | foreman (0.82.0) 122 | thor (~> 0.19.1) 123 | fspath (3.1.0) 124 | gc_tracer (1.5.1) 125 | globalid (0.3.7) 126 | activesupport (>= 4.1.0) 127 | guess_html_encoding (0.0.11) 128 | hashdiff (0.3.2) 129 | hashie (3.5.5) 130 | highline (1.7.8) 131 | hiredis (0.6.1) 132 | htmlentities (4.3.4) 133 | http-cookie (1.0.2) 134 | domain_name (~> 0.5) 135 | http_accept_language (2.0.5) 136 | i18n (0.8.1) 137 | image_size (1.5.0) 138 | in_threads (1.4.0) 139 | jmespath (1.3.1) 140 | jquery-rails (4.2.1) 141 | rails-dom-testing (>= 1, < 3) 142 | railties (>= 4.2.0) 143 | thor (>= 0.14, < 2.0) 144 | jwt (1.5.6) 145 | kgio (2.11.0) 146 | libv8 (5.3.332.38.5) 147 | listen (3.1.5) 148 | rb-fsevent (~> 0.9, >= 0.9.4) 149 | rb-inotify (~> 0.9, >= 0.9.7) 150 | ruby_dep (~> 1.2) 151 | logster (1.2.7) 152 | loofah (2.0.3) 153 | nokogiri (>= 1.5.9) 154 | lru_redux (1.1.0) 155 | mail (2.6.5) 156 | mime-types (>= 1.16, < 4) 157 | memory_profiler (0.9.7) 158 | message_bus (2.0.2) 159 | rack (>= 1.1.3) 160 | metaclass (0.0.4) 161 | method_source (0.8.2) 162 | mime-types (2.99.3) 163 | mini_portile2 (2.1.0) 164 | mini_racer (0.1.9) 165 | libv8 (~> 5.3) 166 | minitest (5.10.1) 167 | mocha (1.1.0) 168 | metaclass (~> 0.0.1) 169 | mock_redis (0.15.4) 170 | moneta (1.0.0) 171 | msgpack (1.1.0) 172 | multi_json (1.12.1) 173 | multi_xml (0.6.0) 174 | multipart-post (2.0.0) 175 | mustache (1.0.5) 176 | netrc (0.11.0) 177 | nokogiri (1.7.2) 178 | mini_portile2 (~> 2.1.0) 179 | nokogumbo (1.4.10) 180 | nokogiri 181 | oauth (0.5.1) 182 | oauth2 (1.3.1) 183 | faraday (>= 0.8, < 0.12) 184 | jwt (~> 1.0) 185 | multi_json (~> 1.3) 186 | multi_xml (~> 0.5) 187 | rack (>= 1.2, < 3) 188 | oj (3.0.5) 189 | omniauth (1.6.1) 190 | hashie (>= 3.4.6, < 3.6.0) 191 | rack (>= 1.6.2, < 3) 192 | omniauth-facebook (4.0.0) 193 | omniauth-oauth2 (~> 1.2) 194 | omniauth-github-discourse (1.1.2) 195 | omniauth (~> 1.0) 196 | omniauth-oauth2 (~> 1.1) 197 | omniauth-google-oauth2 (0.3.1) 198 | jwt (~> 1.0) 199 | multi_json (~> 1.3) 200 | omniauth (>= 1.1.1) 201 | omniauth-oauth2 (>= 1.3.1) 202 | omniauth-instagram (1.0.2) 203 | omniauth (~> 1) 204 | omniauth-oauth2 (~> 1) 205 | omniauth-oauth (1.1.0) 206 | oauth 207 | omniauth (~> 1.0) 208 | omniauth-oauth2 (1.4.0) 209 | oauth2 (~> 1.0) 210 | omniauth (~> 1.2) 211 | omniauth-openid (1.0.1) 212 | omniauth (~> 1.0) 213 | rack-openid (~> 1.3.1) 214 | omniauth-twitter (1.3.0) 215 | omniauth-oauth (~> 1.1) 216 | rack 217 | onebox (1.8.6) 218 | fast_blank (>= 1.0.0) 219 | htmlentities (~> 4.3) 220 | moneta (~> 1.0) 221 | multi_json (~> 1.11) 222 | mustache 223 | nokogiri (~> 1.7) 224 | sanitize 225 | openid-redis-store (0.0.2) 226 | redis 227 | ruby-openid 228 | pg (0.19.0) 229 | progress (3.3.1) 230 | pry (0.10.4) 231 | coderay (~> 1.1.0) 232 | method_source (~> 0.8.1) 233 | slop (~> 3.4) 234 | pry-nav (0.2.4) 235 | pry (>= 0.9.10, < 0.11.0) 236 | pry-rails (0.3.4) 237 | pry (>= 0.9.10) 238 | public_suffix (2.0.5) 239 | puma (3.6.0) 240 | r2 (0.2.6) 241 | rack (1.6.8) 242 | rack-mini-profiler (0.10.4) 243 | rack (>= 1.2.0) 244 | rack-openid (1.3.1) 245 | rack (>= 1.1.0) 246 | ruby-openid (>= 2.1.8) 247 | rack-protection (1.5.3) 248 | rack 249 | rack-test (0.6.3) 250 | rack (>= 1.0) 251 | rails (4.2.8) 252 | actionmailer (= 4.2.8) 253 | actionpack (= 4.2.8) 254 | actionview (= 4.2.8) 255 | activejob (= 4.2.8) 256 | activemodel (= 4.2.8) 257 | activerecord (= 4.2.8) 258 | activesupport (= 4.2.8) 259 | bundler (>= 1.3.0, < 2.0) 260 | railties (= 4.2.8) 261 | sprockets-rails 262 | rails-deprecated_sanitizer (1.0.3) 263 | activesupport (>= 4.2.0.alpha) 264 | rails-dom-testing (1.0.8) 265 | activesupport (>= 4.2.0.beta, < 5.0) 266 | nokogiri (~> 1.6) 267 | rails-deprecated_sanitizer (>= 1.0.1) 268 | rails-html-sanitizer (1.0.3) 269 | loofah (~> 2.0) 270 | rails_multisite (1.0.6) 271 | rails (> 4.2, < 5) 272 | railties (4.2.8) 273 | actionpack (= 4.2.8) 274 | activesupport (= 4.2.8) 275 | rake (>= 0.8.7) 276 | thor (>= 0.18.1, < 2.0) 277 | raindrops (0.18.0) 278 | rake (11.3.0) 279 | rake-compiler (0.9.9) 280 | rake 281 | rb-fsevent (0.9.7) 282 | rb-inotify (0.9.7) 283 | ffi (>= 0.5.0) 284 | rbtrace (0.4.8) 285 | ffi (>= 1.0.6) 286 | msgpack (>= 0.4.3) 287 | trollop (>= 1.16.2) 288 | redis (3.3.3) 289 | redis-namespace (1.5.2) 290 | redis (~> 3.0, >= 3.0.4) 291 | rest-client (1.8.0) 292 | http-cookie (>= 1.0.2, < 2.0) 293 | mime-types (>= 1.16, < 3.0) 294 | netrc (~> 0.7) 295 | rinku (2.0.0) 296 | rmmseg-cpp (0.2.9) 297 | rspec (3.4.0) 298 | rspec-core (~> 3.4.0) 299 | rspec-expectations (~> 3.4.0) 300 | rspec-mocks (~> 3.4.0) 301 | rspec-core (3.4.4) 302 | rspec-support (~> 3.4.0) 303 | rspec-expectations (3.4.0) 304 | diff-lcs (>= 1.2.0, < 2.0) 305 | rspec-support (~> 3.4.0) 306 | rspec-html-matchers (0.7.0) 307 | nokogiri (~> 1) 308 | rspec (~> 3) 309 | rspec-mocks (3.4.1) 310 | diff-lcs (>= 1.2.0, < 2.0) 311 | rspec-support (~> 3.4.0) 312 | rspec-rails (3.4.2) 313 | actionpack (>= 3.0, < 4.3) 314 | activesupport (>= 3.0, < 4.3) 315 | railties (>= 3.0, < 4.3) 316 | rspec-core (~> 3.4.0) 317 | rspec-expectations (~> 3.4.0) 318 | rspec-mocks (~> 3.4.0) 319 | rspec-support (~> 3.4.0) 320 | rspec-support (3.4.1) 321 | rtlit (0.0.5) 322 | ruby-openid (2.7.0) 323 | ruby-readability (0.7.0) 324 | guess_html_encoding (>= 0.0.4) 325 | nokogiri (>= 1.6.0) 326 | ruby_dep (1.5.0) 327 | safe_yaml (1.0.4) 328 | sanitize (4.4.0) 329 | crass (~> 1.0.2) 330 | nokogiri (>= 1.4.4) 331 | nokogumbo (~> 1.4.1) 332 | sass (3.4.23) 333 | sassc (1.11.2) 334 | bundler 335 | ffi (~> 1.9.6) 336 | sass (>= 3.3.0) 337 | seed-fu (2.3.5) 338 | activerecord (>= 3.1, < 4.3) 339 | activesupport (>= 3.1, < 4.3) 340 | shoulda (3.5.0) 341 | shoulda-context (~> 1.0, >= 1.0.1) 342 | shoulda-matchers (>= 1.4.1, < 3.0) 343 | shoulda-context (1.2.2) 344 | shoulda-matchers (2.8.0) 345 | activesupport (>= 3.0.0) 346 | sidekiq (4.2.4) 347 | concurrent-ruby (~> 1.0) 348 | connection_pool (~> 2.2, >= 2.2.0) 349 | rack-protection (>= 1.5.0) 350 | redis (~> 3.2, >= 3.2.1) 351 | simple-rss (1.3.1) 352 | sinatra (1.4.6) 353 | rack (~> 1.4) 354 | rack-protection (~> 1.4) 355 | tilt (>= 1.3, < 3) 356 | slop (3.6.0) 357 | spork (1.0.0rc4) 358 | spork-rails (4.0.0) 359 | rails (>= 3.0.0, < 5) 360 | spork (>= 1.0rc0) 361 | sprockets (3.7.1) 362 | concurrent-ruby (~> 1.0) 363 | rack (> 1, < 3) 364 | sprockets-rails (3.2.0) 365 | actionpack (>= 4.0) 366 | activesupport (>= 4.0) 367 | sprockets (>= 3.0.0) 368 | stackprof (0.2.10) 369 | test_after_commit (1.1.0) 370 | activerecord (>= 3.2) 371 | thor (0.19.4) 372 | thread_safe (0.3.6) 373 | tilt (2.0.5) 374 | timecop (0.8.1) 375 | trollop (2.1.2) 376 | tzinfo (1.2.3) 377 | thread_safe (~> 0.1) 378 | uglifier (3.0.2) 379 | execjs (>= 0.3.0, < 3) 380 | unf (0.1.4) 381 | unf_ext 382 | unf_ext (0.0.7.1) 383 | unicorn (5.3.0) 384 | kgio (~> 2.6) 385 | raindrops (~> 0.7) 386 | uniform_notifier (1.10.0) 387 | webmock (3.0.1) 388 | addressable (>= 2.3.6) 389 | crack (>= 0.3.2) 390 | hashdiff 391 | 392 | PLATFORMS 393 | ruby 394 | 395 | DEPENDENCIES 396 | active_model_serializers (~> 0.8.3) 397 | annotate 398 | aws-sdk 399 | babel-transpiler 400 | barber 401 | better_errors 402 | binding_of_caller 403 | bootsnap 404 | bullet 405 | byebug 406 | certified 407 | discourse-qunit-rails 408 | discourse_image_optim 409 | email_reply_trimmer (= 0.1.6) 410 | ember-handlebars-template (= 0.7.5) 411 | ember-rails (= 0.18.5) 412 | ember-source (= 2.10.0) 413 | excon 414 | execjs 415 | fabrication (= 2.9.8) 416 | fakeweb (~> 1.3.0) 417 | fast_blank 418 | fast_xor 419 | fast_xs 420 | fastimage (= 2.1.0) 421 | flamegraph 422 | foreman 423 | gc_tracer 424 | highline 425 | hiredis 426 | htmlentities 427 | http_accept_language (~> 2.0.5) 428 | listen 429 | logster 430 | lru_redux 431 | mail 432 | memory_profiler 433 | message_bus 434 | mime-types 435 | mini_racer 436 | minitest 437 | mocha 438 | mock_redis 439 | multi_json 440 | mustache 441 | nokogiri 442 | oj 443 | omniauth 444 | omniauth-facebook 445 | omniauth-github-discourse 446 | omniauth-google-oauth2 447 | omniauth-instagram 448 | omniauth-oauth2 449 | omniauth-openid 450 | omniauth-twitter 451 | onebox 452 | openid-redis-store 453 | pg 454 | pry-nav 455 | pry-rails 456 | puma 457 | r2 (~> 0.2.5) 458 | rack-mini-profiler 459 | rack-protection 460 | rails (~> 4.2) 461 | rails_multisite 462 | rake 463 | rb-fsevent 464 | rb-inotify (~> 0.9) 465 | rbtrace 466 | redis 467 | redis-namespace 468 | rest-client 469 | rinku 470 | rmmseg-cpp 471 | rspec 472 | rspec-html-matchers 473 | rspec-rails 474 | rtlit 475 | ruby-readability 476 | sanitize 477 | sassc 478 | seed-fu (~> 2.3.5) 479 | shoulda 480 | sidekiq 481 | simple-rss 482 | sinatra 483 | spork-rails 484 | stackprof 485 | test_after_commit 486 | thor 487 | timecop 488 | uglifier 489 | unf 490 | unicorn 491 | webmock 492 | 493 | BUNDLED WITH 494 | 1.14.6 495 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | 3 | Version 2, June 1991 4 | 5 | Copyright (C) 1989, 1991 Free Software Foundation, Inc. 6 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA 7 | 8 | Everyone is permitted to copy and distribute verbatim copies 9 | of this license document, but changing it is not allowed. 10 | Preamble 11 | 12 | The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. 13 | 14 | When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. 15 | 16 | To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. 17 | 18 | For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. 19 | 20 | We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. 21 | 22 | Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. 23 | 24 | Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. 25 | 26 | The precise terms and conditions for copying, distribution and modification follow. 27 | 28 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 29 | 30 | 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". 31 | 32 | Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 33 | 34 | 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. 35 | 36 | You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 37 | 38 | 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: 39 | 40 | a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. 41 | b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. 42 | c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) 43 | These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. 44 | 45 | Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. 46 | 47 | In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 48 | 49 | 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: 50 | 51 | a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, 52 | b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, 53 | c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) 54 | The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. 55 | 56 | If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 57 | 58 | 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 59 | 60 | 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 61 | 62 | 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 63 | 64 | 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. 65 | 66 | If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. 67 | 68 | It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. 69 | 70 | This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 71 | 72 | 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 73 | 74 | 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. 75 | 76 | Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 77 | 78 | 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. 79 | 80 | NO WARRANTY 81 | 82 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 83 | 84 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 85 | -------------------------------------------------------------------------------- /start.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # Start the Rails server and measure time to first request and request processing times. 4 | 5 | require 'optparse' 6 | require 'rest-client' 7 | require 'json' 8 | require 'gabbler' # Require this before requiring Rails' config/environment.rb, which will start Bundler. 9 | require 'get_process_mem' 10 | 11 | # Run this in "profile" environment for Discourse. 12 | ENV['RAILS_ENV'] = 'profile' 13 | require File.expand_path(File.join(File.dirname(__FILE__), "work/discourse/config/environment")) 14 | 15 | startup_iters = 2 16 | random_seed = 16541799507913229037 # Chosen via irb and '(1..20).map { (0..9).to_a.sample }.join("")' 17 | worker_iterations = 1500 # All iterations, spread between load-test worker threads 18 | warmup_iterations = 300 # Need to test warmup iterations properly... 19 | total_restart_iterations = 1 # This is essentially a second (or more) iteration of nearly everything. 6000 iterations with 10 total restarts 20 | # gives 60,000 iterations. Warmup and startup iterations are also repeated. The "bonus" warm start is not. 21 | workers = 30 22 | worker_processes = 1 23 | port_num = 4567 24 | out_dir = "." 25 | out_file = nil 26 | puma_processes = 10 27 | puma_threads = 6 28 | no_warm_start = false 29 | 30 | OptionParser.new do |opts| 31 | opts.banner = "Usage: ruby start.rb [options]" 32 | opts.on("-r", "--random-seed NUMBER", "random seed") do |r| 33 | random_seed = r.to_i 34 | end 35 | opts.on("-i", "--iterations NUMBER", "number of iterations per user simulator") do |n| 36 | worker_iterations = n.to_i 37 | end 38 | opts.on("-t", "--total-restart-iterations NUMBER", "number of total repetitions of all non-warmup iterations without shutdown") do |n| 39 | total_restart_iterations = n.to_i 40 | end 41 | opts.on("-n", "--num-workers NUMBER", "number of user simulator worker threads per process") do |n| 42 | workers = n.to_i 43 | end 44 | opts.on("-l", "--num-load-processes NUMBER", "number of user simulator work processes") do |n| 45 | worker_processes = n.to_i 46 | end 47 | opts.on("-s", "--num-startup-iters NUMBER", "number of startup/shutdown iterations") do |n| 48 | startup_iters = n.to_i 49 | end 50 | opts.on("-w", "--warmup NUMBER", "number of warm-up iterations") do |n| 51 | warmup_iterations = n.to_i 52 | end 53 | opts.on("-p", "--port NUMBER", "port number for test Rails server") do |n| 54 | port_num = n.to_i 55 | end 56 | opts.on("-o", "--out-dir DIRECTORY", "directory to write JSON output to") do |d| 57 | out_dir = d 58 | end 59 | opts.on("-f", "--out-file FILENAME", "filename to write JSON output to") do |f| 60 | out_file = f 61 | end 62 | opts.on("-t", "--threads-per-server NUMBER", "number of Puma threads per server process") do |t| 63 | puma_threads = t.to_i 64 | end 65 | opts.on("-c", "--cluster-processes NUMBER", "number of Puma processes in cluster mode") do |c| 66 | puma_processes = c.to_i 67 | end 68 | opts.on("-a", "--no-warm-start", "Do not do the normal automatic start/stop warmup iteration") do 69 | no_warm_start = true 70 | end 71 | end.parse! 72 | 73 | raise "No such output directory as #{out_dir.inspect}!" unless File.directory?(out_dir) 74 | 75 | # Make the constant accessible inside the method definitions 76 | PORT_NUM = port_num 77 | PUMA_THREADS = puma_threads 78 | PUMA_PROCESSES = puma_processes 79 | RANDOM_SEED = random_seed 80 | 81 | CONTROL_PORT = 9939 82 | CONTROL_TOKEN = "VeryModelOfAModernMajorGeneral" 83 | 84 | class BenchmarkSystemError < RuntimeError; end 85 | 86 | # Checked system - error if the command fails 87 | def csystem(cmd, err, opts = {}) 88 | out = `#{cmd}` 89 | print "Running command: #{cmd.inspect}\n" if opts[:debug] || opts["debug"] 90 | unless $?.success? || opts[:fail_ok] || opts["fail_ok"] 91 | print "Error running command:\n#{cmd.inspect}\nOutput:\n#{out}\n=====\n" 92 | raise BenchmarkSystemError.new(err) 93 | end 94 | print "Command output:\n#{out}\n=====\n" if opts[:debug] || opts["debug"] 95 | out 96 | end 97 | 98 | def last_pid 99 | @started_pid 100 | end 101 | 102 | def get_server_rss 103 | GetProcessMem.new(@started_pid).bytes 104 | end 105 | 106 | def get_puma_worker_rss 107 | out = `ps -o pid=,rss=,command=` 108 | rss = [] 109 | lines = out.split 110 | lines.each do |line| 111 | pid, rss, command = line.split("\t", 3) 112 | if command =~ Regexp.new("cluster worker (\\d+): #{@started_pid} [discourse]") 113 | offset = $1 114 | rss.push([pid, rss, offset]) 115 | end 116 | end 117 | end 118 | 119 | def get_server_gc_stats 120 | # NOTE: This won't work until a version of Puma later than 3.9.1 (3.11.0 has it). So for now, don't use this. 121 | output = `bundle exec pumactl --control-url tcp://127.0.0.1:#{CONTROL_PORT} --control-token #{CONTROL_TOKEN} gc-stats` 122 | output.sub!(/^[^{]+/, "") 123 | JSON.parse(output.chomp) 124 | end 125 | 126 | def server_start 127 | # Start the server 128 | @started_pid = fork do 129 | STDERR.print "In PID #{Process.pid}, starting server on port #{PORT_NUM}\n" 130 | Dir.chdir "work/discourse" 131 | # Start Puma in a new process group to easily kill subprocesses if necessary 132 | exec({ "RAILS_ENV" => "profile" }, "bundle", "exec", "puma", "--config", "config/puma.rb", "--control", "tcp://127.0.0.1:#{CONTROL_PORT}", "--control-token", CONTROL_TOKEN, "-p", PORT_NUM.to_s, "-w", PUMA_PROCESSES.to_s, "-t", "1:#{PUMA_THREADS}", :pgroup => true) 133 | end 134 | end 135 | 136 | def server_stop 137 | begin 138 | csystem "RAILS_ENV=profile bundle exec pumactl --control-token #{CONTROL_TOKEN} --control-url tcp://127.0.0.1:#{CONTROL_PORT} halt", "Error trying to stop Puma via pumactl!" 139 | rescue BenchmarkSystemError 140 | # Error stopping w/ pumactl, try just killing the process 141 | Process.kill("-INT", @started_pid) 142 | end 143 | print "server_stop: Asked Puma to stop, expected PID #{@started_pid.inspect}.\n" 144 | loop do 145 | # Verify that server we started is sufficiently dead before we restart 146 | STDERR.print "Waiting for dead PID expecting #{@started_pid.inspect}\n" 147 | dead_pid = Process.waitpid(0) 148 | STDERR.print "Got dead pid: #{dead_pid.inspect}\n" 149 | break if dead_pid == @started_pid 150 | end 151 | @started_pid = nil 152 | rescue Errno::ECHILD 153 | # Found no child processes... Which means that whatever we're attempting to wait for, it's already dead. 154 | print "No child processes, moving on with our day.\n" 155 | end 156 | 157 | def read_all_from_pipe(pipe) 158 | out = "" 159 | loop do 160 | chunk = pipe.read 161 | return out if chunk == "" || !chunk 162 | out += chunk 163 | end 164 | raise "You really shouldn't be able to break out of that loop..." 165 | end 166 | 167 | def coordinator_main_body(num_processes, top_pipe) 168 | # Open N processes, with N pipes to and from them. 169 | processes = [] 170 | pipes = [] 171 | num_processes.times do |worker_num| 172 | pipe_out, pipe_in = IO.pipe 173 | 174 | # Inside each process, run the block, print the result and exit. 175 | started_pid = fork do 176 | $0 += "(WORKER #{worker_num + 1})" 177 | pipe_out.close 178 | val = yield 179 | pipe_in.write(JSON.dump val) 180 | exit! 181 | end 182 | pipe_in.close 183 | processes.push(started_pid) 184 | pipes.push(pipe_out) 185 | end 186 | 187 | # Now we get all the output. 188 | result = [] 189 | pipes.each do |pipe| 190 | out = read_all_from_pipe(pipe) 191 | data = JSON.parse(out) 192 | result.concat(data) 193 | pipe.close 194 | end 195 | 196 | # Okay, now clear out all the dead process IDs. Unix won't let them die until they're explicitly waited for. 197 | until processes.empty? 198 | begin 199 | dead_pid = Process.waitpid(0) 200 | processes -= [ dead_pid ] 201 | STDERR.puts "Finished process with pid #{dead_pid.inspect}, waiting for #{processes.inspect}" 202 | rescue Errno::ECHILD 203 | STDERR.puts "ECHILD while waiting for child processes! I don't think this should happen..." 204 | raise 205 | end 206 | end 207 | 208 | result 209 | end 210 | 211 | # Run the block in N processes. The array result in each process will 212 | # be serialized as JSON, then passed back as a string and concatenated 213 | # in the parent process. 214 | def jsonable_with_num_processes(num_processes, &block) 215 | # Only one process? Great, skip all the hard parts. 216 | if num_processes == 1 217 | val = yield 218 | return val 219 | end 220 | 221 | # Okay, this is to talk to a coordinator process... 222 | coordinator_pipe_out, coordinator_pipe_in = IO.pipe 223 | 224 | # Okay, first open a "coordinator" process with its own process 225 | # group ID (pgid). Then we can do cleanup with a "kill -9" type 226 | # solution and get the whole process subtree without killing 227 | # ourself. 228 | coordinator_pid = fork do 229 | coordinator_pipe_out.close # For parent use, not coordinator use 230 | pgid = Process.pid # Get child's own pid 231 | Process.setpgid(pgid, pgid) # Detach into new process group 232 | 233 | combined_output = coordinator_main_body(num_processes, coordinator_pipe_in, &block) 234 | $0 += " (COORDINATOR)" 235 | coordinator_pipe_in.write(JSON.dump combined_output) 236 | sleep 0.01 237 | exit! 238 | end 239 | 240 | coordinator_pipe_in.close # For coordinator use, not parent use 241 | json_result = read_all_from_pipe coordinator_pipe_out 242 | 243 | # Now that we have all output from the coordinator, we'll kill it 244 | # along with all child processes... First "friendly" kill, then "no 245 | # really, go away" 246 | Process.kill("-HUP", coordinator_pid) 247 | sleep 0.1 248 | Process.kill(-9, coordinator_pid) 249 | 250 | # Ordinarily the coordinator shouldn't return its data until all 251 | # child processes have completed and waited for. So we shouldn't get 252 | # zombie processes unless there's an error, which should (I hope) 253 | # kill this process too... 254 | Process.waitpid(coordinator_pid) 255 | 256 | JSON.parse json_result 257 | end 258 | 259 | def single_run_benchmark_output_and_time 260 | t0 = Time.now 261 | loop do 262 | sleep 0.01 263 | output = `curl -f http://localhost:#{PORT_NUM}/ 2>/dev/null` 264 | next unless $?.success? 265 | return [output, Time.now - t0] 266 | end 267 | end 268 | 269 | def with_started_server 270 | server_start 271 | yield 272 | ensure 273 | server_stop 274 | end 275 | 276 | def with_running_server 277 | with_started_server do 278 | failed_iters = 0 279 | loop do 280 | sleep 0.1 281 | output = `curl -f http://localhost:#{PORT_NUM}/ 2>/dev/null` 282 | if $?.success? 283 | yield 284 | return 285 | else 286 | failed_iters += 1 287 | if failed_iters % 10 == 0 288 | puts "Tenth failed iter output:\n#{output}\n===========" 289 | end 290 | raise "Too many failed iterations!" if failed_iters > 50 291 | end 292 | end 293 | end 294 | end 295 | 296 | def full_iteration_start_stop 297 | elapsed = nil 298 | with_started_server do 299 | print "Server is started, running start/stop iteration...\n" 300 | server_output, elapsed = single_run_benchmark_output_and_time 301 | #print "Output:\n#{server_output}\n" 302 | end 303 | elapsed.to_f 304 | end 305 | 306 | def basic_iteration_get_http 307 | t0 = Time.now 308 | RestClient.get "http://localhost:#{PORT_NUM}/benchmark/simple_request" 309 | (Time.now - t0).to_f 310 | end 311 | 312 | require_relative "user_simulator" 313 | 314 | Signal.trap("HUP") do 315 | print "Ignoring SIGHUP...\n" 316 | end 317 | 318 | # One Burn-in Start/Stop Iteration 319 | unless no_warm_start 320 | print "Starting and stopping server to preload caches...\n" 321 | full_iteration_start_stop 322 | end 323 | 324 | worker_times = [] 325 | warmup_times = [] 326 | 327 | # Make sure these are in scope 328 | loaded_rss = nil 329 | final_rss = nil 330 | first_gc_stat = nil 331 | last_gc_stat = nil 332 | 333 | print "Running start-time benchmarks for #{startup_iters} iterations...\n" 334 | startup_times = (1..startup_iters).map { full_iteration_start_stop } 335 | request_times = nil 336 | 337 | with_running_server do 338 | loaded_rss = GetProcessMem.new(last_pid).bytes 339 | #first_gc_stat = get_server_gc_stats 340 | 341 | # By randomizing all "real" actions before all warmups, we guarantee 342 | # that multiple runs with different numbers of warmups but the same 343 | # number of worker iterations will always run the same worker 344 | # *actions* for that number of iterations. But we always want to 345 | # *run* warmup actions *first*, even if we *randomize* them 346 | # *second.* 347 | worker_actions = actions_for_iterations(worker_iterations) 348 | warmup_actions = actions_for_iterations(warmup_iterations) 349 | 350 | # First, warmup iterations. 351 | print "Warmup iterations: #{warmup_iterations}\n" 352 | unless warmup_iterations == 0 353 | warmup_times = jsonable_with_num_processes(worker_processes) { multithreaded_actions(warmup_actions, workers, PORT_NUM) } 354 | end 355 | # Second, real iterations. 356 | print "Benchmark iterations: #{worker_iterations}\n" 357 | unless worker_iterations == 0 358 | worker_times = jsonable_with_num_processes(worker_processes) { multithreaded_actions(worker_actions, workers, PORT_NUM) } 359 | end 360 | final_rss = GetProcessMem.new(last_pid).bytes 361 | #last_gc_stat = get_server_gc_stats 362 | end # Stop the Rails server after all interactions have finished. 363 | 364 | # TODO: Fix these thread run times. process.rb was fixed, the just-to-console times weren't. 365 | print "===== Startup Benchmarks =====\n" 366 | print "Longest run: #{startup_times.max}\n" 367 | print "Shortest run: #{startup_times.min}\n" 368 | print "Mean: #{startup_times.inject(0.0, &:+) / startup_times.size}\n" 369 | print "Median: #{startup_times.sort[ startup_times.size / 2 ] }\n" 370 | print "Raw times: #{startup_times.inspect}\n" 371 | 372 | print "===== Startup Benchmarks =====\n" 373 | worker_times_max = worker_times.map(&:max) 374 | print "Slowest thread run: #{worker_times_max.max}\n" 375 | print "Fastest thread run: #{worker_times_max.min}\n" 376 | print "Mean thread run: #{worker_times_max.inject(0.0, &:+) / worker_times.size}\n" 377 | print "Median thread run: #{worker_times_max.sort[ worker_times.size / 2 ] }\n" 378 | print "Raw times: #{worker_times.inspect}\n" 379 | 380 | env_vars = ENV.keys 381 | important_env_vars = env_vars.select { |name| name.downcase["ruby"] || name.downcase["gem"] || name.downcase["rrb"]} + [ "LD_PRELOAD" ] 382 | env_hash = {} 383 | important_env_vars.each { |var| env_hash["env-#{var}"] = ENV[var] } 384 | 385 | test_data = { 386 | "version" => 3, # Last breaking revision: added total restart iterations, so number of data points may not match worker_iterations. 387 | "settings" => { 388 | "startup_iters" => startup_iters, 389 | "random_seed" => random_seed, 390 | "worker_iterations" => worker_iterations, 391 | "warmup_iterations" => warmup_iterations, 392 | "total_restart_iterations" => total_restart_iterations, 393 | "workers" => workers, 394 | "worker_processes" => worker_processes, 395 | "puma_processes" => puma_processes, 396 | "puma_threads" => puma_threads, 397 | "port_num" => port_num, 398 | "out_dir" => out_dir, 399 | "out_file" => out_file || false, 400 | "no_warm_start" => no_warm_start, 401 | "discourse_revision" => `cd work/discourse && git rev-parse HEAD`.chomp, 402 | }, 403 | "environment" => { 404 | "RUBY_VERSION" => RUBY_VERSION, 405 | "RUBY_DESCRIPTION" => RUBY_DESCRIPTION, 406 | "rvm current" => `rvm current 2>&1`.strip, 407 | "discourse git status" => `cd work/discourse && git status`, 408 | "discourse git sha" => `cd work/discourse && git rev-parse HEAD`.chomp, 409 | "rails_ruby_bench git status" => `git status`, 410 | "rails_ruby_bench git sha" => `git rev-parse HEAD`, 411 | "ec2 instance id" => `wget -q -O - http://169.254.169.254/latest/meta-data/instance-id`, 412 | "ec2 instance type" => `wget -q -O - http://169.254.169.254/latest/meta-data/instance-type`, 413 | }.merge(env_hash), 414 | "startup" => { 415 | "times" => startup_times 416 | }, 417 | "warmup" => { 418 | "times" => warmup_times 419 | }, 420 | "requests" => { 421 | "times" => worker_times 422 | }, 423 | "memory" => { 424 | "master_puma_process" => { 425 | "loaded_rss" => loaded_rss, 426 | "final_rss" => final_rss, 427 | } 428 | #"gc_stat_last" => last_gc_stat, 429 | #"gc_stat_first" => first_gc_stat, 430 | }, 431 | } 432 | 433 | json_filename = File.join(out_dir, out_file || "rails_ruby_bench_#{Time.now.to_i}.json") 434 | print "Writing run data to #{json_filename}...\n" 435 | File.open(json_filename, "w") do |f| 436 | f.print JSON.pretty_generate(test_data) 437 | f.print "\n" 438 | end 439 | print "Wrote run data to #{json_filename}\n" 440 | --------------------------------------------------------------------------------