├── .github └── dependabot.yml ├── .gitignore ├── .rubocop.yml ├── .travis.yml ├── Gemfile ├── LICENSE ├── Readme.md ├── aws ├── api │ └── swagger.json ├── ec2 │ ├── boot │ ├── doCheck │ └── install ├── lambda │ ├── Readme.md │ └── index.js ├── s3 │ └── generate-and-upload-site ├── secretsmanager │ └── get └── sqs │ ├── policy.json │ └── process ├── bench ├── docs ├── 404.md ├── _config.yml ├── _layouts │ └── master.html └── index.md ├── github ├── checks │ ├── complete │ ├── fail │ └── start └── token ├── report └── site-list /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: bundler 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 10 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /Gemfile.lock 2 | /jekyll 3 | /results.csv 4 | /sites 5 | /docs/_site 6 | 7 | .sass-cache/ 8 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | inherit_gem: 2 | jekyll: .rubocop.yml 3 | 4 | AllCops: 5 | TargetRubyVersion: 2.3 6 | Include: 7 | - report 8 | - github/jwt 9 | Exclude: 10 | - sites/**/* 11 | - vendor/**/* 12 | 13 | Jekyll/NoPutsAllowed: 14 | Enabled: false 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 2.4 3 | cache: bundler 4 | 5 | branches: 6 | only: 7 | - master 8 | 9 | before_install: 10 | - gem update --system 11 | - gem install bundler 12 | 13 | before_script: 14 | - bundle update 15 | 16 | matrix: 17 | include: 18 | - name: Rubocop 19 | script: bundle exec rubocop -D 20 | - name: bench 21 | script: ./bench 22 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | repo = ENV.fetch("REPO", "https://github.com/jekyll/jekyll.git") 4 | 5 | if ENV["REF"] 6 | gem "jekyll", git: repo, ref: ENV["REF"] 7 | elsif ENV["PR"] 8 | gem "jekyll", git: repo, ref: "refs/pull/#{ENV['PR']}/head" 9 | else 10 | gem "jekyll", git: repo, branch: "master" 11 | end 12 | 13 | # GitHub Pages 14 | gem "jekyll-avatar" 15 | gem "jekyll-coffeescript" 16 | gem "jekyll-commonmark-ghpages" 17 | gem "jekyll-default-layout" 18 | gem "jekyll-feed" 19 | gem "jekyll-gist" 20 | gem "jekyll-github-metadata" 21 | gem "jekyll-mentions" 22 | gem "jekyll-optional-front-matter" 23 | gem "jekyll-paginate" 24 | gem "jekyll-readme-index" 25 | gem "jekyll-redirect-from" 26 | gem "jekyll-relative-links" 27 | gem "jekyll-remote-theme" 28 | gem "jekyll-sass-converter" 29 | gem "jekyll-seo-tag" 30 | gem "jekyll-sitemap" 31 | gem "jekyll-swiss" 32 | gem "jekyll-theme-architect" 33 | gem "jekyll-theme-cayman" 34 | gem "jekyll-theme-dinky" 35 | gem "jekyll-theme-hacker" 36 | gem "jekyll-theme-leap-day" 37 | gem "jekyll-theme-merlot" 38 | gem "jekyll-theme-midnight" 39 | gem "jekyll-theme-minimal" 40 | gem "jekyll-theme-modernist" 41 | gem "jekyll-theme-primer" 42 | gem "jekyll-theme-slate" 43 | gem "jekyll-theme-tactile" 44 | gem "jekyll-theme-time-machine" 45 | gem "jekyll-titles-from-headings" 46 | gem "jemoji" 47 | gem "kramdown" 48 | 49 | # CloudCannon/base-jekyll-template 50 | gem "jekyll-archives" 51 | gem "jekyll-extract-element" 52 | 53 | # DirtyF/frank.taillandier.me 54 | gem "classifier-reborn" 55 | gem "jekyll-cloudinary", group: :jekyll_plugins 56 | gem "jekyll-include-cache" 57 | gem "jekyll-last-modified-at" 58 | gem "jekyll-tidy" 59 | 60 | # 18F/federalist-docs 61 | gem "jekyll_pages_api_search", group: :jekyll_plugins 62 | gem "redcarpet" 63 | gem "uswds-jekyll" 64 | 65 | group :dev do 66 | gem "rubocop", "~> 0.66.0" 67 | end 68 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Pat Hawks and contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Utterson 2 | 3 | Utterson investigates Jekyll's performance. Compare different versions of Jekyll 4 | (or even specific PRs/commits) to see how changes might affect performance. 5 | 6 | ## Usage 7 | 8 | ### Running tests 9 | 10 | To test the current Jekyll `master` branch: 11 | 12 | ```sh 13 | ./bench 14 | ``` 15 | 16 | To test a Pull Request: 17 | 18 | ```sh 19 | PR=1234 ./bench 20 | ``` 21 | 22 | To test a version: 23 | 24 | ```sh 25 | REF=v3.8.2 ./bench 26 | ``` 27 | 28 | ### Creating reports 29 | 30 | Once multiple tests have been run, generate a report showing differences in 31 | total build time with the command: 32 | 33 | ```sh 34 | ./report 35 | ``` 36 | 37 | Reports will show total build time for all sites using each tested version: 38 | 39 | ```text 40 | | ref | build time in seconds | 41 | |:-----------------------------------------|----------------------:| 42 | | `v3.8.2` | 287.54 | 43 | | `v3.7.3` | 317.74 | 44 | | `v3.6.2` | 295.49 | 45 | | `v3.5.2` | 297.94 | 46 | ``` 47 | -------------------------------------------------------------------------------- /aws/api/swagger.json: -------------------------------------------------------------------------------- 1 | { 2 | "swagger": "2.0", 3 | "info": { 4 | "version": "2018-06-08T18:49:56Z", 5 | "title": "Utterson" 6 | }, 7 | "host": "t5c8f4djff.execute-api.us-east-1.amazonaws.com", 8 | "basePath": "/prod", 9 | "schemes": [ 10 | "https" 11 | ], 12 | "paths": { 13 | "/": { 14 | "post": { 15 | "produces": [ 16 | "application/json" 17 | ], 18 | "responses": { 19 | "200": { 20 | "description": "200 response", 21 | "schema": { 22 | "$ref": "#/definitions/Empty" 23 | } 24 | } 25 | }, 26 | "x-amazon-apigateway-integration": { 27 | "uri": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:631571985519:function:utterson/invocations", 28 | "responses": { 29 | "default": { 30 | "statusCode": "200" 31 | } 32 | }, 33 | "passthroughBehavior": "when_no_match", 34 | "httpMethod": "POST", 35 | "contentHandling": "CONVERT_TO_TEXT", 36 | "type": "aws_proxy" 37 | } 38 | } 39 | } 40 | }, 41 | "definitions": { 42 | "Empty": { 43 | "type": "object", 44 | "title": "Empty Schema" 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /aws/ec2/boot: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # chkconfig: 2345 20 80 3 | 4 | . /etc/init.d/functions 5 | 6 | checkloop() { 7 | cd /home/ec2-user/Utterson 8 | git pull 9 | while ( sudo -u ec2-user -Hi -- /home/ec2-user/Utterson/aws/ec2/doCheck ); do 10 | echo "Checking for more messages." 11 | done 12 | local USERS=$( who | wc -l ) 13 | if (( USERS = 0 )); then 14 | wall "Shutting down." 15 | shutdown -h now 16 | fi 17 | } 18 | 19 | start() { 20 | checkloop & 21 | } 22 | 23 | case "$1" in 24 | start) 25 | start 26 | ;; 27 | stop) 28 | ;; 29 | restart) 30 | ;; 31 | status) 32 | ;; 33 | *) 34 | echo "Usage: $0 {start|stop|status|restart}" 35 | esac 36 | -------------------------------------------------------------------------------- /aws/ec2/doCheck: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd ~/Utterson 4 | aws/sqs/process 5 | -------------------------------------------------------------------------------- /aws/ec2/install: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RUBY="2.4" 4 | 5 | cd ~/ 6 | 7 | # Upgrade AWS-CLI 8 | sudo pip install --upgrade awscli 9 | 10 | # Allow installing Node.js 11 | curl --silent --location https://rpm.nodesource.com/setup_8.x | sudo bash - 12 | 13 | # Install dependencies with Yum 14 | sudo yum -y update 15 | sudo yum -y groupinstall "Development Tools" 16 | sudo yum -y install git-core jq nodejs openssl-devel readline-devel 17 | 18 | # Install rbenv 19 | export PATH="$HOME/.rbenv/bin:$PATH" 20 | echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile 21 | curl -fsSL https://github.com/rbenv/rbenv-installer/raw/master/bin/rbenv-installer | bash 22 | rbenv install 2.5.1 23 | ruby global 2.5.1 24 | ~/.rbenv/bin/rbenv init 25 | 26 | # Install Bundler 27 | gem install bundler 28 | 29 | # Install jwt & octokit for communicating with GitHub 30 | gem install jwt octokit 31 | 32 | # Install rbspy 33 | mkdir ~/rbspy 34 | pushd ~/rbspy 35 | curl -L https://github.com/rbspy/rbspy/releases/download/v0.2.10/rbspy-v0.2.10-x86_64-unknown-linux-musl.tar.gz > rbspy.tar.gz 36 | tar -xvzf rbspy.tar.gz 37 | sudo mv rbspy /usr/bin/ 38 | popd 39 | rm -rf ~/rbspy 40 | 41 | # Download Utterson 42 | git clone https://github.com/jekyll/Utterson.git Utterson 43 | 44 | # Link EC2 boot script to init.d 45 | sudo ln Utterson/aws/ec2/boot /etc/rc.d/init.d/utterson 46 | -------------------------------------------------------------------------------- /aws/lambda/Readme.md: -------------------------------------------------------------------------------- 1 | # Lambda Function 2 | 3 | This Lambda function needs two environment variables set: 4 | 5 | * `QUEUE_URL`: The SQS Queue where build requests will be sent 6 | * `INSTANCE_ID`: The EC2 instance where builds will run 7 | -------------------------------------------------------------------------------- /aws/lambda/index.js: -------------------------------------------------------------------------------- 1 | var AWS = require('aws-sdk'); 2 | 3 | AWS.config.apiVersions = { 4 | ec2: '2016-11-15', 5 | sqs: '2012-11-05', 6 | }; 7 | 8 | var ec2 = new AWS.EC2(); 9 | var sqs = new AWS.SQS(); 10 | 11 | const StringValue = function(str) { 12 | return { 13 | DataType: "String", 14 | StringValue: String(str) 15 | }; 16 | } 17 | 18 | const AllowedActions = ["synchronize", "opened"]; 19 | 20 | exports.handler = async function(event, context) { 21 | context.callbackWaitsForEmptyEventLoop = false; 22 | 23 | try { 24 | var response = { 25 | "isBase64Encoded": false, 26 | "statusCode": 200, 27 | "headers": {}, 28 | "body": "" 29 | }; 30 | 31 | var json = event.body; 32 | if (typeof json === "string") { 33 | json = JSON.parse(json); 34 | } 35 | 36 | if (json["action"] === "created" || "zen" in json) { 37 | response["statusCode"] = 204; 38 | context.succeed(response); 39 | return; 40 | } 41 | 42 | var pull_request; 43 | if (!AllowedActions.includes(json["action"])) { 44 | response["statusCode"] = 204; 45 | context.succeed(response); 46 | return; 47 | } 48 | 49 | pull_request = json["pull_request"]; 50 | 51 | var title = "New PR #" + pull_request["number"] + 52 | "\nwith base " + pull_request["base"]["ref"] + 53 | "\nand head " + pull_request["head"]["sha"]; 54 | 55 | var params = { 56 | InstanceIds: [ 57 | process.env["INSTANCE_ID"], 58 | ] 59 | }; 60 | 61 | var message = { 62 | MessageBody: title, 63 | QueueUrl: process.env["QUEUE_URL"], 64 | MessageGroupId: "0", 65 | MessageAttributes: { 66 | "base-branch": StringValue(pull_request["base"]["ref"]), 67 | "base-sha": StringValue(pull_request["base"]["sha"]), 68 | "head-branch": StringValue(pull_request["head"]["ref"]), 69 | "head-sha": StringValue(pull_request["head"]["sha"]), 70 | "installation": StringValue(json["installation"]["id"]), 71 | "pr": StringValue(pull_request["number"]), 72 | "url": StringValue(pull_request["base"]["repo"]["url"]) 73 | } 74 | }; 75 | if ("clone_url" in pull_request["base"]["repo"]) { 76 | message["MessageAttributes"]["base-repo"] = StringValue(pull_request["base"]["repo"]["clone_url"]); 77 | message["MessageAttributes"]["head-repo"] = StringValue(pull_request["head"]["repo"]["clone_url"]); 78 | message["MessageAttributes"]["html-url"] = StringValue(pull_request["html_url"].replace(/\/pull\/\d+/, '')); 79 | } 80 | else { 81 | message["MessageAttributes"]["base-repo"] = StringValue(json["repository"]["clone_url"]); 82 | message["MessageAttributes"]["head-repo"] = StringValue(json["repository"]["clone_url"]); 83 | message["MessageAttributes"]["html-url"] = StringValue(json["repository"]["html_url"]); 84 | } 85 | response.body = JSON.stringify(message, null, 2); 86 | 87 | await new Promise((resolve, reject) => { 88 | sqs.sendMessage(message, function(err, data) { 89 | if (err) { 90 | console.log("Error", err); 91 | } 92 | else { 93 | console.log("SQS Success"); 94 | } 95 | resolve(); 96 | }); 97 | }); 98 | 99 | await new Promise((resolve, reject) => { 100 | ec2.startInstances(params, function(err, data) { 101 | if (err) { 102 | console.log("Error", err); 103 | } 104 | else { 105 | console.log("EC2 Success"); 106 | } 107 | resolve(); 108 | }); 109 | }); 110 | 111 | context.succeed(response); 112 | } 113 | catch (err) { 114 | response.statusCode = 500; 115 | response.body = err.message + "\n\n" + JSON.stringify(event.body); 116 | context.succeed(response); 117 | } 118 | }; 119 | -------------------------------------------------------------------------------- /aws/s3/generate-and-upload-site: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # set -e 3 | 4 | # Generate the `docs/` site and upload to S3 5 | 6 | echo "Generating docs site" 7 | 8 | SOURCE="docs/" 9 | DESTINATION="sites/destination/docs" 10 | mkdir -p $DESTINATION 11 | 12 | bundle update 13 | bundle exec jekyll build -s $SOURCE -d $DESTINATION 14 | 15 | aws s3 cp --exclude "api/*" --recursive $DESTINATION s3://$S3_BUCKET/ 16 | -------------------------------------------------------------------------------- /aws/secretsmanager/get: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | trap checkSuccess Exit 6 | 7 | SECRET_ID=$1 8 | 9 | function checkSuccess { 10 | STATUS=$? 11 | if [[ $SECRET_ID == "" ]]; then 12 | >&2 echo "Please specify a secret to get:" 13 | >&2 echo " $0 [secret ID]" 14 | exit 1 15 | elif (( $STATUS != 0 )); then 16 | >&2 echo "Could not get secret '$SECRET_ID'" 17 | >&2 echo "Please make sure that the current user has 'secretsmanager:GetSecretValue' permission" 18 | >&2 echo " for the resource arn:aws:secretsmanager:*:*:secret:*" 19 | exit 1 20 | fi 21 | } 22 | 23 | JSON=`aws secretsmanager get-secret-value --secret-id "$SECRET_ID"` 24 | jq -r '.SecretString' <<< $JSON 25 | -------------------------------------------------------------------------------- /aws/sqs/policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Id": "arn:aws:sqs:us-east-1:631571985519:utterson.fifo/SQSDefaultPolicy", 4 | "Statement": [ 5 | { 6 | "Sid": "Sid1528213389225", 7 | "Effect": "Allow", 8 | "Principal": { 9 | "AWS": "arn:aws:iam::631571985519:user/Utterson" 10 | }, 11 | "Action": [ 12 | "SQS:DeleteMessage", 13 | "SQS:SendMessage", 14 | "SQS:ReceiveMessage", 15 | "SQS:ChangeMessageVisibility" 16 | ], 17 | "Resource": "arn:aws:sqs:us-east-1:631571985519:utterson.fifo" 18 | }, 19 | { 20 | "Sid": "Sid1528222156130", 21 | "Effect": "Allow", 22 | "Principal": { 23 | "AWS": "arn:aws:iam::631571985519:role/service-role/utterson-lambda" 24 | }, 25 | "Action": "SQS:SendMessage", 26 | "Resource": "arn:aws:sqs:us-east-1:631571985519:utterson.fifo" 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /aws/sqs/process: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # set -e 3 | 4 | # Builds a PR based on a message received from SQS 5 | 6 | function reportFailure { 7 | if [[ $? != 0 ]]; then 8 | echo "Build Failure" 9 | URL=$URL HEAD_BRANCH=$HEAD_BRANCH HEAD_SHA=$HEAD_SHA ./github/checks/fail 10 | aws sqs delete-message --receipt-handle "$RECEIPT_HANDLE" --queue-url "$QUEUE_URL" 11 | exit 12 | fi 13 | } 14 | 15 | JSON_RESPONSE=$(aws sqs receive-message --queue-url "$QUEUE_URL" --message-attribute-names All) 16 | 17 | if [[ -z $JSON_RESPONSE ]]; then 18 | exit 1 19 | fi 20 | 21 | echo "JSON = $JSON_RESPONSE" 22 | 23 | PR=$(jq -r '.Messages[0].MessageAttributes.pr.StringValue' <<< $JSON_RESPONSE) 24 | URL=$(jq -r '.Messages[0].MessageAttributes.url.StringValue' <<< $JSON_RESPONSE) 25 | BASE_SHA=$(jq -r '.Messages[0].MessageAttributes["base-sha"].StringValue' <<< $JSON_RESPONSE) 26 | HEAD_SHA=$(jq -r '.Messages[0].MessageAttributes["head-sha"].StringValue' <<< $JSON_RESPONSE) 27 | BASE_REPO=$(jq -r '.Messages[0].MessageAttributes["base-repo"].StringValue' <<< $JSON_RESPONSE) 28 | HEAD_REPO=$(jq -r '.Messages[0].MessageAttributes["head-repo"].StringValue' <<< $JSON_RESPONSE) 29 | BASE_BRANCH=$(jq -r '.Messages[0].MessageAttributes["base-branch"].StringValue' <<< $JSON_RESPONSE) 30 | HEAD_BRANCH=$(jq -r '.Messages[0].MessageAttributes["head-branch"].StringValue' <<< $JSON_RESPONSE) 31 | HTML_URL=$(jq -r '.Messages[0].MessageAttributes["html-url"].StringValue' <<< $JSON_RESPONSE) 32 | export INSTALLATION=$(jq -r '.Messages[0].MessageAttributes.installation.StringValue' <<< $JSON_RESPONSE) 33 | RECEIPT_HANDLE=$(jq -r '.Messages[0].ReceiptHandle' <<< $JSON_RESPONSE) 34 | 35 | export STARTED_AT=$(date -u "+%Y-%m-%dT%H:%M:%SZ") 36 | URL=$URL HEAD_BRANCH=$HEAD_BRANCH HEAD_SHA=$HEAD_SHA ./github/checks/start 37 | 38 | # If there are already any previous results, delete them 39 | if [[ -f results.csv ]]; then 40 | rm results.csv 41 | fi 42 | 43 | # Test the new PR, and the base branch 44 | # Before each build, we lock the message for 20 minutes + 10 seconds 45 | # (our maximum allowed build time + a grace period) 46 | aws sqs change-message-visibility --receipt-handle "$RECEIPT_HANDLE" --queue-url "$QUEUE_URL" --visibility-timeout 1210 47 | FLAMEGRAPH="true" PR="$PR" REF="$HEAD_SHA" REPO="$HEAD_REPO" ./bench || reportFailure 48 | aws sqs change-message-visibility --receipt-handle "$RECEIPT_HANDLE" --queue-url "$QUEUE_URL" --visibility-timeout 1210 49 | BRANCH="$BASE_BRANCH" REF="$BASE_SHA" REPO="$BASE_REPO" ./bench || reportFailure 50 | 51 | # Now that we are done building, we only need to lock the message for long 52 | # enough to generate our report. If something fails, we want the message to 53 | # quickly make its way back into the queue to try again 54 | aws sqs change-message-visibility --receipt-handle "$RECEIPT_HANDLE" --queue-url "$QUEUE_URL" --visibility-timeout 30 55 | 56 | # Generate report based on tests 57 | PR="$PR" REF="$HEAD_SHA" HTML_URL="$HTML_URL" ./report > "docs/$HEAD_SHA.md" 58 | cat "docs/$HEAD_SHA.md" 59 | ./aws/s3/generate-and-upload-site 60 | # Delete report 61 | rm "docs/$HEAD_SHA.md" 62 | # Delete flamegraphs associated with a report 63 | rm -rf "docs/$HEAD_SHA" 64 | 65 | # If everything was successful, we can delete this message from SQS 66 | aws sqs delete-message --receipt-handle "$RECEIPT_HANDLE" --queue-url "$QUEUE_URL" 67 | 68 | URL=$URL HEAD_BRANCH=$HEAD_BRANCH HEAD_SHA=$HEAD_SHA ./github/checks/complete 69 | -------------------------------------------------------------------------------- /bench: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | ulimit -t 1200 4 | 5 | function checkFailedBuild { 6 | if [[ $? != 0 ]]; then 7 | echo "Build failed" 8 | exit 1 9 | fi 10 | } 11 | 12 | trap checkFailedBuild Exit 13 | 14 | export BUNDLE_GEMFILE="$(pwd)/Gemfile" 15 | bundle update 16 | 17 | NUMBER_OF_RUNS=3 18 | RESULTS="$(pwd)/results.csv" 19 | if [[ ! -s $RESULTS ]]; then 20 | echo "Jekyll version, user time in seconds, site" > $RESULTS 21 | fi 22 | 23 | if [[ -n $PR ]]; then 24 | VERSION="#$PR" 25 | elif [[ -n $BRANCH ]]; then 26 | VERSION="$BRANCH" 27 | elif [[ -n $REF ]]; then 28 | VERSION="$REF" 29 | else 30 | VERSION="master" 31 | fi 32 | 33 | if command -v gtime; then 34 | TIME=$(which gtime) 35 | else 36 | TIME=$(which time) 37 | fi 38 | 39 | # Create tmp/ directory 40 | TMPDIR="$(pwd)/sites" 41 | if [[ -d $TMPDIR ]]; then 42 | rm -rf "$TMPDIR" 43 | fi 44 | mkdir -p "$TMPDIR/source" 45 | mkdir -p "$TMPDIR/destination" 46 | 47 | # Flush SASS cache 48 | if [[ -d "$(pwd)/.sass-cache" ]]; then 49 | rm -rf "$(pwd)/.sass-cache" 50 | fi 51 | 52 | # Create a directory for flamegraphs 53 | if [[ -n $FLAMEGRAPH ]]; then 54 | mkdir -p "docs/$REF" 55 | fi 56 | 57 | for SITE in $(cat "site-list"); do 58 | echo " 59 | ________________________________________________________________________________ 60 | Sampling: $SITE" 61 | SOURCE="$TMPDIR/source/${SITE##*/}" 62 | DESTINATION=${SOURCE/source/destination} 63 | SVG_PATH="docs/$REF/${SITE##*/}" 64 | if [[ ! -d $SOURCE ]]; then 65 | git clone --recurse-submodules -q "$SITE" "$SOURCE" 66 | fi 67 | for ((i=0; i 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | {{ page.title | smartify }} 10 | 23 | 24 | 25 |
26 | 44 |
45 |
46 |

47 | 48 | Jekyll 49 | Jekyll Logo 50 | 51 |

52 |
53 | 71 |
72 |
73 |
74 |
75 |
76 |
77 |

{{ page.title | smartify }}

78 | {{ content }} 79 |
80 |
81 |
82 |
83 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Utterson 3 | --- 4 | 5 | Utterson investigates Jekyll's performance 6 | -------------------------------------------------------------------------------- /github/checks/complete: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require "json" 5 | require "net/http" 6 | require "uri" 7 | require "date" 8 | 9 | token = File.expand_path("../token", __dir__) 10 | report = File.expand_path("../../report", __dir__) 11 | 12 | date = Time.now.strftime("%Y-%m-%dT%H:%M:%SZ") 13 | 14 | json = JSON( 15 | :name => "Performance Check", 16 | :status => "completed", 17 | :conclusion => "success", 18 | :completed_at => date, 19 | :details_url => "https://utterson.pathawks.com/#{ENV["HEAD_SHA"]}.html", 20 | :head_branch => ENV["HEAD_BRANCH"], 21 | :head_sha => ENV["HEAD_SHA"], 22 | :started_at => ENV.fetch("STARTED_AT", date), 23 | :output => { 24 | :title => "Build Complete", 25 | :summary => "The build completed", 26 | :text => `#{report}`, 27 | } 28 | ) 29 | 30 | uri = URI("#{ENV["URL"]}/check-runs") 31 | req = Net::HTTP::Post.new(uri) 32 | req["Authorization"] = "token #{`#{token}`.chomp}" 33 | req["Accept"] = "application/vnd.github.antiope-preview+json" 34 | req.body = json 35 | 36 | http = Net::HTTP.new(uri.hostname, uri.port) 37 | http.use_ssl = (uri.scheme == "https") 38 | http.request(req) 39 | -------------------------------------------------------------------------------- /github/checks/fail: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require "json" 5 | require "net/http" 6 | require "uri" 7 | require "date" 8 | 9 | token = File.expand_path("../token", __dir__) 10 | date = Time.now.strftime("%Y-%m-%dT%H:%M:%SZ") 11 | 12 | json = JSON( 13 | :name => "Performance Check", 14 | :status => "completed", 15 | :conclusion => "failure", 16 | :completed_at => date, 17 | :started_at => ENV.fetch("STARTED_AT", date), 18 | :head_branch => ENV["HEAD_BRANCH"], 19 | :head_sha => ENV["HEAD_SHA"] 20 | ) 21 | 22 | uri = URI("#{ENV["URL"]}/check-runs") 23 | req = Net::HTTP::Post.new(uri) 24 | req["Authorization"] = "token #{`#{token}`.chomp}" 25 | req["Accept"] = "application/vnd.github.antiope-preview+json" 26 | req.body = json 27 | 28 | http = Net::HTTP.new(uri.hostname, uri.port) 29 | http.use_ssl = (uri.scheme == "https") 30 | http.request(req) 31 | -------------------------------------------------------------------------------- /github/checks/start: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require "json" 5 | require "net/http" 6 | require "uri" 7 | require "date" 8 | 9 | token = File.expand_path("../token", __dir__) 10 | 11 | date = Time.now.strftime("%Y-%m-%dT%H:%M:%SZ") 12 | 13 | json = JSON( 14 | :name => "Performance Check", 15 | :status => "in_progress", 16 | :started_at => ENV.fetch("STARTED_AT", date), 17 | :head_branch => ENV["HEAD_BRANCH"], 18 | :head_sha => ENV["HEAD_SHA"] 19 | ) 20 | 21 | uri = URI("#{ENV["URL"]}/check-runs") 22 | req = Net::HTTP::Post.new(uri) 23 | req["Authorization"] = "token #{`#{token}`.chomp}" 24 | req["Accept"] = "application/vnd.github.antiope-preview+json" 25 | req.body = json 26 | 27 | http = Net::HTTP.new(uri.hostname, uri.port) 28 | http.use_ssl = (uri.scheme == "https") 29 | http.request(req) 30 | -------------------------------------------------------------------------------- /github/token: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require "English" 5 | require "jwt" # https://rubygems.org/gems/jwt 6 | require "octokit" 7 | require "openssl" 8 | 9 | # Private key contents 10 | get_secret = File.expand_path("../aws/secretsmanager/get", __dir__) 11 | private_pem = `#{get_secret} GitHub` 12 | exit unless $CHILD_STATUS.success? 13 | private_key = OpenSSL::PKey::RSA.new(private_pem) 14 | 15 | # Generate the JWT 16 | payload = { 17 | # issued at time 18 | :iat => Time.now.to_i, 19 | # JWT expiration time (10 minute maximum) 20 | :exp => Time.now.to_i + (10 * 60), 21 | # GitHub App's identifier 22 | :iss => 13_446, 23 | } 24 | 25 | jwt = JWT.encode(payload, private_key, "RS256") 26 | client = Octokit::Client.new(:bearer_token => jwt) 27 | installation = ENV.fetch("INSTALLATION") 28 | token = client.create_installation_access_token(installation)[:token] 29 | 30 | puts token 31 | -------------------------------------------------------------------------------- /report: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require "csv" 5 | require "yaml" 6 | 7 | def linkify_pr(text, repo) 8 | "[##{text}](#{File.join(repo, "pull", text)})" 9 | end 10 | 11 | def linkify_sha(ref, repo) 12 | "[`#{ref[0...7]}`](#{File.join(repo, "commit", ref)})" 13 | end 14 | 15 | def linkify_flamegraph(site) 16 | site = site.split("/").last 17 | "[🔥️](https://utterson.pathawks.com/#{ENV["REF"]}/#{site}.svg)" 18 | end 19 | 20 | def linkify_site(site) 21 | "[#{site.split("/").last}](#{site})" 22 | end 23 | 24 | def markdown_header 25 | repo_url = ENV.fetch("HTML_URL", "https://github.com/jekyll/jekyll") 26 | ref = ENV["REF"] 27 | pr = ENV["PR"] 28 | frontmatter = { 29 | "PR" => pr, 30 | "REF" => ref, 31 | "title" => "Performance Check", 32 | "permalink" => "/#{ref}.html", 33 | } 34 | <<~HEADER 35 | #{frontmatter.to_yaml}--- 36 | 37 | The following report was generated for PR #{linkify_pr(pr, repo_url)}, 38 | on commit #{linkify_sha(ref, repo_url)} 39 | 40 | HEADER 41 | end 42 | 43 | begin 44 | results = CSV.table("results.csv") 45 | rescue Errno::ENOENT 46 | puts <<~MSG 47 | Cannot open file results.csv 48 | You must run ./bench before running ./report 49 | MSG 50 | exit 1 51 | end 52 | 53 | puts markdown_header if ENV.key?("REF") && ENV.key?("PR") 54 | 55 | puts "| ref | build time in seconds |" 56 | puts "|:-----------------------------------------|----------------------:|" 57 | 58 | summed_results = {} 59 | site_results = {} 60 | results.each do |row| 61 | summed_results[row[0]] = 0.0 unless summed_results.key?(row[0]) 62 | site_results[row[2]] ||= { 63 | :flamegraph => linkify_flamegraph(row[2]), 64 | :site => linkify_site(row[2]), 65 | :time => 0.0, 66 | } 67 | summed_results[row[0]] += row[1] 68 | site_results[row[2]][:time] += row[1] if row[0] == "##{ENV["PR"]}" 69 | end 70 | 71 | summed_results.each do |ref, time| 72 | ref = "`#{ref}`" unless ref =~ %r!\A(?:[0-9a-f]+|#\d+)\Z! 73 | puts format("| %-40s | %21.2f |", ref, time) 74 | end 75 | 76 | if ENV.key?("REF") && ENV.key?("PR") 77 | puts <<~HEADER 78 | 79 | Below is a table with the total build time for each site (summed total of all builds against this commit) 80 | 81 | | | site | build time in seconds | 82 | |--|:-----|----------------------:| 83 | HEADER 84 | site_results.values.each do |data| 85 | puts format "| %s | %s | %