├── .gitignore
├── README.md
├── config.py
├── generate_upstream_list.py
├── repository.py
├── requirements.txt
├── templates
    ├── base.html
    └── upstreamed_commits.html
└── update_overview.sh


/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | *.git
3 | *.swp
4 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | Opera upstreamtools
 2 | ===================
 3 | 
 4 | A script to create [a web page](https://operasoftware.github.io/upstreamtools/) tracking
 5 | upstreamed code to the Chromium project from Opera.
 6 | 
 7 | 
 8 | Getting started
 9 | ---------------
10 | 
11 | Get the source:
12 | 
13 |     git clone https://github.com/operasoftware/upstreamtools
14 |     cd upstreamtools
15 | 
16 | Install the requirements (possibly inside a virtualenv):
17 | 
18 |     pip install -r requirements.txt
19 | 
20 | Get the required upstream repositories:
21 | 
22 |     git clone --bare https://chromium.googlesource.com/chromium/src.git chromium.src.git
23 |     git clone --bare https://github.com/v8/v8.git v8.git
24 |     git clone --bare https://chromium.googlesource.com/skia.git skia.git
25 |     git clone --bare https://boringssl.googlesource.com/boringssl.git boringssl.git
26 | 
27 | Generate the file:
28 | 
29 |     python generate_upstream_list.py > upstream.html
30 | 
31 | 
32 | Updating the overview
33 | ---------------------
34 | 
35 | There's a simple script which should be able to do the work for you.  It will
36 | generate the list, switch to the `gh-pages` branch and commit it.  If you run
37 | it with the `push` argument it will even try to push it for you.
38 | 
39 |     ./update_overview.sh [push]
40 | 
41 | 
42 | Updating the overview manually
43 | ------------------------------
44 | 
45 | Update the Git checkouts.
46 | 
47 |     for p in chromium.src v8; do git --git-dir=$p.git fetch; done
48 | 
49 | Generate the file, switch branch and check in there.
50 | 
51 |     python generate_upstream_list.py > upstream.html
52 |     git checkout gh-pages
53 |     mv upstream.html index.html
54 |     git commit -m "Update upstream overview." index.html
55 | 
56 | NOTE: It should normally only add new entries. Check if it does something else.
57 | If you did not clone the upstreames like mirrors, the master branch won't
58 | always be poining at the newest point in history. You will have to do a `git
59 | pull` or similar in your working directory.
60 | 
61 | 
62 | License
63 | -------
64 | 
65 | Copyright 2014-2018 Opera Software
66 | 
67 | Licensed under the Apache License, Version 2.0 (the "License");
68 | you may not use this file except in compliance with the License.
69 | You may obtain a copy of the License at
70 | 
71 |    http://www.apache.org/licenses/LICENSE-2.0
72 | 
73 | Unless required by applicable law or agreed to in writing, software
74 | distributed under the License is distributed on an "AS IS" BASIS,
75 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
76 | See the License for the specific language governing permissions and
77 | limitations under the License.
78 | 


--------------------------------------------------------------------------------
/config.py:
--------------------------------------------------------------------------------
 1 | AUTHOR_REGEX = r'(?P<name>[^@\s<]+)@opera\.com'
 2 | BODY_REGEX = r'((Contributed|Patch) (from|by)|Author:).*@opera\.com'
 3 | 
 4 | # Must be the actual '.git'-folder, not a checkout.
 5 | REPOS = [
 6 |     {
 7 |       'name': 'Chromium',
 8 |       'gitdir': './chromium.src.git',
 9 |       'viewvc_url': 'https://chromium.googlesource.com/chromium/src/+/{rev}',
10 |     },
11 |     {
12 |       'name': 'V8',
13 |       'gitdir': './v8.git',
14 |       'viewvc_url': 'https://github.com/v8/v8/commit/{rev}',
15 |       'author_in_commit_body': True,
16 |     },
17 |     {
18 |       'name': 'Skia',
19 |       'gitdir': './skia.git',
20 |       'viewvc_url': 'https://chromium.googlesource.com/skia.git/+/{rev}',
21 |       'author_in_commit_body': True,
22 |     },
23 |     {
24 |       'name': 'BoringSSL',
25 |       'gitdir': './boringssl.git',
26 |       'viewvc_url': 'https://boringssl.googlesource.com/boringssl.git/+/{rev}',
27 |     },
28 | ]
29 | 


--------------------------------------------------------------------------------
/generate_upstream_list.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python2
 2 | # coding: utf-8
 3 | 
 4 | import re
 5 | from jinja2 import Environment, FileSystemLoader
 6 | 
 7 | import config
 8 | from repository import Repo
 9 | 
10 | def get_commit_log(git_repo, viewvc_url, author_in_commit_body=False):
11 |     repo = Repo(git_repo)
12 |     log = repo.commits(config.BODY_REGEX, search_body=author_in_commit_body)
13 | 
14 |     for commit in log[:]:
15 |         author_match = re.search(config.AUTHOR_REGEX, commit.author)
16 |         if author_match:
17 |             commit.stripped_author = author_match.group('name')
18 |         else:
19 |             body_match = re.search(config.AUTHOR_REGEX, commit.body)
20 |             if not body_match:
21 |                 # Might be someone at Vewd calling themselves Opera by mistake
22 |                 # in the code review system. The regexps differ in that one
23 |                 # checks with <email> and the other do not so there is no
24 |                 # guarantee that both hits.
25 |                 continue
26 |             guessed_author = body_match.group('name')
27 |             commit.stripped_author = guessed_author
28 | 
29 |         commit.viewvc = viewvc_url.format(rev=commit.sha)
30 |     return log
31 | 
32 | project_logs = []
33 | for repo in config.REPOS:
34 |     commit_log = get_commit_log(repo['gitdir'], repo['viewvc_url'],
35 |                                 repo.get('author_in_commit_body', False))
36 |     commit_log.sort(key=lambda x: x.date, reverse=True) # FIXME: sort by time also
37 |     project_logs.append({'name': repo['name'], 'log': commit_log})
38 | 
39 | env = Environment(loader=FileSystemLoader('templates'))
40 | tmpl = env.get_template('upstreamed_commits.html')
41 | print tmpl.render(projects=project_logs).encode('utf-8')
42 | 


--------------------------------------------------------------------------------
/repository.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import subprocess
 3 | 
 4 | 
 5 | class InvalidStateError(Exception):
 6 |     pass
 7 | 
 8 | 
 9 | class Repo(object):
10 |     dirty = False
11 | 
12 |     def __init__(self, gitdir, git="git"):
13 |         self.git = git
14 |         self.gitdir = gitdir
15 |         if not os.path.isdir(self.gitdir):
16 |             raise InvalidStateError("Couldn't find gitdir: {0}".format(self.gitdir))
17 | 
18 |     def _git(self, args, check=True, input=None):
19 |         if input:
20 |             inp = subprocess.PIPE
21 |         else:
22 |             inp = None
23 |         cmd = [self.git, '--git-dir='+self.gitdir] + args
24 |         p = subprocess.Popen(cmd,
25 |             stdin=inp,
26 |             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
27 | 
28 |         if input:
29 |             stdout, stderr = p.communicate(input)
30 |         else:
31 |             stdout, stderr = p.communicate()
32 | 
33 |         return p.returncode, stdout, stderr
34 | 
35 |     def commits(self, search, search_body=False):
36 |         raw_commits = self.get_log(search, search_body)
37 |         commits = []
38 |         for c in raw_commits:
39 |             parts = c.split('\n', 5)
40 |             if len(parts) == 6:
41 |                 commit = Commit(*parts)
42 |                 commits.append(commit)
43 |         return commits
44 | 
45 |     def get_log(self, search, search_body):
46 |         commits = []
47 | 
48 |         # Before v8 had a commit queue, upstreamed changes were landed by
49 |         # project members, with a commit message that referred to the patch
50 |         # author.  AFAIK git log does not provide a way to filter by
51 |         # "--author=@opera.com OR --grep=@opera.com" so we need to run git log
52 |         # twice then concatenate and sort the results.
53 | 
54 |         if search_body:
55 |             # commits with a message body that mentions the author
56 |             cmd = [ 'log', '--format=%H%n%an%n%ad%n%ar%n%B',
57 |                     '--extended-regexp', '--date=short', '-z',
58 |                     '--grep={search}'.format(search=search), 'master']
59 |             _rv, bcommits, _stderr = self._git(cmd)
60 |             commits += bcommits.decode('utf-8').strip('\0').split('\0')
61 |             num_commits = str(len(commits))
62 | 
63 |         # commits with an @opera.com author
64 |         cmd = [ 'log', '--format=%H%n%ae%n%ad%n%ar%n%B',
65 |                 '--author=@opera.com', '--date=short', '-z', 'master' ]
66 |         _rv, acommits, _stderr = self._git(cmd)
67 |         commits += acommits.decode('utf-8').strip('\0').split('\0')
68 | 
69 |         return commits
70 | 
71 | 
72 | class Commit(object):
73 |     def __init__(self, sha, author, date, date_relative, subject, body):
74 |         self.sha = sha
75 |         self.author = author
76 |         self.date = date
77 |         self.date_relative = date_relative
78 |         self.subject = subject
79 |         self.body = body
80 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | Jinja2 >= 2.7, < 3.0
2 | 


--------------------------------------------------------------------------------
/templates/base.html:
--------------------------------------------------------------------------------
 1 | <!doctype html>
 2 | <meta charset=utf-8>
 3 | <title>Upstreamed commits &ndash; Opera</title>
 4 | <style>
 5 | body { font-family: sans-serif; }
 6 | table { border-collapse: collapse; border-spacing: 0; width: 100%; }
 7 | tbody tr:hover { background: #f5f5f5; }
 8 | td { vertical-align: top; }
 9 | td, th { padding: 7px; }
10 | th { text-align: left; }
11 | tr { border-bottom: 1px dashed #aaa; }
12 | a { color: #900; transition: 0.4s color; }
13 | a:visited { color: #755; }
14 | a:hover { color: #f00; }
15 | .body { white-space: pre-wrap; }
16 | summary { font-weight: bold; }
17 | main { box-shadow: 0 0 80px -20px rgba(0,0,0,0.5); display: block; margin: 20px auto; max-width: 960px; padding: 20px; }
18 | h1 { margin-top: 0; text-align: center; }
19 | .hint { color: #666; font-size: 0.8em; }
20 | .col-author { width: 160px; }
21 | .col-when { width: 130px; }
22 | .message { cursor: pointer; }
23 | </style>
24 | 
25 | {% block content %}
26 |   <p>Nothing here
27 | {% endblock %}
28 | 


--------------------------------------------------------------------------------
/templates/upstreamed_commits.html:
--------------------------------------------------------------------------------
 1 | {% extends "base.html" %}
 2 | 
 3 | {% block content %}
 4 | <main>
 5 |   <h1>Opera Software upstreamed commits</h1>
 6 |   <p>Upstreamed commits in
 7 |   {% for project in projects %}
 8 |     <a href="#{{ project.name|lower|e }}">{{ project.name|e }}</a>: {{ project.log|length }}{% if project != projects[-1] %},{% else %}.{% endif %}
 9 |   {% endfor %}
10 |   <p class=hint>Click message to expand
11 |   {% for project in projects %}
12 |     <section id="{{ project.name|lower|e }}" class="{{ project.name|lower|e }}">
13 |       <h2>{{ project.name|e }}</h2>
14 |       <table>
15 |       <colgroup class=col-author>
16 |       <colgroup class=col-message>
17 |       <colgroup class=col-when>
18 |       <thead><tr><th>Author <th>Message <th>When
19 |       <tbody>
20 |       {% for c in project.log %}
21 |         <tr>
22 |           <td>{{ c.stripped_author|e }}
23 |           <td class=message>
24 |             <details>
25 |               <summary>{{ c.subject|e }}</summary>
26 |               <pre class=body>{{ c.body|e|urlize }}</pre>
27 |             </details>
28 |           <td><a href="{{ c.viewvc|e }}"><time datetime="{{ c.date|e }}">{{ c.date|e }}</time></a>
29 |       {%- endfor %}
30 |       </table>
31 |     </section>
32 |   {% endfor %}
33 | </main>
34 | <script>
35 |   /* Fallback for Presto */
36 |   if (!('open' in document.createElement('details'))) {
37 |     var messages = document.querySelectorAll('.message');
38 |     for (var i=0; i<messages.length; i++)
39 |     {
40 |       messages[i].querySelector('.body').style.display = 'none';
41 |       /* Yes, creating lots of event listeners ... No delegation here! :P */
42 |       messages[i].addEventListener('click', function()
43 |       {
44 |         var body = this.querySelector('.body');
45 |         if (body.style.display == 'none')
46 |             body.style.display = 'block';
47 |         else
48 |             body.style.display = 'none';
49 |       });
50 |     }
51 |   }
52 | </script>
53 | {% endblock %}
54 | 


--------------------------------------------------------------------------------
/update_overview.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | #set -x  # Display commands as they are run
 4 | 
 5 | for p in chromium.src v8; do
 6 |     git --git-dir=$p.git fetch -q
 7 | done
 8 | 
 9 | python generate_upstream_list.py > upstream.html || {
10 |     echo "Python script errored out. Quitting." 1>&2; exit 1; }
11 | 
12 | # Check that we didn't just delete the file
13 | if [[ $(du upstream.html|cut -f 1) -lt 128 ]]; then
14 |     echo
15 |     echo "Upstream file less than 128 chars. Not committing." 1>&2
16 |     exit 1
17 | fi
18 | 
19 | git checkout -q gh-pages || {
20 |     echo "Trouble checking out gh-pages. Quitting." 1>&2; exit 1; }
21 | mv upstream.html index.html
22 | 
23 | git add index.html
24 | git diff-index --quiet HEAD --
25 | ret=$?
26 | if [ "$ret" -eq 1 ]
27 | then
28 |     git commit -m "Update upstream overview." index.html > /dev/null
29 |     if [[ $1 == 'push' ]]; then
30 |         git push -q
31 |         git checkout -q master
32 |         ret=0
33 |     else
34 |         echo
35 |         echo "Check it with 'git show', then push it:"
36 |         echo
37 |         echo "git push && git checkout master"
38 |         exit 0
39 |     fi
40 | else
41 |     echo "No changes."
42 |     git checkout -q master
43 | fi
44 | 
45 | exit $ret
46 | 


--------------------------------------------------------------------------------