').attr('id', 'intro-top').text('Guess')
63 | .appendTo('body')[0];
64 | var bottom = $('
').attr('id', 'intro-bottom').text('Hub')
65 | .appendTo('body')[0];
66 |
67 | $(top, bottom).show();
68 | animate.in(top, 'bounce-down');
69 | setTimeout(animate.in.bind(animate, bottom, 'bounce-up', function () {
70 | animate.out(top, 'bounce-up', function() { $(top).remove() });
71 | setTimeout(animate.out.bind(animate, bottom, 'bounce-down', function () {
72 | $(bottom).remove();
73 | ready();
74 | }.bind(this)), 100);
75 | }.bind(this)), 150);
76 | };
77 |
78 | Tutorial.prototype._showRepoSelectStep = function() {
79 | this._click_overlay.show();
80 | this.game.timer.stop();
81 |
82 | this._unshade(this.game.$repos, this.game.$commitDisplay);
83 |
84 | var tip = new Tip('Select which repository this commit comes from.');
85 | tip.position('south');
86 | tip.show(this.game.$repos[0]);
87 | animate.in(tip.el, 'fade-up');
88 |
89 | this.$_click_overlay.click(function () {
90 | animate.out(tip.el, 'fade-down', false, function () {
91 | $(tip.el).remove();
92 | this._showTimerStep();
93 | }.bind(this));
94 | }.bind(this));
95 |
96 | audio.play('tutorial-tip');
97 | };
98 |
99 | Tutorial.prototype._showTimerStep = function () {
100 | this._unshade(this.game.$timer);
101 |
102 | var tip = new Tip('Quick! Choose before the time runs out!');
103 | tip.position('west');
104 | tip.show(this.game.$timer[0]);
105 | animate.in(tip.el, 'fade-left');
106 |
107 | this.$_click_overlay.click(function () {
108 | animate.out(tip.el, 'fade-right', function () {
109 | $(tip.el).remove();
110 | this._unshadeAll();
111 | this._overlay.hide();
112 | this._click_overlay.hide();
113 | this.game.startTime = Date.now();
114 | this.game.timer.start();
115 | this.game.user.seen_tutorial(true);
116 | }.bind(this));
117 | }.bind(this));
118 |
119 | audio.play('tutorial-tip');
120 | };
121 |
122 | Tutorial.prototype._unshadeAll = function () {
123 | this._unshaded.forEach(function ($el) {
124 | $el.removeClass('unshade');
125 | });
126 | };
127 |
128 | Tutorial.prototype.showPowerHint = function() {
129 | this._overlay.show();
130 | this._click_overlay.show();
131 | this._unshade(this.game.$powerList);
132 | this._unshade(this.game.$scoreCard);
133 |
134 | var tip = new Tip('Purchase power-ups with score gained during levels.');
135 | tip.position('south');
136 | tip.show(this.game.$powerList[0]);
137 | animate.in(tip.el, 'fade-down');
138 |
139 | this.$_click_overlay.click(function () {
140 | animate.out(tip.el, 'fade-down', function () {
141 | $(tip.el).remove();
142 | this._unshadeAll();
143 | this._overlay.hide();
144 | this._click_overlay.hide();
145 | this.game.user.seen_power_hint(true);
146 | }.bind(this));
147 | }.bind(this));
148 |
149 | audio.play('tutorial-tip');
150 | };
151 |
--------------------------------------------------------------------------------
/app/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "app",
3 | "version": "0.0.0",
4 | "description": "Install [component](https://github.com/component)",
5 | "main": "index.js",
6 | "scripts": {
7 | "test": "echo \"Error: no test specified\" && exit 1"
8 | },
9 | "author": "",
10 | "license": "BSD-2-Clause",
11 | "devDependencies": {
12 | "jake": "~0.7.3",
13 | "glob": "~3.2.6",
14 | "component-builder": "~0.10.0",
15 | "component-minify": "~1.1.1",
16 | "mkdirp": "~0.3.5",
17 | "rimraf": "~2.2.2"
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/backend/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | model.json
3 | commit_index.pickle
4 |
--------------------------------------------------------------------------------
/backend/app.wsgi:
--------------------------------------------------------------------------------
1 | import os, sys
2 |
3 | # Override import path.
4 | ROOT = os.path.dirname(os.path.realpath(__file__))
5 | sys.path.insert(0, ROOT)
6 |
7 | from server import APP as application
8 |
--------------------------------------------------------------------------------
/backend/config.py:
--------------------------------------------------------------------------------
1 | # GitHub auth.
2 | GITHUB_CLIENT_ID = '6e91d029d2eeca74bf24'
3 | GITHUB_CLIENT_SECRET = ''
4 |
5 | # The user agent to use for GitHub requests.
6 | CLIENT_USER_AGENT = 'max99x/guesshub'
7 |
8 | # DB auth.
9 | DB_HOST = 'localhost'
10 | DB_USER = 'root'
11 | DB_PASSWORD = 'root'
12 | DB_NAME = 'commit_game'
13 |
--------------------------------------------------------------------------------
/backend/crawl.py:
--------------------------------------------------------------------------------
1 | import config
2 | import github
3 | import model
4 | import MySQLdb as mysql
5 |
6 | # How many pages of commits to look at (100 per page).
7 | PAGES = 1
8 |
9 | # The maximum number of rows to insert at a time.
10 | BATCH_SIZE = 10
11 |
12 |
13 | def encode(row):
14 | """UTF8-encodes all strings in an iterable, returning a tuple."""
15 | return tuple([i.encode('utf8') if isinstance(i, unicode)
16 | else i for i in row])
17 |
18 |
19 | def insert_repository(cursor, repo):
20 | """Inserts a Repository object into the database using a given cursor."""
21 | sql = ('REPLACE INTO repository VALUES ' +
22 | '(DEFAULT, %s, %s, %s, %s, %s, %s, %s, %s, %s)')
23 | row = (repo.id,
24 | repo.name,
25 | repo.author,
26 | repo.author_avatar_url,
27 | repo.description.encode('utf8'),
28 | repo.is_private,
29 | repo.is_fork,
30 | repo.watcher_count,
31 | repo.star_count)
32 | cursor.execute(sql, encode(row))
33 |
34 |
35 | def insert_commit(cursor, commits):
36 | """Inserts a Commit object into the database using a given cursor."""
37 | sql = ('REPLACE INTO commit VALUES'
38 | '(DEFAULT, %s, %s, %s, %s, %s, %s,'
39 | ' %s, %s, %s, %s, %s, %s, %s, %s)')
40 | rows = []
41 |
42 | def flush():
43 | try:
44 | cursor.executemany(sql, rows)
45 | except Exception as e:
46 | print commits[0].repository, [commit.sha for commit in commits]
47 | raise e
48 |
49 | for commit in commits:
50 | rows.append(encode((commit.sha,
51 | commit.patch_number,
52 | commit.message,
53 | commit.author_login,
54 | commit.author_name,
55 | commit.author_avatar_url,
56 | commit.repository,
57 | commit.filename,
58 | commit.additions,
59 | commit.deletions,
60 | commit.old_start_line,
61 | commit.new_start_line,
62 | commit.block_name,
63 | u'\n'.join(commit.diff_lines))))
64 | if len(rows) >= BATCH_SIZE:
65 | flush()
66 | rows = []
67 | if len(rows) > 0:
68 | flush()
69 |
70 |
71 | def commit_exists(cursor, sha):
72 | """Returns whether at least one commit with a given exists in the database."""
73 | cursor.execute('SELECT COUNT(*) FROM commit WHERE sha = %s', sha)
74 | return cursor.fetchone()[0] > 0
75 |
76 |
77 | def crawl():
78 | """Crawls the top repositories, collecting commits."""
79 | db = mysql.connect(
80 | host=config.DB_HOST,
81 | user=config.DB_USER,
82 | passwd=config.DB_PASSWORD,
83 | db=config.DB_NAME)
84 | cursor = db.cursor()
85 | gh = github.GitHub(
86 | config.GITHUB_CLIENT_ID,
87 | config.GITHUB_CLIENT_SECRET,
88 | config.CLIENT_USER_AGENT)
89 |
90 | try:
91 | for repo in gh.GetTopRepositories():
92 | print 'Starting repo ', repo.name
93 | insert_repository(cursor, repo)
94 | for commit_sha in gh.GetCommitsList(repo.name, PAGES):
95 | if commit_exists(cursor, commit_sha):
96 | print ' Skipping SHA ' + commit_sha
97 | else:
98 | # TODO(max99x): Log and skip errors.
99 | print ' Getting SHA ' + commit_sha, '->',
100 | commits = list(gh.GetCommits(repo.name, commit_sha))
101 | print '%d commit(s)' % len(commits)
102 | insert_commit(cursor, commits)
103 | yield
104 | finally:
105 | cursor.close()
106 | db.close()
107 |
108 |
109 | def main():
110 | """Crawls the top repositories, collecting commits."""
111 | crawler = crawl()
112 | while True:
113 | crawler.next()
114 |
115 |
116 | if __name__ == '__main__':
117 | main()
118 |
--------------------------------------------------------------------------------
/backend/equalize.py:
--------------------------------------------------------------------------------
1 | import config
2 | import MySQLdb as mysql
3 |
4 | MAX_ROWS = 20000
5 |
6 | def main():
7 | db = mysql.connect(
8 | host=config.DB_HOST,
9 | user=config.DB_USER,
10 | passwd=config.DB_PASSWORD,
11 | db=config.DB_NAME)
12 | cursor = db.cursor()
13 | cursor.execute('SELECT repository, COUNT(*) as count '
14 | 'FROM commit '
15 | 'WHERE grade >= 0 '
16 | 'GROUP BY repository '
17 | 'HAVING count >= %s '
18 | 'ORDER BY count DESC',
19 | (MAX_ROWS,))
20 |
21 | for repo, count in cursor.fetchall():
22 | cursor.execute('UPDATE commit '
23 | 'SET grade=-grade '
24 | 'WHERE grade >= 0 AND repository = %s '
25 | 'LIMIT %s',
26 | (repo, count - MAX_ROWS))
27 |
28 |
29 | if __name__ == '__main__':
30 | main()
31 |
--------------------------------------------------------------------------------
/backend/github.py:
--------------------------------------------------------------------------------
1 | import itertools
2 | import re
3 | import requests
4 | import time
5 |
6 | import config
7 | import model
8 |
9 | # The maximum number of entries that can be requested per page.
10 | MAX_PAGE_SIZE = 100
11 |
12 | # Implementation constants.
13 | NEXT_PAGE_REGEX = re.compile(r'<([^<>]+)>; rel="next"')
14 | TOTAL_PAGES_REGEX = re.compile(r'<[^<>]+=(\d+)>; rel="last"')
15 |
16 |
17 | class GitHub(object):
18 | def __init__(self, client_id, client_secret, user_agent):
19 | self.client_id = client_id
20 | self.client_secret = client_secret
21 | self.user_agent = user_agent
22 |
23 | def Fetch(self, url, **params):
24 | """
25 | Performs an authenticated GitHub request, retrying on server errors and
26 | sleeping if the rate limit is exceeded.
27 |
28 | Args:
29 | url: The URL to request. An absolute path or full URL. Can include a
30 | query string.
31 | prams: The query parameters to send.
32 |
33 | Returns:
34 | A requests.Response object.
35 | """
36 | full_params = params.copy()
37 | full_params['client_id'] = self.client_id
38 | full_params['client_secret'] = self.client_secret
39 | request_headers = {'User-Agent': self.user_agent}
40 | if not url.startswith('https://api.github.com/'):
41 | url = 'https://api.github.com/' + url
42 | result = requests.get(url, params=full_params, headers=request_headers)
43 |
44 | # Explode on client errors and retry on server error.
45 | status = result.status_code
46 | if status >= 500:
47 | return self.Fetch(url, **params)
48 | elif status == 403:
49 | # Sleep and retry when we hit the rate limit.
50 | print 'Hit rate limit.'
51 | response_headers = result.headers
52 | reset_time = int(response_headers['X-RateLimit-Reset'])
53 | delay_time = int(reset_time - time.time()) + 1
54 | if delay_time > 0: # Time sync issues may result in negative delay.
55 | print 'Sleeping for', delay_time, 'seconds...'
56 | time.sleep(delay_time)
57 | return self.Fetch(url, **params)
58 | elif status >= 400 and status < 500:
59 | raise RuntimeError('Client error, HTTP %s.\n'
60 | 'Path: %s\nParams: %s\nResponse: %s' %
61 | (status, url, params, result.json()))
62 |
63 | # All's well that ends well.
64 | return result
65 |
66 | def List(self, url, pages=None, **params):
67 | """Fetches and yields each item from a GH listing."""
68 | if pages is None:
69 | page_range = itertools.repeat(1)
70 | else:
71 | page_range = range(pages)
72 |
73 | for page in page_range:
74 | # Fetch current page.
75 | response = self.Fetch(url, per_page=MAX_PAGE_SIZE, **params)
76 | json = response.json()
77 | if isinstance(json, dict):
78 | assert 'items' in json, json
79 | json = json['items']
80 | for item_json in json:
81 | yield item_json
82 |
83 | # Get next page.
84 | if 'Link' in response.headers:
85 | next_page_links = NEXT_PAGE_REGEX.findall(response.headers['Link'])
86 | if next_page_links:
87 | assert len(next_page_links) == 1, response.headers
88 | url = next_page_links[0]
89 |
90 | def GetCommitsList(self, repo, pages_count):
91 | """Yields the SHAs of recent commits given a repo name."""
92 | for commit_json in self.List('repos/%s/commits' % repo, pages_count):
93 | yield commit_json['sha']
94 |
95 | def GetCommits(self, repo, sha):
96 | """Yields Commit objects for each patch in a given commit.
97 |
98 | The commit is specified by a repo name and a SHA hash.
99 | """
100 | commit_json = self.Fetch('repos/%s/commits/%s' % (repo, sha)).json()
101 | for commit in model.Commit.split_from_json(commit_json):
102 | yield commit
103 |
104 | def GetStarCount(self, repo):
105 | """Returns the number stars for a repository given its name."""
106 | response = self.Fetch('repos/%s/stargazers' % repo)
107 | first_page_count = len(response.json())
108 |
109 | if 'Link' in response.headers:
110 | # Multiple pages. Estimate.
111 | matches = TOTAL_PAGES_REGEX.findall(response.headers['Link'])
112 | assert matches and len(matches) == 1, response.headers
113 | total_pages = int(matches[0])
114 | return (total_pages - 1) * first_page_count + first_page_count / 2
115 | else:
116 | # One page. Exact number.
117 | return first_page_count
118 |
119 | def GetUserRespositories(self, username):
120 | """Yields the repositories of a given user as Repository objects."""
121 | for repository_json in self.List('users/%s/repos' % username):
122 | yield model.Repository(repository_json,
123 | self.GetStarCount(repository_json['full_name']))
124 |
125 | def GetTopUsers(self):
126 | """Yields the usernames of the top 10,000 users."""
127 | search_url = 'search/users?q=followers%3A%3E%3D0&sort=followers'
128 | for user_json in self.List(search_url):
129 | yield user_json['login']
130 |
131 | def GetTopRepositories(self):
132 | """Yields the top 10,000 repositories as Repository objects."""
133 | search_url = 'search/repositories?q=stars%3A>%3D0&sort=stars'
134 | for repository_json in self.List(search_url):
135 | yield model.Repository(repository_json,
136 | self.GetStarCount(repository_json['full_name']))
137 |
--------------------------------------------------------------------------------
/backend/grade.py:
--------------------------------------------------------------------------------
1 | import config
2 | import collections
3 | import math
4 | import json
5 | import re
6 | import sys
7 | import MySQLdb as mysql
8 | import MySQLdb.cursors as mysql_cursors
9 | from os import path
10 |
11 | COUNT_SQL = 'SELECT MAX(order_id) FROM commit'
12 | BATCH_SQL = 'SELECT * FROM commit WHERE %s <= order_id AND order_id < %s'
13 | UPDATE_SQL = 'UPDATE commit SET grade = %s WHERE order_id = %s'
14 | BATCH_SIZE = 1000
15 | WORD_REGEX = re.compile(r'\w+')
16 | SNAKE_REGEX = re.compile(r'[a-zA-Z]{3,}')
17 | CAMEL_REGEX = re.compile(r'(?:[A-Z]|\b)[a-zA-Z]{2,}')
18 | TOKEN_REGEX = re.compile(r'[\x21-\x2f\x3a-\x40\x5B-\x60]+|'
19 | r'(?:[A-Z]|\b)[a-zA-Z]{2,}|'
20 | r'[a-z]{3,}')
21 |
22 | MIN_LINE_COUNT = 3
23 | MAX_LINE_COUNT = 25
24 | MIN_DIFF_COUNT = 2
25 | MAX_LINE_LENGTH = 80
26 | MIN_SPACE_RATIO = 0.01
27 | MIN_WORDS_COUNT = 10
28 | MIN_CONTEXT_LINE_COUNT = 1
29 | MAX_KEYWORDS_COUNT = 12
30 |
31 | SMALL_LINE_COUNT = 5
32 | LARGE_LINE_COUNT = 20
33 |
34 | ALLOWED_TYPES = {
35 | 'c', 'h',
36 | 'cc', 'cpp', 'cxx', 'hh', 'hpp', 'hxx',
37 | 'coffee', 'coffeescript', 'litcoffee',
38 | 'cs'
39 | 'd',
40 | 'go',
41 | 'hs', 'lhs',
42 | 'java',
43 | 'scala',
44 | 'js', 'ts',
45 | 'lua',
46 | 'php', 'phtml',
47 | 'py', 'pyw',
48 | 'r',
49 | 'rb',
50 | 'scm',
51 | 'sh', 'bash', 'zsh',
52 | 'sql',
53 | 'groovy', 'gvy', 'gy',
54 | 'gsh', 'vsh', 'fsh', 'shader',
55 | }
56 | STOP_WORDS = {
57 | "a", "about", "above", "after", "again", "against", "all", "am", "an", "and",
58 | "any", "are", "aren\'t", "as", "at", "be", "because", "been", "before",
59 | "being", "below", "between", "both", "but", "by", "can\'t", "cannot",
60 | "could", "couldn\'t", "did", "didn\'t", "do", "does", "doesn\'t", "doing",
61 | "don\'t", "down", "during", "each", "few", "for", "from", "further", "had",
62 | "hadn\'t", "has", "hasn\'t", "have", "haven\'t", "having", "he", "he\'d",
63 | "he\'ll", "he\'s", "her", "here", "here\'s", "hers", "herself", "him",
64 | "himself", "his", "how", "how\'s", "i", "i\'d", "i\'ll", "i\'m", "i\'ve",
65 | "if", "in", "into", "is", "isn\'t", "it", "it\'s", "its", "itself", "let\'s",
66 | "me", "more", "most", "mustn\'t", "my", "myself", "no", "nor", "not", "of",
67 | "off", "on", "once", "only", "or", "other", "ought", "our", "ours ",
68 | "ourselves", "out", "over", "own", "same", "shan\'t", "she", "she\'d",
69 | "she\'ll", "she\'s", "should", "shouldn\'t", "so", "some", "such", "than",
70 | "that", "that\'s", "the", "their", "theirs", "them", "themselves", "then",
71 | "there", "there\'s", "these", "they", "they\'d", "they\'ll", "they\'re",
72 | "they\'ve", "this", "those", "through", "to", "too", "under", "until", "up",
73 | "very", "was", "wasn\'t", "we", "we\'d", "we\'ll", "we\'re", "we\'ve", "were",
74 | "weren\'t", "what", "what\'s", "when", "when\'s", "where", "where\'s",
75 | "which", "while", "who", "who\'s", "whom", "why", "why\'s", "with", "won\'t",
76 | "would", "wouldn\'t", "you", "you\'d", "you\'ll", "you\'re", "you\'ve",
77 | "your", "yours", "yourself", "yourselves"
78 | }
79 |
80 | DB = mysql.connect(
81 | host=config.DB_HOST,
82 | user=config.DB_USER,
83 | passwd=config.DB_PASSWORD,
84 | db=config.DB_NAME,
85 | cursorclass=mysql_cursors.DictCursor)
86 |
87 |
88 | class Model(object):
89 | """A naive Bayesian text classifier."""
90 |
91 | def __init__(self):
92 | self.samples = collections.defaultdict(
93 | lambda: collections.defaultdict(lambda: 1))
94 | self.label_counts = collections.defaultdict(lambda: 0)
95 |
96 | def probability(self, features, label):
97 | """Returns the probability of a feature set being labeled with a given label.
98 |
99 | The features a list of strings. The returned value is between 0 and 1.
100 | """
101 | total_labels = sum(self.label_counts.values())
102 | log_prob = self._scaled_log_probability(features, label, total_labels)
103 | total_log_prob = log_prob
104 | for other_label in self.label_counts:
105 | if other_label != label:
106 | other_log_prob = self._scaled_log_probability(
107 | features, other_label, total_labels)
108 | total_log_prob = self._log_add(total_log_prob, other_log_prob)
109 | return math.exp(log_prob - total_log_prob)
110 |
111 | def classify(self, features):
112 | """Returns the most likely label for a given feature set (string list)."""
113 | total_labels = sum(self.label_counts.values())
114 | max_prob = -1
115 | max_val = None
116 | for label in self.label_counts:
117 | prob = self._scaled_log_probability(features, label, total_labels)
118 | if prob > max_prob:
119 | max_prob = prob
120 | max_val = label
121 | return max_val
122 |
123 | def dump(self, filename):
124 | """Saves the model to a file in JSON format."""
125 | f = open(filename, 'w')
126 | json.dump([m.samples, m.label_counts], f)
127 | f.close()
128 |
129 | @staticmethod
130 | def load(filename):
131 | """Loads a model previously saved by Model.dump()."""
132 | data = json.load(open(filename))
133 | m = Model()
134 | m.samples = data[0]
135 | m.label_counts = data[1]
136 | return m
137 |
138 | @staticmethod
139 | def build():
140 | """Generates a classifier based on all the commits in the DB."""
141 | m = Model()
142 | for batch in fetch_commits():
143 | for commit in batch:
144 | tokens = tokenize(
145 | commit['diff_lines'] + ' ' + (commit['block_name'] or ''))
146 | tokens = [i.lower() for i in tokens]
147 | counts = collections.defaultdict(int)
148 | for token in tokens:
149 | if token not in STOP_WORDS:
150 | counts[token] += 1
151 | top_tokens = sorted(counts.iteritems(), key=lambda x:-x[1])[:20]
152 | features = [i[0] for i in top_tokens]
153 | features.append(path.splitext(commit['filename'])[1])
154 | m._add_sample(features, commit['repository'])
155 |
156 | return m
157 |
158 | def _add_sample(self, features, label):
159 | """Adds a feature set (string list) and its label to the samples."""
160 | for feature in features:
161 | self.samples[feature][label] += 1
162 | self.label_counts[label] += 1
163 |
164 | def _scaled_log_probability(self, features, label, total_labels):
165 | """Returns the likelihood of features to be labeled with the given label.
166 |
167 | The returned value is scaled by the probability of the features, so
168 | it is only meaningful when compared to values returned from calls to
169 | this method that pass the same features (but a different label).
170 |
171 | The value is in log space because it may exceed the range of a double
172 | precision floating point number.
173 | """
174 | label_count = float(self.label_counts[label])
175 | log_class_probability = math.log(label_count / total_labels)
176 | log_correlation_probability = 0
177 | default_log_prob = math.log(1 / label_count)
178 | for feature in features:
179 | log_prob = default_log_prob
180 | feature_counts = self.samples.get(feature)
181 | if feature_counts is not None:
182 | count = feature_counts.get(label, 1)
183 | if count != 1:
184 | log_prob = count / label_count
185 | log_correlation_probability += log_prob
186 | return log_class_probability + log_correlation_probability
187 |
188 | @staticmethod
189 | def _log_add(log_x, log_y):
190 | """Returns log(x + y) given log(x) and log(y)."""
191 | if log_y > log_x:
192 | log_x, log_y = log_y, log_x
193 | if log_x == float('-inf'):
194 | return log_x
195 | neg_diff = log_y - log_x
196 | if neg_diff < -20:
197 | return log_x
198 | return log_x + math.log(1.0 + math.exp(neg_diff))
199 |
200 |
201 | def compute_grade(commit, classifier):
202 | """Computes a grade for a given commit, either -1 or between 0 and 100.
203 |
204 | -1 signifies invalid commits. Otherwise harder commits have higher grades.
205 | """
206 |
207 | # Skip non code diffs.
208 | (_, ext) = path.splitext(commit['filename'])
209 | if ext[1:] not in ALLOWED_TYPES:
210 | return -1
211 |
212 | lines = commit['diff_lines'].split('\n')
213 |
214 | # Skip diffs with too few lines.
215 | if len(lines) < MIN_LINE_COUNT:
216 | return -1
217 |
218 | # Skip diffs with too many lines.
219 | if len(lines) > MAX_LINE_COUNT:
220 | return -1
221 |
222 | # Skip diffs with long lines.
223 | if any([len(l) > MAX_LINE_LENGTH for l in lines]):
224 | return -1
225 |
226 | # Skip diffs with too little spacing.
227 | space_count = commit['diff_lines'].replace('\t', ' ').count(' ')
228 | if space_count / float(len(commit['diff_lines'])) < MIN_SPACE_RATIO:
229 | return -1
230 |
231 | # Skip diffs with too few diff lines.
232 | diff_count = len([i for i in lines if i[:1] in ('+', '-')])
233 | if diff_count < MIN_DIFF_COUNT:
234 | return -1
235 |
236 | # Skip diffs with too few context lines.
237 | context_count = len([i for i in lines if i[:1] not in ('+', '-', '\\')])
238 | if context_count < MIN_CONTEXT_LINE_COUNT:
239 | return -1
240 |
241 | # Skip diffs with too few words.
242 | words = extract_words(commit['diff_lines'], commit['block_name'])
243 | if len(words) < MIN_WORDS_COUNT:
244 | return -1
245 |
246 | # How many keywords do we have that reference the repo name?
247 | repo_keywords = set(extract_words(commit['repository']))
248 | keywords_count = len([i for i in words if i in repo_keywords])
249 |
250 | # Skip diffs with too many keywords.
251 | if keywords_count > MAX_KEYWORDS_COUNT:
252 | return -1
253 |
254 | # How many keywords do we have that reference the metadata?
255 | metadata = (commit['filename'], commit['author_login'], commit['author_name'])
256 | metadata_keywords = set(extract_words(*metadata))
257 | metadata_keyword_matches = [i for i in words if i in metadata_keywords]
258 | metadata_keywords_count = len(metadata_keyword_matches)
259 |
260 | # Calculate the grade, starting from perfectly hard.
261 | if keywords_count > 0:
262 | # Has repo keywords. Difficulty <= 50.
263 | grade = 42 - min(keywords_count, 6) * 7
264 | grade += min(len(lines) - MIN_LINE_COUNT, 8)
265 | else:
266 | # No repo keywords. Difficulty > 50.
267 | grade = 100
268 |
269 | # Maybe we have metadata keywords?
270 | grade -= min(metadata_keywords_count, 5) * 2
271 |
272 | # Classify.
273 | tokens = tokenize(commit['diff_lines'] + ' ' + (commit['block_name'] or ''))
274 | probability = classifier.probability(tokens, commit['repository'])
275 | if probability < 0.001 and probability != 0:
276 | # Switch to log scaling here.
277 | grade -= min(-math.log(probability) / 15, 40)
278 | else:
279 | grade -= (1 - probability) * 40
280 |
281 | return int(round(grade))
282 |
283 |
284 | def tokenize(s):
285 | """Extracts tokesn from a string."""
286 | return TOKEN_REGEX.findall(s)
287 |
288 |
289 | def extract_words(*strings):
290 | """Extracts words from all strings in a list."""
291 | text = ' '.join([i or '' for i in strings])
292 | words = WORD_REGEX.findall(text)
293 |
294 | tokens = []
295 | for word in words:
296 | word_lower = word.lower()
297 | tokens.append(word_lower)
298 | # Extract snake- and camel-case words.
299 | snakes = SNAKE_REGEX.findall(word)
300 | camels = CAMEL_REGEX.findall(word)
301 | for subword in set(snakes + camels):
302 | subword_lower = subword.lower()
303 | if subword_lower != word_lower:
304 | tokens.append(subword_lower)
305 |
306 | return tokens
307 |
308 |
309 | def fetch_commits():
310 | """Yields batches of commits from the DB."""
311 | count_cursor = DB.cursor()
312 | count_cursor.execute(COUNT_SQL)
313 | count = count_cursor.fetchone()['MAX(order_id)']
314 |
315 | read_cursor = DB.cursor()
316 | for start in range(0, count + BATCH_SIZE, BATCH_SIZE):
317 | print 'Starting at', start
318 | read_cursor.execute(BATCH_SQL, (start, start + BATCH_SIZE))
319 | yield read_cursor.fetchall()
320 |
321 |
322 | def update_grades(classifier):
323 | """Updates all commits with newly-computed grades."""
324 | write_cursor = DB.cursor()
325 | for batch in fetch_commits():
326 | ids = [c['order_id'] for c in batch]
327 | grades = map(lambda c: compute_grade(c, classifier), batch)
328 | write_cursor.executemany(UPDATE_SQL, zip(grades, ids))
329 | pos_grades = [i for i in grades if i >= 0]
330 | print ' invalid=%d, min=%s, max=%s, mean=%s, median=%s' % (
331 | len(grades) - len(pos_grades),
332 | min(pos_grades) if pos_grades else None,
333 | max(pos_grades) if pos_grades else None,
334 | sum(pos_grades) / float(len(pos_grades)) if pos_grades else None,
335 | sorted(pos_grades)[len(pos_grades) / 2] if pos_grades else None)
336 |
337 |
338 | def show_grade_histogram():
339 | """Prints a histogram of positive grades."""
340 | read_cursor = DB.cursor()
341 | read_cursor.execute('SELECT grade, COUNT(grade) FROM commit '
342 | 'WHERE grade >= 0 GROUP BY grade')
343 | histogram = dict([(i['grade'], i['COUNT(grade)'])
344 | for i in read_cursor.fetchall()])
345 | print 'Histogram:'
346 | for n in range(0, max(histogram.keys()) + 1):
347 | print '%-2d: %d' % (n, histogram.get(n, 0))
348 | print 'Total:', sum(histogram.values())
349 |
350 |
351 | if __name__ == '__main__':
352 | mode = sys.argv[1]
353 | if mode == 'build':
354 | m = Model.build()
355 | m.dump('model.json')
356 | elif mode == 'grade':
357 | m = Model.load('model.json')
358 | update_grades(m)
359 | show_grade_histogram()
360 | else:
361 | print 'Invalid mode. Call "grade.py build" or "grade.py grade".'
362 |
--------------------------------------------------------------------------------
/backend/model.py:
--------------------------------------------------------------------------------
1 | import re
2 |
3 | # A regex to pull the repository name from a commit URL.
4 | REPOSITORY_REGEX = re.compile(
5 | r'^https://api.github.com/repos/([^/]+/[^/]+)/commits/[a-fA-F0-9]+$')
6 |
7 | # A regex to pull the repository name from a commit URL.
8 | PATCH_HEADER_REGEX = re.compile(
9 | r'^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@(?: (\S.*))?$')
10 |
11 |
12 | class Commit(object):
13 | """A single patch hunk linked to a GitHub commit."""
14 |
15 | MAX_LINES_PER_PATCH = 25
16 |
17 | def __init__(self, commit_json,
18 | patch_number, patch_filename,
19 | patch_start_old, patch_start_new,
20 | patch_header, patch_lines):
21 | self.sha = commit_json['sha']
22 | self.patch_number = patch_number
23 | self.message = commit_json['commit']['message']
24 | if commit_json['author'] is not None and 'login' in commit_json['author']:
25 | self.author_login = commit_json['author']['login']
26 | self.author_avatar_url = commit_json['author']['avatar_url']
27 | else:
28 | self.author_login = None
29 | self.author_avatar_url = None
30 | self.author_name = commit_json['commit']['author']['name']
31 |
32 | repository_matches = REPOSITORY_REGEX.findall(commit_json['url'])
33 | assert repository_matches and len(repository_matches) == 1, commit_json
34 | self.repository = repository_matches[0]
35 |
36 | self.filename = patch_filename
37 | self.additions = len([i for i in patch_lines if i.startswith('+')])
38 | self.deletions = len([i for i in patch_lines if i.startswith('-')])
39 | self.old_start_line = patch_start_old
40 | self.new_start_line = patch_start_new
41 | self.block_name = patch_header or None
42 | self.diff_lines = [i.replace('\t', ' ') for i in patch_lines]
43 |
44 | @staticmethod
45 | def split_from_json(json):
46 | """Given a GitHub commit JSON, yield Commit objects for all patch hunks."""
47 | assert json
48 | patch_number = 0
49 | if 'files' in json:
50 | for patch_json in json['files']:
51 | if 'patch' in patch_json:
52 | for hunk in Commit.split_patch(patch_json['patch']):
53 | for patch_block in Commit.split_hunk(hunk):
54 | yield Commit(json,
55 | patch_number,
56 | patch_json['filename'],
57 | *patch_block)
58 | patch_number += 1
59 |
60 | @staticmethod
61 | def split_patch(patch):
62 | """Given a full patch, yield all hunks.
63 |
64 | The hunks are tuples of:
65 | old_start_line, new_start_line, optional_header, list_of_lines
66 | """
67 | current_header = None
68 | current_lines = []
69 | def assemble():
70 | matches = PATCH_HEADER_REGEX.findall(current_header)
71 | assert matches and len(matches) == 1, current_header
72 | old_start, new_start, header = matches[0]
73 | return old_start, new_start, header, current_lines
74 |
75 | for line in patch.splitlines():
76 | if line.startswith('@@'):
77 | # Header line.
78 | if current_header is not None:
79 | # Flush current block.
80 | yield assemble()
81 | current_header = line
82 | else:
83 | # Regular line.
84 | current_lines.append(line)
85 |
86 | assert current_header is not None, patch
87 | yield assemble()
88 |
89 | @staticmethod
90 | def split_hunk(input):
91 | """Given a patch hunk, split it into pieces.
92 |
93 | The yielded results are hunks in the same format as the input.
94 | """
95 | old_start_line, new_start_line, optional_header, lines = input
96 | if len(lines) <= Commit.MAX_LINES_PER_PATCH:
97 | yield input
98 | else:
99 | def is_context_line(i):
100 | return lines[i].startswith(' ')
101 |
102 | last = 0
103 | cur_old_start_line = old_start_line
104 | cur_new_start_line = new_start_line
105 | seen_a_diff = False
106 | for i in range(0, len(lines)):
107 | if lines[i].startswith('-'):
108 | cur_old_start_line += 1
109 | seen_a_diff = True
110 | elif lines[i].startswith('+'):
111 | cur_new_start_line += 1
112 | seen_a_diff = True
113 | elif lines[i].startswith(' '):
114 | cur_old_start_line += 1
115 | cur_new_start_line += 1
116 |
117 | if (i != len(lines) - 1 and
118 | is_context_line(i - 1) and
119 | is_context_line(i) and
120 | is_context_line(i + 1) and
121 | seen_a_diff):
122 | yield old_start_line, new_start_line, optional_header, lines[last:i]
123 | seen_a_diff = False
124 | last = i
125 | old_start_line = cur_old_start_line
126 | new_start_line = cur_new_start_line
127 |
128 | if last != i:
129 | yield old_start_line, new_start_line, optional_header, lines[last:i]
130 |
131 |
132 | class Repository(object):
133 | """A GitHub repository."""
134 |
135 | def __init__(self, repository_json, star_count):
136 | self.id = repository_json['id']
137 | self.name = repository_json['full_name']
138 | self.author = repository_json['owner']['login']
139 | self.author_avatar_url = repository_json['owner']['avatar_url']
140 | self.description = repository_json['description']
141 | self.is_private = repository_json['private']
142 | self.is_fork = repository_json['fork']
143 | self.watcher_count = repository_json['watchers']
144 | self.star_count = star_count
145 |
--------------------------------------------------------------------------------
/backend/schema.sql:
--------------------------------------------------------------------------------
1 | CREATE SCHEMA IF NOT EXISTS `commit_game`;
2 | USE `commit_game`;
3 |
4 | CREATE TABLE `commit`(
5 | `order_id` INT(11) NOT NULL AUTO_INCREMENT,
6 | `sha` CHAR(40) NOT NULL,
7 | `patch_number` INT NOT NULL,
8 | `message` TEXT NOT NULL,
9 | `author_login` VARCHAR(255),
10 | `author_name` VARCHAR(255),
11 | `author_avatar_url` TEXT,
12 | `repository` VARCHAR(127) NOT NULL,
13 | `filename` VARCHAR(255) NOT NULL,
14 | `additions` INT UNSIGNED NOT NULL,
15 | `deletions` INT UNSIGNED NOT NULL,
16 | `old_start_line` INT UNSIGNED NOT NULL,
17 | `new_start_line` INT UNSIGNED NOT NULL,
18 | `block_name` VARCHAR(255),
19 | `diff_lines` LONGTEXT NOT NULL,
20 | `grade` INT NOT NULL DEFAULT -100,
21 | PRIMARY KEY (`order_id`),
22 | UNIQUE KEY `sha_patch_number_index` (`sha`,`patch_number`),
23 | KEY `grade_index` (`grade`)
24 | ) ENGINE = MyISAM;
25 |
26 | CREATE TABLE `repository`(
27 | `order_id` INT(11) NOT NULL AUTO_INCREMENT,
28 | `id` BIGINT NOT NULL,
29 | `name` VARCHAR(255) NOT NULL,
30 | `author` VARCHAR(255) NOT NULL,
31 | `author_avatar_url` TEXT NOT NULL,
32 | `description` TEXT NOT NULL,
33 | `is_private` BIT(1) NOT NULL,
34 | `is_fork` BIT(1) NOT NULL,
35 | `watcher_count` INT NOT NULL,
36 | `star_count` INT NOT NULL,
37 | PRIMARY KEY(`name`),
38 | UNIQUE INDEX `name_UNIQUE`(`name` ASC),
39 | UNIQUE INDEX `order_id_UNIQUE` (`order_id` ASC)
40 | ) ENGINE = MyISAM;
41 |
--------------------------------------------------------------------------------
/backend/server.py:
--------------------------------------------------------------------------------
1 | import collections
2 | import config
3 | import flask
4 | import json
5 | import gc
6 | import array
7 | import random
8 | import os
9 | import mimetypes
10 | import MySQLdb as mysql
11 | import MySQLdb.cursors
12 | import cPickle as pickle
13 | import platform
14 |
15 | APP = flask.Flask(__name__,
16 | static_folder='../app' if os.environ.get('ENV') == 'DEV' else '../app/dist',
17 | static_url_path='')
18 |
19 | ROOT = os.path.dirname(os.path.realpath(__file__))
20 | INDEX_CACHE_FILENAME = 'commit_index.pickle'
21 |
22 |
23 | def connect_to_db():
24 | return mysql.connect(
25 | host=config.DB_HOST,
26 | user=config.DB_USER,
27 | passwd=config.DB_PASSWORD,
28 | db=config.DB_NAME,
29 | cursorclass=MySQLdb.cursors.DictCursor)
30 |
31 |
32 | def initialize():
33 | print 'Initializing...'
34 | with connect_to_db() as cursor:
35 | # Get all repositories.
36 | cursor.execute('SELECT * FROM repository')
37 | first_row = cursor.fetchone()
38 | repo_class = collections.namedtuple('Repository',
39 | ' '.join(first_row.keys()))
40 | repos = {}
41 | repos[first_row['name']] = repo_class(**first_row)
42 | for row in cursor.fetchall():
43 | repos[row['name']] = repo_class(**row)
44 |
45 | # Get a commit index.
46 | index_class = collections.namedtuple('Index', 'grades ids')
47 | cache_path = os.path.join(ROOT, INDEX_CACHE_FILENAME)
48 | if os.path.exists(cache_path):
49 | print ' ...from cache.'
50 | cache_file = open(cache_path, 'r')
51 | commit_index_raw = pickle.load(cache_file)
52 | commit_index = index_class(*commit_index_raw)
53 | cache_file.close()
54 | else:
55 | print ' ...from DB.'
56 | cursor.execute('SELECT grade, order_id '
57 | 'FROM commit WHERE grade >= 0 '
58 | 'ORDER BY grade ASC')
59 | grades = []
60 | ids = array.array('i')
61 | last_grade = -1
62 | for _ in xrange(0, cursor.rowcount / 1000):
63 | for commit in cursor.fetchmany(1000):
64 | grade = commit['grade']
65 | if last_grade != grade:
66 | for _ in xrange(last_grade, grade):
67 | grades.append(len(ids))
68 | last_grade = grade
69 | ids.append(commit['order_id'])
70 | commit_index = index_class(tuple(grades), tuple(ids))
71 |
72 | cache_file = open(cache_path, 'w')
73 | pickle.dump([tuple(grades), tuple(ids)], cache_file)
74 | cache_file.close()
75 |
76 | # Try to GC, since iterating over the table uses up a lot of RAM.
77 | gc.collect()
78 |
79 | print 'Initialization done.'
80 | return repos, commit_index
81 |
82 |
83 | # Assume we have enough memory to keep all repositories and a commit index loaded.
84 | REPOS, COMMIT_INDEX = initialize()
85 | REPO_NAMES = REPOS.keys()
86 |
87 |
88 | @APP.route("/")
89 | @APP.route("/level/
")
90 | def homepage(level_id=None):
91 | return open(os.path.join(ROOT, '../app/index.html'), 'r').read()
92 |
93 |
94 | @APP.route("/level///")
95 | def level(length, min_grade, max_grade):
96 | length = min(int(length), 256)
97 | max_grade = int(max_grade)
98 | min_grade = int(min_grade)
99 |
100 | start = COMMIT_INDEX.grades[min_grade]
101 | end = COMMIT_INDEX.grades[max_grade]
102 | ids = [COMMIT_INDEX.ids[i] for i in random.sample(xrange(start, end), length)]
103 |
104 | sql = 'SELECT * FROM commit WHERE order_id IN (%s)' % ','.join(map(str, ids))
105 | with connect_to_db() as cursor:
106 | cursor.execute(sql)
107 | commits = cursor.fetchall()
108 |
109 | levels = []
110 | for commit in commits:
111 | repo_names = random.sample(REPO_NAMES, 4)
112 | if commit['repository'] not in repo_names:
113 | repo_names = repo_names[:3] + [commit['repository']]
114 | repos = [REPOS[i]._asdict() for i in repo_names]
115 | random.shuffle(repos)
116 | levels.append({'commit': commit, 'repos': repos})
117 |
118 | return flask.Response(json.dumps({'rounds': levels}), mimetype='text/json')
119 |
120 | if platform.system() == 'Windows':
121 | # FontAwesome files cause a weird bug on Windows. Hack around it.
122 | @APP.route('/build/FortAwesome-Font-Awesome/fonts/')
123 | @APP.route('/build//FortAwesome-Font-Awesome/fonts/')
124 | def custom_static(path):
125 | path = path.replace('../', '') # Make sure we don't try to reach outside.
126 | fs_path = os.path.join(
127 | ROOT, '../app/build/FortAwesome-Font-Awesome/fonts/', path)
128 | print fs_path
129 | if os.path.exists(fs_path):
130 | mime = mimetypes.guess_type(fs_path)[0]
131 | return flask.Response(open(fs_path, 'rb'), mimetype=mime)
132 | else:
133 | return 'Not Found', 404
134 |
135 |
136 | if __name__ == "__main__":
137 | APP.run(debug=True if os.environ.get('ENV') == 'DEV' else False)
138 |
--------------------------------------------------------------------------------
/mocks/game-screen-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/max99x/guesshub/74f0160d27dc02ff8e411bd07fd5c5448fd513e0/mocks/game-screen-1.png
--------------------------------------------------------------------------------
/mocks/game-screen-1.xcf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/max99x/guesshub/74f0160d27dc02ff8e411bd07fd5c5448fd513e0/mocks/game-screen-1.xcf
--------------------------------------------------------------------------------
/mocks/game-screen-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/max99x/guesshub/74f0160d27dc02ff8e411bd07fd5c5448fd513e0/mocks/game-screen-2.png
--------------------------------------------------------------------------------
/mocks/game-screen-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/max99x/guesshub/74f0160d27dc02ff8e411bd07fd5c5448fd513e0/mocks/game-screen-3.png
--------------------------------------------------------------------------------
/mocks/hub-with-powers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/max99x/guesshub/74f0160d27dc02ff8e411bd07fd5c5448fd513e0/mocks/hub-with-powers.png
--------------------------------------------------------------------------------
/mocks/hub.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/max99x/guesshub/74f0160d27dc02ff8e411bd07fd5c5448fd513e0/mocks/hub.png
--------------------------------------------------------------------------------
/mocks/intro.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/max99x/guesshub/74f0160d27dc02ff8e411bd07fd5c5448fd513e0/mocks/intro.gif
--------------------------------------------------------------------------------
/mocks/progress-bar-mock.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/max99x/guesshub/74f0160d27dc02ff8e411bd07fd5c5448fd513e0/mocks/progress-bar-mock.png
--------------------------------------------------------------------------------
/screenshot.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/max99x/guesshub/74f0160d27dc02ff8e411bd07fd5c5448fd513e0/screenshot.png
--------------------------------------------------------------------------------
/todo-priorities:
--------------------------------------------------------------------------------
1 | Nice to have
2 | Add animation on level unlock.
3 | ./power-list/power-list.js:16: // TODO: Make choices respond to QWER keyboards keys.
4 | ./repo-list/repo-list.js:30: // TODO: Make choices respond to 1-4 keyboards keys.
5 | Include commits from the game repository itself.
6 |
7 | Post-launch
8 | Achievements
9 | Keep track of achievements.
10 | Show a lists of achievements when the user clicks on the trophy icon.
11 | Show achievements on the finish page.
12 | User Statistics
13 | Keep track of all players' statistics.
14 | Show the user's statistics in the achievements UI.
15 | Show a graph comparison against other players on the finish screen.
16 | Bonus levels.
17 | Background music.
18 |
19 | Misc
20 | ./boot/game.js:131: // TODO: Maybe move these into Power.use()?
21 | ./models/level.js:28: // TODO: Retry with exponential backoff on errors.
22 | ./crawl.py:98: # TODO(max99x): Log and skip errors.
23 | ./boot/game.js:67: // TODO: Properly destroy widgets?
24 |
--------------------------------------------------------------------------------
41 | {{name}}
42 |