├── .github
    └── workflows
    │   └── mirror-pull-requests.yaml
├── .gitignore
├── .travis.yml
├── AUTHORS
├── CONTRIBUTING.md
├── CONTRIBUTORS
├── LICENSE
├── README.md
├── app
    ├── admin
    │   ├── app.yaml
    │   ├── cron.yaml
    │   ├── index.html
    │   ├── main.go
    │   ├── operations.go
    │   └── persistent.go
    ├── dispatch.yaml
    └── hooks
    │   ├── Dockerfile
    │   ├── app.yaml
    │   ├── ephemeral.go
    │   ├── main.go
    │   └── persistent.go
├── auth
    └── auth.go
├── batch
    └── batch.go
├── go.mod
├── go.sum
└── mirror
    ├── conversions.go
    ├── conversions_test.go
    ├── doc.go
    ├── output.go
    ├── output_test.go
    ├── readall.go
    └── readall_test.go


/.github/workflows/mirror-pull-requests.yaml:
--------------------------------------------------------------------------------
 1 | name: Mirror pull requests into git-notes
 2 | on: [pull_request, issue_comment, pull_request_review, pull_request_review_comment, status]
 3 | jobs:
 4 |   build:
 5 |     name: Mirror
 6 |     runs-on: ubuntu-latest
 7 |     steps:
 8 |       - name: Check out the repo
 9 |         uses: actions/checkout@v2
10 | 
11 |       - name: Set up Go 1.14
12 |         uses: actions/setup-go@v1
13 |         with:
14 |           go-version: 1.14
15 | 
16 |       - name: Setup go modules
17 |         run: |
18 |           export GO111MODULE=on
19 |           go mod init workflow || true 
20 |           go get github.com/google/git-appraise/git-appraise
21 |           go get github.com/google/git-pull-request-mirror/batch
22 | 
23 |       - name: Configure git for the PR mirror
24 |         run: |
25 |           git config --global user.email "${{ github.repository }}@github.com"
26 |           git config --global user.name "Pull Request Mirror"
27 | 
28 |       - name: Fetch upstream refs
29 |         run: |
30 |           git fetch origin --unshallow
31 |           git fetch origin '+refs/heads/*:refs/remotes/origin/*'
32 |           git fetch origin '+refs/heads/master:refs/heads/master' || git pull
33 |           git fetch origin '+refs/tags/*:refs/tags/*'
34 |           git fetch origin '+refs/pull/*:refs/pull/*'
35 |           git fetch origin '+refs/devtools/*:refs/devtools/*'
36 | 
37 |       - name: Pull existing reviews
38 |         run: go run github.com/google/git-appraise/git-appraise pull
39 | 
40 |       - name: Mirror pull requests into local reviews
41 |         run: go run github.com/google/git-pull-request-mirror/batch --target '${{ github.repository }}' --local ./ --auth-token '${{ secrets.PR_MIRROR_TOKEN }}'
42 | 
43 |       - name: Merge any upstream review changes
44 |         run: go run github.com/google/git-appraise/git-appraise pull
45 | 
46 |       - name: Push updated reviews back upstream
47 |         run: go run github.com/google/git-appraise/git-appraise push
48 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *~


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | sudo: false
2 | language: go
3 | 
4 | go:
5 | - 1.13.x
6 | 


--------------------------------------------------------------------------------
/AUTHORS:
--------------------------------------------------------------------------------
1 | # This is the official list of [Project Name] authors for copyright purposes.
2 | # This file is distinct from the CONTRIBUTORS files.
3 | # See the latter for an explanation.
4 | # Names should be added to this file as:
5 | # Name or Organization <email address>
6 | # The email address is not required for organizations.
7 | Google Inc.


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | Want to contribute? Great! First, read this page (including the small print at the end).
 2 | 
 3 | ### Before you contribute
 4 | Before we can use your code, you must sign the
 5 | [Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual?csw=1)
 6 | (CLA), which you can do online. The CLA is necessary mainly because you own the
 7 | copyright to your changes, even after your contribution becomes part of our
 8 | codebase.  Therefore, we need your permission to use and distribute your code.
 9 | We also need to be sure of various other things—for instance that you'll tell
10 | us if you know that your code infringes on other people's patents. You don't
11 | have to sign the CLA until after you've submitted your code for review and a
12 | member has approved it, but you must do it before we can put your code into our
13 | codebase. Before you start working on a larger contribution, you should get in
14 | touch with us first through the issue tracker with your idea so that we can
15 | help out and possibly guide you. Coordinating up front avoids frustrations later.
16 | 
17 | ### Code reviews
18 | All submissions, including submissions by project members, require review. You
19 | may use a Github pull request to start such a review, but the review itself
20 | will be conducted using the git-appraise tool.
21 | 
22 | ### The small print
23 | Contributions made by corporations are covered by a different agreement than
24 | the one above, the Software Grant and Corporate Contributor License Agreement.
25 | 


--------------------------------------------------------------------------------
/CONTRIBUTORS:
--------------------------------------------------------------------------------
 1 | # People who have agreed to one of the CLAs and can contribute patches.
 2 | # The AUTHORS file lists the copyright holders; this file
 3 | # lists people.  For example, Google employees are listed here
 4 | # but not in AUTHORS, because Google holds the copyright.
 5 | #
 6 | # https://developers.google.com/open-source/cla/individual
 7 | # https://developers.google.com/open-source/cla/corporate
 8 | #
 9 | # Names should be added to this file as:
10 | #     Name <email address>
11 | James Gilles <jhgilles@google.com>
12 | Omar Jarjur <ojarjur@google.com>


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 | 
  2 |                                  Apache License
  3 |                            Version 2.0, January 2004
  4 |                         http://www.apache.org/licenses/
  5 | 
  6 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  7 | 
  8 |    1. Definitions.
  9 | 
 10 |       "License" shall mean the terms and conditions for use, reproduction,
 11 |       and distribution as defined by Sections 1 through 9 of this document.
 12 | 
 13 |       "Licensor" shall mean the copyright owner or entity authorized by
 14 |       the copyright owner that is granting the License.
 15 | 
 16 |       "Legal Entity" shall mean the union of the acting entity and all
 17 |       other entities that control, are controlled by, or are under common
 18 |       control with that entity. For the purposes of this definition,
 19 |       "control" means (i) the power, direct or indirect, to cause the
 20 |       direction or management of such entity, whether by contract or
 21 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 22 |       outstanding shares, or (iii) beneficial ownership of such entity.
 23 | 
 24 |       "You" (or "Your") shall mean an individual or Legal Entity
 25 |       exercising permissions granted by this License.
 26 | 
 27 |       "Source" form shall mean the preferred form for making modifications,
 28 |       including but not limited to software source code, documentation
 29 |       source, and configuration files.
 30 | 
 31 |       "Object" form shall mean any form resulting from mechanical
 32 |       transformation or translation of a Source form, including but
 33 |       not limited to compiled object code, generated documentation,
 34 |       and conversions to other media types.
 35 | 
 36 |       "Work" shall mean the work of authorship, whether in Source or
 37 |       Object form, made available under the License, as indicated by a
 38 |       copyright notice that is included in or attached to the work
 39 |       (an example is provided in the Appendix below).
 40 | 
 41 |       "Derivative Works" shall mean any work, whether in Source or Object
 42 |       form, that is based on (or derived from) the Work and for which the
 43 |       editorial revisions, annotations, elaborations, or other modifications
 44 |       represent, as a whole, an original work of authorship. For the purposes
 45 |       of this License, Derivative Works shall not include works that remain
 46 |       separable from, or merely link (or bind by name) to the interfaces of,
 47 |       the Work and Derivative Works thereof.
 48 | 
 49 |       "Contribution" shall mean any work of authorship, including
 50 |       the original version of the Work and any modifications or additions
 51 |       to that Work or Derivative Works thereof, that is intentionally
 52 |       submitted to Licensor for inclusion in the Work by the copyright owner
 53 |       or by an individual or Legal Entity authorized to submit on behalf of
 54 |       the copyright owner. For the purposes of this definition, "submitted"
 55 |       means any form of electronic, verbal, or written communication sent
 56 |       to the Licensor or its representatives, including but not limited to
 57 |       communication on electronic mailing lists, source code control systems,
 58 |       and issue tracking systems that are managed by, or on behalf of, the
 59 |       Licensor for the purpose of discussing and improving the Work, but
 60 |       excluding communication that is conspicuously marked or otherwise
 61 |       designated in writing by the copyright owner as "Not a Contribution."
 62 | 
 63 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 64 |       on behalf of whom a Contribution has been received by Licensor and
 65 |       subsequently incorporated within the Work.
 66 | 
 67 |    2. Grant of Copyright License. Subject to the terms and conditions of
 68 |       this License, each Contributor hereby grants to You a perpetual,
 69 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 70 |       copyright license to reproduce, prepare Derivative Works of,
 71 |       publicly display, publicly perform, sublicense, and distribute the
 72 |       Work and such Derivative Works in Source or Object form.
 73 | 
 74 |    3. Grant of Patent License. Subject to the terms and conditions of
 75 |       this License, each Contributor hereby grants to You a perpetual,
 76 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 77 |       (except as stated in this section) patent license to make, have made,
 78 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 79 |       where such license applies only to those patent claims licensable
 80 |       by such Contributor that are necessarily infringed by their
 81 |       Contribution(s) alone or by combination of their Contribution(s)
 82 |       with the Work to which such Contribution(s) was submitted. If You
 83 |       institute patent litigation against any entity (including a
 84 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 85 |       or a Contribution incorporated within the Work constitutes direct
 86 |       or contributory patent infringement, then any patent licenses
 87 |       granted to You under this License for that Work shall terminate
 88 |       as of the date such litigation is filed.
 89 | 
 90 |    4. Redistribution. You may reproduce and distribute copies of the
 91 |       Work or Derivative Works thereof in any medium, with or without
 92 |       modifications, and in Source or Object form, provided that You
 93 |       meet the following conditions:
 94 | 
 95 |       (a) You must give any other recipients of the Work or
 96 |           Derivative Works a copy of this License; and
 97 | 
 98 |       (b) You must cause any modified files to carry prominent notices
 99 |           stating that You changed the files; and
100 | 
101 |       (c) You must retain, in the Source form of any Derivative Works
102 |           that You distribute, all copyright, patent, trademark, and
103 |           attribution notices from the Source form of the Work,
104 |           excluding those notices that do not pertain to any part of
105 |           the Derivative Works; and
106 | 
107 |       (d) If the Work includes a "NOTICE" text file as part of its
108 |           distribution, then any Derivative Works that You distribute must
109 |           include a readable copy of the attribution notices contained
110 |           within such NOTICE file, excluding those notices that do not
111 |           pertain to any part of the Derivative Works, in at least one
112 |           of the following places: within a NOTICE text file distributed
113 |           as part of the Derivative Works; within the Source form or
114 |           documentation, if provided along with the Derivative Works; or,
115 |           within a display generated by the Derivative Works, if and
116 |           wherever such third-party notices normally appear. The contents
117 |           of the NOTICE file are for informational purposes only and
118 |           do not modify the License. You may add Your own attribution
119 |           notices within Derivative Works that You distribute, alongside
120 |           or as an addendum to the NOTICE text from the Work, provided
121 |           that such additional attribution notices cannot be construed
122 |           as modifying the License.
123 | 
124 |       You may add Your own copyright statement to Your modifications and
125 |       may provide additional or different license terms and conditions
126 |       for use, reproduction, or distribution of Your modifications, or
127 |       for any such Derivative Works as a whole, provided Your use,
128 |       reproduction, and distribution of the Work otherwise complies with
129 |       the conditions stated in this License.
130 | 
131 |    5. Submission of Contributions. Unless You explicitly state otherwise,
132 |       any Contribution intentionally submitted for inclusion in the Work
133 |       by You to the Licensor shall be under the terms and conditions of
134 |       this License, without any additional terms or conditions.
135 |       Notwithstanding the above, nothing herein shall supersede or modify
136 |       the terms of any separate license agreement you may have executed
137 |       with Licensor regarding such Contributions.
138 | 
139 |    6. Trademarks. This License does not grant permission to use the trade
140 |       names, trademarks, service marks, or product names of the Licensor,
141 |       except as required for reasonable and customary use in describing the
142 |       origin of the Work and reproducing the content of the NOTICE file.
143 | 
144 |    7. Disclaimer of Warranty. Unless required by applicable law or
145 |       agreed to in writing, Licensor provides the Work (and each
146 |       Contributor provides its Contributions) on an "AS IS" BASIS,
147 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 |       implied, including, without limitation, any warranties or conditions
149 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 |       PARTICULAR PURPOSE. You are solely responsible for determining the
151 |       appropriateness of using or redistributing the Work and assume any
152 |       risks associated with Your exercise of permissions under this License.
153 | 
154 |    8. Limitation of Liability. In no event and under no legal theory,
155 |       whether in tort (including negligence), contract, or otherwise,
156 |       unless required by applicable law (such as deliberate and grossly
157 |       negligent acts) or agreed to in writing, shall any Contributor be
158 |       liable to You for damages, including any direct, indirect, special,
159 |       incidental, or consequential damages of any character arising as a
160 |       result of this License or out of the use or inability to use the
161 |       Work (including but not limited to damages for loss of goodwill,
162 |       work stoppage, computer failure or malfunction, or any and all
163 |       other commercial damages or losses), even if such Contributor
164 |       has been advised of the possibility of such damages.
165 | 
166 |    9. Accepting Warranty or Additional Liability. While redistributing
167 |       the Work or Derivative Works thereof, You may choose to offer,
168 |       and charge a fee for, acceptance of support, warranty, indemnity,
169 |       or other liability obligations and/or rights consistent with this
170 |       License. However, in accepting such obligations, You may act only
171 |       on Your own behalf and on Your sole responsibility, not on behalf
172 |       of any other Contributor, and only if You agree to indemnify,
173 |       defend, and hold each Contributor harmless for any liability
174 |       incurred by, or claims asserted against, such Contributor by reason
175 |       of your accepting any such warranty or additional liability.
176 | 
177 |    END OF TERMS AND CONDITIONS
178 | 
179 |    APPENDIX: How to apply the Apache License to your work.
180 | 
181 |       To apply the Apache License to your work, attach the following
182 |       boilerplate notice, with the fields enclosed by brackets "[]"
183 |       replaced with your own identifying information. (Don't include
184 |       the brackets!)  The text should be enclosed in the appropriate
185 |       comment syntax for the file format. We also recommend that a
186 |       file or class name and description of purpose be included on the
187 |       same "printed page" as the copyright notice for easier
188 |       identification within third-party archives.
189 | 
190 |    Copyright [yyyy] [name of copyright owner]
191 | 
192 |    Licensed under the Apache License, Version 2.0 (the "License");
193 |    you may not use this file except in compliance with the License.
194 |    You may obtain a copy of the License at
195 | 
196 |        http://www.apache.org/licenses/LICENSE-2.0
197 | 
198 |    Unless required by applicable law or agreed to in writing, software
199 |    distributed under the License is distributed on an "AS IS" BASIS,
200 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 |    See the License for the specific language governing permissions and
202 |    limitations under the License.
203 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Mirror Github Pull Requests into the git-appraise formats
 2 | 
 3 | This repo contains a tool to mirror pull requests metadata into the corresponding git
 4 | repository using a feature of git called [git-notes](http://git-scm.com/docs/git-notes).
 5 | 
 6 | The format written is the one defined by the
 7 | [git-appraise code review system](https://github.com/google/git-appraise), so pull
 8 | requests that are mirrored using this tool can be reviewed using git-appraise.
 9 | 
10 | ## Disclaimer
11 | 
12 | This is not an officially supported Google product.
13 | 
14 | ## Organization
15 | 
16 | There are 3 packages in this repo:
17 | - `mirror` is a go library for mirroring the pull request metadata into git-notes.
18 | - `batch` is a batch processor to mirror Github data into a local repository.
19 | - `app` is a webapp/bot that sets up Github webhooks and mirrors data incrementally
20 |   whenever an interesting event happens on the Github repo.
21 | 
22 | ### The Batch Tool
23 | 
24 | The batch tool performs a single pass of reading all of the pull request metadata for
25 | a repo, and mirroring it into your local clone of that repo.
26 | 
27 | The tool can support running unauthenticated, but will be extremely rate-limited, so
28 | it is better if you create a [personal access token](https://help.github.com/articles/creating-an-access-token-for-command-line-use/),
29 | with the `repo` scope, for it to use.
30 | 
31 | Setup:
32 | 
33 | ```shell
34 | go get github.com/google/git-pull-request-mirror/batch
35 | go build -o ~/bin/pr-mirror "${GOPATH}/src/github.com/google/git-pull-request-mirror/batch/batch.go"
36 | ```
37 | 
38 | Example Usage (after you've cloned the repo to mirror):
39 | 
40 | ```shell
41 | git fetch origin '+refs/pull/*:refs/pull/*'
42 | git appraise pull
43 | ~/bin/pr-mirror --target ${GITHUB_USER}/${GITHUB_REPO} --local ./ -auth-token ${YOUR_AUTH_TOKEN}
44 | git appraise pull
45 | git appraise push
46 | ```
47 | 
48 | ### The Github Mirror App
49 | 
50 | This app allows users to continually update their git repositories with github
51 | metadata (pull requests and build statuses). It runs in an AppEngine app, and
52 | should expose a web interface at <PROJECT>.appspot.com.
53 | 
54 | It uses the app engine datastore to store its configuration.
55 | 
56 | To deploy:
57 | 
58 | ```shell
59 | gcloud app deploy ./app/admin/*.yaml
60 | gcloud app deploy ./app/hooks/*.yaml
61 | gcloud app deploy ./app/*.yaml
62 | ```
63 | 


--------------------------------------------------------------------------------
/app/admin/app.yaml:
--------------------------------------------------------------------------------
 1 | # Copyright 2019 Google LLC
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     https://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | 
15 | runtime: go
16 | api_version: go1
17 | 
18 | handlers:
19 | - url: /add
20 |   script: _go_app
21 |   login: admin
22 | 
23 | - url: /delete
24 |   script: _go_app
25 |   login: admin
26 | 
27 | - url: /restartOperations
28 |   script: _go_app
29 | 
30 | - url: /
31 |   script: _go_app
32 |   login: admin
33 | 


--------------------------------------------------------------------------------
/app/admin/cron.yaml:
--------------------------------------------------------------------------------
1 | cron:
2 | - description: "hourly restart abandoned operations"
3 |   url: /restartOperations
4 |   schedule: every 60 mins
5 | 


--------------------------------------------------------------------------------
/app/admin/index.html:
--------------------------------------------------------------------------------
 1 | <!doctype html>
 2 | <!-- Copyright 2015 Google Inc. All Rights Reserved.
 3 | 
 4 | Licensed under the Apache License, Version 2.0 (the "License");
 5 | you may not use this file except in compliance with the License.
 6 | You may obtain a copy of the License at
 7 | 
 8 |     http://www.apache.org/licenses/LICENSE-2.0
 9 | 
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.-->
15 | 
16 | <html>
17 | <head>
18 | 	<title>Github Mirror Configuration</title>
19 | </head>
20 | <body>
21 | 	<h1>Github Mirror Configuration</h1>
22 | 	<table>
23 | 		<tr>
24 | 			<td>Repository</td>
25 | 			<td>Status</td>
26 | 		</tr>
27 | 		{{ range $repo := .Repos }}
28 | 		<tr>
29 | 			<td>
30 | 				<code>{{ $repo.Name }}</code>
31 | 			</td>
32 | 			<td>
33 | 				<code>{{ $repo.Status }}</code>
34 | 			</td>
35 | 			<td>
36 | 				{{ if $repo.ErrorCause }}
37 | 				<code>({{ $repo.ErrorCause }})</code>
38 | 				{{ end }}
39 | 			</td>
40 | 			<td>
41 | 				<form method="post" action="/delete">
42 | 					<input type="hidden" name="repoName" value="{{ $repo.Name }}"/>
43 | 					<button type="submit">Disable</button>
44 | 				</form>
45 | 			</td>
46 | 		</tr>
47 | 		{{ end }}
48 | 	</table>
49 | 	<p>Add new:</p>
50 | 	<form method="post" action="/add">
51 | 		<label for="name">
52 | 			<span>Repository: </span>
53 | 			<input type="text" id="repoName" name="repoName" text="user/repo" required/>
54 | 		</label>
55 | 		<label for="key">
56 | 			<span>Access Token: </span>
57 | 			<input type="text" id="repoToken" name="repoToken" required/>
58 | 		</label>
59 | 		<button>Submit</button>
60 | 	</form>
61 | 	<p>Note:</p>
62 | 	<p>Generate access keys in the 
63 | 	<a href="https://github.com/settings/tokens" target="_blank">Github Personal access tokens</a>
64 | 	control panel. Make sure they have the <code>repo</code>, <code>public_repo</code>,
65 | 	<code>write:repo_hook</code>, and <code>repo:status</code> scopes; alternatively,
66 | 	leave all scopes deactivated.</p>
67 | </body>
68 | </html>
69 | 
70 | 


--------------------------------------------------------------------------------
/app/admin/main.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package main
 18 | 
 19 | import (
 20 | 	"fmt"
 21 | 	"html/template"
 22 | 	"net/http"
 23 | 	"strings"
 24 | 
 25 | 	"google.golang.org/appengine"
 26 | 	"google.golang.org/appengine/log"
 27 | 	"google.golang.org/appengine/user"
 28 | )
 29 | 
 30 | // Code for the web control panel
 31 | 
 32 | const (
 33 | 	// idRepoName is the id used in an http form for a repository
 34 | 	idRepoName = "repoName"
 35 | 	// idRepoToken is the id used in an http form for a github API key
 36 | 	idRepoToken = "repoToken"
 37 | )
 38 | 
 39 | var configTemplate = template.Must(template.ParseFiles("index.html"))
 40 | 
 41 | // renderRepo represents a single repository to be rendered on the page
 42 | type renderRepo struct {
 43 | 	Name       string
 44 | 	Status     string
 45 | 	ErrorCause string
 46 | }
 47 | 
 48 | // renderConfig is the top-level struct passed to rendering
 49 | type renderConfig struct {
 50 | 	Repos []renderRepo
 51 | }
 52 | 
 53 | // configHandler renders a configuration page
 54 | func configHandler(w http.ResponseWriter, req *http.Request) {
 55 | 	ctx := appengine.NewContext(req)
 56 | 
 57 | 	repos, err := getAllRepoData(appengine.NewContext(req))
 58 | 
 59 | 	if err != nil {
 60 | 		log.Errorf(ctx, "Error fetching repos: %s", err.Error())
 61 | 		http.Error(w, err.Error(), http.StatusInternalServerError)
 62 | 		return
 63 | 	}
 64 | 
 65 | 	conf := renderConfig{}
 66 | 
 67 | 	for _, repo := range repos {
 68 | 		conf.Repos = append(conf.Repos, renderRepo{
 69 | 			Name:       fmt.Sprintf("%s/%s", repo.User, repo.Repo),
 70 | 			Status:     repo.Status,
 71 | 			ErrorCause: repo.ErrorCause,
 72 | 		})
 73 | 	}
 74 | 
 75 | 	configTemplate.Execute(w, &conf)
 76 | }
 77 | 
 78 | // addHandler handles POSTs to the /add endpoint
 79 | func addHandler(w http.ResponseWriter, req *http.Request) {
 80 | 	defer http.Redirect(w, req, "/", http.StatusSeeOther)
 81 | 	ctx := appengine.NewContext(req)
 82 | 
 83 | 	if req.Method != "POST" {
 84 | 		log.Errorf(ctx, "Incorrect method for /add endpoint: %s", req.Method)
 85 | 		return
 86 | 	}
 87 | 
 88 | 	err := req.ParseForm()
 89 | 	if err != nil {
 90 | 		log.Errorf(ctx, "Couldn't parse form for /add endpoint: %s", err.Error())
 91 | 		return
 92 | 	}
 93 | 
 94 | 	repoName := req.PostForm.Get(idRepoName)
 95 | 	if repoName == "" {
 96 | 		log.Errorf(ctx, "No repoName for /add endpoint: %v", req.PostForm)
 97 | 		return
 98 | 	}
 99 | 
100 | 	repoToken := req.PostForm.Get(idRepoToken)
101 | 	if repoToken == "" {
102 | 		log.Errorf(ctx, "No repoToken for /add endpoint: %v", req.PostForm)
103 | 		return
104 | 	}
105 | 
106 | 	splitName := strings.Split(repoName, "/")
107 | 	if len(splitName) != 2 {
108 | 		log.Errorf(ctx, "Invalid repository name (can't split on '/'): %s", repoName)
109 | 		return
110 | 	}
111 | 
112 | 	log.Infof(ctx, "Adding repository %s", repoName)
113 | 
114 | 	err = initRepoData(ctx, splitName[0], splitName[1], repoToken)
115 | 
116 | 	if err != nil {
117 | 		log.Errorf(ctx, "Couldn't store repository %s: %s", repoName, err.Error())
118 | 		return
119 | 	}
120 | 
121 | 	validate(ctx, splitName[0], splitName[1])
122 | }
123 | 
124 | // deleteHandler handles POSTS to the /delete endpoint
125 | func deleteHandler(w http.ResponseWriter, req *http.Request) {
126 | 	defer http.Redirect(w, req, "/", http.StatusSeeOther)
127 | 	ctx := appengine.NewContext(req)
128 | 
129 | 	if req.Method != "POST" {
130 | 		log.Errorf(ctx, "Incorrect method for /delete endpoint: %s", req.Method)
131 | 		return
132 | 	}
133 | 
134 | 	err := req.ParseForm()
135 | 	if err != nil {
136 | 		log.Errorf(ctx, "Couldn't parse form for /delete endpoint: %s", err.Error())
137 | 		return
138 | 	}
139 | 
140 | 	fullRepoName := req.PostForm.Get(idRepoName)
141 | 	if fullRepoName == "" {
142 | 		log.Errorf(ctx, "No repoName for /delete endpoint: %v", req.PostForm)
143 | 		return
144 | 	}
145 | 
146 | 	splitName := strings.Split(fullRepoName, "/")
147 | 	if len(splitName) != 2 {
148 | 		log.Errorf(ctx, "Invalid repository name (can't split on '/'): %s", fullRepoName)
149 | 		return
150 | 	}
151 | 
152 | 	deactivate(ctx, splitName[0], splitName[1])
153 | }
154 | 
155 | func restartOperationsHandler(w http.ResponseWriter, req *http.Request) {
156 | 	ctx := appengine.NewContext(req)
157 | 	restartAbandonedOperations(ctx)
158 | 	w.Write([]byte("done"))
159 | }
160 | 
161 | // enforceLoginHandler wraps another handler, returning a handler that will
162 | // enforce user login and then pass off control down the chain.
163 | func enforceLoginHandler(next http.Handler) http.Handler {
164 | 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
165 | 		ctx := appengine.NewContext(req)
166 | 		u := user.Current(ctx)
167 | 		if u == nil {
168 | 			// Not logged in
169 | 			url, err := user.LoginURL(ctx, req.URL.String())
170 | 			if err != nil {
171 | 				http.Error(w, err.Error(), http.StatusInternalServerError)
172 | 				return
173 | 			}
174 | 			http.Redirect(w, req, url, http.StatusSeeOther)
175 | 			return
176 | 		}
177 | 
178 | 		// Ensure that the persistent storage is set up before continuing...
179 | 		initStorage(ctx)
180 | 
181 | 		// Pass off control
182 | 		next.ServeHTTP(w, req)
183 | 	})
184 | }
185 | 
186 | func setupHandlers() {
187 | 	http.Handle("/add", enforceLoginHandler(http.HandlerFunc(addHandler)))
188 | 	http.Handle("/delete", enforceLoginHandler(http.HandlerFunc(deleteHandler)))
189 | 	http.Handle("/restartOperations", http.HandlerFunc(restartOperationsHandler))
190 | 	http.Handle("/", enforceLoginHandler(http.HandlerFunc(configHandler)))
191 | }
192 | 
193 | func main() {
194 | 	setupHandlers()
195 | 	appengine.Main()
196 | }
197 | 


--------------------------------------------------------------------------------
/app/admin/operations.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package main
 18 | 
 19 | import (
 20 | 	"context"
 21 | 	"crypto/rand"
 22 | 	"encoding/hex"
 23 | 	"errors"
 24 | 	"fmt"
 25 | 	"strings"
 26 | 	"sync"
 27 | 	"time"
 28 | 
 29 | 	"github.com/google/go-github/github"
 30 | 	"golang.org/x/oauth2"
 31 | 	"google.golang.org/appengine"
 32 | 	"google.golang.org/appengine/log"
 33 | )
 34 | 
 35 | const (
 36 | 	maxRetries   = 200
 37 | 	scopesHeader = "X-OAuth-Scopes"
 38 | 	secretSize   = 64
 39 | 
 40 | 	githubEventHeader     = "X-Github-Event"
 41 | 	githubSignatureHeader = "X-Hub-Signature"
 42 | 
 43 | 	eventPing         = "ping"
 44 | 	eventStatus       = "status"
 45 | 	eventPullRequest  = "pull_request"
 46 | 	eventDiffComment  = "pull_request_review_comment"
 47 | 	eventIssueComment = "issue_comment"
 48 | )
 49 | 
 50 | var errTooManyRetries = errors.New("Too many retries!")
 51 | 
 52 | // retry reduces github api-retrying boilerplate for when we run out of requests.
 53 | // It will call the given function until it succeeds or errors out, or until it
 54 | // has retried more than $maxRetries times.
 55 | // Use like so:
 56 | //
 57 | //	var zen string
 58 | //
 59 | //	err = retry(ctx, func() (resp *github.Response, err error) {
 60 | //		zen, resp, err = githubClient.Zen()
 61 | //		return
 62 | //	})
 63 | //
 64 | func retry(ctx context.Context, f func() (*github.Response, error)) error {
 65 | 	for i := 0; i < maxRetries; i++ {
 66 | 		resp, err := f()
 67 | 
 68 | 		if resp != nil && resp.Rate.Remaining == 0 {
 69 | 			// Timeout problems
 70 | 			waitDuration := resp.Rate.Reset.Sub(time.Now())
 71 | 			log.Infof(ctx, "Ran out of github API requests; sleeping %v (until %v)",
 72 | 				waitDuration,
 73 | 				resp.Rate.Reset.Time)
 74 | 			time.Sleep(waitDuration)
 75 | 			continue
 76 | 		}
 77 | 		if err != nil {
 78 | 			// Error unrelated to timeout
 79 | 			return err
 80 | 		}
 81 | 		// operation performed successfully
 82 | 		return nil
 83 | 	}
 84 | 	log.Errorf(ctx, "Too many retries, abandoning operation")
 85 | 	return errTooManyRetries
 86 | }
 87 | 
 88 | // Each repository goes through the following lifecycle states:
 89 | //
 90 | //   [validating]
 91 | //     |
 92 | //     | (validate access to the repo)
 93 | //     |
 94 | //     V
 95 | //   [hooks initializing]
 96 | //     |
 97 | //     | (create the web hook, and then receive the web hook "ping")
 98 | //     |
 99 | //     V
100 | //   [initializing]
101 | //     |
102 | //     | (mirror the pull requests)
103 | //     |
104 | //     V
105 | //   [ready]
106 | //     | ^
107 | //     | | (recieve any web hook and mirror the pull requests)
108 | //     | |
109 | //     +-+
110 | 
111 | // validate ensures that the repo is accessible
112 | func validate(ctx context.Context, user, repo string) {
113 | 	log.Infof(ctx, "Validating repo %s/%s", user, repo)
114 | 
115 | 	errorf := makeErrorf(ctx, user, repo)
116 | 
117 | 	repoData, err := getRepoData(ctx, user, repo)
118 | 	if err != nil {
119 | 		errorf("Can't load repo to validate: %s", err.Error())
120 | 		return
121 | 	}
122 | 
123 | 	httpClient := oauth2.NewClient(ctx, oauth2.StaticTokenSource(
124 | 		&oauth2.Token{AccessToken: repoData.Token},
125 | 	))
126 | 
127 | 	githubClient := github.NewClient(httpClient)
128 | 
129 | 	var resp *github.Response
130 | 	err = retry(ctx, func() (*github.Response, error) {
131 | 		// APIMeta will always succeed and will tell us what scopes
132 | 		// we have.
133 | 		_, resp, err = githubClient.APIMeta(ctx)
134 | 		return resp, err
135 | 	})
136 | 
137 | 	if err != nil {
138 | 		errorf("Can't validate repo %s/%s: %s", user, repo, err.Error())
139 | 		return
140 | 	}
141 | 
142 | 	scopesHeader := resp.Header["X-Oauth-Scopes"]
143 | 
144 | 	if len(scopesHeader) == 0 {
145 | 		// No scopes means that a token has access to all *public* repositories.
146 | 		// It's simplest to just require private access.
147 | 		errorf("Invalid token, missing scopes: `repo`, `write:repo_hook`")
148 | 		return
149 | 	}
150 | 
151 | 	// The token has scopes.
152 | 	// Let's make sure it has all the ones we need enabled.
153 | 	// Note that strictly speaking, we need the repo, public_repo,
154 | 	// write:repo_hook, and repo:status scopes, but repo and
155 | 	// write:repo_hook subsume the others.
156 | 
157 | 	// Necessary because github makes things comma-delimited instead
158 | 	// of semicolon-delimited for some reason.
159 | 	scopes := strings.Split(scopesHeader[0], ", ")
160 | 
161 | 	var hasRepo bool
162 | 	var hasWriteRepoHook bool
163 | 	for _, scope := range scopes {
164 | 		switch scope {
165 | 		case "repo":
166 | 			hasRepo = true
167 | 		case "admin:repo_hook":
168 | 			hasWriteRepoHook = true
169 | 		case "write:repo_hook":
170 | 			hasWriteRepoHook = true
171 | 		}
172 | 	}
173 | 
174 | 	if !hasRepo || !hasWriteRepoHook {
175 | 		var missingScopes string
176 | 		if !hasRepo && !hasWriteRepoHook {
177 | 			missingScopes = "repo, write:repo_hook"
178 | 		} else if !hasRepo {
179 | 			missingScopes = "repo"
180 | 		} else {
181 | 			missingScopes = "write:repo_hook"
182 | 		}
183 | 		errorf("Invalid token for %s/%s, missing scopes: %s... had: %v",
184 | 			user,
185 | 			repo,
186 | 			missingScopes,
187 | 			scopes)
188 | 		return
189 | 	}
190 | 
191 | 	log.Infof(ctx, "Validated repo %s/%s", user, repo)
192 | 
193 | 	err = retry(ctx, func() (resp *github.Response, err error) {
194 | 		_, resp, err = githubClient.Repositories.Get(ctx, user, repo)
195 | 		return
196 | 	})
197 | 
198 | 	if err != nil {
199 | 		errorf("Can't validate repo %s/%s: %s", user, repo, err.Error())
200 | 	}
201 | 
202 | 	err = modifyRepoData(ctx, user, repo, func(item *repoStorageData) {
203 | 		item.Status = statusHooksInitializing
204 | 	})
205 | 
206 | 	if err != nil {
207 | 		errorf("Can't change repo status: %s", err.Error())
208 | 	}
209 | 
210 | 	createHooks(ctx, user, repo)
211 | }
212 | 
213 | // hook sets up webhooks for a given repository
214 | func createHooks(ctx context.Context, userName, repoName string) {
215 | 	errorf := makeErrorf(ctx, userName, repoName)
216 | 	repoData, err := getRepoData(ctx, userName, repoName)
217 | 	if err != nil {
218 | 		errorf("Can't load repo to hook: %s", err.Error())
219 | 		return
220 | 	}
221 | 
222 | 	client := github.NewClient(oauth2.NewClient(ctx, oauth2.StaticTokenSource(
223 | 		&oauth2.Token{AccessToken: repoData.Token},
224 | 	)))
225 | 
226 | 	active := true
227 | 
228 | 	secret := make([]byte, secretSize)
229 | 	_, err = rand.Read(secret)
230 | 	if err != nil {
231 | 		errorf("Can't create secret key: %s", err.Error())
232 | 		return
233 | 	}
234 | 	secretHex := hex.EncodeToString(secret)
235 | 
236 | 	// TODO allow non-appspot urls?
237 | 	url := fmt.Sprintf("https://github-mirror-dot-%s.appspot.com/hook/%s/%s", appengine.AppID(ctx), userName, repoName)
238 | 
239 | 	log.Infof(ctx, "Creating hook for %s/%s: url `%s`", userName, repoName, url)
240 | 
241 | 	var hook *github.Hook
242 | 	err = retry(ctx, func() (resp *github.Response, err error) {
243 | 		hook, resp, err = client.Repositories.CreateHook(ctx, userName, repoName, &github.Hook{
244 | 			Events: []string{
245 | 				eventPing,
246 | 				eventStatus,
247 | 				eventPullRequest,
248 | 				eventDiffComment,
249 | 				eventIssueComment,
250 | 			},
251 | 			Active: &active,
252 | 			Config: map[string]interface{}{
253 | 				"url":          url,
254 | 				"content_type": "json",
255 | 				"secret":       secretHex,
256 | 				"insecure_ssl": false,
257 | 			},
258 | 		})
259 | 		return
260 | 	})
261 | 	if err != nil {
262 | 		errorf("Can't create hook: %s", err.Error())
263 | 		return
264 | 	}
265 | 
266 | 	if hook.ID == nil {
267 | 		errorf("No hook ID for new hook")
268 | 		return
269 | 	}
270 | 
271 | 	log.Infof(ctx, "Hook creation for %s/%s successful", userName, repoName)
272 | 
273 | 	err = modifyRepoData(ctx, userName, repoName, func(item *repoStorageData) {
274 | 		item.HookSecret = secretHex
275 | 		item.HookID = *hook.ID
276 | 	})
277 | 
278 | 	if err != nil {
279 | 		errorf("Can't set repo status to ready: %s", err.Error())
280 | 		return
281 | 	}
282 | 
283 | 	log.Infof(ctx, "Repo waiting for hook ping: %s/%s", userName, repoName)
284 | }
285 | 
286 | // deactivate deletes webhooks and forgets data for a given repository
287 | func deactivate(ctx context.Context, userName, repoName string) {
288 | 	errorf := makeErrorf(ctx, userName, repoName)
289 | 
290 | 	repoData, err := getRepoData(ctx, userName, repoName)
291 | 	if err != nil {
292 | 		errorf("Can't load repo to deactivate: %s", err.Error())
293 | 		return
294 | 	}
295 | 
296 | 	client := github.NewClient(oauth2.NewClient(ctx, oauth2.StaticTokenSource(
297 | 		&oauth2.Token{AccessToken: repoData.Token},
298 | 	)))
299 | 
300 | 	log.Infof(ctx, "Deleting hook for repository %s/%s", userName, repoName)
301 | 	err = retry(ctx, func() (resp *github.Response, err error) {
302 | 		resp, err = client.Repositories.DeleteHook(ctx, userName, repoName, repoData.HookID)
303 | 		return
304 | 	})
305 | 	if err != nil {
306 | 		errorf("Can't delete webhook: %s", err.Error())
307 | 		// Keep going; we should still delete the repository data
308 | 	} else {
309 | 		log.Infof(ctx, "Deleting hook for repository %s/%s succeeded", userName, repoName)
310 | 	}
311 | 
312 | 	log.Infof(ctx, "Deleting repository data for %s/%s", userName, repoName)
313 | 	err = deleteRepoData(ctx, userName, repoName)
314 | 	if err != nil {
315 | 		errorf("Can't delete repository data: %s", err.Error())
316 | 		return
317 | 	}
318 | }
319 | 
320 | // restartAbandonedOperations runs when the web server starts.
321 | // It goes through the repos in the data store and checks their statuses.
322 | // If they're validating or initializing, those processes will restart.
323 | // If they actually finished validating / initializing but didn't write
324 | // to the store that's fine, since all operations are indempotent; we
325 | // can redo it.
326 | func restartAbandonedOperations(ctx context.Context) {
327 | 	ctx, done := context.WithCancel(ctx)
328 | 	defer done()
329 | 
330 | 	log.Infof(ctx, "Restarting abandoned operations...")
331 | 
332 | 	repos, err := getAllRepoData(ctx)
333 | 	if err != nil {
334 | 		log.Errorf(ctx, "Can't load repos: %s", err.Error())
335 | 		return
336 | 	}
337 | 
338 | 	var wg sync.WaitGroup
339 | 	for _, repo := range repos {
340 | 		wg.Add(1)
341 | 		go func(repo repoStorageData) {
342 | 			switch repo.Status {
343 | 			case statusReady:
344 | 				log.Infof(ctx, "Repo ready: %s/%s", repo.User, repo.Repo)
345 | 			case statusError:
346 | 				log.Infof(ctx, "Repo errored out: %s/%s", repo.User, repo.Repo)
347 | 			case statusValidating:
348 | 				log.Infof(ctx, "Repo requires validation: %s/%s", repo.User, repo.Repo)
349 | 				validate(ctx, repo.User, repo.Repo)
350 | 			case statusInitializing:
351 | 				log.Infof(ctx, "Repo requires initialization: %s/%s", repo.User, repo.Repo)
352 | 			case statusHooksInitializing:
353 | 				log.Infof(ctx, "Repo requires hook initialization: %s/%s", repo.User, repo.Repo)
354 | 				createHooks(ctx, repo.User, repo.Repo)
355 | 			default:
356 | 				log.Errorf(ctx, "Unrecognized status for repo %s/%s: %s", repo.User, repo.Repo, repo.Status)
357 | 			}
358 | 			wg.Done()
359 | 		}(repo)
360 | 	}
361 | 	wg.Wait()
362 | }
363 | 
364 | // makeErrorf returns a utility function that logs a given error and then sets the repo's error information to that error
365 | func makeErrorf(ctx context.Context, userName, repoName string) func(string, ...interface{}) {
366 | 	return func(format string, params ...interface{}) {
367 | 		errText := fmt.Sprintf(format, params...)
368 | 		log.Errorf(ctx, "%s/%s: %s", userName, repoName, errText)
369 | 		err := setRepoError(ctx, userName, repoName, errText)
370 | 		if err != nil {
371 | 			log.Errorf(ctx, "Can't set repo error status for %s/%s: %s",
372 | 				userName,
373 | 				repoName,
374 | 				err.Error(),
375 | 			)
376 | 		}
377 | 	}
378 | }
379 | 


--------------------------------------------------------------------------------
/app/admin/persistent.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package main
 18 | 
 19 | // Storage for persistent repository metadata: what they are, what their keys
 20 | // are, etc.
 21 | // Uses the appengine datastore.
 22 | 
 23 | import (
 24 | 	"fmt"
 25 | 
 26 | 	"golang.org/x/net/context"
 27 | 	"google.golang.org/appengine/datastore"
 28 | )
 29 | 
 30 | type repoStorageData struct {
 31 | 	User       string
 32 | 	Repo       string
 33 | 	Token      string // TODO(jhgilles): add another layer of encryption here?
 34 | 	HookID     int64
 35 | 	HookSecret string
 36 | 	Status     string
 37 | 	ErrorCause string
 38 | }
 39 | 
 40 | type repoExistsError struct {
 41 | 	User string
 42 | 	Repo string
 43 | }
 44 | 
 45 | func (e *repoExistsError) Error() string {
 46 | 	return fmt.Sprintf("Already tracking repo: %s/%s, can't initialize",
 47 | 		e.User,
 48 | 		e.Repo,
 49 | 	)
 50 | }
 51 | 
 52 | const (
 53 | 	repoKind  = "repo"
 54 | 	emptyKind = "empty"
 55 | 
 56 | 	storageReposPath = "repos"
 57 | 
 58 | 	statusValidating        = "Validating"         // Verifying repo w/ github
 59 | 	statusInitializing      = "Initializing"       // Performing initial pull-all
 60 | 	statusHooksInitializing = "Hooks Initializing" // Setting up hooks
 61 | 	statusReady             = "Ready"              // Ready and waiting for hooks
 62 | 	statusError             = "Error"              // Hit an unrecoverable error
 63 | )
 64 | 
 65 | func initStorage(ctx context.Context) error {
 66 | 	ctx, done := context.WithCancel(ctx)
 67 | 	defer done()
 68 | 
 69 | 	rootKey := makeReposRootKey(ctx)
 70 | 	_, err := datastore.Put(ctx, rootKey, &struct{}{})
 71 | 	return err
 72 | }
 73 | 
 74 | // initRepoData is called to declare a new active repository in the
 75 | // datastore. It should run after the repo has been verified to work.
 76 | func initRepoData(ctx context.Context, user, repo, token string) error {
 77 | 	item := repoStorageData{
 78 | 		User:   user,
 79 | 		Repo:   repo,
 80 | 		Token:  token,
 81 | 		Status: statusValidating,
 82 | 	}
 83 | 	key := makeRepoKey(ctx, user, repo)
 84 | 	return datastore.RunInTransaction(ctx, func(ctx context.Context) error {
 85 | 		var currentItem repoStorageData
 86 | 		err := datastore.Get(ctx, key, &currentItem)
 87 | 
 88 | 		if err != datastore.ErrNoSuchEntity {
 89 | 			if err != nil {
 90 | 				return err
 91 | 			}
 92 | 			return &repoExistsError{
 93 | 				User: user,
 94 | 				Repo: repo,
 95 | 			}
 96 | 		}
 97 | 
 98 | 		_, err = datastore.Put(ctx, key, &item)
 99 | 		return err
100 | 	}, &datastore.TransactionOptions{})
101 | }
102 | 
103 | func modifyRepoData(ctx context.Context, user, repo string, f func(*repoStorageData)) error {
104 | 	return datastore.RunInTransaction(ctx, func(ctx context.Context) error {
105 | 		key := makeRepoKey(ctx, user, repo)
106 | 
107 | 		var item repoStorageData
108 | 
109 | 		err := datastore.Get(ctx, key, &item)
110 | 		if err != nil {
111 | 			return err
112 | 		}
113 | 
114 | 		f(&item)
115 | 
116 | 		_, err = datastore.Put(ctx, key, &item)
117 | 
118 | 		return err
119 | 	}, &datastore.TransactionOptions{})
120 | }
121 | 
122 | // setRepoError sets a repo to statusErrpr with the given cause
123 | func setRepoError(ctx context.Context, user, repo, errorCause string) error {
124 | 	return modifyRepoData(ctx, user, repo, func(item *repoStorageData) {
125 | 		item.Status = statusError
126 | 		item.ErrorCause = errorCause
127 | 	})
128 | }
129 | 
130 | // deleteRepoData does exactly what you'd expect.
131 | func deleteRepoData(ctx context.Context, user, repo string) error {
132 | 	key := makeRepoKey(ctx, user, repo)
133 | 	return datastore.Delete(ctx, key)
134 | }
135 | 
136 | // getRepoData returns the data for a single repo
137 | func getRepoData(ctx context.Context, user, repo string) (result repoStorageData, err error) {
138 | 	key := makeRepoKey(ctx, user, repo)
139 | 	err = datastore.Get(ctx, key, &result)
140 | 	return
141 | }
142 | 
143 | // getAllRepoData returns all active or errored repos.
144 | func getAllRepoData(ctx context.Context) ([]repoStorageData, error) {
145 | 	rootKey := makeReposRootKey(ctx)
146 | 	q := datastore.NewQuery(repoKind).Ancestor(rootKey)
147 | 	it := q.Run(ctx)
148 | 	current := new(repoStorageData)
149 | 	result := []repoStorageData{}
150 | 
151 | 	var err error
152 | 
153 | 	for _, err = it.Next(current); err == nil; _, err = it.Next(current) {
154 | 		result = append(result, *current)
155 | 	}
156 | 
157 | 	if err != datastore.Done {
158 | 		return nil, err
159 | 	}
160 | 
161 | 	return result, nil
162 | }
163 | 
164 | func makeReposRootKey(ctx context.Context) *datastore.Key {
165 | 	return datastore.NewKey(
166 | 		ctx,
167 | 		emptyKind,
168 | 		storageReposPath,
169 | 		0,
170 | 		nil,
171 | 	)
172 | }
173 | 
174 | func makeRepoKey(ctx context.Context, user, repo string) *datastore.Key {
175 | 	return datastore.NewKey(
176 | 		ctx,
177 | 		repoKind,
178 | 		fmt.Sprintf("%s/%s", user, repo),
179 | 		0,
180 | 		makeReposRootKey(ctx),
181 | 	)
182 | }
183 | 


--------------------------------------------------------------------------------
/app/dispatch.yaml:
--------------------------------------------------------------------------------
 1 | # Copyright 2019 Google Inc. All Rights Reserved.
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | 
15 | dispatch:
16 | - url: "/hooks/*"
17 |   service: github-mirror-hooks
18 | 
19 | - url: "/*"
20 |   service: default
21 | 


--------------------------------------------------------------------------------
/app/hooks/Dockerfile:
--------------------------------------------------------------------------------
 1 | # Copyright 2015 Google Inc. All Rights Reserved.
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | 
15 | # First stage that builds the go binary...
16 | FROM gcr.io/gcp-runtimes/go1-builder:1.12 as builder
17 | 
18 | WORKDIR /go/src/app
19 | COPY . .
20 | 
21 | RUN apt-get update -yq && \
22 |     apt-get install -yq git-core && \
23 |     export PATH="${PATH}:/usr/local/go/bin" && \
24 |     export GOPATH="/go/" && \
25 |     mkdir -p "${GOPATH}" && \
26 |     go get github.com/google/go-github/github && \
27 |     go get github.com/google/git-appraise/git-appraise && \
28 |     go get github.com/google/git-pull-request-mirror/mirror && \
29 |     go get google.golang.org/appengine && \
30 |     go get golang.org/x/oauth2 && \
31 |     go get cloud.google.com/go/compute/metadata && \
32 |     go get cloud.google.com/go/datastore && \
33 |     cp ${GOPATH}/bin/git-appraise /usr/local/bin/git-appraise
34 | 
35 | RUN export GOPATH="/go/" && \
36 |     /usr/local/go/bin/go build -o app .
37 | 
38 | # Second stage that defines the serving app...
39 | FROM gcr.io/distroless/base:latest
40 | 
41 | COPY --from=builder /usr/bin/* /usr/bin/
42 | COPY --from=builder /usr/local/bin/* /usr/local/bin/
43 | COPY --from=builder /usr/lib/git-core/* /usr/lib/git-core/
44 | COPY --from=builder /usr/share/git-core/* /usr/share/git-core/
45 | COPY --from=builder /lib/x86_64-linux-gnu/* /lib/x86_64-linux-gnu/
46 | COPY --from=builder /usr/lib/x86_64-linux-gnu/* /usr/lib/x86_64-linux-gnu/
47 | COPY --from=builder /go/src/app/app /usr/local/bin/app
48 | 
49 | CMD ["/usr/local/bin/app"]
50 | 
51 | 
52 | 


--------------------------------------------------------------------------------
/app/hooks/app.yaml:
--------------------------------------------------------------------------------
 1 | # Copyright 2015 Google Inc. All Rights Reserved.
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | 
15 | service: github-mirror
16 | runtime: custom
17 | env: flex
18 | 
19 | health_check:
20 |   enable_health_check: False
21 | 
22 | manual_scaling:
23 |   instances: 1
24 | 


--------------------------------------------------------------------------------
/app/hooks/ephemeral.go:
--------------------------------------------------------------------------------
 1 | /*
 2 | Copyright 2015 Google Inc. All Rights Reserved.
 3 | 
 4 | Licensed under the Apache License, Version 2.0 (the "License");
 5 | you may not use this file except in compliance with the License.
 6 | You may obtain a copy of the License at
 7 | 
 8 |     http://www.apache.org/licenses/LICENSE-2.0
 9 | 
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | package main
17 | 
18 | import (
19 | 	"fmt"
20 | 	"io/ioutil"
21 | 	"os"
22 | 	"os/exec"
23 | 
24 | 	"github.com/google/git-appraise/repository"
25 | 	"golang.org/x/net/context"
26 | )
27 | 
28 | const (
29 | 	remoteName      = "origin"
30 | 	notesRefPattern = "refs/notes/devtools/*"
31 | 	fetchSpec       = "+refs/pull/*:refs/pull/*"
32 | 	retryAttempts   = 10
33 | )
34 | 
35 | // Clone creates a local copy of the repository accessible at
36 | // github.com/user/repo with token, in a system temp directory.
37 | func clone(c context.Context, repoOwner, repoName, token string) (repository.Repo, error) {
38 | 	dir, err := ioutil.TempDir("", fmt.Sprintf("%s-%s", repoOwner, repoName))
39 | 	if err != nil {
40 | 		return nil, fmt.Errorf("failure creating the temporary directory for cloning: %v", err)
41 | 	}
42 | 	cloneCmd := exec.Command("git", "clone", makeRemoteURL(token, repoOwner, repoName), dir)
43 | 	if out, err := cloneCmd.CombinedOutput(); err != nil {
44 | 		return nil, fmt.Errorf("failure issuing the clone command, %v: %q", err, out)
45 | 	}
46 | 	repo, err := repository.NewGitRepo(dir)
47 | 	if err != nil {
48 | 		return nil, fmt.Errorf("failure loading the cloned repository: %v", err)
49 | 	}
50 | 	if err := repo.PullNotes(remoteName, notesRefPattern); err != nil {
51 | 		return nil, fmt.Errorf("failure pulling the git-notes: %v", err)
52 | 	}
53 | 	fetchCmd := exec.Command("git", "fetch", "origin", fetchSpec)
54 | 	fetchCmd.Dir = dir
55 | 	if _, err := fetchCmd.CombinedOutput(); err != nil {
56 | 		return nil, fmt.Errorf("failure fetching pull requests from the remote: %v", err)
57 | 	}
58 | 	configUserCmd := exec.Command("git", "config", "--local", "--add", "user.name", "Github Mirror")
59 | 	configUserCmd.Dir = dir
60 | 	if _, err := configUserCmd.CombinedOutput(); err != nil {
61 | 		return nil, fmt.Errorf("failure configuring the local git user: %v", err)
62 | 	}
63 | 	userEmail := os.Getenv("GOOGLE_CLOUD_PROJECT") + "@appspot.gserviceaccount.com"
64 | 	configEmailCmd := exec.Command("git", "config", "--local", "--add", "user.email", userEmail)
65 | 	configEmailCmd.Dir = dir
66 | 	if out, err := configEmailCmd.CombinedOutput(); err != nil {
67 | 		return nil, fmt.Errorf("failure configuring the local get user email address: %v, %q", err, out)
68 | 	}
69 | 	return repo, nil
70 | }
71 | 
72 | func syncNotes(repo repository.Repo) error {
73 | 	var err error
74 | 	for attempt := 0; attempt < retryAttempts; attempt++ {
75 | 		err = repo.PullNotes(remoteName, notesRefPattern)
76 | 		if err == nil {
77 | 			err = repo.PushNotes(remoteName, notesRefPattern)
78 | 			if err == nil {
79 | 				return err
80 | 			}
81 | 		}
82 | 	}
83 | 	return err
84 | }
85 | 
86 | // makeRemoteURL computes a URL to use with git
87 | func makeRemoteURL(token, repoOwner, repo string) string {
88 | 	return fmt.Sprintf("https://%s@github.com/%s/%s", token, repoOwner, repo)
89 | }
90 | 


--------------------------------------------------------------------------------
/app/hooks/main.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | package main
 17 | 
 18 | import (
 19 | 	"bytes"
 20 | 	"context"
 21 | 	"crypto/hmac"
 22 | 	"crypto/sha1"
 23 | 	"encoding/hex"
 24 | 	"encoding/json"
 25 | 	"fmt"
 26 | 	"io/ioutil"
 27 | 	"log"
 28 | 	"net/http"
 29 | 	"strings"
 30 | 
 31 | 	"cloud.google.com/go/datastore"
 32 | 	"github.com/google/git-pull-request-mirror/mirror"
 33 | 	"github.com/google/go-github/github"
 34 | 	"golang.org/x/oauth2"
 35 | 	"google.golang.org/appengine"
 36 | 
 37 | 	"cloud.google.com/go/compute/metadata"
 38 | )
 39 | 
 40 | const (
 41 | 	githubEventHeader     = "X-Github-Event"
 42 | 	githubSignatureHeader = "X-Hub-Signature"
 43 | 
 44 | 	eventPing         = "ping"
 45 | 	eventStatus       = "status"
 46 | 	eventPullRequest  = "pull_request"
 47 | 	eventDiffComment  = "pull_request_review_comment"
 48 | 	eventIssueComment = "issue_comment"
 49 | )
 50 | 
 51 | // makeErrorf returns a utility function that logs a given error and then sets the repo's error information to that error
 52 | func makeErrorf(ctx context.Context, c *datastore.Client, userName, repoName string) func(string, ...interface{}) {
 53 | 	return func(format string, params ...interface{}) {
 54 | 		errText := fmt.Sprintf(format, params...)
 55 | 		log.Printf("%s/%s: %s", userName, repoName, errText)
 56 | 		err := setRepoError(ctx, c, userName, repoName, errText)
 57 | 		if err != nil {
 58 | 			log.Printf("Can't set repo error status for %s/%s: %s",
 59 | 				userName,
 60 | 				repoName,
 61 | 				err.Error(),
 62 | 			)
 63 | 		}
 64 | 	}
 65 | }
 66 | 
 67 | // initialize performs initial reading and commiting for the repository
 68 | func initialize(ctx context.Context, c *datastore.Client, userName, repoName string) {
 69 | 	errorf := makeErrorf(ctx, c, userName, repoName)
 70 | 	repoData, err := getRepoData(ctx, c, userName, repoName)
 71 | 	if err != nil {
 72 | 		errorf("Can't load repo to initialize: %s", err.Error())
 73 | 		return
 74 | 	}
 75 | 
 76 | 	repo, err := clone(ctx, userName, repoName, repoData.Token)
 77 | 	if err != nil {
 78 | 		errorf("Can't clone repo: %v", err)
 79 | 		return
 80 | 	}
 81 | 
 82 | 	client := github.NewClient(oauth2.NewClient(ctx, oauth2.StaticTokenSource(
 83 | 		&oauth2.Token{AccessToken: repoData.Token},
 84 | 	)))
 85 | 
 86 | 	errChan := make(chan error, 1000)
 87 | 	nErrors := 0
 88 | 	go func() {
 89 | 		for err := range errChan {
 90 | 			errorf(err.Error())
 91 | 			nErrors++
 92 | 		}
 93 | 	}()
 94 | 
 95 | 	reviews, err := mirror.GetAllPullRequests(repo, userName, repoName, client, errChan)
 96 | 	if err != nil {
 97 | 		errorf("Can't get PRs: %s", err.Error())
 98 | 		return
 99 | 	}
100 | 
101 | 	statuses, err := mirror.GetAllStatuses(userName, repoName, client, errChan)
102 | 	if err != nil {
103 | 		errorf("Can't get statuses: %s", err.Error())
104 | 		return
105 | 	}
106 | 	close(errChan)
107 | 
108 | 	nStatuses := len(statuses)
109 | 	nReviews := len(reviews)
110 | 	logChan := make(chan string, 1000)
111 | 	go func() {
112 | 		for msg := range logChan {
113 | 			log.Printf(msg)
114 | 		}
115 | 	}()
116 | 	log.Printf("Done reading! Read %d statuses, %d PRs", nStatuses, nReviews)
117 | 	log.Printf("Committing...\n")
118 | 	if err := mirror.WriteNewReports(statuses, repo, logChan); err != nil {
119 | 		errorf(err.Error())
120 | 		return
121 | 	}
122 | 	if err := mirror.WriteNewReviews(reviews, repo, logChan); err != nil {
123 | 		errorf(err.Error())
124 | 		return
125 | 	}
126 | 	close(logChan)
127 | 	err = syncNotes(repo)
128 | 	if err != nil {
129 | 		errorf("Error pushing initialization changes for %s/%s: %s",
130 | 			userName,
131 | 			repoName,
132 | 			err.Error())
133 | 		return
134 | 	}
135 | 	log.Printf("Success initializing %s/%s", userName, repoName)
136 | 
137 | 	err = modifyRepoData(ctx, c, userName, repoName, func(item *repoStorageData) {
138 | 		item.Status = statusReady
139 | 		item.ErrorCause = ""
140 | 	})
141 | 
142 | 	if err != nil {
143 | 		errorf("Can't change repo status for %s/%s: %s",
144 | 			userName,
145 | 			repoName,
146 | 			err.Error(),
147 | 		)
148 | 	}
149 | }
150 | 
151 | // All webhooks are sent a "ping" event on creation
152 | func pingHook(ctx context.Context, c *datastore.Client, userName, repoName string, repoData repoStorageData, content []byte) {
153 | 	var payload struct {
154 | 		Zen    string `json:"zen"`
155 | 		HookID int    `json:"hook_id"`
156 | 	}
157 | 
158 | 	err := json.Unmarshal(content, &payload)
159 | 	if err != nil {
160 | 		log.Printf("Can't parse payload for ping hook: %s, %s", err.Error(), content)
161 | 		return
162 | 	}
163 | 
164 | 	err = modifyRepoData(ctx, c, userName, repoName, func(item *repoStorageData) {
165 | 		item.Status = statusInitializing
166 | 	})
167 | 
168 | 	if err != nil {
169 | 		log.Printf("Can't set repo %s/%s to initializing: %s", userName, repoName, err.Error())
170 | 	}
171 | 
172 | 	// Pass off to initialization
173 | 	initialize(ctx, c, userName, repoName)
174 | }
175 | 
176 | type hookHandler struct {
177 | 	projectID string
178 | }
179 | 
180 | func (h *hookHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
181 | 	ctx := req.Context()
182 | 
183 | 	sigHex := req.Header.Get(githubSignatureHeader)
184 | 	if !strings.HasPrefix(sigHex, "sha1=") || strings.TrimPrefix(sigHex, "sha1=") == "" {
185 | 		log.Printf("Hook hit with no signature")
186 | 		http.Error(w, "Webhook requires "+githubSignatureHeader+" header", http.StatusBadRequest)
187 | 		return
188 | 	}
189 | 
190 | 	sig, err := hex.DecodeString(strings.TrimPrefix(sigHex, "sha1="))
191 | 	if err != nil {
192 | 		log.Printf("Hook can't decode hex signature `%s`: %s", sigHex, err.Error())
193 | 		http.Error(w, "Can't decode signature", http.StatusBadRequest)
194 | 		return
195 | 	}
196 | 
197 | 	content, err := ioutil.ReadAll(req.Body)
198 | 	if err != nil {
199 | 		log.Printf("Hook request error: %s", err.Error())
200 | 		http.Error(w, "Can't read request body", http.StatusInternalServerError)
201 | 		return
202 | 	}
203 | 
204 | 	event := req.Header.Get(githubEventHeader)
205 | 	if event == "" {
206 | 		log.Printf("Hook hit with no event type")
207 | 		http.Error(w, "Webhook requires "+githubEventHeader+" header", http.StatusBadRequest)
208 | 		return
209 | 	}
210 | 
211 | 	pathParts := strings.Split(req.URL.Path, "/")
212 | 	if len(pathParts) != 4 {
213 | 		log.Printf("Hook hit with invalid path length: %d", len(pathParts))
214 | 		http.Error(w, "Invalid /hook/:user/:repo URL", http.StatusBadRequest)
215 | 		return
216 | 	}
217 | 
218 | 	userName := pathParts[2]
219 | 	repoName := pathParts[3]
220 | 
221 | 	c, err := datastore.NewClient(ctx, h.projectID)
222 | 	if err != nil {
223 | 		log.Printf("Hook cannot connect to the datastore: %v", err)
224 | 		http.Error(w, "Cannot connect to the datastore", http.StatusInternalServerError)
225 | 		return
226 | 	}
227 | 
228 | 	repo, err := getRepoData(ctx, c, userName, repoName)
229 | 	if err != nil {
230 | 		log.Printf("Hook can't retrieve repo: %s", err.Error())
231 | 		http.Error(w, "Can't retrieve repo information", http.StatusInternalServerError)
232 | 		return
233 | 	}
234 | 
235 | 	mac := hmac.New(sha1.New, []byte(repo.HookSecret))
236 | 	mac.Write(content)
237 | 	expectedSig := mac.Sum(nil)
238 | 	if !bytes.Equal(expectedSig, sig) {
239 | 		log.Printf("Hook hit with invalid signature; '%x' vs. '%x'", expectedSig, sig)
240 | 		http.Error(w, "Invalid signature", http.StatusBadRequest)
241 | 		return
242 | 	}
243 | 
244 | 	go func() {
245 | 		ctx, done := context.WithCancel(context.Background())
246 | 		defer done()
247 | 
248 | 		if event == eventPing {
249 | 			pingHook(ctx, c, userName, repoName, repo, content)
250 | 			return
251 | 		}
252 | 		initialize(ctx, c, userName, repoName)
253 | 	}()
254 | 	w.WriteHeader(http.StatusOK)
255 | }
256 | 
257 | func main() {
258 | 	projectID, err := metadata.ProjectID()
259 | 	if err != nil {
260 | 		log.Fatalf("Failed to read the project ID from the metadata server: %v", err)
261 | 	}
262 | 
263 | 	http.Handle("/hook/", &hookHandler{
264 | 		projectID: projectID,
265 | 	})
266 | 
267 | 	appengine.Main()
268 | }
269 | 


--------------------------------------------------------------------------------
/app/hooks/persistent.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package main
 18 | 
 19 | // Storage for persistent repository metadata: what they are, what their keys
 20 | // are, etc.
 21 | // Uses standarad datastore client library.
 22 | 
 23 | import (
 24 | 	"fmt"
 25 | 
 26 | 	"cloud.google.com/go/datastore"
 27 | 	"golang.org/x/net/context"
 28 | )
 29 | 
 30 | type repoStorageData struct {
 31 | 	User       string
 32 | 	Repo       string
 33 | 	Token      string // TODO(jhgilles): add another layer of encryption here?
 34 | 	HookID     int
 35 | 	HookSecret string
 36 | 	Status     string
 37 | 	ErrorCause string
 38 | }
 39 | 
 40 | const (
 41 | 	repoKind  = "repo"
 42 | 	emptyKind = "empty"
 43 | 
 44 | 	storageReposPath = "repos"
 45 | 
 46 | 	statusValidating        = "Validating"         // Verifying repo w/ github
 47 | 	statusInitializing      = "Initializing"       // Performing initial pull-all
 48 | 	statusHooksInitializing = "Hooks Initializing" // Setting up hooks
 49 | 	statusReady             = "Ready"              // Ready and waiting for hooks
 50 | 	statusError             = "Error"              // Hit an unrecoverable error
 51 | )
 52 | 
 53 | // setRepoError sets a repo to statusErrpr with the given cause
 54 | func setRepoError(ctx context.Context, c *datastore.Client, user, repo, errorCause string) error {
 55 | 	return modifyRepoData(ctx, c, user, repo, func(item *repoStorageData) {
 56 | 		item.Status = statusError
 57 | 		item.ErrorCause = errorCause
 58 | 	})
 59 | }
 60 | 
 61 | func modifyRepoData(ctx context.Context, c *datastore.Client, user, repo string, f func(*repoStorageData)) error {
 62 | 	_, err := c.RunInTransaction(ctx, func(txn *datastore.Transaction) error {
 63 | 		key := makeRepoKey(user, repo)
 64 | 
 65 | 		var item repoStorageData
 66 | 		if err := c.Get(ctx, key, &item); err != nil {
 67 | 			return err
 68 | 		}
 69 | 
 70 | 		f(&item)
 71 | 		if _, err := c.Put(ctx, key, &item); err != nil {
 72 | 			return err
 73 | 		}
 74 | 		return nil
 75 | 	})
 76 | 	return err
 77 | }
 78 | 
 79 | // getRepoData returns the data for a single repo
 80 | func getRepoData(ctx context.Context, c *datastore.Client, user, repo string) (result repoStorageData, err error) {
 81 | 	key := makeRepoKey(user, repo)
 82 | 	err = c.Get(ctx, key, &result)
 83 | 	return result, err
 84 | }
 85 | 
 86 | func makeReposRootKey() *datastore.Key {
 87 | 	return datastore.NameKey(
 88 | 		emptyKind,
 89 | 		storageReposPath,
 90 | 		nil,
 91 | 	)
 92 | }
 93 | 
 94 | func makeRepoKey(user, repo string) *datastore.Key {
 95 | 	return datastore.NameKey(
 96 | 		repoKind,
 97 | 		fmt.Sprintf("%s/%s", user, repo),
 98 | 		makeReposRootKey(),
 99 | 	)
100 | }
101 | 


--------------------------------------------------------------------------------
/auth/auth.go:
--------------------------------------------------------------------------------
 1 | /*
 2 | Copyright 2015 Google Inc. All Rights Reserved.
 3 | 
 4 | Licensed under the Apache License, Version 2.0 (the "License");
 5 | you may not use this file except in compliance with the License.
 6 | You may obtain a copy of the License at
 7 | 
 8 |     http://www.apache.org/licenses/LICENSE-2.0
 9 | 
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | 
17 | // Package auth provides helper methods for generating clients for the GitHub API.
18 | //
19 | // This includes building authentication into the generated client (when applicable).
20 | //
21 | // Note that we don't provide username/password authentication; It's both insecure
22 | // and more complex to implement.
23 | package auth
24 | 
25 | import (
26 | 	"context"
27 | 	"fmt"
28 | 	"os"
29 | 
30 | 	"github.com/google/go-github/github"
31 | 	"golang.org/x/oauth2"
32 | )
33 | 
34 | const (
35 | 	// TokenHelp is a human-friendly string that can be used to describe token requirements in usage messages.
36 | 	TokenHelp = `You can generate a token at: https://github.com/settings/tokens
37 | Note that the 'public_repo' scope is needed for public repositories,
38 | And the 'repo' scope is needed for private repositories.
39 | `
40 | )
41 | 
42 | // UnauthenticatedClient builds a github client that uses http.Client's default
43 | // HTTP transport.
44 | // The client will be insecure and extremely rate-limited; non-authenticated
45 | // users are limited to 60 requests / hour.
46 | func UnauthenticatedClient() *github.Client {
47 | 	return github.NewClient(nil)
48 | }
49 | 
50 | // TokenClient takes an oauth token and returns an authenticated github client.
51 | // The client is guaranteed to work.
52 | func TokenClient(token string) *github.Client {
53 | 	httpClient := oauth2.NewClient(
54 | 		oauth2.NoContext,
55 | 		oauth2.StaticTokenSource(
56 | 			&oauth2.Token{AccessToken: token},
57 | 		),
58 | 	)
59 | 
60 | 	githubClient := github.NewClient(httpClient)
61 | 
62 | 	_, _, err := githubClient.Users.Get(context.TODO(), "")
63 | 
64 | 	if err != nil {
65 | 		fmt.Println("Token error: ", err)
66 | 		fmt.Println(TokenHelp)
67 | 		os.Exit(1)
68 | 	}
69 | 
70 | 	return githubClient
71 | }
72 | 


--------------------------------------------------------------------------------
/batch/batch.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | // Package batch is the source for a command-line tool to pull all of the data
 18 | // from a Github repository and write it into a local clone of that repository.
 19 | //
 20 | // You need to clone the repository yourself (including the "refs/pull/*" refs)
 21 | // before running the tool.
 22 | //
 23 | // Example Usage:
 24 | //    git clone https://github.com/google/git-appraise git-appraise/
 25 | //    cd git-appraise
 26 | //    git fetch origin '+refs/pull/*:refs/pull/*'
 27 | //    ~/bin/github-mirror --target google/git-appraise --local ./ -auth-token <YOUR_AUTH_TOKEN>
 28 | //
 29 | // Note that the "-auth-token" flag is optional, but highly recommended. Without it
 30 | // your API requests will be throttled to 60 per hour.
 31 | 
 32 | package main
 33 | 
 34 | import (
 35 | 	"context"
 36 | 	"flag"
 37 | 	"fmt"
 38 | 	"io/ioutil"
 39 | 	"log"
 40 | 	"os"
 41 | 	"strings"
 42 | 
 43 | 	"github.com/google/git-appraise/repository"
 44 | 	"github.com/google/go-github/github"
 45 | 
 46 | 	"github.com/google/git-pull-request-mirror/auth"
 47 | 	"github.com/google/git-pull-request-mirror/mirror"
 48 | )
 49 | 
 50 | var remoteRepository = flag.String("target", "", "Github repository to read data from")
 51 | var localRepositoryDir = flag.String("local", ".", "Local repository to write notes to")
 52 | var token = flag.String("auth-token", "", "Github OAuth token with either the `repo' or `public_repo' scopes: https://github.com/settings/tokens")
 53 | var quiet = flag.Bool("quiet", false, "Don't log information to stdout")
 54 | 
 55 | func usage(errorMessage string) {
 56 | 	fmt.Fprintln(os.Stderr, errorMessage)
 57 | 	flag.Usage()
 58 | 	os.Exit(1)
 59 | }
 60 | 
 61 | func main() {
 62 | 	flag.Parse()
 63 | 	splitTarget := strings.Split(*remoteRepository, "/")
 64 | 	if len(splitTarget) != 2 {
 65 | 		usage("Target repository is required, in the format `user/repo'")
 66 | 	}
 67 | 	userName := splitTarget[0]
 68 | 	repoName := splitTarget[1]
 69 | 
 70 | 	localDirInfo, err := os.Stat(*localRepositoryDir)
 71 | 	if err != nil {
 72 | 		log.Fatal(err)
 73 | 	}
 74 | 	if !localDirInfo.IsDir() {
 75 | 		usage("Local repository must be a directory")
 76 | 	}
 77 | 
 78 | 	local, err := repository.NewGitRepo(*localRepositoryDir)
 79 | 	if err != nil {
 80 | 		log.Fatal("Couldn't open local repository: ", err.Error(), "\n",
 81 | 			"Make sure you clone the remote repository locally first!")
 82 | 	}
 83 | 
 84 | 	tokenAuth := *token != ""
 85 | 	if !tokenAuth {
 86 | 		fmt.Fprintln(os.Stderr, "Not using authentication. Note that this will be EXTREMELY SLOW;")
 87 | 		fmt.Fprintln(os.Stderr, "you get 60 requests to the github API per hour.")
 88 | 		fmt.Fprint(os.Stderr, auth.TokenHelp)
 89 | 	}
 90 | 
 91 | 	var client *github.Client
 92 | 	if tokenAuth {
 93 | 		client = auth.TokenClient(*token)
 94 | 	} else {
 95 | 		client = auth.UnauthenticatedClient()
 96 | 	}
 97 | 
 98 | 	_, _, err = client.Repositories.Get(context.TODO(), userName, repoName)
 99 | 	if err != nil {
100 | 		log.Fatal("Error fetching repository info: ", err.Error())
101 | 	}
102 | 
103 | 	errOutput := make(chan error, 1000)
104 | 	nErrors := 0
105 | 	go func() {
106 | 		for err := range errOutput {
107 | 			if !*quiet {
108 | 				log.Println(err)
109 | 			}
110 | 			nErrors++
111 | 		}
112 | 	}()
113 | 	statuses, err := mirror.GetAllStatuses(userName, repoName, client, errOutput)
114 | 	if err != nil {
115 | 		log.Fatal("Error reading statuses: ", err.Error())
116 | 	}
117 | 	reviews, err := mirror.GetAllPullRequests(local, userName, repoName, client, errOutput)
118 | 	if err != nil {
119 | 		log.Fatal("Error reading pull requests: ", err.Error())
120 | 	}
121 | 	close(errOutput)
122 | 
123 | 	nStatuses := len(statuses)
124 | 	nReviews := len(reviews)
125 | 	var l *log.Logger
126 | 	if *quiet {
127 | 		l = log.New(ioutil.Discard, "", 0)
128 | 	} else {
129 | 		l = log.New(os.Stdout, "", 0)
130 | 	}
131 | 	logChan := make(chan string, 1000)
132 | 	go func() {
133 | 		for msg := range logChan {
134 | 			l.Println(msg)
135 | 		}
136 | 	}()
137 | 
138 | 	l.Printf("Done reading! Read %d statuses, %d PRs", nStatuses, nReviews)
139 | 	l.Printf("Committing...\n")
140 | 	if err := mirror.WriteNewReports(statuses, local, logChan); err != nil {
141 | 		log.Fatal(err)
142 | 	}
143 | 	if err := mirror.WriteNewReviews(reviews, local, logChan); err != nil {
144 | 		log.Fatal(err)
145 | 	}
146 | 	close(logChan)
147 | 
148 | 	l.Printf("Done! Hit %d errors", nErrors)
149 | 	if nErrors > 0 {
150 | 		os.Exit(1)
151 | 	}
152 | }
153 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/google/git-pull-request-mirror
 2 | 
 3 | go 1.14
 4 | 
 5 | require (
 6 | 	cloud.google.com/go v0.57.0
 7 | 	cloud.google.com/go/datastore v1.1.0
 8 | 	github.com/google/git-appraise v0.0.0-20200404013623-45703e83847b
 9 | 	github.com/google/go-github v17.0.0+incompatible
10 | 	github.com/google/go-querystring v1.0.0 // indirect
11 | 	golang.org/x/net v0.0.0-20200506145744-7e3656a0809f
12 | 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
13 | 	google.golang.org/appengine v1.6.6
14 | )
15 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
  1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
  2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
  3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
  4 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
  5 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
  6 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
  7 | cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
  8 | cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
  9 | cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
 10 | cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
 11 | cloud.google.com/go v0.57.0 h1:EpMNVUorLiZIELdMZbCYX/ByTFCdoYopYAGxaGVz9ms=
 12 | cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 13 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 14 | cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 15 | cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 16 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 17 | cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ=
 18 | cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 19 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 20 | cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 21 | cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
 22 | cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 23 | cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 24 | cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 25 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 26 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 27 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 28 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 29 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 30 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 31 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 32 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 33 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 34 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 35 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 36 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 37 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 38 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 39 | github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 40 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 41 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 42 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 43 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 44 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 45 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 46 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 47 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 48 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 49 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 50 | github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 51 | github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 52 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 53 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 54 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 55 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 56 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 57 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 58 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 59 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 60 | github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
 61 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 62 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 63 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 64 | github.com/google/git-appraise v0.0.0-20200404013623-45703e83847b h1:2L5JXjTR6chAmgAkSZdPsrEiTkm0ln36d3VzcGeVjAQ=
 65 | github.com/google/git-appraise v0.0.0-20200404013623-45703e83847b/go.mod h1:KfFFhDMvZpwTFSo3XF/SgiVVESM4vbCd0IVtvYjneF8=
 66 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 67 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 68 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 69 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 70 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 71 | github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 72 | github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 73 | github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 74 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 75 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 76 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 77 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 78 | github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 79 | github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 80 | github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 81 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 82 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 83 | github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
 84 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 85 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 86 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 87 | github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 88 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 89 | github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 90 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 91 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 92 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 93 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 94 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 95 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 96 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 97 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 98 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 99 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
100 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
101 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
102 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
103 | go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
104 | go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
105 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
106 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
107 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
108 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
109 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
110 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
111 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
112 | golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
113 | golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
114 | golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
115 | golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
116 | golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
117 | golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
118 | golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
119 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
120 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
121 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
122 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
123 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
124 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
125 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
126 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
127 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
128 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
129 | golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
130 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
131 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
132 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
133 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
134 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
135 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
136 | golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
137 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
138 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
139 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
140 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
141 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
142 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
143 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
144 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
145 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
146 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
147 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
148 | golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
149 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
150 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
151 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
152 | golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
153 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
154 | golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
155 | golang.org/x/net v0.0.0-20200506145744-7e3656a0809f h1:QBjCr1Fz5kw158VqdE9JfI9cJnl/ymnJWAdMuinqL7Y=
156 | golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
157 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
158 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
159 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
160 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
161 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
162 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
163 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
164 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
165 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
166 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
167 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
168 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
169 | golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
170 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
171 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
172 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
173 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
174 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
175 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
176 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
177 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
178 | golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
179 | golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
180 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
181 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
182 | golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
183 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
184 | golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
185 | golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
186 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
187 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
188 | golang.org/x/sys v0.0.0-20200501052902-10377860bb8e h1:hq86ru83GdWTlfQFZGO4nZJTU4Bs2wfHl8oFHRaXsfc=
189 | golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
190 | golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
191 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
192 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
193 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
194 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
195 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
196 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
197 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
198 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
199 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
200 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
201 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
202 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
203 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
204 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
205 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
206 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
207 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
208 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
209 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
210 | golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
211 | golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
212 | golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
213 | golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
214 | golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
215 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
216 | golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
217 | golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
218 | golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
219 | golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
220 | golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
221 | golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
222 | golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
223 | golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
224 | golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
225 | golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
226 | golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
227 | golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
228 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
229 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
230 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
231 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
232 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
233 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
234 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
235 | google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
236 | google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
237 | google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
238 | google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
239 | google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
240 | google.golang.org/api v0.22.0 h1:J1Pl9P2lnmYFSJvgs70DKELqHNh8CNWXPbud4njEE2s=
241 | google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
242 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
243 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
244 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
245 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
246 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
247 | google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
248 | google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
249 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
250 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
251 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
252 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
253 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
254 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
255 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
256 | google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
257 | google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
258 | google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
259 | google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
260 | google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
261 | google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
262 | google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
263 | google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
264 | google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
265 | google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
266 | google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84 h1:pSLkPbrjnPyLDYUO2VM9mDLqo2V6CFBY84lFSZAfoi4=
267 | google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
268 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
269 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
270 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
271 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
272 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
273 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
274 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
275 | google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
276 | google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4=
277 | google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
278 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
279 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
280 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
281 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
282 | google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
283 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
284 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
285 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
286 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
287 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
288 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
289 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
290 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
291 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
292 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
293 | honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
294 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
295 | rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
296 | rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
297 | 


--------------------------------------------------------------------------------
/mirror/conversions.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package mirror
 18 | 
 19 | import (
 20 | 	"errors"
 21 | 	"fmt"
 22 | 	"regexp"
 23 | 	"strconv"
 24 | 	"strings"
 25 | 	"time"
 26 | 
 27 | 	"github.com/google/git-appraise/repository"
 28 | 	"github.com/google/git-appraise/review"
 29 | 	"github.com/google/git-appraise/review/ci"
 30 | 	"github.com/google/git-appraise/review/comment"
 31 | 	"github.com/google/git-appraise/review/request"
 32 | 	github "github.com/google/go-github/github"
 33 | )
 34 | 
 35 | var (
 36 | 	// ErrNoTimestamp is exactly what it sounds like.
 37 | 	ErrNoTimestamp = errors.New("Github status contained no timestamp")
 38 | 	// ErrInvalidState is returned when a github repository status has an
 39 | 	// invalid state.
 40 | 	ErrInvalidState = errors.New(`Github status state was not "success", "failure", "error", "pending", or null`)
 41 | 	// ErrInsufficientInfo is returned when not enough information is given
 42 | 	// to perform a valid conversion.
 43 | 	ErrInsufficientInfo = errors.New("insufficient data for meaningful conversion")
 44 | )
 45 | 
 46 | // ConvertTime converts a Time instance into the serialized string used in the git-appraise JSON formats.
 47 | func ConvertTime(t time.Time) string {
 48 | 	return fmt.Sprintf("%10d", t.Unix())
 49 | }
 50 | 
 51 | // ConvertStatus converts a commit status fetched from the GitHub API into a CI report.
 52 | func ConvertStatus(repoStatus *github.RepoStatus) (*ci.Report, error) {
 53 | 	result := ci.Report{}
 54 | 	if repoStatus.UpdatedAt != nil {
 55 | 		result.Timestamp = ConvertTime(*repoStatus.UpdatedAt)
 56 | 	} else if repoStatus.CreatedAt != nil {
 57 | 		result.Timestamp = ConvertTime(*repoStatus.CreatedAt)
 58 | 	} else {
 59 | 		return nil, ErrNoTimestamp
 60 | 	}
 61 | 
 62 | 	if repoStatus.State != nil {
 63 | 		if *repoStatus.State == "success" {
 64 | 			result.Status = ci.StatusSuccess
 65 | 		} else if *repoStatus.State == "failure" || *repoStatus.State == "error" {
 66 | 			result.Status = ci.StatusFailure
 67 | 		} else if *repoStatus.State != "pending" {
 68 | 			return nil, ErrInvalidState
 69 | 		}
 70 | 	}
 71 | 
 72 | 	if repoStatus.TargetURL != nil {
 73 | 		result.URL = *repoStatus.TargetURL
 74 | 	}
 75 | 
 76 | 	if repoStatus.Context != nil {
 77 | 		result.Agent = *repoStatus.Context
 78 | 	}
 79 | 	return &result, nil
 80 | }
 81 | 
 82 | // ConvertPullRequest converts a pull request fetched from the GitHub API into a review request.
 83 | func ConvertPullRequest(pr *github.PullRequest) (*request.Request, error) {
 84 | 	if pr.Number == nil || pr.User.Login == nil ||
 85 | 		pr.Base == nil || pr.Base.Ref == nil || pr.Base.SHA == nil ||
 86 | 		(pr.CreatedAt == nil && pr.UpdatedAt == nil) {
 87 | 		return nil, ErrInsufficientInfo
 88 | 	}
 89 | 
 90 | 	var timestamp string
 91 | 	if pr.UpdatedAt != nil {
 92 | 		timestamp = ConvertTime(*pr.UpdatedAt)
 93 | 	} else {
 94 | 		timestamp = ConvertTime(*pr.CreatedAt)
 95 | 	}
 96 | 
 97 | 	var targetRef string
 98 | 	if strings.HasPrefix(*pr.Base.Ref, "refs/heads") {
 99 | 		targetRef = *pr.Base.Ref
100 | 	} else {
101 | 		targetRef = fmt.Sprintf("refs/heads/%s", *pr.Base.Ref)
102 | 	}
103 | 
104 | 	var description string
105 | 	if pr.Title != nil {
106 | 		description = *pr.Title
107 | 	}
108 | 	if pr.Body != nil && *pr.Body != "" {
109 | 		description += "\n\n" + *pr.Body
110 | 	}
111 | 
112 | 	r := request.Request{
113 | 		Timestamp:   timestamp,
114 | 		ReviewRef:   fmt.Sprintf("refs/pull/%d/head", *pr.Number),
115 | 		TargetRef:   targetRef,
116 | 		Requester:   *pr.User.Login,
117 | 		Description: description,
118 | 	}
119 | 	return &r, nil
120 | }
121 | 
122 | // ConvertIssueComment converts a comment on the issue associated with a pull request into a git-appraise review comment.
123 | func ConvertIssueComment(issueComment *github.IssueComment) (*comment.Comment, error) {
124 | 	if issueComment.User == nil || issueComment.User.Login == nil || issueComment.Body == nil ||
125 | 		(issueComment.UpdatedAt == nil && issueComment.CreatedAt == nil) {
126 | 		return nil, ErrInsufficientInfo
127 | 	}
128 | 
129 | 	var timestamp string
130 | 	if issueComment.UpdatedAt != nil {
131 | 		timestamp = ConvertTime(*issueComment.UpdatedAt)
132 | 	}
133 | 	if issueComment.CreatedAt != nil {
134 | 		timestamp = ConvertTime(*issueComment.CreatedAt)
135 | 	}
136 | 
137 | 	c := comment.Comment{
138 | 		Timestamp:   timestamp,
139 | 		Author:      *issueComment.User.Login,
140 | 		Description: *issueComment.Body,
141 | 	}
142 | 	return &c, nil
143 | }
144 | 
145 | // ConvertDiffComment converts a comment on the diff associated with a pull request into a git-appraise review comment.
146 | func ConvertDiffComment(diffComment *github.PullRequestComment) (*comment.Comment, error) {
147 | 	if diffComment.User == nil || diffComment.User.Login == nil || diffComment.Body == nil ||
148 | 		(diffComment.UpdatedAt == nil && diffComment.CreatedAt == nil) ||
149 | 		diffComment.OriginalCommitID == nil {
150 | 		return nil, ErrInsufficientInfo
151 | 	}
152 | 
153 | 	var timestamp string
154 | 	if diffComment.UpdatedAt != nil {
155 | 		timestamp = ConvertTime(*diffComment.UpdatedAt)
156 | 	}
157 | 	if diffComment.CreatedAt != nil {
158 | 		timestamp = ConvertTime(*diffComment.CreatedAt)
159 | 	}
160 | 
161 | 	c := comment.Comment{
162 | 		Timestamp:   timestamp,
163 | 		Author:      *diffComment.User.Login,
164 | 		Description: *diffComment.Body,
165 | 		Location: &comment.Location{
166 | 			Commit: *diffComment.OriginalCommitID,
167 | 		},
168 | 	}
169 | 	if diffComment.Path != nil {
170 | 		c.Location.Path = *diffComment.Path
171 | 		if diffComment.DiffHunk != nil {
172 | 			startLine, err := commentStartLine(diffComment)
173 | 			if err != nil {
174 | 				return nil, err
175 | 			}
176 | 			c.Location.Range = &comment.Range{
177 | 				StartLine: startLine,
178 | 			}
179 | 		}
180 | 	}
181 | 	return &c, nil
182 | }
183 | 
184 | // ConvertPullRequestToReview converts a pull request from the GitHub API into a git-appraise review.
185 | //
186 | // Since the GitHub API returns pull request data in three different places (the PullRequest
187 | // object, the list of comments on the corresponding issue, and the list of diff comments),
188 | // all three must be supplied.
189 | //
190 | // This method requires a local clone of the repository in order to compute the locations of
191 | // the different commits in the review.
192 | func ConvertPullRequestToReview(pr *github.PullRequest, issueComments []*github.IssueComment, diffComments []*github.PullRequestComment, repo repository.Repo) (*review.Review, error) {
193 | 	request, err := ConvertPullRequest(pr)
194 | 	if err != nil {
195 | 		return nil, err
196 | 	}
197 | 	revision, err := computeReviewStartingCommit(pr, repo)
198 | 	if err != nil {
199 | 		return nil, err
200 | 	}
201 | 	mergeBase, err := repo.MergeBase(request.TargetRef, revision)
202 | 	if err != nil {
203 | 		return nil, err
204 | 	}
205 | 	if mergeBase != revision {
206 | 		request.BaseCommit = mergeBase
207 | 	}
208 | 
209 | 	var comments []review.CommentThread
210 | 	for _, issueComment := range issueComments {
211 | 		c, err := ConvertIssueComment(issueComment)
212 | 		if err != nil {
213 | 			return nil, err
214 | 		}
215 | 		hash, err := c.Hash()
216 | 		if err != nil {
217 | 			return nil, err
218 | 		}
219 | 		comments = append(comments, review.CommentThread{
220 | 			Hash:    hash,
221 | 			Comment: *c,
222 | 		})
223 | 	}
224 | 	for _, diffComment := range diffComments {
225 | 		c, err := ConvertDiffComment(diffComment)
226 | 		if err != nil {
227 | 			return nil, err
228 | 		}
229 | 		hash, err := c.Hash()
230 | 		if err != nil {
231 | 			return nil, err
232 | 		}
233 | 		comments = append(comments, review.CommentThread{
234 | 			Hash:    hash,
235 | 			Comment: *c,
236 | 		})
237 | 	}
238 | 	r := review.Review{
239 | 		Summary: &review.Summary{
240 | 			Repo:     repo,
241 | 			Revision: revision,
242 | 			Request:  *request,
243 | 			Comments: comments,
244 | 		},
245 | 	}
246 | 
247 | 	return &r, nil
248 | }
249 | 
250 | // commentStartLine takes a PullRequestComment and returns the comment's start line.
251 | func commentStartLine(diffComment *github.PullRequestComment) (uint32, error) {
252 | 	// This takes some contortions to figure out. The diffComment has a "position"
253 | 	// field, but that is not the position of the comment. Instead, that is the
254 | 	// position of the comment within the diff. Furthermore, this diff is a unified diff,
255 | 	// so that position number includes lines which the diff removes. On the flip side,
256 | 	// the diff included is only the portion of the diff that precedes the comment, so
257 | 	// we don't actually need the position field at all.
258 | 	//
259 | 	// As such, to get the actual line number we have to:
260 | 	// 1. Parse the diff to find out at which line it starts.
261 | 	// 2. Split the diff by lines.
262 | 	// 3. Remove all of the lines that start with "-" (indicating a removal).
263 | 	// 4. Count the number of lines left after #3.
264 | 	//
265 | 	// Finally, we add the results from #1 and #4 to get the actual start line.
266 | 	diffLines := strings.Split(*diffComment.DiffHunk, "\n")
267 | 	if len(diffLines) < 2 {
268 | 		// This shouldn't happen; it means we recieved an invalid hunk from GitHub.
269 | 		return 0, fmt.Errorf("Insufficient comment diff-hunk: %q", *diffComment.DiffHunk)
270 | 	}
271 | 
272 | 	// The first line of the hunk should have the following format:
273 | 	//  @@ -lhs-start-line[,lhs-end-line] +rhs-start-line[,rhs-end-line] @@...
274 | 	// ... what we care about is the rhs-start-line.
275 | 	hunkStartPattern := regexp.MustCompile("@@ -([[:digit:]]+)(,[[:digit:]]+)? \\+([[:digit:]]+)(,[[:digit:]]+)? @@")
276 | 	hunkStartParts := hunkStartPattern.FindStringSubmatch(diffLines[0])
277 | 	if len(hunkStartParts) < 4 {
278 | 		// This shouldn't happen; it means the start of the hunk is malformed
279 | 		return 0, fmt.Errorf("Mallformed diff-hunk first line: %q", diffLines[0])
280 | 	}
281 | 	rhsStartLineString := hunkStartParts[3]
282 | 	diffPosition, err := strconv.Atoi(rhsStartLineString)
283 | 	if err != nil {
284 | 		return 0, err
285 | 	}
286 | 	if len(diffLines) > 1 {
287 | 		for _, line := range diffLines[1:] {
288 | 			if !strings.HasPrefix(line, "-") {
289 | 				diffPosition = diffPosition + 1
290 | 			}
291 | 		}
292 | 	}
293 | 	return uint32(diffPosition), nil
294 | }
295 | 
296 | // computeReviewStartingCommit computes the first commit in the review.
297 | func computeReviewStartingCommit(pr *github.PullRequest, repo repository.Repo) (string, error) {
298 | 	if pr.Base == nil || pr.Base.SHA == nil ||
299 | 		pr.Head == nil || pr.Head.SHA == nil {
300 | 		return "", ErrInsufficientInfo
301 | 	}
302 | 
303 | 	prCommits, err := repo.ListCommitsBetween(*pr.Base.SHA, *pr.Head.SHA)
304 | 	if err != nil {
305 | 		return "", err
306 | 	}
307 | 	if len(prCommits) == 0 {
308 | 		return *pr.Head.SHA, nil
309 | 	}
310 | 	return prCommits[0], nil
311 | }
312 | 


--------------------------------------------------------------------------------
/mirror/conversions_test.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | package mirror
 17 | 
 18 | import (
 19 | 	"fmt"
 20 | 	"strconv"
 21 | 	"strings"
 22 | 	"testing"
 23 | 	"time"
 24 | 
 25 | 	"github.com/google/git-appraise/repository"
 26 | 	"github.com/google/git-appraise/review"
 27 | 	"github.com/google/git-appraise/review/ci"
 28 | 	github "github.com/google/go-github/github"
 29 | )
 30 | 
 31 | var (
 32 | 	// Even though these are constants, we define them as variables so we can take their addresses.
 33 | 	repoOwner        = "example_org"
 34 | 	repoName         = "example_repo"
 35 | 	contributorLogin = "helpful_contributor"
 36 | )
 37 | 
 38 | func TestConvertStatus(t *testing.T) {
 39 | 	state := "success"
 40 | 	targetURL := "https://ci.example.com/build"
 41 | 	context := "ci/example"
 42 | 	createdAt := time.Now().Truncate(time.Second)
 43 | 	input := github.RepoStatus{
 44 | 		State:     &state,
 45 | 		TargetURL: &targetURL,
 46 | 		Context:   &context,
 47 | 		CreatedAt: &createdAt,
 48 | 	}
 49 | 	result, err := ConvertStatus(&input)
 50 | 	if err != nil || result == nil {
 51 | 		t.Fatal(err)
 52 | 	}
 53 | 	if result.Status != ci.StatusSuccess {
 54 | 		t.Errorf("%v != %v", result.Status, ci.StatusSuccess)
 55 | 	}
 56 | 	if result.URL != targetURL {
 57 | 		t.Errorf("%v != %v", result.URL, targetURL)
 58 | 	}
 59 | 	if result.Agent != context {
 60 | 		t.Errorf("%v != %v", result.Agent, context)
 61 | 	}
 62 | 	resultTimestamp, err := strconv.ParseInt(result.Timestamp, 10, 64)
 63 | 	if err != nil {
 64 | 		t.Fatal(err)
 65 | 	}
 66 | 	resultTime := time.Unix(resultTimestamp, 0)
 67 | 	if resultTime != createdAt {
 68 | 		t.Errorf("%v != %v", resultTime, createdAt)
 69 | 	}
 70 | }
 71 | 
 72 | func buildTestPullRequest(testRepo repository.Repo, reqNum int) *github.PullRequest {
 73 | 	reqTime := time.Now().Add(-3 * time.Hour)
 74 | 	reqTitle := "Bug fixes."
 75 | 	reqBody := "Fix some bugs."
 76 | 
 77 | 	baseRef := repository.TestTargetRef
 78 | 	baseCommitSHA := repository.TestCommitE
 79 | 	headRef := repository.TestReviewRef
 80 | 	headCommitSHA := repository.TestCommitG
 81 | 	return &github.PullRequest{
 82 | 		CreatedAt: &reqTime,
 83 | 		Body:      &reqBody,
 84 | 		Title:     &reqTitle,
 85 | 		Number:    &reqNum,
 86 | 		Base: &github.PullRequestBranch{
 87 | 			Ref: &baseRef,
 88 | 			SHA: &baseCommitSHA,
 89 | 			Repo: &github.Repository{
 90 | 				Name: &repoName,
 91 | 				Owner: &github.User{
 92 | 					Login: &repoOwner,
 93 | 				},
 94 | 			},
 95 | 		},
 96 | 		Head: &github.PullRequestBranch{
 97 | 			Ref: &headRef,
 98 | 			SHA: &headCommitSHA,
 99 | 			Repo: &github.Repository{
100 | 				Name: &repoName,
101 | 				Owner: &github.User{
102 | 					Login: &repoOwner,
103 | 				},
104 | 			},
105 | 		},
106 | 		User: &github.User{
107 | 			Login: &contributorLogin,
108 | 		},
109 | 	}
110 | }
111 | 
112 | func TestConvertPullRequest(t *testing.T) {
113 | 	testRepo := repository.NewMockRepoForTest()
114 | 	reqNum := 4
115 | 	pullRef := fmt.Sprintf("refs/pull/%d/head", reqNum)
116 | 	pr := buildTestPullRequest(testRepo, reqNum)
117 | 	r, err := ConvertPullRequest(pr)
118 | 	if err != nil {
119 | 		t.Fatal(err)
120 | 	}
121 | 	if r == nil {
122 | 		t.Fatal("Unexpected nil request")
123 | 	}
124 | 	if r.ReviewRef != pullRef || r.TargetRef != *pr.Base.Ref || r.Requester != contributorLogin ||
125 | 		!strings.Contains(r.Description, *pr.Title) || !strings.Contains(r.Description, *pr.Body) ||
126 | 		r.BaseCommit != "" || r.Timestamp != ConvertTime(*pr.CreatedAt) {
127 | 		t.Errorf("Unexpected request %v", r)
128 | 	}
129 | }
130 | 
131 | func verifyCommentPresent(r *review.Review, message, author string) bool {
132 | 	for _, thread := range r.Comments {
133 | 		if thread.Comment.Description == message && thread.Comment.Author == author {
134 | 			return true
135 | 		}
136 | 	}
137 | 	return false
138 | }
139 | 
140 | func verifyCommentPresentAtLine(r *review.Review, message, author string, lineNumber uint32) bool {
141 | 	for _, thread := range r.Comments {
142 | 		if thread.Comment.Description == message && thread.Comment.Author == author &&
143 | 			thread.Comment.Location.Range.StartLine == lineNumber {
144 | 			return true
145 | 		}
146 | 	}
147 | 	return false
148 | }
149 | 
150 | func TestConvertPullRequestToReview(t *testing.T) {
151 | 	testRepo := repository.NewMockRepoForTest()
152 | 	reqNum := 4
153 | 	pr := buildTestPullRequest(testRepo, reqNum)
154 | 	now := time.Now()
155 | 
156 | 	issueComment1 := "Please sign our CLA"
157 | 	issueTime1 := now.Add(-2 * time.Hour)
158 | 	issueComment2 := "Done"
159 | 	issueTime2 := now.Add(-1 * time.Hour)
160 | 	issueComments := []*github.IssueComment{
161 | 		&github.IssueComment{
162 | 			Body: &issueComment1,
163 | 			User: &github.User{
164 | 				Login: &repoOwner,
165 | 			},
166 | 			CreatedAt: &issueTime1,
167 | 		},
168 | 		&github.IssueComment{
169 | 			Body: &issueComment2,
170 | 			User: &github.User{
171 | 				Login: &contributorLogin,
172 | 			},
173 | 			CreatedAt: &issueTime2,
174 | 		},
175 | 	}
176 | 
177 | 	filePath := "example.go"
178 | 	diffHunk := "@@ -4,6 +10,10 @@ func changedMethod() {\n \t// This is an existing line\n \t// This is another existing line\n-\t//This is a removed line\n+\t//This is a new line\n+\t//This is a second new line, with a comment\")"
179 | 	var commentLineNumber uint32 = 14
180 | 	diffComment1 := "Comment on line 14"
181 | 	diffTime1 := now.Add(-2 * time.Hour)
182 | 	diffComment2 := "Reply to comment on line 14"
183 | 	diffTime2 := now.Add(-2 * time.Hour)
184 | 	diffCommit := repository.TestCommitG
185 | 	diffComments := []*github.PullRequestComment{
186 | 		&github.PullRequestComment{
187 | 			Body:             &diffComment1,
188 | 			Path:             &filePath,
189 | 			OriginalCommitID: &diffCommit,
190 | 			DiffHunk:         &diffHunk,
191 | 			User: &github.User{
192 | 				Login: &repoOwner,
193 | 			},
194 | 			CreatedAt: &diffTime1,
195 | 		},
196 | 		&github.PullRequestComment{
197 | 			Body:             &diffComment2,
198 | 			Path:             &filePath,
199 | 			OriginalCommitID: &diffCommit,
200 | 			DiffHunk:         &diffHunk,
201 | 			User: &github.User{
202 | 				Login: &contributorLogin,
203 | 			},
204 | 			CreatedAt: &diffTime2,
205 | 		},
206 | 	}
207 | 
208 | 	r, err := ConvertPullRequestToReview(pr, issueComments, diffComments, testRepo)
209 | 	if err != nil {
210 | 		t.Fatal(err)
211 | 	}
212 | 	if r == nil {
213 | 		t.Fatal("Unexpected nil review")
214 | 	}
215 | 	if r.Revision != repository.TestCommitG {
216 | 		t.Fatal("Failed to compute the review commit")
217 | 	}
218 | 	if r.Request.BaseCommit != repository.TestCommitE {
219 | 		t.Fatal("Failed to compute the base commit")
220 | 	}
221 | 
222 | 	reviewJSON, err := r.GetJSON()
223 | 	if err != nil {
224 | 		t.Fatal(err)
225 | 	}
226 | 	if !verifyCommentPresent(r, issueComment1, repoOwner) ||
227 | 		!verifyCommentPresent(r, issueComment2, contributorLogin) {
228 | 		t.Fatal("Missing expected top-level comments")
229 | 	}
230 | 	if !verifyCommentPresentAtLine(r, diffComment1, repoOwner, commentLineNumber) ||
231 | 		!verifyCommentPresentAtLine(r, diffComment2, contributorLogin, commentLineNumber) {
232 | 		t.Errorf("Missing expected line comments: %s", reviewJSON)
233 | 	}
234 | }
235 | 


--------------------------------------------------------------------------------
/mirror/doc.go:
--------------------------------------------------------------------------------
 1 | /*
 2 | Copyright 2015 Google Inc. All Rights Reserved.
 3 | 
 4 | Licensed under the Apache License, Version 2.0 (the "License");
 5 | you may not use this file except in compliance with the License.
 6 | You may obtain a copy of the License at
 7 | 
 8 |     http://www.apache.org/licenses/LICENSE-2.0
 9 | 
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 | 
17 | // Package mirror handles pulling data from Github and feeding it into the
18 | // git-appraise metadata formats. It's used by both the batch processing tool
19 | // and the webapp.
20 | package mirror
21 | 


--------------------------------------------------------------------------------
/mirror/output.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package mirror
 18 | 
 19 | import (
 20 | 	"encoding/json"
 21 | 	"fmt"
 22 | 	"github.com/google/git-appraise/repository"
 23 | 	"github.com/google/git-appraise/review"
 24 | 	"github.com/google/git-appraise/review/ci"
 25 | 	"github.com/google/git-appraise/review/comment"
 26 | 	"github.com/google/git-appraise/review/request"
 27 | )
 28 | 
 29 | // WriteNewReports takes a list of CI reports read from GitHub, and writes to the repo any that are new.
 30 | //
 31 | // The passed in logChan variable is used as our intermediary for logging, and allows us to
 32 | // use the same logic for logging messages in either our CLI or our App Engine apps, even though
 33 | // the two have different logging frameworks.
 34 | func WriteNewReports(reportsMap map[string][]ci.Report, repo repository.Repo, logChan chan<- string) error {
 35 | 	for commit, commitReports := range reportsMap {
 36 | 		existingReports := ci.ParseAllValid(repo.GetNotes(ci.Ref, commit))
 37 | 		for _, report := range commitReports {
 38 | 			bytes, err := json.Marshal(report)
 39 | 			note := repository.Note(bytes)
 40 | 			if err != nil {
 41 | 				return err
 42 | 			}
 43 | 			missing := true
 44 | 			for _, existing := range existingReports {
 45 | 				if existing == report {
 46 | 					missing = false
 47 | 				}
 48 | 			}
 49 | 			if missing {
 50 | 				logChan <- fmt.Sprintf("Found a new report for %.12s: %q", commit, string(bytes))
 51 | 				if err := repo.AppendNote(ci.Ref, commit, note); err != nil {
 52 | 					return err
 53 | 				}
 54 | 			}
 55 | 		}
 56 | 	}
 57 | 	return nil
 58 | }
 59 | 
 60 | // WriteNewComments takes a list of review comments read from GitHub, and writes to the repo any that are new.
 61 | //
 62 | // The passed in logChan variable is used as our intermediary for logging, and allows us to
 63 | // use the same logic for logging messages in either our CLI or our App Engine apps, even though
 64 | // the two have different logging frameworks.
 65 | func WriteNewComments(r review.Review, repo repository.Repo, logChan chan<- string) error {
 66 | 	existingComments := comment.ParseAllValid(repo.GetNotes(comment.Ref, r.Revision))
 67 | 	for _, commentThread := range r.Comments {
 68 | 		commentNote, err := commentThread.Comment.Write()
 69 | 		if err != nil {
 70 | 			return err
 71 | 		}
 72 | 		missing := true
 73 | 		for _, existing := range existingComments {
 74 | 			if CommentsOverlap(existing, commentThread.Comment) {
 75 | 				missing = false
 76 | 			}
 77 | 		}
 78 | 		if missing {
 79 | 			logChan <- fmt.Sprintf("Found a new comment: %q", string(commentNote))
 80 | 			if err := repo.AppendNote(comment.Ref, r.Revision, commentNote); err != nil {
 81 | 				return err
 82 | 			}
 83 | 		}
 84 | 	}
 85 | 	return nil
 86 | }
 87 | 
 88 | func quoteComment(c comment.Comment) string {
 89 | 	return fmt.Sprintf("%s:\n\n%s", c.Author, c.Description)
 90 | }
 91 | 
 92 | func commentDescriptionsMatch(a, b comment.Comment) bool {
 93 | 	return a.Author == b.Author && a.Description == b.Description
 94 | }
 95 | 
 96 | func commentDescriptionsOverlap(a, b comment.Comment) bool {
 97 | 	return commentDescriptionsMatch(a, b) ||
 98 | 		a.Description == quoteComment(b) ||
 99 | 		quoteComment(a) == b.Description
100 | }
101 | 
102 | func commentLocationPathsMatch(a, b comment.Location) bool {
103 | 	return a == b ||
104 | 		(a.Commit == b.Commit && a.Path == b.Path)
105 | }
106 | 
107 | func commentLocationsOverlap(a, b comment.Comment) bool {
108 | 	return a.Location == b.Location ||
109 | 		(a.Location == nil && b.Location.Path == "") ||
110 | 		(b.Location == nil && a.Location.Path == "") ||
111 | 		(a.Location != nil && b.Location != nil && commentLocationPathsMatch(*a.Location, *b.Location))
112 | }
113 | 
114 | // CommentsOverlap determines if two review comments are sufficiently similar that one is a good-enough replacement for the other.
115 | //
116 | // The purpose of this method is to account for the semantic differences between the comments on a GitHub
117 | // pull request and the comments on a git-appraise review.
118 | //
119 | // More specifically, GitHub issue comments roughly correspond to git-appraise review-level comments, and
120 | // GitHub pull request comments roughly correspond to git-appraise line-level comments, but with the
121 | // following differences:
122 | //
123 | // 1. None of the GitHub comments can have a parent, but any of the git-appraise ones can.
124 | // 2. The author and timestamp of a GitHub comment is based on the call to the API, so if we want to
125 | //    mirror a comment from git-appraise into GitHub, then when we read that new comment back out this
126 | //    metadata will be different.
127 | // 3. Review-level comments in git-appraise can have a specificed commit, but issue comments can not.
128 | // 4. File-level comments in git-appraise have no corresponding equivalent in GitHub.
129 | // 5. Line-level comments in GitHub have to be anchored in part of the diff, while in git-appraise
130 | //    they can occur anywhere within the file.
131 | //
132 | // To work around these issues, we build in the following leeway:
133 | // 1. We treat two comment descriptions as equivalent if one looks like a quote of the other.
134 | // 2. We treat two locations as equivalent if one of the following holds:
135 | //    0. They actually are the same
136 | //    1. Both are review-level comments, and one of them does not have a commit set
137 | //    2. They are either file-level or line-level comments and occur in the same file
138 | //
139 | // This definition of equivalence does allow some information to be lost when converting from one
140 | // format to the other, but the important information (who said what) gets maintained and we avoid
141 | // accidentally mirroring the same comment back and forth multiple times.
142 | func CommentsOverlap(a, b comment.Comment) bool {
143 | 	return a == b ||
144 | 		(commentLocationsOverlap(a, b) && commentDescriptionsOverlap(a, b))
145 | }
146 | 
147 | // WriteNewReviews takes a list of reviews read from GitHub, and writes to the repo any review
148 | // data that has not already been written to it.
149 | //
150 | // The passed in logChan variable is used as our intermediary for logging, and allows us to
151 | // use the same logic for logging messages in either our CLI or our App Engine apps, even though
152 | // the two have different logging frameworks.
153 | func WriteNewReviews(reviews []review.Review, repo repository.Repo, logChan chan<- string) error {
154 | 	existingReviews := review.ListAll(repo)
155 | 	for _, r := range reviews {
156 | 		requestNote, err := r.Request.Write()
157 | 		if err != nil {
158 | 			return err
159 | 		}
160 | 		if err != nil {
161 | 			return err
162 | 		}
163 | 		alreadyPresent := false
164 | 		if existing := findMatchingExistingReview(r, existingReviews); existing != nil {
165 | 			alreadyPresent = RequestsOverlap(existing.Request, r.Request)
166 | 			r.Revision = existing.Revision
167 | 		}
168 | 		if !alreadyPresent {
169 | 			requestJSON, err := r.GetJSON()
170 | 			if err != nil {
171 | 				return err
172 | 			}
173 | 			logChan <- fmt.Sprintf("Found a new review for %.12s:\n%s\n", r.Revision, requestJSON)
174 | 			if err := repo.AppendNote(request.Ref, r.Revision, requestNote); err != nil {
175 | 				return err
176 | 			}
177 | 		}
178 | 		if err := WriteNewComments(r, repo, logChan); err != nil {
179 | 			return err
180 | 		}
181 | 	}
182 | 	return nil
183 | }
184 | 
185 | // findMatchingExistingReview determines if the given list of existing reviews includes
186 | // one that overlaps with the given new review.
187 | func findMatchingExistingReview(r review.Review, existingReviews []review.Summary) *review.Summary {
188 | 	for _, existing := range existingReviews {
189 | 		// This code assumes that a review ref is never reused. That is not
190 | 		// true in general for git-appraise (which does support reusing
191 | 		// developer branches), but *is* true for reviews created by this tool.
192 | 		// The reason for this is that we always set the review ref to the
193 | 		// "refs/pull/<PR#>/head" ref, and GitHub does not reuse pull request
194 | 		// numbers.
195 | 		if existing.Request.ReviewRef == r.Request.ReviewRef {
196 | 			return &existing
197 | 		}
198 | 	}
199 | 	return nil
200 | }
201 | 
202 | // RequestsOverlap determines if two review requests are sufficiently similar that one is a good-enough replacement for the other.
203 | //
204 | // The purpose of this method is to account for the semantic differences between a GitHub pull request and a
205 | // git-appraise request. More specifically, a GitHub pull request can only have a single "assignee", but a
206 | // git-appraise review can have multiple reviewers. As such, when we compare two requests to see if they are
207 | // "close enough", we ignore the reviewers field.
208 | func RequestsOverlap(a, b request.Request) bool {
209 | 	return a.ReviewRef == b.ReviewRef &&
210 | 		a.TargetRef == b.TargetRef &&
211 | 		a.Description == b.Description &&
212 | 		(a.BaseCommit == b.BaseCommit || a.BaseCommit == "" || b.BaseCommit == "")
213 | }
214 | 


--------------------------------------------------------------------------------
/mirror/output_test.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package mirror
 18 | 
 19 | import (
 20 | 	"testing"
 21 | 
 22 | 	"github.com/google/git-appraise/review/comment"
 23 | 	"github.com/google/git-appraise/review/request"
 24 | )
 25 | 
 26 | func TestCommentsOverlap(t *testing.T) {
 27 | 	reviewLevelComment := comment.Comment{
 28 | 		Timestamp: "00000000",
 29 | 		Author:    "user@example.com",
 30 | 		Location: &comment.Location{
 31 | 			Commit: "ABCDEFG",
 32 | 		},
 33 | 		Description: "Please fix so and so...",
 34 | 	}
 35 | 	if !CommentsOverlap(reviewLevelComment, reviewLevelComment) {
 36 | 		t.Fatal("Erroneous distinction drawn between identical review-level comments")
 37 | 	}
 38 | 
 39 | 	repeatedReviewLevelComment := comment.Comment{
 40 | 		Timestamp: "00000000",
 41 | 		Author:    "user@example.com",
 42 | 		Location: &comment.Location{
 43 | 			Commit: "ABCDEFH",
 44 | 		},
 45 | 		Description: "Please fix so and so...",
 46 | 	}
 47 | 	if CommentsOverlap(reviewLevelComment, repeatedReviewLevelComment) {
 48 | 		t.Fatal("Failed to distinguish between review comments at different commits")
 49 | 	}
 50 | 
 51 | 	issueComment := comment.Comment{
 52 | 		Timestamp:   "FFFFFFFF",
 53 | 		Author:      "user@example.com",
 54 | 		Description: "Please fix so and so...",
 55 | 	}
 56 | 	if !CommentsOverlap(reviewLevelComment, issueComment) {
 57 | 		t.Fatal("Erroneous distinction drawn between a review-level comment and an issue comment")
 58 | 	}
 59 | 	reviewLevelCommentHash, err := reviewLevelComment.Hash()
 60 | 	if err != nil {
 61 | 		t.Fatal(err)
 62 | 	}
 63 | 	reviewLevelChildComment := comment.Comment{
 64 | 		Timestamp: "FFFFFFFG",
 65 | 		Author:    "user2@example.com",
 66 | 		Parent:    reviewLevelCommentHash,
 67 | 		Location: &comment.Location{
 68 | 			Commit: "ABCDEFG",
 69 | 		},
 70 | 		Description: "Done",
 71 | 	}
 72 | 	issueChildComment := comment.Comment{
 73 | 		Timestamp:   "FFFFFFFH",
 74 | 		Author:      "user2@example.com",
 75 | 		Description: "Done",
 76 | 	}
 77 | 	if !CommentsOverlap(reviewLevelChildComment, issueChildComment) {
 78 | 		t.Fatal("Erroneous distinction drawn between a review-level child comment and an issue comment")
 79 | 	}
 80 | }
 81 | 
 82 | func TestRequestsOverlap(t *testing.T) {
 83 | 	request1 := request.Request{
 84 | 		Timestamp:   "00000000",
 85 | 		Requester:   "user@example.com",
 86 | 		TargetRef:   "refs/heads/dev",
 87 | 		ReviewRef:   "refs/pull/42/head",
 88 | 		Description: "Bug fixes",
 89 | 		BaseCommit:  "ABCDEFG",
 90 | 	}
 91 | 	if !RequestsOverlap(request1, request1) {
 92 | 		t.Fatal("Identical requests were not determined to overlap")
 93 | 	}
 94 | 
 95 | 	request2 := request.Request{
 96 | 		Timestamp:   "FFFFFFFF",
 97 | 		Requester:   request1.Requester,
 98 | 		TargetRef:   request1.TargetRef,
 99 | 		ReviewRef:   request1.ReviewRef,
100 | 		Description: request1.Description,
101 | 		BaseCommit:  request1.BaseCommit,
102 | 	}
103 | 	if !RequestsOverlap(request1, request2) {
104 | 		t.Fatal("Timestamps should not be used for determining overlap")
105 | 	}
106 | 
107 | 	request3 := request.Request{
108 | 		Timestamp:   request1.Timestamp,
109 | 		Requester:   request1.Requester,
110 | 		TargetRef:   "refs/heads/master",
111 | 		ReviewRef:   request1.ReviewRef,
112 | 		Description: request1.Description,
113 | 		BaseCommit:  request1.BaseCommit,
114 | 	}
115 | 	if RequestsOverlap(request1, request3) {
116 | 		t.Fatal("Requests with different targets should not overlap")
117 | 	}
118 | }
119 | 


--------------------------------------------------------------------------------
/mirror/readall.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package mirror
 18 | 
 19 | import (
 20 | 	"context"
 21 | 	"errors"
 22 | 	"fmt"
 23 | 	"log"
 24 | 	"net/http"
 25 | 	"time"
 26 | 
 27 | 	"github.com/google/git-appraise/repository"
 28 | 	"github.com/google/git-appraise/review"
 29 | 	"github.com/google/git-appraise/review/ci"
 30 | 	github "github.com/google/go-github/github"
 31 | )
 32 | 
 33 | const (
 34 | 	maxRetryAttempts = 100
 35 | )
 36 | 
 37 | var (
 38 | 	// ErrInvalidRemoteRepo is returned when a given github repo is missing
 39 | 	// required information.
 40 | 	ErrInvalidRemoteRepo = errors.New("github repo requires name and owner login")
 41 | )
 42 | 
 43 | // Utilities for reading all of the pull request data for a specific repository.
 44 | 
 45 | // Can be stubbed out in testing; satisfied by github.Client.Repositories
 46 | type repositoriesService interface {
 47 | 	ListStatuses(ctx context.Context, owner, repo, ref string, opt *github.ListOptions) ([]*github.RepoStatus, *github.Response, error)
 48 | }
 49 | 
 50 | type pullRequestsService interface {
 51 | 	List(ctx context.Context, owner string, repo string, opt *github.PullRequestListOptions) ([]*github.PullRequest, *github.Response, error)
 52 | 	ListComments(ctx context.Context, owner string, repo string, number int, opt *github.PullRequestListCommentsOptions) ([]*github.PullRequestComment, *github.Response, error)
 53 | }
 54 | 
 55 | type issuesService interface {
 56 | 	ListComments(ctx context.Context, owner string, repo string, number int, opt *github.IssueListCommentsOptions) ([]*github.IssueComment, *github.Response, error)
 57 | }
 58 | 
 59 | type retryableRequest func() (*github.Response, error)
 60 | 
 61 | func executeRequest(request retryableRequest) error {
 62 | 	for i := 0; i < maxRetryAttempts; i++ {
 63 | 		resp, err := request()
 64 | 		if err == nil || resp.StatusCode != http.StatusForbidden || resp.Rate.Remaining != 0 {
 65 | 			return err
 66 | 		}
 67 | 		waitDuration := resp.Rate.Reset.Sub(time.Now())
 68 | 		log.Printf("Ran out of github API requests; sleeping %v (until %v)",
 69 | 			waitDuration,
 70 | 			resp.Rate.Reset.Time)
 71 | 		time.Sleep(waitDuration)
 72 | 	}
 73 | 	return fmt.Errorf("Exceeded the maximum of %d retry attempts", maxRetryAttempts)
 74 | }
 75 | 
 76 | // A retryableListRequest is a procedure that executes a list request in a way that is safe to retry.
 77 | //
 78 | // The contract for such a procedure is that it performs *exactly* one of the following:
 79 | //  1. Returns an error
 80 | // or
 81 | //  2. Captures the returned results in some internal state and returns a response with the LastPage field set.
 82 | type retryableListRequest func(github.ListOptions) (*github.Response, error)
 83 | 
 84 | // executeListRequest takes a retryableListRequest, and runs it for every page of
 85 | // results returned by the GitHub API.
 86 | func executeListRequest(request retryableListRequest) error {
 87 | 	for page, maxPage := 1, 1; page <= maxPage; page++ {
 88 | 		listOpts := github.ListOptions{
 89 | 			Page:    page,
 90 | 			PerPage: 100, // The maximum number of results per page
 91 | 		}
 92 | 		err := executeRequest(func() (*github.Response, error) {
 93 | 			resp, err := request(listOpts)
 94 | 			if err == nil {
 95 | 				maxPage = resp.LastPage
 96 | 			}
 97 | 			return resp, err
 98 | 		})
 99 | 		if err != nil {
100 | 			return err
101 | 		}
102 | 	}
103 | 	return nil
104 | }
105 | 
106 | // GetAllStatuses iterates through all of the head commits in the remote
107 | // repository, reads their statuses from Github, and returns the git-appraise equivalents.
108 | //
109 | // Errors processing individual channels will be passed through the supplied
110 | // error channel; errors that prevent all processing will be returned directly.
111 | func GetAllStatuses(remoteUser, remoteRepo string, client *github.Client, errOutput chan<- error) (map[string][]ci.Report, error) {
112 | 	if remoteUser == "" || remoteRepo == "" {
113 | 		return nil, ErrInvalidRemoteRepo
114 | 	}
115 | 	commits, err := iterateRemoteCommits(remoteUser, remoteRepo, client)
116 | 	if err != nil {
117 | 		return nil, err
118 | 	}
119 | 
120 | 	return fetchStatuses(commits, remoteUser, remoteRepo, client.Repositories, errOutput)
121 | }
122 | 
123 | // iterateRemoteCommits returns a slice of the head commits for every ref in the remote repo.
124 | func iterateRemoteCommits(remoteUser, remoteRepo string, client *github.Client) ([]string, error) {
125 | 	var remoteCommits []string
126 | 	err := executeListRequest(func(listOpts github.ListOptions) (*github.Response, error) {
127 | 		opts := &github.ReferenceListOptions{
128 | 			ListOptions: listOpts,
129 | 		}
130 | 		refs, response, err := client.Git.ListRefs(context.TODO(), remoteUser, remoteRepo, opts)
131 | 		if err == nil {
132 | 			for _, ref := range refs {
133 | 				remoteCommits = append(remoteCommits, *ref.Object.SHA)
134 | 			}
135 | 		}
136 | 		return response, err
137 | 	})
138 | 	if err != nil {
139 | 		return nil, err
140 | 	}
141 | 	return remoteCommits, nil
142 | }
143 | 
144 | func fetchReportsForCommit(commitSHA, remoteUser, remoteRepo string, repoService repositoriesService, errOutput chan<- error) ([]ci.Report, error) {
145 | 	var reports []ci.Report
146 | 	err := executeListRequest(func(listOpts github.ListOptions) (*github.Response, error) {
147 | 		statuses, resp, err := repoService.ListStatuses(context.TODO(), remoteUser, remoteRepo, commitSHA, &listOpts)
148 | 		if err == nil {
149 | 			for _, status := range statuses {
150 | 				report, err := ConvertStatus(status)
151 | 				if err != nil {
152 | 					errOutput <- err
153 | 				} else {
154 | 					reports = append(reports, *report)
155 | 				}
156 | 			}
157 | 		}
158 | 		return resp, err
159 | 	})
160 | 	if err != nil {
161 | 		return nil, err
162 | 	}
163 | 	return reports, nil
164 | }
165 | 
166 | func fetchStatuses(commits []string, remoteUser, remoteRepo string, repoService repositoriesService, errOutput chan<- error) (map[string][]ci.Report, error) {
167 | 	reportsByCommitHash := make(map[string][]ci.Report)
168 | 	for _, commitSHA := range commits {
169 | 		reports, err := fetchReportsForCommit(commitSHA, remoteUser, remoteRepo, repoService, errOutput)
170 | 		if err != nil {
171 | 			return nil, err
172 | 		}
173 | 		reportsByCommitHash[commitSHA] = reports
174 | 	}
175 | 	return reportsByCommitHash, nil
176 | }
177 | 
178 | // GetAllPullRequests reads all of the pull requests from the given repository.
179 | // It returns successful conversions and encountered errors in a channel.
180 | // Errors processing individual channels will be passed through the supplied
181 | // error channel; errors that prevent all processing will be returned directly.
182 | func GetAllPullRequests(local repository.Repo, remoteUser, remoteRepo string, client *github.Client, errOutput chan<- error) ([]review.Review, error) {
183 | 	if remoteUser == "" || remoteRepo == "" {
184 | 		return nil, ErrInvalidRemoteRepo
185 | 	}
186 | 
187 | 	prs, err := fetchPullRequests(remoteUser, remoteRepo, client.PullRequests)
188 | 	if err != nil {
189 | 		return nil, err
190 | 	}
191 | 	var output []review.Review
192 | 	for _, pr := range prs {
193 | 		issueComments, diffComments, err := fetchComments(pr, remoteUser, remoteRepo, client.PullRequests, client.Issues)
194 | 		if err != nil {
195 | 			errOutput <- err
196 | 		} else {
197 | 			review, err := ConvertPullRequestToReview(pr, issueComments, diffComments, local)
198 | 			if err != nil {
199 | 				errOutput <- err
200 | 			} else {
201 | 				output = append(output, *review)
202 | 			}
203 | 		}
204 | 	}
205 | 	return output, nil
206 | }
207 | 
208 | func fetchPullRequests(remoteUser, remoteRepo string, prs pullRequestsService) ([]*github.PullRequest, error) {
209 | 	var results []*github.PullRequest
210 | 	err := executeListRequest(func(listOpts github.ListOptions) (*github.Response, error) {
211 | 		opts := &github.PullRequestListOptions{
212 | 			State:       "all",
213 | 			ListOptions: listOpts,
214 | 		}
215 | 		pullRequests, response, err := prs.List(context.TODO(), remoteUser, remoteRepo, opts)
216 | 		if err == nil {
217 | 			results = append(results, pullRequests...)
218 | 		}
219 | 		return response, err
220 | 	})
221 | 	if err != nil {
222 | 		return nil, err
223 | 	}
224 | 	return results, nil
225 | }
226 | 
227 | // fetchComments fetches all of the comments for each issue it gets and then converts them.
228 | func fetchComments(pr *github.PullRequest, remoteUser, remoteRepo string, prs pullRequestsService, is issuesService) ([]*github.IssueComment, []*github.PullRequestComment, error) {
229 | 	var issueComments []*github.IssueComment
230 | 	err := executeListRequest(func(listOpts github.ListOptions) (*github.Response, error) {
231 | 		listOptions := &github.IssueListCommentsOptions{
232 | 			ListOptions: listOpts,
233 | 		}
234 | 		cs, resp, err := is.ListComments(context.TODO(), remoteUser, remoteRepo, *pr.Number, listOptions)
235 | 		if err == nil {
236 | 			issueComments = append(issueComments, cs...)
237 | 		}
238 | 		return resp, err
239 | 	})
240 | 	if err != nil {
241 | 		return nil, nil, err
242 | 	}
243 | 	var diffComments []*github.PullRequestComment
244 | 	err = executeListRequest(func(listOpts github.ListOptions) (*github.Response, error) {
245 | 		listOptions := &github.PullRequestListCommentsOptions{
246 | 			ListOptions: listOpts,
247 | 		}
248 | 		cs, resp, err := prs.ListComments(context.TODO(), remoteUser, remoteRepo, *pr.Number, listOptions)
249 | 		if err == nil {
250 | 			diffComments = append(diffComments, cs...)
251 | 		}
252 | 		return resp, err
253 | 	})
254 | 	if err != nil {
255 | 		return nil, nil, err
256 | 	}
257 | 	return issueComments, diffComments, nil
258 | }
259 | 


--------------------------------------------------------------------------------
/mirror/readall_test.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package mirror
 18 | 
 19 | import (
 20 | 	"context"
 21 | 	"fmt"
 22 | 	"net/http"
 23 | 	"testing"
 24 | 	"time"
 25 | 
 26 | 	"github.com/google/git-appraise/review/ci"
 27 | 	github "github.com/google/go-github/github"
 28 | )
 29 | 
 30 | var (
 31 | 	stateSuccess             = "success"
 32 | 	stateFailure             = "failure"
 33 | 	statusTargetURLFormat    = "ci.example.com/%d"
 34 | 	statusContext            = "CI Runner"
 35 | 	pageCount                = 10
 36 | 	statusSuccessfulResponse = github.Response{
 37 | 		Response: &http.Response{
 38 | 			StatusCode: http.StatusOK,
 39 | 		},
 40 | 		LastPage: pageCount,
 41 | 		Rate: github.Rate{
 42 | 			Remaining: 1,
 43 | 		},
 44 | 	}
 45 | 	statusThrottledResponse = github.Response{
 46 | 		Response: &http.Response{
 47 | 			StatusCode: http.StatusForbidden,
 48 | 		},
 49 | 		LastPage: pageCount,
 50 | 		Rate: github.Rate{
 51 | 			Remaining: 0,
 52 | 		},
 53 | 	}
 54 | )
 55 | 
 56 | type repoServiceResponse struct {
 57 | 	Results  []*github.RepoStatus
 58 | 	Response github.Response
 59 | 	Error    error
 60 | }
 61 | 
 62 | type repoServiceStub struct {
 63 | 	Index     int
 64 | 	Responses []repoServiceResponse
 65 | }
 66 | 
 67 | func (s *repoServiceStub) ListStatuses(ctx context.Context, owner, repo, ref string, opt *github.ListOptions) ([]*github.RepoStatus, *github.Response, error) {
 68 | 	if s.Index >= len(s.Responses) {
 69 | 	}
 70 | 	r := s.Responses[s.Index]
 71 | 	s.Index++
 72 | 	return r.Results, &r.Response, r.Error
 73 | }
 74 | 
 75 | func TestFetchReports(t *testing.T) {
 76 | 	var responses []repoServiceResponse
 77 | 	var expectedReports []ci.Report
 78 | 
 79 | 	now := time.Now()
 80 | 	for i := 0; i < pageCount; i++ {
 81 | 		successURL := fmt.Sprintf(statusTargetURLFormat, i*2)
 82 | 		successResult := &github.RepoStatus{
 83 | 			CreatedAt: &now,
 84 | 			State:     &stateSuccess,
 85 | 			TargetURL: &successURL,
 86 | 			Context:   &statusContext,
 87 | 		}
 88 | 		successReport, err := ConvertStatus(successResult)
 89 | 		if err != nil {
 90 | 			t.Fatal(err)
 91 | 		}
 92 | 		failureURL := fmt.Sprintf(statusTargetURLFormat, i*2+1)
 93 | 		failureResult := &github.RepoStatus{
 94 | 			CreatedAt: &now,
 95 | 			State:     &stateFailure,
 96 | 			TargetURL: &failureURL,
 97 | 			Context:   &statusContext,
 98 | 		}
 99 | 		failureReport, err := ConvertStatus(failureResult)
100 | 		if err != nil {
101 | 			t.Fatal(err)
102 | 		}
103 | 		expectedReports = append(expectedReports, *successReport)
104 | 		expectedReports = append(expectedReports, *failureReport)
105 | 		responses = append(responses, repoServiceResponse{
106 | 			Results: []*github.RepoStatus{
107 | 				successResult,
108 | 				failureResult,
109 | 			},
110 | 			Response: statusSuccessfulResponse,
111 | 			Error:    nil,
112 | 		})
113 | 		responses = append(responses, repoServiceResponse{
114 | 			Results:  nil,
115 | 			Response: statusThrottledResponse,
116 | 			Error:    fmt.Errorf("Too many requests, for now"),
117 | 		})
118 | 	}
119 | 	serviceStub := &repoServiceStub{
120 | 		Index:     0,
121 | 		Responses: responses,
122 | 	}
123 | 
124 | 	errOut := make(chan error, 1000)
125 | 	resultingReports, err := fetchReportsForCommit("ABCDEF", "user", "repo", serviceStub, errOut)
126 | 	if err != nil || len(errOut) > 0 {
127 | 		t.Fatal(err, errOut)
128 | 	}
129 | 	if len(resultingReports) != len(expectedReports) {
130 | 		t.Errorf("Unexpected reports: %v vs. %v", resultingReports, expectedReports)
131 | 	}
132 | 	for i := range resultingReports {
133 | 		if resultingReports[i] != expectedReports[i] {
134 | 			t.Errorf("Unexpected reports: %v vs. %v", resultingReports, expectedReports)
135 | 		}
136 | 	}
137 | }
138 | 


--------------------------------------------------------------------------------