├── .github └── workflows │ └── runtests.yml ├── .gitignore ├── LICENSE ├── README.md ├── noxfile.py ├── requirements_test.txt ├── runtests.py ├── setup.cfg ├── setup.py └── tracext ├── __init__.py └── github └── __init__.py /.github/workflows/runtests.yml: -------------------------------------------------------------------------------- 1 | name: runtests 2 | run-name: Run test suite for trac-github 3 | on: [pull_request] 4 | jobs: 5 | runtests-py2: 6 | runs-on: ubuntu-latest 7 | container: 8 | image: python:2.7.18-buster 9 | 10 | steps: 11 | - uses: actions/checkout@v4 12 | - run: pip install nox-py2 13 | - run: git config --global user.name runtest 14 | - run: git config --global user.email runtest@localhost 15 | - run: nox --non-interactive --error-on-missing-interpreter --session runtests -- --git-default-branch=master 16 | 17 | # runtests-py3: 18 | # runs-on: ubuntu-latest 19 | # steps: 20 | # - uses: wntrblm/nox@2022.8.7 21 | # with: 22 | # python-versions: "3.7" 23 | # - uses: actions/checkout@v4 24 | # - run: git config --global user.name runtest 25 | # - run: git config --global user.email runtest@localhost 26 | # - run: git config --global init.defaultBranch main 27 | # - run: nox --non-interactive --error-on-missing-interpreter --session runtests 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.pyc 3 | .coverage 4 | build 5 | dist 6 | htmlcov 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2016, Aymeric Augustin. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | * Neither the name of trac-github nor the names of its contributors may 13 | be used to endorse or promote products derived from this software without 14 | specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Trac - GitHub integration 2 | ========================= 3 | 4 | Features 5 | -------- 6 | 7 | This Trac plugin performs four functions: 8 | 9 | 1. update the local git mirror used by Trac after each push to GitHub, and 10 | notify the new changesets to Trac; 11 | 2. authenticate users with their GitHub account; 12 | 3. direct changeset TracLinks to GitHub's repository browser. 13 | 4. sync GitHub teams to Trac permission groups 14 | 15 | The notification of new changesets is strictly equivalent to the command 16 | described in Trac's setup guide: 17 | 18 | trac-admin TRAC_ENV changeset added ... 19 | 20 | Each feature is implemented in its own component and can be enabled or 21 | disabled (almost) independently. 22 | 23 | Requirements 24 | ------------ 25 | 26 | trac-github requires Trac >= 0.12 and the git plugin. 27 | 28 | The git plugin [is included](http://trac.edgewall.org/wiki/TracGit) in Trac >= 29 | 1.0 — you only have to enable it in `trac.ini`. For Trac 0.12 you have to 30 | [install it](http://trac-hacks.org/wiki/GitPlugin): 31 | 32 | pip install git+https://github.com/hvr/trac-git-plugin.git 33 | 34 | Then install trac-github itself: 35 | 36 | pip install trac-github 37 | 38 | `requests_oauthlib` is also a requirement if you plan to use `GitHubLoginModule`: 39 | 40 | pip install requests_oauthlib 41 | 42 | Setup 43 | ----- 44 | 45 | _Warning: the commands below are provided for illustrative purposes. You'll 46 | have to adapt them to your setup._ 47 | 48 | ### Post-commit hook 49 | 50 | **`tracext.github.GitHubPostCommitHook`** implements a post-commit hook called 51 | by GitHub after each push. 52 | 53 | It updates the git mirror used by Trac, triggers a cache update and notifies 54 | components of the new changesets. Notifications are used by Trac's [commit 55 | ticket updater](http://trac.edgewall.org/wiki/CommitTicketUpdater) and 56 | [notifications](http://trac.edgewall.org/wiki/TracNotification). 57 | 58 | First, you need a mirror of your GitHub repository, writable by the webserver, 59 | for Trac's use: 60 | 61 | cd /home/trac 62 | git clone --mirror git://github.com//.git 63 | chown -R www-data:www-data .git 64 | 65 | Ensure that the user under which your web server runs can update the mirror: 66 | 67 | su www-data 68 | git --git-dir=/home/trac/.git remote update --prune 69 | 70 | Now edit your `trac.ini` as follows to configure both the git and the 71 | trac-github plugins: 72 | 73 | [components] 74 | tracext.github.GitHubPostCommitHook = enabled 75 | tracext.github.GitHubMixin = enabled 76 | tracopt.ticket.commit_updater.* = enabled 77 | tracopt.versioncontrol.git.* = enabled 78 | 79 | [git] 80 | trac_user_rlookup = enabled 81 | 82 | [github] 83 | repository = / 84 | 85 | [trac] 86 | repository_sync_per_request = # Trac < 1.2 87 | 88 | [repositories] 89 | .dir = /home/trac/.git 90 | .type = git 91 | .sync_per_request = false # Trac >= 1.2 92 | 93 | In Trac 0.12, use `tracext.git.* = enabled` instead of 94 | `tracopt.versioncontrol.git.* = enabled`. 95 | 96 | `tracopt.ticket.commit_updater.*` activates the [commit ticket 97 | updater](http://trac.edgewall.org/wiki/CommitTicketUpdater). It isn't 98 | required, but it's the most useful feature enabled by trac-github. 99 | 100 | The author names that Trac caches are of the pattern 101 | `Full Name `. The `trac_user_rlookup` option enables 102 | reverse mapping from email address to Trac user id. This is necessary 103 | for commit ticket updater to function, and for `[trac]` options like 104 | [show_full_names](https://trac.edgewall.org/wiki/TracIni#trac-show_full_names-option) 105 | and 106 | [show_email_addresses](https://trac.edgewall.org/wiki/TracIni#trac-show_email_addresses-option) 107 | to be effective. 108 | 109 | Reload the web server and your repository should appear in Trac. 110 | 111 | Perform an initial synchronization of the cache. 112 | 113 | trac-admin $env repository resync "(default)" 114 | 115 | Note that `"(default")` will need to be replaced with the repository 116 | name if a named repository is used. See the 117 | [Trac documentation](https://trac.edgewall.org/wiki/TracRepositoryAdmin#ReposTracIni) 118 | for more information. 119 | 120 | Browse to the home page of your project in Trac and append `/github` to the 121 | URL. Append `/github/` if you have a named repository 122 | (see [multiple repositories](#multiple-repositories)). You should see the 123 | following message: 124 | 125 | Endpoint is ready to accept GitHub notifications. 126 | 127 | This is the URL of the endpoint. 128 | 129 | If you get a Trac error page saying "No handler matched request to /github" 130 | instead, the plugin isn't installed properly. Make sure you've followed the 131 | installation instructions correctly and 132 | [search Trac's logs](https://trac.edgewall.org/wiki/TracTroubleshooting#ChecktheLogs) 133 | for errors. 134 | 135 | Now go to your project's settings page on GitHub. In the "Webhooks & Services" 136 | tab, click "Add webhook". Put the URL of the endpoint in the "Payload URL" 137 | field and set the "Content type" to `application/json`. Click "Add webhook". 138 | 139 | If you click on the webhook you just created, at the bottom of the page, you 140 | should see that a "ping" payload was successufully delivered to Trac 141 | 142 | Optionally, you can run additional actions every time GitHub triggers a webhook 143 | by placing a custom executable script at `.git/hooks/trac-github-update`. 144 | 145 | ### Authentication 146 | 147 | **`tracext.github.GitHubLoginModule`** provides authentication through 148 | GitHub's OAuth API. It obtains users' names and email addresses after a 149 | successful login if they're public and saves them in the preferences. 150 | 151 | To use this module, your Trac instance must be served over HTTPS. This is a 152 | requirement of the OAuth2 standard. 153 | 154 | Go to your accounts settings page on GitHub. From the *OAuth Application* 155 | page, click the *Developer applications* tab and *Register new application*. 156 | For the "Authorization callback URL", put the URL of the homepage of your 157 | project in Trac, starting with `https://`, and append `/github/oauth`. 158 | In other words, this is the URL of the endpoint you used above plus `/oauth` 159 | Then click *Register application*. 160 | 161 | You're redirected to your newly created application's page, which provides a 162 | Client ID and a Client Secret. 163 | 164 | Now edit edit `trac.ini` as follows: 165 | 166 | [components] 167 | trac.web.auth.LoginModule = disabled 168 | tracext.github.GitHubLoginModule = enabled 169 | 170 | [github] 171 | client_id = 172 | client_secret = 173 | 174 | This example disables `trac.web.auth.LoginModule`. Otherwise different users 175 | could authenticate with the same username through different systems! 176 | 177 | If it's impractical to set the Client ID and Client Secret in the Trac 178 | configuration file, you have some alternatives: 179 | 180 | - If `client_secret` matches `[A-Z_]+` (uppercase only), trac-github will use 181 | the content of the corresponding environment variable as client secret. 182 | - If `client_secret` starts with '/' or './', trac-github will interpret it as 183 | a file name and use the contents of that file as client secret. 184 | - If `client_secret` is anything else, trac-github will use it as is. 185 | 186 | By default the preferences will use the public email address of the 187 | authenticated GitHub user. If the public email address is not set, the field 188 | will be empty. If the email address is important for your Trac installation 189 | (for example for notifications), the `request_email` option can be set to 190 | always request access to all email addresses from GitHub. The primary address 191 | will be stored in the preferences on the first login. 192 | 193 | [github] 194 | request_email = true 195 | preferred_email_domain = example.org 196 | 197 | if specified, the first address matching the optional `preferred_email_domain` 198 | will be used instead of the primary address. 199 | 200 | Note that the Trac mail address will only be initialized on the first login. 201 | Users can still change or remove the email address from their Trac account. 202 | 203 | ### Browser 204 | 205 | **`tracext.github.GitHubBrowser`** redirects changeset TracLinks to 206 | the GitHub repositor browser. It requires the post-commit hook. 207 | 208 | To enable it, edit `trac.ini` as follows: 209 | 210 | [components] 211 | trac.versioncontrol.web_ui.browser.BrowserModule = disabled 212 | trac.versioncontrol.web_ui.changeset.ChangesetModule = disabled 213 | trac.versioncontrol.web_ui.log.LogModule = disabled 214 | tracext.github.GitHubBrowser = enabled 215 | tracext.github.GitHubMixin = enabled 216 | 217 | Since it replaces standard URLs of Trac, you must disable three components in 218 | `trac.versioncontrol.web_ui`, as shown above. 219 | 220 | With the `BROWSER_MODULE` disabled the `BROWSER_VIEW` and `FILE_VIEW` permissions will no longer be available. The permissions are checked when rendering files in the timeline, when `[timeline]` `changeset_show_files` is non-zero. Enabling the permission policy will make the list of files visible in the timeline for users that possess `CHANGESET_VIEW`. 221 | 222 | Add the permission policy before `DefaultPermissionsPolicy`. It is usually correct to make it the first entry in the list. 223 | 224 | The following will be correct for a Trac 1.2 installation that had the default value for `permission_policies`. 225 | 226 | [trac] 227 | permission_policies = GitHubPolicy, ReadonlyWikiPolicy, DefaultPermissionPolicy, LegacyAttachmentPolicy 228 | 229 | ### Group Synchronization 230 | 231 | GitHub teams can be synced to Trac permission groups using 232 | **`tracext.github.GitHubGroupsProvider`**. It uses a dedicated GitHub user and 233 | their personal access token to synchronize group memberships. Note that this 234 | user must have permission to read all your organization's teams. Additionally, 235 | this module implements a Webhook endpoint to keep the groups synchronized at 236 | all times. 237 | 238 | Register a new user for the synchronization or re-use an existing bot user. 239 | Make sure the bot user has owner privileges for your organization. Go to 240 | *Settings* > *Developer settings* > *Personal access tokens* and click 241 | *Generate a new token*. Make sure `read:org` under `admin:org` is checked and 242 | submit. Copy the displayed hex string. 243 | 244 | Now edit edit `trac.ini` as follows: 245 | 246 | [components] 247 | tracext.github.GitHubGroupsProvider = enabled 248 | tracext.github.GitHubMixin = enabled 249 | 250 | [github] 251 | organization = 252 | username = 253 | access_token = 254 | 255 | This should give you an initial working synchronization of your organization's 256 | teams, but no automatic update. Because the cache does not expire, restarting 257 | trac is your only option to force a resync. If the synchronization does not 258 | work as expected, enable debug logging in Trac and check the logfile. 259 | 260 | Next, you should configure a Webhook to keep your groups up to date. Browse to 261 | the home page of your project in Trac and append `/github-groups` to the URL. 262 | You should see the following message: 263 | 264 | Endpoint is ready to accept GitHub Organization membership notifications. 265 | 266 | This is the URL of the endpoint. 267 | 268 | Log in as an organization owner and find the *Webhooks* panel in the 269 | organization's settings. Add a new webhook and use the endpoint URL in the 270 | *Payload URL* field. Use `application/json` as *Content type*. Leave the secret 271 | empty for now, and select *Membership* from the list of individual events. 272 | Disable the *Push* event, since this endpoint will not handle it. Add the 273 | webhook, open it and check the list of recent deliveries. It should have sent 274 | a successful ping event. 275 | 276 | Finally, you should secure your webhook. Generate a random shared secret, for 277 | example using `/dev/urandom` and a hash algorithm: 278 | 279 | dd if=/dev/urandom of=/dev/stdout bs=16 count=16 | openssl dgst -sha256 280 | 281 | Copy the secret, edit `trac.ini` and add 282 | 283 | [github] 284 | webhook_secret = 285 | 286 | Go to your webook's settings on GitHub again and paste the secret in the 287 | *Secret* field. After saving, select the ping event from the recent deliveries 288 | list and click *Redeliver* to make sure the shared secret works. 289 | 290 | The synchronized groups will be named `github-${orgname}-${team_slug}`, e.g. 291 | for the *extraordinary league* team of the *people* organization, the group in 292 | Trac will be named `github-people-extraordinary-league`. 293 | 294 | An additional `github-${orgname}` group will contain all members of all teams 295 | in your organization. Note that members of your organization that are not part 296 | of a team will not be part of this group. This limitation is necessary because 297 | GitHub does not (yet) provide a notification mechanism for changes in 298 | organization membership. 299 | 300 | If you do not want to store the API secrets for `access_token` and 301 | `webhook_secret` in trac.ini, you can use the same alternatives as for 302 | `client_id` and `client_secret` documented [above](#authentication). 303 | 304 | 305 | Advanced setup 306 | -------------- 307 | 308 | ### Branches 309 | 310 | By default, trac-github notifies all commits to Trac. But you may not wish 311 | to trigger notifications for commits on experimental branches until they're 312 | merged, for example. 313 | 314 | You can configure trac-github to only notify commits on some branches: 315 | 316 | [github] 317 | branches = master 318 | 319 | You can provide more than one branch name, and you can use 320 | [shell-style wildcards](https://docs.python.org/2.7/library/fnmatch.html): 321 | 322 | [github] 323 | branches = master stable/* 324 | 325 | This option also restricts which branches are shown in the timeline. 326 | 327 | Besides, trac-github uses relies on the 'distinct' flag set by GitHub to 328 | prevent duplicate notifications when you merge branches. 329 | 330 | ### Multiple repositories 331 | 332 | If you have multiple repositories, you must tell Trac how they're called on 333 | GitHub: 334 | 335 | [github] 336 | repository = / # default repository 337 | .repository = / # for each extra repository 338 | .branches = # optional 339 | 340 | When you configure the webhook URLs, append the name used by Trac to identify 341 | the repository: 342 | 343 | http:///github/ 344 | 345 | ### Private repositories 346 | 347 | If you're deploying trac-github on a private Trac instance to manage private 348 | repositories, you have to take a few extra steps to allow Trac to pull changes 349 | from GitHub. The trick is to have Trac authenticate with a SSH key referenced 350 | as a deployment key on GitHub. 351 | 352 | All the commands shown below must be run by the webserver's user, eg www-data: 353 | 354 | $ su www-data 355 | 356 | Generate a dedicated SSH key with an empty passphrase and obtain the public 357 | key: 358 | 359 | $ ssh-keygen -f ~/.ssh/id_rsa_trac 360 | $ cat ~/.ssh/id_rsa_trac.pub 361 | 362 | Make sure you've obtained the public key (`.pub`). It should begin with 363 | `ssh-rsa`. If you're seeing an armored blob of data, it's the private key! 364 | 365 | Go to your project's settings page on GitHub. In the Deploy Keys tab, add the 366 | public key. 367 | 368 | Edit the SSH configuration for the `www-data` user: 369 | 370 | $ vi ~/.ssh/config 371 | 372 | Append the following lines: 373 | 374 | Host github-trac 375 | Hostname github.com 376 | IdentityFile ~/.ssh/id_rsa_trac 377 | 378 | Edit the git configuration for the repository: 379 | 380 | $ cd /home/trac/.git 381 | $ vi config 382 | 383 | Replace `github.com` in the `url` parameter by the `Host` value you've added 384 | to the SSH configuration: 385 | 386 | url = git@github-trac:/.git 387 | 388 | Make sure the authentication works: 389 | 390 | $ git remote update --prune 391 | 392 | Since GitHub doesn't allow reusing SSH keys across repositories, you have to 393 | generate a new key and pick a new `Host` value for each new repository. 394 | 395 | Development 396 | ----------- 397 | 398 | In a [virtualenv](https://virtualenv.pypa.io/en/stable/), install the 399 | requirements: 400 | 401 | pip install trac 402 | pip install coverage # if you want to run the tests under coverage 403 | pip install -e . 404 | 405 | or, instead of `pip install trac`: 406 | 407 | pip install trac==0.12.7 408 | pip install -e git+https://github.com/hvr/trac-git-plugin.git 409 | 410 | *The version of PyGIT bundled with `trac-git-plugin` doesn't work with 411 | the `git` binary shipped with OS X. To fix it, in the virtualenv, edit 412 | `src/tracgit/tracext/git/PyGIT.py` and replace `_, _, version = 413 | v.strip().split()` with `version = v.strip().split()[2]`.* 414 | 415 | Run the tests with: 416 | 417 | ./runtests.py 418 | 419 | Display Trac's log during the tests with: 420 | 421 | ./runtests.py --with-trac-log 422 | 423 | Run the tests under coverage with: 424 | 425 | coverage erase 426 | ./runtests.py --with-coverage 427 | coverage html 428 | 429 | If you put a breakpoint in the test suite, you can interact with Trac's web 430 | interface at [http://localhost:8765/](http://localhost:8765/) and with the git 431 | repositories through the command line. 432 | 433 | Running `tracd` ([TracStandalone](https://trac.edgewall.org/wiki/TracStandalone)) 434 | is the most convenient way to develop Trac from your workstation. Your local 435 | instance of `tracd` can be exposed to the internet using 436 | [ngrok](https://ngrok.com/). Download, extract and run `ngrok`: 437 | 438 | unzip ngrok-*.zip 439 | ngrok http 8000 --log ngrok.log 440 | 441 | The `ngrok` window will display a forwarding URL, for example: 442 | 443 | Forwarding https://abd75d3e.ngrok.io -> localhost:8000 444 | 445 | The URL will be used for configuring the webhook and will change 446 | each time you restart ngrok. See the 447 | [ngrok docs](https://ngrok.com/docs) for additional configuration options. 448 | 449 | Run `tracd` on the port you specified to `ngrok`: 450 | 451 | tracd -r -s -p 8000 /path/to/trac/env 452 | 453 | Complete the standard configuration steps in [setup](#setup). See the 454 | [Trac docs](https://trac.edgewall.org/wiki/TracDev/DevelopmentEnvironmentSetup) 455 | for additional information on setting up a Trac development environment. 456 | 457 | 458 | Release Steps 459 | ------------- 460 | 461 | You need to be an owner of the 462 | [package on PyPI](https://pypi.python.org/pypi/trac-github) to create a release. 463 | The steps assume you've configured a 464 | [.pypirc file](https://packaging.python.org/distributing/#create-an-account). 465 | 466 | 1. Update the [changelog](#changelog). 467 | 2. Set `tag_build = ` in 468 | [setup.cfg](https://github.com/trac-hacks/trac-github/blob/master/setup.cfg) 469 | 3. Create the release: 470 | 471 | ``` 472 | $ virtualenv pve 473 | $ . pve/bin/activate 474 | $ pip install -U pip wheel setuptools twine 475 | $ git clone https://github.com/trac-hacks/trac-github.git 476 | $ cd trac-github 477 | $ git tag 478 | $ git push --tags 479 | $ rm -r dist # if reusing virtualenv, but using a new virtualenv is advised 480 | $ python setup.py sdist bdist_wheel 481 | $ twine upload dist/*.tar.gz dist/*.whl 482 | ``` 483 | 484 | Known issues 485 | ------------ 486 | 487 | Once in a while, a notification doesn't appear in Trac. 488 | 489 | Usually, that happens when Trac fails to find the commit that triggered the 490 | notification, even though it just synchronized the git repository with GitHub. 491 | 492 | You can confirm that in your webhook's configuration page on GitHub. Scroll 493 | down to "Recent Deliveries" and look at the delivery that failed. In the 494 | "Response" tab, you should see a response body such as: 495 | 496 | Running hook on (default) 497 | * Updating clone 498 | * Synchronizing with clone 499 | * Unknown commit ... 500 | 501 | Simply click "Redeliver". Then missing notification should appear in Trac and 502 | the response body should change to: 503 | 504 | Running hook on (default) 505 | * Updating clone 506 | * Synchronizing with clone 507 | * Adding commit ... 508 | 509 | This problem isn't well understood. It may be related to Trac's access layer 510 | for git repositories. If you have an idea to fix it, please submit a patch! 511 | 512 | Changelog 513 | --------- 514 | 515 | ### 2.4 (not yet released) 516 | 517 | * Fix improperly configured namespace package. (#131) 518 | * Add configuration option for path prefix of login and logout. (#127) 519 | * Add `GitHubPolicy` permission policy to make `[timeline]` 520 | `changeset_show_file` option work correctly. (#126) 521 | 522 | ### 2.3 523 | 524 | * Support webhook signature verification for post commit hooks. (#114) 525 | * Allow passing a GitHub push webhook payload to a custom script per repository 526 | that will receive GitHub's JSON on stdin for further postprocessing. (#114) 527 | * Improve interaction with both GitHub and non-GitHub repositories on a single 528 | instance by delegating /changeset to the original ChangesetModule if enabled 529 | and the GitHub module did not match. (#110) 530 | * Optionally request access to non-public email addresses from GitHub and allow 531 | selection of an address by specifying a preferred domain. (#105) 532 | * Support synchronizing GitHub teams to Trac permission groups. (#104) 533 | 534 | ### 2.2 535 | 536 | * CSRF security fix: add verification of OAuth state parameter. 537 | 538 | ### 2.1.5 539 | 540 | * Support reading the GitHub OAuth secret from a file. 541 | * Trap `MissingTokenError` and add a warning. 542 | 543 | ### 2.1.4 544 | 545 | * Make `requests-oauthlib` a requirement for `GitHubLoginModule`. 546 | * Improve description of functionality provided by plugin. 547 | 548 | ### 2.1.3 549 | 550 | * Fix GitHub login failure with recent versions of oauthlib. 551 | * Fix logout after GitHub login on Trac >= 1.0.2. 552 | * Update configuration example to reflect Trac's current best practice. 553 | * Move the project to the trac-hacks organization on GitHub. 554 | 555 | ### 2.1.2 556 | 557 | * Make `tracext` a namespace package to support installation as an egg. 558 | * Improve responses when there's no repository at a requests's target URL. 559 | 560 | ### 2.1.1 561 | 562 | * Fix GitHub login failure when a user has no email on GitHub. 563 | 564 | ### 2.1 565 | 566 | * Add support for GitHub login. 567 | 568 | ### 2.0 569 | 570 | * Adapt to GitHub's new webhooks. 571 | 572 | When you upgrade from 1.x, you must change your webhooks settings on GitHub to 573 | use the application/vnd.github.v3+json format. 574 | 575 | ### 1.2 576 | 577 | * Add support for cached repositories. 578 | 579 | ### 1.1 580 | 581 | * Add support for multiple repositories. 582 | * Add an option to restrict notifications to some branches. 583 | * Try to avoid duplicate notifications (GitHub doesn't document the payload). 584 | * Use GitHub's generic webhook URLs. 585 | * Use a git mirror instead of a bare clone. 586 | 587 | ### 1.0 588 | 589 | * Public release. 590 | 591 | License 592 | ------- 593 | 594 | This plugin is released under the BSD license. 595 | 596 | It was initially written for [Django's Trac](https://code.djangoproject.com/). 597 | Prominent users include [jQuery Trac](https://bugs.jquery.com), [jQuery UI 598 | Trac](https://bugs.jqueryui.com) and [MacPorts 599 | Trac](https://trac.macports.org). 600 | -------------------------------------------------------------------------------- /noxfile.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import nox 4 | 5 | if sys.version_info.major == 2: 6 | TRAC_VERSIONS = ["1.4.4", "1.2.6"] 7 | else: 8 | TRAC_VERSIONS = ["1.6"] 9 | 10 | 11 | @nox.session 12 | @nox.parametrize("trac", TRAC_VERSIONS) 13 | def runtests(session, trac): 14 | session.install("-r", "requirements_test.txt") 15 | session.install("Trac==%s" % trac) 16 | session.run("python", "runtests.py", *session.posargs) 17 | -------------------------------------------------------------------------------- /requirements_test.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | requests-oauthlib==1.3.1 3 | lxml==5.0.1 4 | # Obviously Trac is also needed, but because we want to test several versions 5 | # then we install it manually 6 | -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This software is licensed as described in the file LICENSE, which 5 | # you should have received as part of this distribution. 6 | 7 | """Functional tests for the Trac-GitHub plugin. 8 | 9 | Trac's testing framework isn't well suited for plugins, so we NIH'd a bit. 10 | """ 11 | 12 | import argparse 13 | import BaseHTTPServer 14 | import ConfigParser 15 | import json 16 | import os 17 | import random 18 | import re 19 | import shutil 20 | import signal 21 | import subprocess 22 | import sys 23 | import tempfile 24 | import threading 25 | import time 26 | import traceback 27 | import unittest 28 | import urllib2 29 | import urlparse 30 | 31 | from lxml import html 32 | 33 | from trac.env import Environment 34 | from trac.ticket.model import Ticket 35 | from trac.util.translation import _ 36 | 37 | import requests 38 | 39 | 40 | GIT = 'test-git-foo' 41 | ALTGIT = 'test-git-bar' 42 | NOGHGIT = 'test-git-nogithub' 43 | TESTDIR = '.' 44 | 45 | ENV = 'test-trac-github' 46 | CONF = '%s/conf/trac.ini' % ENV 47 | HTDIGEST = '%s/passwd' % ENV 48 | URL = 'http://localhost:8765/%s' % ENV 49 | SECRET = 'test-secret' 50 | HEADERS = {'Content-Type': 'application/json', 'X-GitHub-Event': 'push'} 51 | UPDATEHOOK = '%s-mirror/hooks/trac-github-update' % GIT 52 | 53 | # Global variables overriden when running the module (see very bottom of file) 54 | COVERAGE = False 55 | SHOW_LOG = False 56 | TRAC_ADMIN_BIN = 'trac-admin' 57 | TRACD_BIN = 'tracd' 58 | COVERAGE_BIN = 'coverage' 59 | GIT_DEFAULT_BRANCH = 'main' 60 | 61 | 62 | class HttpNoRedirectHandler(urllib2.HTTPRedirectHandler): 63 | 64 | def redirect_request(self, req, fp, code, msg, headers, newurl): 65 | raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp) 66 | 67 | urllib2.install_opener(urllib2.build_opener(HttpNoRedirectHandler())) 68 | 69 | 70 | def d(*args): 71 | """ 72 | Return an absolute path where the given arguments are joined and 73 | prepended with the TESTDIR. 74 | """ 75 | return os.path.join(TESTDIR, *args) 76 | 77 | 78 | def git_check_output(*args, **kwargs): 79 | """ 80 | Run the given git command (`*args`), optionally on the given 81 | repository (`kwargs['repo']`), return the output of that command 82 | as a string. 83 | """ 84 | repo = kwargs.pop('repo', None) 85 | 86 | if repo is None: 87 | cmdargs = ["git"] + list(args) 88 | else: 89 | cmdargs = ["git", "-C", d(repo)] + list(args) 90 | 91 | return subprocess.check_output(cmdargs, **kwargs) 92 | 93 | 94 | class TracGitHubTests(unittest.TestCase): 95 | 96 | cached_git = False 97 | 98 | @classmethod 99 | def setUpClass(cls): 100 | cls.createGitRepositories() 101 | cls.createTracEnvironment() 102 | cls.startTracd() 103 | cls.env = Environment(d(ENV)) 104 | 105 | @classmethod 106 | def tearDownClass(cls): 107 | cls.env.shutdown() 108 | cls.stopTracd() 109 | cls.removeTracEnvironment() 110 | cls.removeGitRepositories() 111 | 112 | @classmethod 113 | def createGitRepositories(cls): 114 | git_check_output('init', d(GIT)) 115 | git_check_output('init', d(ALTGIT)) 116 | git_check_output('init', d(NOGHGIT)) 117 | cls.makeGitCommit(GIT, 'README', 'default git repository\n') 118 | cls.makeGitCommit(ALTGIT, 'README', 'alternative git repository\n') 119 | cls.makeGitCommit(NOGHGIT, 'README', 'git repository not on GitHub\n') 120 | git_check_output('clone', '--quiet', '--mirror', d(GIT), d('%s-mirror' % GIT)) 121 | git_check_output('clone', '--quiet', '--mirror', d(ALTGIT), d('%s-mirror' % ALTGIT)) 122 | 123 | @classmethod 124 | def removeGitRepositories(cls): 125 | shutil.rmtree(d(GIT)) 126 | shutil.rmtree(d(ALTGIT)) 127 | shutil.rmtree(d(NOGHGIT)) 128 | shutil.rmtree(d('%s-mirror' % GIT)) 129 | shutil.rmtree(d('%s-mirror' % ALTGIT)) 130 | 131 | @classmethod 132 | def createTracEnvironment(cls, **kwargs): 133 | subprocess.check_output([TRAC_ADMIN_BIN, d(ENV), 'initenv', 134 | 'Trac - GitHub tests', 'sqlite:db/trac.db']) 135 | subprocess.check_output([TRAC_ADMIN_BIN, d(ENV), 'permission', 136 | 'add', 'anonymous', 'TRAC_ADMIN']) 137 | 138 | conf = ConfigParser.ConfigParser() 139 | with open(d(CONF), 'rb') as fp: 140 | conf.readfp(fp) 141 | 142 | conf.add_section('components') 143 | conf.set('components', 'trac.versioncontrol.web_ui.browser.BrowserModule', 'disabled') 144 | conf.set('components', 'trac.versioncontrol.web_ui.changeset.ChangesetModule', 'disabled') 145 | conf.set('components', 'trac.versioncontrol.web_ui.log.LogModule', 'disabled') 146 | conf.set('components', 'trac.versioncontrol.svn_fs.*', 'disabled') # avoid spurious log messages 147 | conf.set('components', 'tracext.git.*', 'enabled') # Trac 0.12.4 148 | conf.set('components', 'tracext.github.*', 'enabled') 149 | conf.set('components', 'tracopt.ticket.commit_updater.*', 'enabled') 150 | conf.set('components', 'tracopt.versioncontrol.git.*', 'enabled') # Trac 1.0 151 | 152 | cached_git = cls.cached_git 153 | if 'cached_git' in kwargs: 154 | cached_git = kwargs['cached_git'] 155 | if cached_git: 156 | conf.add_section('git') 157 | conf.set('git', 'cached_repository', 'true') 158 | conf.set('git', 'persistent_cache', 'true') 159 | 160 | if not conf.has_section('github'): 161 | conf.add_section('github') 162 | client_id = '01234567890123456789' 163 | if 'client_id' in kwargs: 164 | client_id = kwargs['client_id'] 165 | conf.set('github', 'client_id', client_id) 166 | client_secret = '0123456789abcdef0123456789abcdef012345678' 167 | if 'client_secret' in kwargs: 168 | client_secret = kwargs['client_secret'] 169 | conf.set('github', 'client_secret', client_secret) 170 | conf.set('github', 'repository', 'aaugustin/trac-github') 171 | conf.set('github', 'alt.repository', 'follower/trac-github') 172 | conf.set('github', 'alt.branches', '%s stable/*' % GIT_DEFAULT_BRANCH) 173 | if 'request_email' in kwargs: 174 | conf.set('github', 'request_email', kwargs['request_email']) 175 | if 'preferred_email_domain' in kwargs: 176 | conf.set('github', 'preferred_email_domain', kwargs['preferred_email_domain']) 177 | if 'organization' in kwargs: 178 | conf.set('github', 'organization', kwargs['organization']) 179 | if 'username' in kwargs and 'access_token' in kwargs: 180 | conf.set('github', 'username', kwargs['username']) 181 | conf.set('github', 'access_token', kwargs['access_token']) 182 | if 'webhook_secret' in kwargs: 183 | conf.set('github', 'webhook_secret', kwargs['webhook_secret']) 184 | if 'username_prefix' in kwargs: 185 | conf.set('github', 'username_prefix', kwargs['username_prefix']) 186 | 187 | if SHOW_LOG: 188 | # The [logging] section already exists in the default trac.ini file. 189 | conf.set('logging', 'log_type', 'stderr') 190 | else: 191 | # Write debug log so you can read it on crashes 192 | conf.set('logging', 'log_type', 'file') 193 | conf.set('logging', 'log_file', 'trac.log') 194 | conf.set('logging', 'log_level', 'DEBUG') 195 | 196 | conf.add_section('repositories') 197 | conf.set('repositories', '.dir', d('%s-mirror' % GIT)) 198 | conf.set('repositories', '.type', 'git') 199 | conf.set('repositories', 'alt.dir', d('%s-mirror' % ALTGIT)) 200 | conf.set('repositories', 'alt.type', 'git') 201 | conf.set('repositories', 'nogh.dir', d(NOGHGIT, '.git')) 202 | conf.set('repositories', 'nogh.type', 'git') 203 | 204 | # Show changed files in timeline, which will trigger the 205 | # IPermissionPolicy code paths 206 | conf.set('timeline', 'changeset_show_files', '-1') 207 | old_permission_policies = conf.get('trac', 'permission_policies') 208 | if 'GitHubPolicy' not in old_permission_policies: 209 | conf.set('trac', 'permission_policies', 210 | 'GitHubPolicy, %s' % old_permission_policies) 211 | 212 | with open(d(CONF), 'wb') as fp: 213 | conf.write(fp) 214 | 215 | with open(d(HTDIGEST), 'w') as fp: 216 | # user: user, pass: pass, realm: realm 217 | fp.write("user:realm:8493fbc53ba582fb4c044c456bdc40eb\n") 218 | 219 | run_resync = kwargs['resync'] if 'resync' in kwargs else True 220 | if run_resync: 221 | # Allow skipping resync for perfomance reasons if not required 222 | subprocess.check_output([TRAC_ADMIN_BIN, d(ENV), 'repository', 'resync', '']) 223 | subprocess.check_output([TRAC_ADMIN_BIN, d(ENV), 'repository', 'resync', 'alt']) 224 | subprocess.check_output([TRAC_ADMIN_BIN, d(ENV), 'repository', 'resync', 'nogh']) 225 | 226 | @classmethod 227 | def removeTracEnvironment(cls): 228 | shutil.rmtree(d(ENV)) 229 | 230 | @classmethod 231 | def startTracd(cls, **kwargs): 232 | if COVERAGE: 233 | tracd = [COVERAGE_BIN, 'run', '--append', '--branch', 234 | '--source=tracext.github', TRACD_BIN] 235 | 236 | else: 237 | tracd = [TRACD_BIN] 238 | if SHOW_LOG: 239 | kwargs['stdout'] = sys.stdout 240 | kwargs['stderr'] = sys.stderr 241 | cls.tracd = subprocess.Popen(tracd + ['--port', '8765', '--auth=*,%s,realm' % d(HTDIGEST), d(ENV)], **kwargs) 242 | 243 | waittime = 0.1 244 | for _ in range(5): 245 | try: 246 | urllib2.urlopen(URL) 247 | except urllib2.URLError: 248 | time.sleep(waittime) 249 | waittime *= 2 250 | else: 251 | break 252 | else: 253 | raise RuntimeError("Can't communicate with tracd running on port 8765") 254 | 255 | @classmethod 256 | def stopTracd(cls): 257 | cls.tracd.send_signal(signal.SIGINT) 258 | cls.tracd.wait() 259 | 260 | @staticmethod 261 | def makeGitBranch(repo, branch): 262 | git_check_output('branch', branch, repo=repo) 263 | 264 | @staticmethod 265 | def makeGitCommit(repo, path, content, message='edit', branch=None): 266 | if branch is None: 267 | branch = GIT_DEFAULT_BRANCH 268 | 269 | if branch != GIT_DEFAULT_BRANCH: 270 | git_check_output('checkout', branch, repo=repo) 271 | with open(d(repo, path), 'wb') as fp: 272 | fp.write(content) 273 | git_check_output('add', path, repo=repo) 274 | git_check_output('commit', '-m', message, repo=repo) 275 | if branch != GIT_DEFAULT_BRANCH: 276 | git_check_output('checkout', GIT_DEFAULT_BRANCH, repo=repo) 277 | changeset = git_check_output('rev-parse', 'HEAD', repo=repo) 278 | return changeset.strip() 279 | 280 | @staticmethod 281 | def makeGitHubHookPayload(n=1, reponame=''): 282 | # See https://developer.github.com/v3/activity/events/types/#pushevent 283 | # We don't reproduce the entire payload, only what the plugin needs. 284 | repo = {'': GIT, 'alt': ALTGIT}[reponame] 285 | 286 | commits = [] 287 | log = git_check_output( 288 | 'log', 289 | '-%d' % n, 290 | '--branches', 291 | '--format=oneline', 292 | '--topo-order', 293 | repo=repo 294 | ) 295 | for line in log.splitlines(): 296 | id, _, message = line.partition(' ') 297 | commits.append({'id': id, 'message': message, 'distinct': True}) 298 | payload = {'commits': commits} 299 | return payload 300 | 301 | @staticmethod 302 | def openGitHubHook(n=1, reponame='', payload=None): 303 | if not payload: 304 | payload = TracGitHubTests.makeGitHubHookPayload(n, reponame) 305 | url = (URL + '/github/' + reponame) if reponame else URL + '/github' 306 | request = urllib2.Request(url, json.dumps(payload), HEADERS) 307 | return urllib2.urlopen(request) 308 | 309 | 310 | class GitHubBrowserTests(TracGitHubTests): 311 | 312 | def testLinkToChangeset(self): 313 | self.makeGitCommit(GIT, 'myfile', 'for browser tests') 314 | changeset = self.openGitHubHook().read().rstrip()[-40:] 315 | try: 316 | urllib2.urlopen(URL + '/changeset/' + changeset) 317 | except urllib2.HTTPError as exc: 318 | self.assertEqual(exc.code, 302) 319 | self.assertEqual(exc.headers['Location'], 320 | 'https://github.com/aaugustin/trac-github/commit/%s' % changeset) 321 | else: 322 | self.fail("URL didn't redirect") 323 | 324 | def testAlternateLinkToChangeset(self): 325 | self.makeGitCommit(ALTGIT, 'myfile', 'for browser tests') 326 | changeset = self.openGitHubHook(1, 'alt').read().rstrip()[-40:] 327 | try: 328 | urllib2.urlopen(URL + '/changeset/' + changeset + '/alt') 329 | except urllib2.HTTPError as exc: 330 | self.assertEqual(exc.code, 302) 331 | self.assertEqual(exc.headers['Location'], 332 | 'https://github.com/follower/trac-github/commit/%s' % changeset) 333 | else: 334 | self.fail("URL didn't redirect") 335 | 336 | def testNonGitHubLinkToChangeset(self): 337 | changeset = self.makeGitCommit(NOGHGIT, 'myfile', 'for browser tests') 338 | subprocess.check_output([TRAC_ADMIN_BIN, d(ENV), 'changeset', 'added', 'nogh', changeset]) 339 | response = requests.get(URL + '/changeset/' + changeset + '/nogh', allow_redirects=False) 340 | self.assertEqual(response.status_code, 200) 341 | 342 | def testLinkToPath(self): 343 | self.makeGitCommit(GIT, 'myfile', 'for more browser tests') 344 | changeset = self.openGitHubHook().read().rstrip()[-40:] 345 | try: 346 | urllib2.urlopen(URL + '/changeset/' + changeset + '/myfile') 347 | except urllib2.HTTPError as exc: 348 | self.assertEqual(exc.code, 302) 349 | self.assertEqual(exc.headers['Location'], 350 | 'https://github.com/aaugustin/trac-github/blob/%s/myfile' % changeset) 351 | else: 352 | self.fail("URL didn't redirect") 353 | 354 | def testAlternateLinkToPath(self): 355 | self.makeGitCommit(ALTGIT, 'myfile', 'for more browser tests') 356 | changeset = self.openGitHubHook(1, 'alt').read().rstrip()[-40:] 357 | try: 358 | urllib2.urlopen(URL + '/changeset/' + changeset + '/alt/myfile') 359 | except urllib2.HTTPError as exc: 360 | self.assertEqual(exc.code, 302) 361 | self.assertEqual(exc.headers['Location'], 362 | 'https://github.com/follower/trac-github/blob/%s/myfile' % changeset) 363 | else: 364 | self.fail("URL didn't redirect") 365 | 366 | def testNonGitHubLinkToPath(self): 367 | changeset = self.makeGitCommit(NOGHGIT, 'myfile', 'for more browser tests') 368 | subprocess.check_output([TRAC_ADMIN_BIN, d(ENV), 'changeset', 'added', 'nogh', changeset]) 369 | response = requests.get(URL + '/changeset/' + changeset + '/nogh/myfile', allow_redirects=False) 370 | self.assertEqual(response.status_code, 200) 371 | 372 | def testBadChangeset(self): 373 | with self.assertRaisesRegexp(urllib2.HTTPError, r'^HTTP Error 404: Not Found$'): 374 | urllib2.urlopen(URL + '/changeset/1234567890') 375 | 376 | def testBadUrl(self): 377 | with self.assertRaisesRegexp(urllib2.HTTPError, r'^HTTP Error 404: Not Found$'): 378 | urllib2.urlopen(URL + '/changesetnosuchurl') 379 | 380 | def testTimelineFiltering(self): 381 | self.makeGitBranch(GIT, 'stable/2.0') 382 | self.makeGitBranch(GIT, 'unstable/2.0') 383 | self.makeGitBranch(ALTGIT, 'stable/2.0') 384 | self.makeGitBranch(ALTGIT, 'unstable/2.0') 385 | self.makeGitCommit(GIT, 'myfile', 'timeline 1\n', 'msg 1') 386 | self.makeGitCommit(GIT, 'myfile', 'timeline 2\n', 'msg 2', 'stable/2.0') 387 | self.makeGitCommit(GIT, 'myfile', 'timeline 3\n', 'msg 3', 'unstable/2.0') 388 | self.makeGitCommit(ALTGIT, 'myfile', 'timeline 4\n', 'msg 4') 389 | self.makeGitCommit(ALTGIT, 'myfile', 'timeline 5\n', 'msg 5', 'stable/2.0') 390 | self.makeGitCommit(ALTGIT, 'myfile', 'timeline 6\n', 'msg 6', 'unstable/2.0') 391 | self.openGitHubHook(3) 392 | self.openGitHubHook(3, 'alt') 393 | html = urllib2.urlopen(URL + '/timeline').read() 394 | self.assertTrue('msg 1' in html) 395 | self.assertTrue('msg 2' in html) 396 | self.assertTrue('msg 3' in html) 397 | self.assertTrue('msg 4' in html) 398 | self.assertTrue('msg 5' in html) 399 | self.assertFalse('msg 6' in html) 400 | 401 | 402 | class GitHubLoginModuleTests(TracGitHubTests): 403 | 404 | @classmethod 405 | def startTracd(cls, **kwargs): 406 | # Disable check for HTTPS to avoid adding complexity to the test setup. 407 | kwargs['env'] = os.environ.copy() 408 | kwargs['env']['OAUTHLIB_INSECURE_TRANSPORT'] = '1' 409 | super(GitHubLoginModuleTests, cls).startTracd(**kwargs) 410 | 411 | def testLogin(self): 412 | response = requests.get(URL + '/github/login', allow_redirects=False) 413 | self.assertEqual(response.status_code, 302) 414 | 415 | redirect_url = urlparse.urlparse(response.headers['Location']) 416 | self.assertEqual(redirect_url.scheme, 'https') 417 | self.assertEqual(redirect_url.netloc, 'github.com') 418 | self.assertEqual(redirect_url.path, '/login/oauth/authorize') 419 | params = urlparse.parse_qs(redirect_url.query, keep_blank_values=True) 420 | state = params['state'][0] # this is a random value 421 | self.assertEqual(params, { 422 | 'client_id': ['01234567890123456789'], 423 | 'redirect_uri': [URL + '/github/oauth'], 424 | 'response_type': ['code'], 425 | 'scope': [''], 426 | 'state': [state], 427 | }) 428 | 429 | def testOauthInvalidState(self): 430 | session = requests.Session() 431 | 432 | # This adds a oauth_state parameter in the Trac session. 433 | response = session.get(URL + '/github/login', allow_redirects=False) 434 | self.assertEqual(response.status_code, 302) 435 | 436 | response = session.get( 437 | URL + '/github/oauth?code=01234567890123456789&state=wrong_state', 438 | allow_redirects=False) 439 | self.assertEqual(response.status_code, 302) 440 | self.assertEqual(response.headers['Location'], URL) 441 | 442 | response = session.get(URL) 443 | self.assertEqual(response.status_code, 200) 444 | self.assertIn( 445 | "Invalid request. Please try to login again.", response.text) 446 | 447 | def testOauthInvalidStateWithoutSession(self): 448 | session = requests.Session() 449 | 450 | # There's no oauth_state parameter in the Trac session. 451 | # OAuth callback requests without state must still fail. 452 | 453 | response = session.get( 454 | URL + '/github/oauth?code=01234567890123456789', 455 | allow_redirects=False) 456 | self.assertEqual(response.status_code, 302) 457 | self.assertEqual(response.headers['Location'], URL) 458 | 459 | response = session.get(URL) 460 | self.assertEqual(response.status_code, 200) 461 | self.assertIn( 462 | "Invalid request. Please try to login again.", response.text) 463 | 464 | def testLogout(self): 465 | response = requests.get(URL + '/github/logout', allow_redirects=False) 466 | self.assertEqual(response.status_code, 302) 467 | self.assertEqual(response.headers['Location'], URL) 468 | 469 | class GitHubLoginModuleConfigurationTests(TracGitHubTests): 470 | # Append custom failure messages to the automatically generated ones 471 | longMessage = True 472 | 473 | @classmethod 474 | def setUpClass(cls): 475 | cls.createGitRepositories() 476 | cls.mockdata = startAPIMock(8768) 477 | 478 | trac_env = os.environ.copy() 479 | trac_env.update({ 480 | 'TRAC_GITHUB_OAUTH_URL': 'http://127.0.0.1:8768/', 481 | 'TRAC_GITHUB_API_URL': 'http://127.0.0.1:8768/', 482 | 'OAUTHLIB_INSECURE_TRANSPORT': '1' 483 | }) 484 | trac_env_broken = trac_env.copy() 485 | trac_env_broken.update({ 486 | 'TRAC_GITHUB_OAUTH_URL': 'http://127.0.0.1:8769/', 487 | 'TRAC_GITHUB_API_URL': 'http://127.0.0.1:8769/', 488 | }) 489 | trac_env_broken_api = trac_env.copy() 490 | trac_env_broken_api.update({ 491 | 'TRAC_GITHUB_API_URL': 'http://127.0.0.1:8769/', 492 | }) 493 | 494 | cls.trac_env = trac_env 495 | cls.trac_env_broken = trac_env_broken 496 | cls.trac_env_broken_api = trac_env_broken_api 497 | 498 | with open(d(SECRET), 'wb') as fp: 499 | fp.write('98765432109876543210') 500 | 501 | 502 | @classmethod 503 | def tearDownClass(cls): 504 | cls.removeGitRepositories() 505 | os.remove(d(SECRET)) 506 | 507 | def testLoginWithReqEmail(self): 508 | """Test that configuring request_email = true requests the user:email scope from GitHub""" 509 | with TracContext(self, request_email=True, resync=False): 510 | response = requests.get(URL + '/github/login', allow_redirects=False) 511 | self.assertEqual(response.status_code, 302) 512 | 513 | redirect_url = urlparse.urlparse(response.headers['Location']) 514 | self.assertEqual(redirect_url.scheme, 'https') 515 | self.assertEqual(redirect_url.netloc, 'github.com') 516 | self.assertEqual(redirect_url.path, '/login/oauth/authorize') 517 | params = urlparse.parse_qs(redirect_url.query, keep_blank_values=True) 518 | state = params['state'][0] # this is a random value 519 | self.assertEqual(params, { 520 | 'client_id': ['01234567890123456789'], 521 | 'redirect_uri': [URL + '/github/oauth'], 522 | 'response_type': ['code'], 523 | 'scope': ['user:email'], 524 | 'state': [state], 525 | }) 526 | 527 | def loginAndVerifyClientId(self, expected_client_id): 528 | """ 529 | Open the login page and check that the client_id in the redirect target 530 | matches the expected value. 531 | """ 532 | response = requests.get(URL + '/github/login', allow_redirects=False) 533 | self.assertEqual(response.status_code, 302) 534 | 535 | redirect_url = urlparse.urlparse(response.headers['Location']) 536 | self.assertEqual(redirect_url.scheme, 'https') 537 | self.assertEqual(redirect_url.netloc, 'github.com') 538 | self.assertEqual(redirect_url.path, '/login/oauth/authorize') 539 | params = urlparse.parse_qs(redirect_url.query, keep_blank_values=True) 540 | state = params['state'][0] # this is a random value 541 | self.assertEqual(params, { 542 | 'client_id': [expected_client_id], 543 | 'redirect_uri': [URL + '/github/oauth'], 544 | 'response_type': ['code'], 545 | 'scope': [''], 546 | 'state': [state], 547 | }) 548 | 549 | def testLoginWithSecretInEnvironment(self): 550 | """Test that passing client_id in environment works""" 551 | 552 | secret_env = os.environ.copy() 553 | secret_env.update({'TRAC_GITHUB_CLIENT_ID': '98765432109876543210'}) 554 | 555 | with TracContext(self, client_id='TRAC_GITHUB_CLIENT_ID', env=secret_env): 556 | self.loginAndVerifyClientId('98765432109876543210') 557 | 558 | def testLoginWithSecretInFile(self): 559 | """Test that passing client_id in absolute path works""" 560 | 561 | path = d(SECRET) 562 | 563 | with TracContext(self, client_id=path): 564 | self.loginAndVerifyClientId('98765432109876543210') 565 | 566 | def testLoginWithSecretInRelativeFile(self): 567 | """Test that passing client_id in relative path works""" 568 | 569 | path = './' + os.path.relpath(d(SECRET)) 570 | 571 | with TracContext(self, client_id=path): 572 | self.loginAndVerifyClientId('98765432109876543210') 573 | 574 | def testLoginWithUnconfiguredClientId(self): 575 | """Test that leaving client_id unconfigured prints a warning""" 576 | with TracContext(self, client_id=''): 577 | session = requests.Session() 578 | 579 | response = session.get(URL + '/github/login', allow_redirects=True) 580 | self.assertEqual(response.status_code, 200) 581 | 582 | tree = html.fromstring(response.content) 583 | errmsg = ''.join(tree.xpath('//div[@id="warning"]/text()')).strip() 584 | self.assertIn( 585 | "GitHubLogin configuration incomplete, missing client_id or " 586 | "client_secret", errmsg, 587 | "An unconfigured GitHubLogin module should redirect and print " 588 | "a warning on login attempts.") 589 | 590 | def attemptHttpAuth(self, testenv, **kwargs): 591 | """ 592 | Helper method that attempts to log in using HTTP authentication in the 593 | given testenv; returns a tuple where the first item is the error 594 | message in the notification box on the trac page loaded after the login 595 | attempt (or an empty string on success) and the second item is the 596 | username as seen by trac. 597 | """ 598 | with TracContext(self, env=testenv, resync=False, **kwargs): 599 | session = requests.Session() 600 | 601 | # This logs into trac using HTTP authentication 602 | # This adds a oauth_state parameter in the Trac session. 603 | response = session.get(URL + '/login', auth=requests.auth.HTTPDigestAuth('user', 'pass')) 604 | self.assertNotEqual(response.status_code, 403) 605 | 606 | response = session.get(URL + '/newticket') # this should trigger IPermissionGroupProvider 607 | self.assertEqual(response.status_code, 200) 608 | tree = html.fromstring(response.content) 609 | warning = ''.join(tree.xpath('//div[@id="warning"]/text()')).strip() 610 | user = '' 611 | match = re.match(r"logged in as (.*)", 612 | ', '.join(tree.xpath('//div[@id="metanav"]/ul/li[@class="first"]/text()'))) 613 | if match: 614 | user = match.group(1) 615 | return (warning, user) 616 | 617 | def attemptValidOauth(self, testenv, callback, **kwargs): 618 | """ 619 | Helper method that runs a valid OAuth2 attempt in the given testenv 620 | with the given callback; returns a tuple where the first item is the 621 | error message in the notification box on the trac page loaded after the 622 | login attempt (or an empty string on success), the second item is 623 | a list of email addresses found in email fields of the preferences 624 | after login and the third item is the username of the user that was 625 | logged in as seen by trac.. 626 | """ 627 | ctxt_kwargs = {} 628 | other_kwargs = {} 629 | for kwarg in kwargs: 630 | if kwarg in TracContext._valid_attrs: 631 | ctxt_kwargs[kwarg] = kwargs[kwarg] 632 | else: 633 | other_kwargs[kwarg] = kwargs[kwarg] 634 | with TracContext(self, env=testenv, resync=False, **ctxt_kwargs): 635 | updateMockData(self.mockdata, postcallback=callback, **other_kwargs) 636 | try: 637 | session = requests.Session() 638 | 639 | # This adds a oauth_state parameter in the Trac session. 640 | response = session.get(URL + '/github/login', allow_redirects=False) 641 | self.assertEqual(response.status_code, 302) 642 | 643 | # Extract the state from the redirect 644 | redirect_url = urlparse.urlparse(response.headers['Location']) 645 | params = urlparse.parse_qs(redirect_url.query, keep_blank_values=True) 646 | state = params['state'][0] # this is a random value 647 | response = session.get( 648 | URL + '/github/oauth', 649 | params={ 650 | 'code': '01234567890123456789', 651 | 'state': state 652 | }, 653 | allow_redirects=False) 654 | self.assertEqual(response.status_code, 302) 655 | 656 | response = session.get(URL + '/prefs') 657 | self.assertEqual(response.status_code, 200) 658 | tree = html.fromstring(response.content) 659 | warning = ''.join(tree.xpath('//div[@id="warning"]/text()')).strip() 660 | email = tree.xpath('//input[@id="email"]/@value') 661 | user = '' 662 | match = re.match(r"logged in as (.*)", 663 | ', '.join(tree.xpath('//div[@id="metanav"]/ul/li[@class="first"]/text()'))) 664 | if match: 665 | user = match.group(1) 666 | return (warning, email, user) 667 | finally: 668 | # disable callback again 669 | updateMockData(self.mockdata, postcallback="") 670 | 671 | def testOauthBackendUnavailable(self): 672 | """ 673 | Test that an OAuth backend that resets the connection does not crash 674 | the login 675 | """ 676 | errmsg, emails, _ = self.attemptValidOauth(self.trac_env_broken, "") 677 | self.assertIn( 678 | "Invalid request. Please try to login again.", 679 | errmsg, 680 | "OAuth Authorization Request with unavailable backend should not succeed.") 681 | 682 | def testOauthBackendFails(self): 683 | """Test that an OAuth backend that fails does not crash the login""" 684 | def cb(path, args): 685 | return 403, {} 686 | errmsg, emails, _ = self.attemptValidOauth(self.trac_env, cb) 687 | self.assertIn( 688 | "Invalid request. Please try to login again.", 689 | errmsg, 690 | "OAuth Authorization Request with failing backend should not succeed.") 691 | 692 | def oauthCallbackSuccess(self, path, args): 693 | """ 694 | GitHubAPIMock POST callback that contains a successful OAuth 695 | Authentication Response 696 | """ 697 | return 200, { 698 | 'access_token': '190c20e9d87de41264749672ccacdd63a0ae2345a63b2703e26e651248c3b50e', 699 | 'token_type': 'bearer' 700 | } 701 | 702 | def testOauthValidButUnavailAPI(self): 703 | """ 704 | Test that accessing an unavailable GitHub API with what seems to be 705 | a valid OAuth2 token does not crash the login 706 | """ 707 | errmsg, emails, _ = self.attemptValidOauth(self.trac_env_broken_api, self.oauthCallbackSuccess) 708 | self.assertIn( 709 | "An error occurred while communicating with the GitHub API", 710 | errmsg, 711 | "Request to unavailable API with valid OAuth token should print an error.") 712 | 713 | def testOauthValidButBrokenAPI(self): 714 | """ 715 | Test that accessing an broken GitHub API with what seems to be a valid 716 | OAuth2 token does not crash the login 717 | """ 718 | errmsg, emails, _ = self.attemptValidOauth(self.trac_env_broken_api, 719 | self.oauthCallbackSuccess, 720 | retcode=403) 721 | self.assertIn( 722 | "An error occurred while communicating with the GitHub API", 723 | errmsg, 724 | "Failing API request with valid OAuth token should print an error.") 725 | 726 | def testOauthValidEmailAPIInvalid(self): 727 | """ 728 | Test that a login with a valid OAuth2 but invalid data returned from 729 | the email request API does not crash 730 | """ 731 | answers = { 732 | '/user': { 733 | 'user': 'trololol', 734 | 'email': 'lololort@example.com', 735 | 'login': 'trololol' 736 | }, 737 | '/user/email': { 738 | 'foo': 'bar' 739 | } 740 | } 741 | 742 | errmsg, emails, _ = self.attemptValidOauth( 743 | self.trac_env, self.oauthCallbackSuccess, retcode=200, 744 | answers=answers, request_email=True) 745 | self.assertIn( 746 | "An error occurred while retrieving your email address from the GitHub API", 747 | errmsg, 748 | "Failing email API request with valid OAuth token should print an error.") 749 | 750 | def getEmail(self, answers, **kwargs): 751 | """Get and return the email address the system has chosen for the given config and answers""" 752 | errmsg, emails, _ = self.attemptValidOauth( 753 | self.trac_env, self.oauthCallbackSuccess, retcode=200, 754 | answers=answers, **kwargs) 755 | self.assertEqual(len(errmsg), 0, 756 | "Successful login should not print an error.") 757 | 758 | return emails 759 | 760 | def getUser(self, answers, **kwargs): 761 | """Get and return the user name the system has chosen for the given config and answers""" 762 | errmsg, _, user = self.attemptValidOauth( 763 | self.trac_env, self.oauthCallbackSuccess, retcode=200, 764 | answers=answers, **kwargs) 765 | self.assertEqual(len(errmsg), 0, 766 | "Successful login should not print an error.") 767 | 768 | return user 769 | 770 | def testOauthValid(self): 771 | """Test that a login with a valid OAuth2 token succeeds""" 772 | answers = { 773 | '/user': { 774 | 'user': 'trololol', 775 | 'email': 'lololort@example.com', 776 | 'login': 'trololol' 777 | } 778 | } 779 | 780 | email = self.getEmail(answers) 781 | self.assertEqual(email, ['lololort@example.com']) 782 | user = self.getUser(answers) 783 | self.assertEqual(user, 'trololol') 784 | 785 | def testUsernamePrefix(self): 786 | """Test that setting a prefix for GitHub usernames works""" 787 | answers = { 788 | '/user': { 789 | 'user': 'trololol', 790 | 'email': 'lololort@example.com', 791 | 'login': 'trololol' 792 | } 793 | } 794 | 795 | user = self.getUser(answers, username_prefix='github-') 796 | self.assertEqual(user, 'github-trololol') 797 | 798 | errmsg, user = self.attemptHttpAuth(self.trac_env, 799 | username_prefix='github-', 800 | organization='org', 801 | username='github-bot-user', 802 | access_token='accesstoken') 803 | self.assertEqual(len(errmsg), 0, 804 | "HTTP authentication should still work.") 805 | self.assertEqual(user, "user", 806 | "Non-GitHub-authentication should yield unprefixed usernames") 807 | 808 | def testOauthEmailIgnoresUnverified(self): 809 | """ 810 | Test that request_email=True ignores unverified email addresses and 811 | prefers primary addresses 812 | """ 813 | answers = { 814 | '/user': { 815 | 'user': 'trololol', 816 | 'login': 'trololol' 817 | }, 818 | '/user/emails': [ 819 | { 820 | 'email': 'torvalds@linux-foundation.org', 821 | 'verified': False, 822 | 'primary': True 823 | }, 824 | { 825 | 'email': 'lololort@example.net', 826 | 'verified': True, 827 | 'primary': False 828 | }, 829 | { 830 | 'email': 'lololort@example.com', 831 | 'verified': True, 832 | 'primary': True 833 | }, 834 | ] 835 | } 836 | 837 | email = self.getEmail(answers, request_email=True) 838 | self.assertEqual(email, ['lololort@example.com']) 839 | 840 | def testPreferredEmailDomain(self): 841 | """ 842 | Test that an email address matching the preferred email domain is 843 | preferred to one marked primary. 844 | """ 845 | answers = { 846 | '/user': { 847 | 'user': 'trololol', 848 | 'login': 'trololol' 849 | }, 850 | '/user/emails': [ 851 | { 852 | 'email': 'torvalds@linux-foundation.org', 853 | 'verified': False, 854 | 'primary': True 855 | }, 856 | { 857 | 'email': 'lololort@example.com', 858 | 'verified': True, 859 | 'primary': True 860 | }, 861 | { 862 | 'email': 'lololort@example.net', 863 | 'verified': True, 864 | 'primary': False 865 | }, 866 | ] 867 | } 868 | 869 | email = self.getEmail(answers, request_email=True, 870 | preferred_email_domain='example.net') 871 | self.assertEqual(email, ['lololort@example.net']) 872 | 873 | def testPreferredEmailFallbackToPrimary(self): 874 | """ 875 | Test that the primary address is chosen if no address matches the 876 | preferred email domain. 877 | """ 878 | answers = { 879 | '/user': { 880 | 'user': 'trololol', 881 | 'login': 'trololol' 882 | }, 883 | '/user/emails': [ 884 | { 885 | 'email': 'lololort@example.com', 886 | 'verified': True, 887 | 'primary': True 888 | }, 889 | { 890 | 'email': 'lololort@example.net', 891 | 'verified': True, 892 | 'primary': False 893 | }, 894 | ] 895 | } 896 | 897 | email = self.getEmail(answers, request_email=True, 898 | preferred_email_domain='example.org') 899 | self.assertEqual(email, ['lololort@example.com']) 900 | 901 | def testPreferredEmailCaseInsensitive(self): 902 | """ 903 | Test that the preferred email domain is honoured regardless of case. 904 | """ 905 | answers = { 906 | '/user': { 907 | 'user': 'trololol', 908 | 'login': 'trololol' 909 | }, 910 | '/user/emails': [ 911 | { 912 | 'email': 'lololort@example.com', 913 | 'verified': True, 914 | 'primary': True 915 | }, 916 | { 917 | 'email': 'lololort@EXAMPLE.NET', 918 | 'verified': True, 919 | 'primary': False 920 | }, 921 | ] 922 | } 923 | 924 | email = self.getEmail(answers, request_email=True, 925 | preferred_email_domain='example.net') 926 | self.assertEqual(email, ['lololort@EXAMPLE.NET']) 927 | 928 | 929 | class GitHubPostCommitHookTests(TracGitHubTests): 930 | 931 | def testDefaultRepository(self): 932 | output = self.openGitHubHook(0).read() 933 | self.assertEqual(output, "Running hook on (default)\n" 934 | "* Updating clone\n" 935 | "* Synchronizing with clone\n") 936 | 937 | def testAlternativeRepository(self): 938 | output = self.openGitHubHook(0, 'alt').read() 939 | self.assertEqual(output, "Running hook on alt\n" 940 | "* Updating clone\n" 941 | "* Synchronizing with clone\n") 942 | 943 | def testCommit(self): 944 | self.makeGitCommit(GIT, 'foo', 'foo content\n') 945 | output = self.openGitHubHook().read() 946 | self.assertRegexpMatches(output, r"Running hook on \(default\)\n" 947 | r"\* Updating clone\n" 948 | r"\* Synchronizing with clone\n" 949 | r"\* Adding commit [0-9a-f]{40}\n") 950 | 951 | def testMultipleCommits(self): 952 | self.makeGitCommit(GIT, 'bar', 'bar content\n') 953 | self.makeGitCommit(GIT, 'bar', 'more bar content\n') 954 | output = self.openGitHubHook(2).read() 955 | self.assertRegexpMatches(output, r"Running hook on \(default\)\n" 956 | r"\* Updating clone\n" 957 | r"\* Synchronizing with clone\n" 958 | r"\* Adding commits [0-9a-f]{40}, [0-9a-f]{40}\n") 959 | 960 | def testCommitOnBranch(self): 961 | self.makeGitBranch(ALTGIT, 'stable/1.0') 962 | self.makeGitCommit(ALTGIT, 'stable', 'stable branch\n', branch='stable/1.0') 963 | self.makeGitBranch(ALTGIT, 'unstable/1.0') 964 | self.makeGitCommit(ALTGIT, 'unstable', 'unstable branch\n', branch='unstable/1.0') 965 | output = self.openGitHubHook(2, 'alt').read() 966 | self.assertRegexpMatches(output, r"Running hook on alt\n" 967 | r"\* Updating clone\n" 968 | r"\* Synchronizing with clone\n" 969 | r"\* Adding commit [0-9a-f]{40}\n" 970 | r"\* Skipping commit [0-9a-f]{40}\n") 971 | 972 | def testUnknownCommit(self): 973 | # Emulate self.openGitHubHook to use a non-existent commit id 974 | random_id = ''.join(random.choice('0123456789abcdef') for _ in range(40)) 975 | payload = {'commits': [{'id': random_id, 'message': '', 'distinct': True}]} 976 | request = urllib2.Request(URL + '/github', json.dumps(payload), HEADERS) 977 | output = urllib2.urlopen(request).read() 978 | self.assertRegexpMatches(output, r"Running hook on \(default\)\n" 979 | r"\* Updating clone\n" 980 | r"\* Synchronizing with clone\n" 981 | r"\* Unknown commit [0-9a-f]{40}\n") 982 | 983 | def testNotification(self): 984 | ticket = Ticket(self.env) 985 | ticket['summary'] = 'I need a commit!' 986 | ticket['status'] = 'new' 987 | ticket['owner'] = '' 988 | ticket_id = ticket.insert() 989 | 990 | ticket = Ticket(self.env, ticket_id) 991 | self.assertEqual(ticket['status'], 'new') 992 | self.assertEqual(ticket['resolution'], '') 993 | 994 | message = "Fix #%d: here you go." % ticket_id 995 | self.makeGitCommit(GIT, 'newfile', 'with some new content', message) 996 | self.openGitHubHook() 997 | 998 | ticket = Ticket(self.env, ticket_id) 999 | self.assertEqual(ticket['status'], 'closed') 1000 | self.assertEqual(ticket['resolution'], 'fixed') 1001 | changelog = ticket.get_changelog() 1002 | self.assertEqual(len(changelog), 4) 1003 | self.assertEqual(changelog[0][2], 'comment') 1004 | self.assertIn("here you go", changelog[0][4]) 1005 | 1006 | def testComplexNotification(self): 1007 | ticket1 = Ticket(self.env) 1008 | ticket1['summary'] = 'Fix please.' 1009 | ticket1['status'] = 'new' 1010 | ticket1_id = ticket1.insert() 1011 | ticket2 = Ticket(self.env) 1012 | ticket2['summary'] = 'This one too, thanks.' 1013 | ticket2['status'] = 'new' 1014 | ticket2_id = ticket2.insert() 1015 | 1016 | ticket1 = Ticket(self.env, ticket1_id) 1017 | self.assertEqual(ticket1['status'], 'new') 1018 | self.assertEqual(ticket1['resolution'], '') 1019 | ticket2 = Ticket(self.env, ticket2_id) 1020 | self.assertEqual(ticket2['status'], 'new') 1021 | self.assertEqual(ticket2['resolution'], '') 1022 | 1023 | message1 = "Fix #%d: you're welcome." % ticket1_id 1024 | self.makeGitCommit(ALTGIT, 'newfile', 'with some new content', message1) 1025 | message2 = "See #%d: you bet." % ticket2_id 1026 | self.makeGitCommit(ALTGIT, 'newfile', 'with improved content', message2) 1027 | self.openGitHubHook(2, 'alt') 1028 | 1029 | ticket1 = Ticket(self.env, ticket1_id) 1030 | self.assertEqual(ticket1['status'], 'closed') 1031 | self.assertEqual(ticket1['resolution'], 'fixed') 1032 | changelog1 = ticket1.get_changelog() 1033 | # Trac 1.2 generates three fields, Trac 1.0 four. 1034 | self.assertGreaterEqual(len(changelog1), 3) 1035 | self.assertEqual(changelog1[0][2], 'comment') 1036 | self.assertIn("you're welcome", changelog1[0][4]) 1037 | ticket2 = Ticket(self.env, ticket2_id) 1038 | self.assertEqual(ticket2['status'], 'new') 1039 | self.assertEqual(ticket2['resolution'], '') 1040 | changelog2 = ticket2.get_changelog() 1041 | self.assertEqual(len(changelog2), 1) 1042 | self.assertEqual(changelog2[0][2], 'comment') 1043 | self.assertIn("you bet", changelog2[0][4]) 1044 | 1045 | def testPing(self): 1046 | payload = {'zen': "Readability counts."} 1047 | headers = {'Content-Type': 'application/json', 'X-GitHub-Event': 'ping'} 1048 | request = urllib2.Request(URL + '/github', json.dumps(payload), headers) 1049 | output = urllib2.urlopen(request).read() 1050 | self.assertEqual(output, "Readability counts.") 1051 | 1052 | def testUnknownEvent(self): 1053 | headers = {'Content-Type': 'application/json', 'X-GitHub-Event': 'pull'} 1054 | request = urllib2.Request(URL + '/github', json.dumps({}), headers) 1055 | with self.assertRaisesRegexp(urllib2.HTTPError, r'^HTTP Error 400: Bad Request$'): 1056 | urllib2.urlopen(request) 1057 | 1058 | def testBadMethod(self): 1059 | with self.assertRaisesRegexp(urllib2.HTTPError, r'^HTTP Error 405: Method Not Allowed$'): 1060 | urllib2.urlopen(URL + '/github') 1061 | 1062 | def testBadPayload(self): 1063 | request = urllib2.Request(URL + '/github', 'foobar', HEADERS) 1064 | with self.assertRaisesRegexp(urllib2.HTTPError, r'^HTTP Error 400: Bad Request$'): 1065 | urllib2.urlopen(request) 1066 | 1067 | def testBadRepository(self): 1068 | request = urllib2.Request(URL + '/github/nosuchrepo', '{}', HEADERS) 1069 | with self.assertRaisesRegexp(urllib2.HTTPError, r'^HTTP Error 400: Bad Request$'): 1070 | urllib2.urlopen(request) 1071 | 1072 | def testBadUrl(self): 1073 | request = urllib2.Request(URL + '/githubnosuchurl', '{}', HEADERS) 1074 | with self.assertRaisesRegexp(urllib2.HTTPError, r'^HTTP Error 404: Not Found$'): 1075 | urllib2.urlopen(request) 1076 | 1077 | 1078 | class GitHubPostCommitHookWithSignedWebHookTests(TracGitHubTests): 1079 | 1080 | @classmethod 1081 | def setUpClass(cls): 1082 | cls.createGitRepositories() 1083 | cls.createTracEnvironment(webhook_secret='6c12713595df9247974fa0f2f99b94c815f242035c49c7f009892bfd7d9f0f98') 1084 | cls.startTracd() 1085 | cls.env = Environment(d(ENV)) 1086 | 1087 | def testUnsignedPing(self): 1088 | payload = {'zen': "Readability counts."} 1089 | headers = {'Content-Type': 'application/json', 'X-GitHub-Event': 'ping'} 1090 | request = urllib2.Request(URL + '/github', json.dumps(payload), headers) 1091 | with self.assertRaisesRegexp(urllib2.HTTPError, r'^HTTP Error 403: Forbidden$'): 1092 | urllib2.urlopen(request).read() 1093 | 1094 | def testSignedPing(self): 1095 | # Correct signature can be generated with OpenSSL: 1096 | # $> printf '{"zen": "Echo me"}\n' | openssl dgst -sha256 -hmac $webhook_secret 1097 | payload = {'zen': "Echo me"} 1098 | signature = "sha256=cacc93c2df1b21313e16d8690fc21e56229b6a9525e7016db38bdf9bad708fed" 1099 | headers = {'Content-Type': 'application/json', 1100 | 'X-GitHub-Event': 'ping', 1101 | 'X-Hub-Signature': signature} 1102 | request = urllib2.Request(URL + '/github', json.dumps(payload) + '\n', headers) 1103 | output = urllib2.urlopen(request).read() 1104 | self.assertEqual(output, "Echo me") 1105 | 1106 | 1107 | class GitHubPostCommitHookWithUpdateHookTests(TracGitHubTests): 1108 | 1109 | @classmethod 1110 | def createUpdateHook(cls): 1111 | with open(d(UPDATEHOOK), 'wb') as fp: 1112 | # simple shell script to echo back all input 1113 | fp.write("""#!/bin/sh\nexec cat""") 1114 | os.fchmod(fp.fileno(), 0o755) 1115 | 1116 | def createFailingUpdateHook(cls): 1117 | with open(d(UPDATEHOOK), 'wb') as fp: 1118 | fp.write("""#!/bin/sh\nexit 1""") 1119 | os.fchmod(fp.fileno(), 0o755) 1120 | 1121 | @classmethod 1122 | def removeUpdateHook(cls): 1123 | os.remove(d(UPDATEHOOK)) 1124 | 1125 | @classmethod 1126 | def setUpClass(cls): 1127 | super(GitHubPostCommitHookWithUpdateHookTests, cls).setUpClass() 1128 | # Make sure the hooks directory exists in the repo (can be disabled in some git configs) 1129 | try: 1130 | os.mkdir(d('%s-mirror' % GIT, 'hooks')) 1131 | except OSError: 1132 | pass 1133 | cls.createUpdateHook() 1134 | 1135 | @classmethod 1136 | def tearDownClass(cls): 1137 | cls.removeUpdateHook() 1138 | super(GitHubPostCommitHookWithUpdateHookTests, cls).tearDownClass() 1139 | 1140 | def testUpdateHook(self): 1141 | self.makeGitCommit(GIT, 'foo', 'foo content\n') 1142 | payload = self.makeGitHubHookPayload() 1143 | output = self.openGitHubHook(payload=payload).read() 1144 | self.assertRegexpMatches(output, r"Running hook on \(default\)\n" 1145 | r"\* Updating clone\n" 1146 | r"\* Synchronizing with clone\n" 1147 | r"\* Adding commit [0-9a-f]{40}\n" 1148 | r"\* Running trac-github-update hook\n") 1149 | self.assertEqual(output.split('\n')[-1], json.dumps(payload)) 1150 | 1151 | def testUpdateHookExecFailure(self): 1152 | os.chmod(d(UPDATEHOOK), 0o644) 1153 | self.makeGitCommit(GIT, 'bar', 'bar content\n') 1154 | payload = self.makeGitHubHookPayload() 1155 | with self.assertRaisesRegexp(urllib2.HTTPError, r'^HTTP Error 500: Internal Server Error$'): 1156 | output = self.openGitHubHook(payload=payload).read() 1157 | 1158 | def testUpdateHookFailure(self): 1159 | self.createFailingUpdateHook() 1160 | self.makeGitCommit(GIT, 'baz', 'baz content\n') 1161 | payload = self.makeGitHubHookPayload() 1162 | with self.assertRaisesRegexp(urllib2.HTTPError, r'^HTTP Error 500: Internal Server Error$'): 1163 | output = self.openGitHubHook(payload=payload).read() 1164 | 1165 | 1166 | class GitHubBrowserWithCacheTests(GitHubBrowserTests): 1167 | 1168 | cached_git = True 1169 | 1170 | 1171 | class GitHubPostCommitHookWithCacheTests(GitHubPostCommitHookTests): 1172 | 1173 | cached_git = True 1174 | 1175 | 1176 | class GitHubAPIMock(BaseHTTPServer.BaseHTTPRequestHandler): 1177 | def log_message(self, format, *args): 1178 | # Visibly differentiate GitHub API mock logging from tracd logs 1179 | sys.stderr.write("%s [%s] %s\n" % 1180 | (self.__class__.__name__, 1181 | self.log_date_time_string(), 1182 | format%args)) 1183 | 1184 | def do_GET(self): 1185 | md = self.server.mockdata 1186 | md['lock'].acquire() 1187 | retcode = md['retcode'] 1188 | contenttype = md['content-type'] 1189 | answers = md['answers'].copy() 1190 | md['lock'].release() 1191 | 1192 | path, _, querystring = self.path.partition('?') 1193 | parameters = {k: v for k, _, v in (q.partition('=') for q in querystring.split('&') if q)} 1194 | 1195 | if path in answers: 1196 | answer = answers[path] 1197 | else: 1198 | answer = {'message': 'No handler for URI %s' % path} 1199 | retcode = 404 1200 | 1201 | self.send_response(retcode) 1202 | 1203 | # Add pagination 1204 | per_page = 5 1205 | if isinstance(answer, list): 1206 | length = len(answer) 1207 | page = int(parameters.get('page', 1)) 1208 | start = (page - 1) * per_page 1209 | end = page * per_page 1210 | answer = answer[start:end] 1211 | 1212 | links = [] 1213 | if page > 1: 1214 | prevparams = parameters.copy() 1215 | prevparams.update({'page': (page - 1)}) 1216 | prev_link = '; rel="prev"'.format( 1217 | self.headers['Host'], 1218 | path, 1219 | '&'.join(('='.join((str(k), str(v))) for k, v in prevparams.iteritems())) 1220 | ) 1221 | links.append(prev_link) 1222 | if length >= end: 1223 | nextparams = parameters.copy() 1224 | nextparams.update({'page': (page + 1)}) 1225 | next_link = '; rel="next"'.format( 1226 | self.headers['Host'], 1227 | path, 1228 | '&'.join(('='.join((str(k), str(v))) for k, v in nextparams.iteritems())) 1229 | ) 1230 | links.append(next_link) 1231 | if len(links) > 0: 1232 | self.send_header("Link", ", ".join(links)) 1233 | 1234 | self.send_header("Content-Type", contenttype) 1235 | self.end_headers() 1236 | 1237 | self.wfile.write(json.dumps(answer)) 1238 | 1239 | def do_POST(self): 1240 | md = self.server.mockdata 1241 | md['lock'].acquire() 1242 | retcode = md['retcode'] 1243 | contenttype = md['content-type'] 1244 | postcallback = md['post-callback'] 1245 | md['lock'].release() 1246 | 1247 | max_chunk_size = 10*1024*1024 1248 | size_remaining = int(self.headers["content-length"]) 1249 | L = [] 1250 | while size_remaining: 1251 | chunk_size = min(size_remaining, max_chunk_size) 1252 | chunk = self.rfile.read(chunk_size) 1253 | if not chunk: 1254 | break 1255 | L.append(chunk) 1256 | size_remaining -= len(L[-1]) 1257 | args = urlparse.parse_qs(''.join(L)) 1258 | 1259 | retcode = 404 1260 | answer = {} 1261 | if postcallback: 1262 | try: 1263 | retcode, answer = postcallback(self.path, args) 1264 | except Exception: 1265 | retcode = 500 1266 | answer = traceback.format_exc() 1267 | 1268 | self.send_response(retcode) 1269 | self.send_header("Content-Type", contenttype) 1270 | self.end_headers() 1271 | self.wfile.write(json.dumps(answer)) 1272 | 1273 | 1274 | class TracContext(object): 1275 | """ 1276 | Context manager that starts and stops a configured tracd instance on port 1277 | 8765. 1278 | """ 1279 | 1280 | _valid_attrs = ('cached_git', 1281 | 'client_id', 1282 | 'client_secret', 1283 | 'request_email', 1284 | 'preferred_email_domain', 1285 | 'organization', 1286 | 'username', 1287 | 'access_token', 1288 | 'webhook_secret', 1289 | 'username_prefix', 1290 | 'resync') 1291 | """ List of all valid attributes to be passed to createTracEnvironment() """ 1292 | 1293 | cached_git = False 1294 | """ Whether to use a persistent repository cache """ 1295 | 1296 | traclock = threading.Lock() 1297 | """ Lock to ensure no two trac instances are started simultaneously. """ 1298 | 1299 | def __init__(self, testobj, env=None, **kwargs): 1300 | """ 1301 | Set up a new Trac context manager. Arguments are: 1302 | 1303 | :param testobj: An instance of `TracGitHubTests` used to create the 1304 | Trac environment and start tracd. 1305 | :param env: Dictionary of environment variables to set when starting 1306 | tracd, or `None` for a copy of the current environment. 1307 | :param client_id: Client ID for the GitHub OAuth application 1308 | :param client_secret: Client Secret for the GitHub OAuth application 1309 | :param request_email: `True` to request access to all email addresses 1310 | from GitHub in the login module; defaults to 1311 | `False`. 1312 | :param cached_git: `True` to use a persistent repository cache; 1313 | defaults to `False`. 1314 | :param organization: Name of the GitHub organization to configure for 1315 | group syncing. Defaults to unset. 1316 | :param username: Username of the GitHub user to use for group syncing. 1317 | Defaults to unset. 1318 | :param access_token: GitHub access token of the GitHub user to use for 1319 | group syncing. Defaults to unset. 1320 | :param webhook_secret: Secret used to validate WebHook API calls if 1321 | present. Defaults to unset. 1322 | :param username_prefix: Prefix for GitHub usernames to allow 1323 | co-existance of non-GitHub with GitHub accounts. 1324 | :param resync: `False` to skip running `trac admin repository resync` 1325 | during environment setup for speed reasons. Defaults to 1326 | `True`. 1327 | """ 1328 | for kwarg in kwargs: 1329 | if kwarg in self._valid_attrs: 1330 | setattr(self, kwarg, kwargs[kwarg]) 1331 | self._env = env 1332 | self._testobj = testobj 1333 | 1334 | def __enter__(self): 1335 | """ 1336 | Create a trac environment and start a tracd instance with this 1337 | environment. Returns an instance of a `trac.env.Environment`. 1338 | """ 1339 | # Only one trac environment at the same time because of the shared port 1340 | # and FS resources 1341 | self.traclock.acquire() 1342 | 1343 | # Set up trac env 1344 | kwargs = {} 1345 | for attr in self._valid_attrs: 1346 | if hasattr(self, attr): 1347 | kwargs[attr] = getattr(self, attr) 1348 | self._testobj.createTracEnvironment(**kwargs) 1349 | self._tracenv = Environment(d(ENV)) 1350 | 1351 | # Start tracd 1352 | self._testobj.startTracd(env=self._env) 1353 | 1354 | return self._tracenv 1355 | 1356 | def __exit__(self, etype, evalue, traceback): 1357 | """ 1358 | Shut down a running tracd instance and clean up the environment. Always 1359 | returns `False` to re-throw any exceptions that might have occurred. 1360 | """ 1361 | # Stop tracd 1362 | self._testobj.stopTracd() 1363 | # Clean up trac env 1364 | self._tracenv.shutdown() 1365 | self._testobj.removeTracEnvironment() 1366 | self.traclock.release() 1367 | return False 1368 | 1369 | class GitHubGroupsProviderTests(TracGitHubTests): 1370 | # Append custom failure messages to the automatically generated ones 1371 | longMessage = True 1372 | # GitHubGroupsProvider configuration values (note that not all of those are always used!) 1373 | organization = 'org' 1374 | username = 'github-test-sync-user' 1375 | access_token = 'e42b79d0a8275cab1f8c8c8ff0e2d99537b54ed9' 1376 | webhook_secret = '6c12713595df9247974fa0f2f99b94c815f242035c49c7f009892bfd7d9f0f98' 1377 | 1378 | @classmethod 1379 | def setUpClass(cls): 1380 | cls.createGitRepositories() 1381 | cls.mockdata = startAPIMock(8766) 1382 | 1383 | # Prepare sets of tracd environment variables 1384 | tracd_env = os.environ.copy() 1385 | tracd_env.update({'TRAC_GITHUB_API_URL': 'http://127.0.0.1:8766/'}) 1386 | tracd_env_debug = tracd_env.copy() 1387 | tracd_env_debug.update({'TRAC_GITHUB_ENABLE_DEBUGGING': '1'}) 1388 | tracd_env_broken = tracd_env_debug.copy() 1389 | tracd_env_broken.update({'TRAC_GITHUB_API_URL': 'http://127.0.0.1:8767/'}) 1390 | 1391 | cls.tracd_env = tracd_env 1392 | cls.tracd_env_debug = tracd_env_debug 1393 | cls.tracd_env_broken = tracd_env_broken 1394 | 1395 | # Prepare sets of trac configuration settings 1396 | trac_env = { 1397 | 'cached_git': True, 1398 | 'resync': False, 1399 | 'organization': cls.organization, 1400 | 'username': cls.username, 1401 | 'access_token': cls.access_token 1402 | } 1403 | trac_env_secured = trac_env.copy() 1404 | trac_env_secured.update({ 1405 | 'webhook_secret': cls.webhook_secret 1406 | }) 1407 | 1408 | cls.trac_env = trac_env 1409 | cls.trac_env_secured = trac_env_secured 1410 | 1411 | @classmethod 1412 | def tearDownClass(cls): 1413 | cls.removeGitRepositories() 1414 | # API Mock server is a daemon thread and will automatically stop 1415 | 1416 | def test_000_api_refuses_connection(self): 1417 | """ 1418 | Test that a request does not fail even if the API refuses connections. 1419 | """ 1420 | with TracContext(self, env=self.tracd_env_broken, **self.trac_env): 1421 | response = requests.get(URL + '/github-groups-dump', allow_redirects=False) 1422 | self.assertEqual(response.status_code, 200, 1423 | "Request with unresponsive API endpoint should not fail") 1424 | self.assertEqual(response.json(), {}, 1425 | "Unavailable API should yield no groups at all.") 1426 | 1427 | def test_001_unconfigured(self): 1428 | """ 1429 | Test whether a request with an unconfigured GitHubGroupsProvider fails. 1430 | """ 1431 | with TracContext(self, resync=False): 1432 | response = requests.get(URL + '/newticket', allow_redirects=False) 1433 | self.assertEqual(response.status_code, 200, 1434 | "Unconfigured GitHubGroupsProvider caused requests to fail") 1435 | 1436 | def test_002_disabled_debugging(self): 1437 | """ 1438 | Test that the debugging functionality does not work if not explicitly enabled. 1439 | """ 1440 | self.assertNotIn('TRAC_GITHUB_ENABLE_DEBUGGING', self.tracd_env, 1441 | "tracd_env enables debugging, but should not; did you export TRAC_GITHUB_ENABLE_DEBUGGING?") 1442 | with TracContext(self, env=self.tracd_env, resync=False): 1443 | response = requests.get(URL + '/github-groups-dump', allow_redirects=False) 1444 | self.assertEqual(response.status_code, 404, 1445 | "Debugging API was not enabled, but did not return HTTP 404") 1446 | 1447 | def test_003_api_returns_500(self): 1448 | """ 1449 | Test that a request with a failing API endpoint still succeeds. 1450 | """ 1451 | updateMockData(self.mockdata, retcode=500, answers={ 1452 | '/orgs/%s/teams' % self.organization: {} 1453 | }) 1454 | with TracContext(self, env=self.tracd_env_debug, **self.trac_env): 1455 | response = requests.get(URL + '/github-groups-dump', allow_redirects=False) 1456 | self.assertEqual(response.status_code, 200, 1457 | "Request with failing API endpoint should not fail") 1458 | self.assertEqual(response.json(), {}, 1459 | "500 on API should yield no groups at all") 1460 | 1461 | def test_004_api_returns_404(self): 1462 | """ 1463 | Test that a request with non-existant API endpoint still succeeds. 1464 | """ 1465 | updateMockData(self.mockdata, retcode=404, answers={}) 1466 | 1467 | with TracContext(self, env=self.tracd_env_debug, **self.trac_env): 1468 | response = requests.get(URL + '/github-groups-dump', allow_redirects=False) 1469 | self.assertEqual(response.status_code, 200, 1470 | "Request with 404 API endpoint should not fail") 1471 | self.assertEqual(response.json(), {}, 1472 | "404 on API should yield no groups at all") 1473 | 1474 | def test_005_org_has_no_teams(self): 1475 | """ 1476 | Test that a GitHub organization without teams is handled correctly. 1477 | """ 1478 | updateMockData(self.mockdata, retcode=200, answers={ 1479 | '/orgs/%s/teams' % self.organization: [] 1480 | }) 1481 | 1482 | with TracContext(self, env=self.tracd_env_debug, **self.trac_env): 1483 | response = requests.get(URL + '/github-groups-dump', allow_redirects=False) 1484 | self.assertEqual(response.status_code, 200, 1485 | "Request with organization without teams should not fail") 1486 | self.assertEqual(response.json(), {}, 1487 | "No github teams should yield no groups, but groups were returned") 1488 | 1489 | def test_006_normal_operation(self): 1490 | """ 1491 | Test results of normal operation and conversion of API results. 1492 | """ 1493 | users = [ 1494 | {"login": u"octocat"}, 1495 | {"login": u"octobird"}, 1496 | {"login": u"octodolphin"}, 1497 | {"login": u"octofox"}, 1498 | {"login": u"octoraccoon"}, 1499 | {"login": u"octokangaroo"}, 1500 | {"login": u"octokoala"}, 1501 | {"login": u"octospider"} 1502 | ] 1503 | team1members = [ 1504 | users[0], 1505 | users[1], 1506 | users[2], 1507 | users[4], 1508 | users[5], 1509 | users[6], 1510 | users[7] 1511 | ] 1512 | team12members = [ 1513 | users[0], 1514 | users[2], 1515 | users[3] 1516 | ] 1517 | 1518 | updateMockData(self.mockdata, retcode=200, answers={ 1519 | '/orgs/%s/teams' % self.organization: [ 1520 | { 1521 | "id": 1, 1522 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1523 | "name": u"Justice League", 1524 | "slug": u"justice-league" 1525 | }, 1526 | { 1527 | "id": 12, 1528 | "url": u"%sorganizations/14143/team/12" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1529 | "name": u"The League of Extraordinary Gentlemen and Gentlewomen", 1530 | "slug": u"gentlepeople" 1531 | } 1532 | ], 1533 | '/organizations/14143/team/1/members': team1members, 1534 | '/organizations/14143/team/12/members': team12members 1535 | }) 1536 | 1537 | with TracContext(self, env=self.tracd_env_debug, **self.trac_env): 1538 | response = requests.get(URL + '/github-groups-dump', allow_redirects=False) 1539 | self.assertEqual(response.status_code, 200) 1540 | 1541 | data = response.json() 1542 | # All users present? 1543 | for user in users: 1544 | login = user['login'] 1545 | self.assertIn(login, data, "user %s expected in groups" % login) 1546 | # All users part of the org? 1547 | for user in users: 1548 | login = user['login'] 1549 | groups = data[login] 1550 | self.assertIn(u"github-%s" % self.organization, 1551 | groups, 1552 | "user %s expected to be in organization group" % login) 1553 | # and in the group exactly once? 1554 | occurrences = len([x for x in groups if x == u"github-%s" % self.organization]) 1555 | self.assertEqual(occurrences, 1, 1556 | "user %s is expected once in organization group" % login) 1557 | 1558 | # Users are in the groups where we expect them? 1559 | for user in users: 1560 | login = user['login'] 1561 | groups = data[login] 1562 | if user in team1members: 1563 | self.assertIn(u"github-%s-justice-league" % self.organization, 1564 | groups, 1565 | "user %s expected in justice-league group" % login) 1566 | else: 1567 | self.assertNotIn(u"github-%s-justice-league" % self.organization, 1568 | groups, 1569 | "user %s not expected in justice-league group" % login) 1570 | if user in team12members: 1571 | self.assertIn(u"github-%s-gentlepeople" % self.organization, 1572 | groups, 1573 | "user %s expected in gentlepeople group" % login) 1574 | else: 1575 | self.assertNotIn(u"github-%s-gentlepeople" % self.organization, 1576 | groups, 1577 | "user %s not expected in gentlepeople group" % login) 1578 | 1579 | # Any unexpected groups? 1580 | allgroups = (u"github-%s" % self.organization, 1581 | u"github-%s-gentlepeople" % self.organization, 1582 | u"github-%s-justice-league" % self.organization) 1583 | for user in users: 1584 | login = user['login'] 1585 | for group in data[login]: 1586 | self.assertIn(group, allgroups, 1587 | "Unexpected group found for user %s" % login) 1588 | 1589 | # Any unexpected users? 1590 | allusers = [x["login"] for x in users] 1591 | for login in data.keys(): 1592 | self.assertIn(login, allusers, "Unexpected user found in result") 1593 | 1594 | def test_007_hook_get_request(self): 1595 | """ 1596 | Test that a GET request to /github-groups/? prints a message and returns HTTP 405. 1597 | """ 1598 | updateMockData(self.mockdata, retcode=200, answers={ 1599 | '/orgs/%s/teams' % self.organization: [] 1600 | }) 1601 | 1602 | with TracContext(self, env=self.tracd_env, **self.trac_env): 1603 | response = requests.get(URL + '/github-groups', allow_redirects=False) 1604 | self.assertEqual(response.status_code, 405, 1605 | "GET /github-groups did not return HTTP 405") 1606 | self.assertEqual(response.text, 1607 | "Endpoint is ready to accept GitHub Organization membership notifications.\n") 1608 | response = requests.get(URL + '/github-groups/', allow_redirects=False) 1609 | self.assertEqual(response.status_code, 405, 1610 | "/github-groups/ did not return 405") 1611 | self.assertEqual(response.text, 1612 | "Endpoint is ready to accept GitHub Organization membership notifications.\n") 1613 | 1614 | def test_008_hook_unsupported_event(self): 1615 | """ 1616 | Test that unsupported events sent to /github-groups/ are handled correctly. 1617 | """ 1618 | updateMockData(self.mockdata, retcode=200, answers={ 1619 | '/orgs/%s/teams' % self.organization: [] 1620 | }) 1621 | 1622 | with TracContext(self, env=self.tracd_env, **self.trac_env): 1623 | response = requests.post(URL + '/github-groups', 1624 | allow_redirects=False, 1625 | headers={'X-GitHub-Event': 'FooEvent'}) 1626 | self.assertEqual(response.status_code, 400, 1627 | "Sending an unsupported event should return HTTP 400") 1628 | self.assertEqual(response.text, "Event type FooEvent is not supported\n") 1629 | 1630 | def test_009_hook_ping_event(self): 1631 | """ 1632 | Test that a ping event sent to /github-groups/ is handled correctly. 1633 | """ 1634 | updateMockData(self.mockdata, retcode=200, answers={ 1635 | '/orgs/%s/teams' % self.organization: [] 1636 | }) 1637 | 1638 | with TracContext(self, env=self.tracd_env, **self.trac_env): 1639 | response = requests.post(URL + '/github-groups', 1640 | allow_redirects=False, 1641 | headers={'X-GitHub-Event': 'ping'}, 1642 | json={'zen': 'Echo me!'}) 1643 | self.assertEqual(response.status_code, 200, 1644 | "Ping event should return HTTP 200") 1645 | self.assertEqual(response.text, "Echo me!") 1646 | 1647 | def test_010_hook_ping_event_nonjson_payload(self): 1648 | """ 1649 | Test that a ping event with non-JSON payload sent to /github-groups/ does not crash the service. 1650 | """ 1651 | updateMockData(self.mockdata, retcode=200, answers={ 1652 | '/orgs/%s/teams' % self.organization: [] 1653 | }) 1654 | 1655 | with TracContext(self, env=self.tracd_env, **self.trac_env): 1656 | response = requests.post(URL + '/github-groups', 1657 | allow_redirects=False, 1658 | headers={'X-GitHub-Event': 'ping'}, 1659 | data="Fail to parse as JSON") 1660 | self.assertEqual(response.status_code, 400, 1661 | "Invalid payloads should return HTTP 400") 1662 | self.assertEqual(response.text, "Invalid payload\n") 1663 | 1664 | def test_011_hook_ping_event_invalid_json_payload(self): 1665 | """ 1666 | Test that a ping event without the expected JSON fields sent to /github-groups/ does not crash the service. 1667 | """ 1668 | updateMockData(self.mockdata, retcode=200, answers={ 1669 | '/orgs/%s/teams' % self.organization: [] 1670 | }) 1671 | 1672 | with TracContext(self, env=self.tracd_env, **self.trac_env): 1673 | response = requests.post(URL + '/github-groups', 1674 | allow_redirects=False, 1675 | headers={'X-GitHub-Event': 'ping'}, 1676 | json=[{'bar': 'baz'}]) 1677 | self.assertEqual(response.status_code, 500, 1678 | "Invalid payloads should return HTTP 500") 1679 | self.assertIn("Exception occurred while handling payload, possible invalid payload\n", response.text) 1680 | self.assertIn("Traceback (most recent call last):", response.text) 1681 | 1682 | def test_012_hook_membership_event_delete_team(self): 1683 | """ 1684 | Test that deleting a team with an event sent to /github-groups/ works. 1685 | """ 1686 | users = [ 1687 | {"login": u"octocat"}, 1688 | {"login": u"octobird"}, 1689 | {"login": u"octodolphin"}, 1690 | {"login": u"octofox"} 1691 | ] 1692 | team1members = [ 1693 | users[0], 1694 | users[1], 1695 | users[2] 1696 | ] 1697 | team12members = [ 1698 | users[0], 1699 | users[2], 1700 | users[3] 1701 | ] 1702 | 1703 | updateMockData(self.mockdata, retcode=200, answers={ 1704 | '/orgs/%s/teams' % self.organization: [ 1705 | { 1706 | "id": 1, 1707 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1708 | "name": u"Justice League", 1709 | "slug": u"justice-league" 1710 | }, 1711 | { 1712 | "id": 12, 1713 | "url": u"%sorganizations/14143/team/12" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1714 | "name": u"The League of Extraordinary Gentlemen and Gentlewomen", 1715 | "slug": u"gentlepeople" 1716 | } 1717 | ], 1718 | '/organizations/14143/team/1/members': team1members, 1719 | '/organizations/14143/team/12/members': team12members 1720 | }) 1721 | 1722 | update = { 1723 | "team": { 1724 | "id": 1, 1725 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1726 | "name": u"Justice League", 1727 | "deleted": True 1728 | } 1729 | } 1730 | 1731 | with TracContext(self, env=self.tracd_env_debug, **self.trac_env): 1732 | # Make sure the to-be-removed group exists 1733 | response = requests.get(URL + '/github-groups-dump', allow_redirects=False) 1734 | self.assertEqual(response.status_code, 200) 1735 | 1736 | data = response.json() 1737 | allgroups = set() 1738 | for groups in data.values(): 1739 | allgroups.update(groups) 1740 | self.assertIn(u"github-%s-justice-league" % self.organization, allgroups, 1741 | "Group to be removed not found in group output, test will be meaningless.") 1742 | 1743 | # Change the Mock API output 1744 | updateMockData(self.mockdata, answers={ 1745 | '/orgs/%s/teams' % self.organization: [ 1746 | { 1747 | "id": 12, 1748 | "url": u"%sorganizations/14143/team/12" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1749 | "name": u"The League of Extraordinary Gentlemen and Gentlewomen", 1750 | "slug": u"gentlepeople" 1751 | } 1752 | ], 1753 | '/organizations/14143/team/12/members': team12members 1754 | }) 1755 | 1756 | # Send the delete event 1757 | response = requests.post(URL + '/github-groups', 1758 | allow_redirects=False, 1759 | headers={'X-GitHub-Event': 'membership'}, 1760 | json=update) 1761 | self.assertEqual(response.status_code, 200, 1762 | "MembershipEvent handling should return HTTP 200") 1763 | self.assertEqual(response.text, "success") 1764 | 1765 | # Check that the group is gone 1766 | response = requests.get(URL + '/github-groups-dump', allow_redirects=False) 1767 | self.assertEqual(response.status_code, 200) 1768 | 1769 | data = response.json() 1770 | self.assertGreater(len(data), 0, "No groups returned after update") 1771 | for login in data: 1772 | groups = data[login] 1773 | self.assertNotIn(u"github-%s-justice-league" % self.organization, 1774 | groups, 1775 | "Deleted group still shows up for user %s" % login) 1776 | self.assertNotIn(users[1]["login"], data, 1777 | "user %s should have been removed completely, but is still present" % users[1]["login"]) 1778 | 1779 | def test_013_hook_membership_event_delete_nonexistant_team(self): 1780 | """ 1781 | Test that a membership event that deletes a non-existant team does not crash anything. 1782 | """ 1783 | 1784 | updateMockData(self.mockdata, retcode=200, answers={ 1785 | '/orgs/%s/teams' % self.organization: [] 1786 | }) 1787 | 1788 | update = { 1789 | "team": { 1790 | "id": 1, 1791 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1792 | "name": u"Justice League", 1793 | "deleted": True 1794 | } 1795 | } 1796 | 1797 | with TracContext(self, env=self.tracd_env_debug, **self.trac_env): 1798 | # Send the delete event 1799 | response = requests.post(URL + '/github-groups', 1800 | allow_redirects=False, 1801 | headers={'X-GitHub-Event': 'membership'}, 1802 | json=update) 1803 | self.assertEqual(response.status_code, 200, 1804 | "Deleting non-existant teams should return HTTP 200") 1805 | self.assertEqual(response.text, "success") 1806 | 1807 | def test_014_hook_membership_event_add_team(self): 1808 | """ 1809 | Test that adding a team with a MembershipEvent works as expected. 1810 | """ 1811 | 1812 | updateMockData(self.mockdata, retcode=200, answers={ 1813 | '/orgs/%s/teams' % self.organization: [] 1814 | }) 1815 | 1816 | update = { 1817 | "team": { 1818 | "id": 1, 1819 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1820 | "name": u"Justice League", 1821 | "slug": u"justice-league" 1822 | } 1823 | } 1824 | 1825 | users = [ 1826 | {"login": u"octocat"}, 1827 | ] 1828 | team1members = [users[0]] 1829 | 1830 | with TracContext(self, env=self.tracd_env_debug, **self.trac_env): 1831 | # Update the API result 1832 | updateMockData(self.mockdata, retcode=200, answers={ 1833 | '/orgs/%s/teams' % self.organization: [ 1834 | { 1835 | "id": 1, 1836 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1837 | "name": u"Justice League", 1838 | "slug": u"justice-league" 1839 | }, 1840 | ], 1841 | '/organizations/14143/team/1/members': team1members, 1842 | }) 1843 | 1844 | # Send the update event 1845 | response = requests.post(URL + '/github-groups', 1846 | allow_redirects=False, 1847 | headers={'X-GitHub-Event': 'membership'}, 1848 | json=update) 1849 | self.assertEqual(response.status_code, 200, 1850 | "Adding members to non-existant teams should return HTTP 200") 1851 | self.assertEqual(response.text, "success") 1852 | 1853 | # Check that the member and group were added 1854 | response = requests.get(URL + '/github-groups-dump', allow_redirects=False) 1855 | self.assertEqual(response.status_code, 200) 1856 | 1857 | data = response.json() 1858 | self.assertGreater(len(data), 0, "No groups returned after update") 1859 | self.assertIn(users[0]["login"], data, 1860 | "User %s expected after update, but not present" % users[0]["login"]) 1861 | self.assertItemsEqual( 1862 | data[users[0]["login"]], 1863 | (u"github-%s-justice-league" % self.organization, u"github-%s" % self.organization), 1864 | "User %s does not have expected groups after update" % users[0]["login"]) 1865 | 1866 | def test_015_hook_membership_event_add_member(self): 1867 | """ 1868 | Test that adding a user to an existing team with a MembershipEvent works. 1869 | """ 1870 | users = [ 1871 | {"login": u"octocat"}, 1872 | {"login": u"octofox"}, 1873 | ] 1874 | team1members = [users[0]] 1875 | 1876 | updateMockData(self.mockdata, retcode=200, answers={ 1877 | '/orgs/%s/teams' % self.organization: [ 1878 | { 1879 | "id": 1, 1880 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1881 | "name": u"Justice League", 1882 | "slug": u"justice-league" 1883 | }, 1884 | ], 1885 | '/organizations/14143/team/1/members': list(team1members) 1886 | }) 1887 | 1888 | update = { 1889 | "team": { 1890 | "id": 1, 1891 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1892 | "name": u"Justice League", 1893 | "slug": u"justice-league" 1894 | } 1895 | } 1896 | 1897 | with TracContext(self, env=self.tracd_env_debug, **self.trac_env): 1898 | # Update the API result 1899 | team1members.append(users[1]) 1900 | updateMockData(self.mockdata, retcode=200, answers={ 1901 | '/orgs/%s/teams' % self.organization: [ 1902 | { 1903 | "id": 1, 1904 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1905 | "name": u"Justice League", 1906 | "slug": u"justice-league" 1907 | }, 1908 | ], 1909 | '/organizations/14143/team/1/members': list(team1members) 1910 | }) 1911 | 1912 | # Send the update event 1913 | response = requests.post(URL + '/github-groups', 1914 | allow_redirects=False, 1915 | headers={'X-GitHub-Event': 'membership'}, 1916 | json=update) 1917 | self.assertEqual(response.status_code, 200, 1918 | "Adding members to existing teams should return HTTP 200") 1919 | self.assertEqual(response.text, "success") 1920 | 1921 | # Check that the member and group were added 1922 | response = requests.get(URL + '/github-groups-dump', allow_redirects=False) 1923 | self.assertEqual(response.status_code, 200) 1924 | 1925 | data = response.json() 1926 | self.assertGreater(len(data), 0, "No groups returned after update") 1927 | self.assertIn(users[1]["login"], data, 1928 | "User %s expected after update, but not present" % users[1]["login"]) 1929 | self.assertItemsEqual( 1930 | data[users[1]["login"]], 1931 | (u"github-%s-justice-league" % self.organization, u"github-%s" % self.organization), 1932 | "User %s does not have expected groups after update" % users[1]["login"]) 1933 | 1934 | def test_016_hook_membership_event_remove_member(self): 1935 | """ 1936 | Test that removing a member from an existing team using a MembershipEvent works. 1937 | """ 1938 | users = [ 1939 | {"login": u"octocat"}, 1940 | {"login": u"octofox"}, 1941 | ] 1942 | team1members = [users[0], users[1]] 1943 | 1944 | updateMockData(self.mockdata, retcode=200, answers={ 1945 | '/orgs/%s/teams' % self.organization: [ 1946 | { 1947 | "id": 1, 1948 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1949 | "name": u"Justice League", 1950 | "slug": u"justice-league" 1951 | }, 1952 | ], 1953 | '/organizations/14143/team/1/members': list(team1members) 1954 | }) 1955 | 1956 | update = { 1957 | "team": { 1958 | "id": 1, 1959 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1960 | "name": u"Justice League", 1961 | "slug": u"justice-league" 1962 | } 1963 | } 1964 | 1965 | with TracContext(self, env=self.tracd_env_debug, **self.trac_env): 1966 | # Update the API result 1967 | team1members.remove(users[1]) 1968 | updateMockData(self.mockdata, retcode=200, answers={ 1969 | '/orgs/%s/teams' % self.organization: [ 1970 | { 1971 | "id": 1, 1972 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 1973 | "name": u"Justice League", 1974 | "slug": u"justice-league" 1975 | }, 1976 | ], 1977 | '/organizations/14143/team/1/members': list(team1members) 1978 | }) 1979 | 1980 | # Send the update event 1981 | response = requests.post(URL + '/github-groups', 1982 | allow_redirects=False, 1983 | headers={'X-GitHub-Event': 'membership'}, 1984 | json=update) 1985 | self.assertEqual(response.status_code, 200, 1986 | "Removing members from existing teams should return HTTP 200") 1987 | self.assertEqual(response.text, "success") 1988 | 1989 | # Check that the member and group were added 1990 | response = requests.get(URL + '/github-groups-dump', allow_redirects=False) 1991 | self.assertEqual(response.status_code, 200) 1992 | 1993 | data = response.json() 1994 | self.assertGreater(len(data), 0, "No groups returned after update") 1995 | self.assertNotIn(users[1]["login"], data, 1996 | "User %s not expected after update, but present" % users[1]["login"]) 1997 | 1998 | def test_017_hook_unsigned_ping_event(self): 1999 | """ 2000 | Test that an unsigned event sent to /github-groups/ is rejected if a webhook secret was configured. 2001 | """ 2002 | updateMockData(self.mockdata, retcode=200, answers={ 2003 | '/orgs/%s/teams' % self.organization: [] 2004 | }) 2005 | 2006 | with TracContext(self, env=self.tracd_env, **self.trac_env_secured): 2007 | response = requests.post(URL + '/github-groups', 2008 | allow_redirects=False, 2009 | headers={'X-GitHub-Event': 'ping'}, 2010 | json={'zen': 'Echo me!'}) 2011 | self.assertEqual(response.status_code, 403, 2012 | "Unsigned ping event should return HTTP 403") 2013 | self.assertEqual(response.text, "Webhook signature verification failed\n") 2014 | 2015 | def test_018_hook_unsupported_sig_algo_ping_event(self): 2016 | """ 2017 | Test that an event sent to /github-groups/ with an unsupported signature algorithm is rejected if a webhook secret was configured. 2018 | """ 2019 | updateMockData(self.mockdata, retcode=200, answers={ 2020 | '/orgs/%s/teams' % self.organization: [] 2021 | }) 2022 | 2023 | with TracContext(self, env=self.tracd_env, **self.trac_env_secured): 2024 | response = requests.post(URL + '/github-groups', 2025 | allow_redirects=False, 2026 | headers={ 2027 | 'X-GitHub-Event': 'ping', 2028 | 'X-Hub-Signature': 'foofoo=barbar' 2029 | }, 2030 | json={'zen': 'Echo me!'}) 2031 | self.assertEqual(response.status_code, 403, 2032 | "Ping event with invalid signature algorithm should return HTTP 403") 2033 | self.assertEqual(response.text, "Webhook signature verification failed\n") 2034 | 2035 | def test_019_hook_invalid_sig_ping_event(self): 2036 | """ 2037 | Test that an event sent to /github-groups/ with an invalid signature is rejected if a webhook secret was configured. 2038 | """ 2039 | updateMockData(self.mockdata, retcode=200, answers={ 2040 | '/orgs/%s/teams' % self.organization: [] 2041 | }) 2042 | 2043 | with TracContext(self, env=self.tracd_env, **self.trac_env_secured): 2044 | response = requests.post(URL + '/github-groups', 2045 | allow_redirects=False, 2046 | headers={ 2047 | 'X-GitHub-Event': 'ping', 2048 | 'X-Hub-Signature': 'sha1=f1d2d2f924e986ac86fdf7b36c94bcdf32beec15' 2049 | }, 2050 | json={'zen': 'Echo me!'}) 2051 | self.assertEqual(response.status_code, 403, 2052 | "Ping event with invalid signature should return HTTP 403") 2053 | self.assertEqual(response.text, "Webhook signature verification failed\n") 2054 | 2055 | def test_020_hook_signed_ping_event(self): 2056 | """ 2057 | Test that a correctly signed ping event sent to /github-groups/ is accepted if a webhook secret was configured. 2058 | """ 2059 | updateMockData(self.mockdata, retcode=200, answers={ 2060 | '/orgs/%s/teams' % self.organization: [] 2061 | }) 2062 | 2063 | with TracContext(self, env=self.tracd_env, **self.trac_env_secured): 2064 | # Correct signature can be generated with OpenSSL: 2065 | # $> printf '{"zen": "Echo me"}\n' | openssl dgst -sha256 -hmac $webhook_secret 2066 | signature = "sha256=cacc93c2df1b21313e16d8690fc21e56229b6a9525e7016db38bdf9bad708fed" 2067 | response = requests.post(URL + '/github-groups', 2068 | allow_redirects=False, 2069 | headers={ 2070 | 'X-GitHub-Event': 'ping', 2071 | 'X-Hub-Signature': signature 2072 | }, 2073 | data='{"zen": "Echo me"}\n') 2074 | self.assertEqual(response.status_code, 200, 2075 | "Ping event with valid signature should return HTTP 200") 2076 | self.assertEqual(response.text, "Echo me") 2077 | 2078 | def test_021_hook_membership_event_api_failure(self): 2079 | """ 2080 | Test that a failing API after a membership event was sent correctly returns a failure state. 2081 | """ 2082 | 2083 | updateMockData(self.mockdata, retcode=200, answers={ 2084 | '/orgs/%s/teams' % self.organization: [] 2085 | }) 2086 | 2087 | update = { 2088 | "team": { 2089 | "id": 1, 2090 | "url": u"%sorganizations/14143/team/1" % self.tracd_env_debug.get('TRAC_GITHUB_API_URL'), 2091 | "name": u"Justice League", 2092 | "deleted": True 2093 | } 2094 | } 2095 | 2096 | with TracContext(self, env=self.tracd_env_debug, **self.trac_env): 2097 | # Change the mock to always fail 2098 | updateMockData(self.mockdata, retcode=403) 2099 | # Send the delete event 2100 | response = requests.post(URL + '/github-groups', 2101 | allow_redirects=False, 2102 | headers={'X-GitHub-Event': 'membership'}, 2103 | json=update) 2104 | self.assertEqual(response.status_code, 500, 2105 | "Sending a membership event with broken API backend should return HTTP 500") 2106 | self.assertEqual(response.text, "failure") 2107 | 2108 | def startAPIMock(port): 2109 | """ 2110 | Start an GitHub API mocking server on the given `port` and return the 2111 | `mockdata` dict as explained in `apiMockServer()`. Use `updateMockData()` 2112 | to change the contents of the mocking data. The mocking server runs in 2113 | a daemon thread and does not need to be stopped. 2114 | 2115 | :param port: The port to which the server should bind 2116 | """ 2117 | mockdata = { 2118 | 'lock': threading.Lock(), 2119 | 'retcode': 500, 2120 | 'content-type': 'application/json', 2121 | 'answers': {}, 2122 | 'postcallback': None 2123 | } 2124 | thread = threading.Thread(target=apiMockServer, 2125 | args=(port, mockdata)) 2126 | thread.daemon = True 2127 | thread.start() 2128 | 2129 | return mockdata 2130 | 2131 | def updateMockData(md, retcode=None, contenttype=None, answers=None, 2132 | postcallback=None): 2133 | """ 2134 | Update a mockdata object using appropriate locking. Each of the keyword 2135 | arguments can be `None` (the same as not passing it), which means it should 2136 | not be changed, or a value, which will be copied into the mockdata dict. 2137 | 2138 | :param md: The mockdata object as returned by `startAPIMock()` 2139 | :param retcode: The HTTP return code for the next requests 2140 | :param contenttype: The Content-Type HTTP header for the next requests 2141 | :param answers: A dictionary mapping paths to objects that will be 2142 | JSON-encoded and returned for requests to the paths. 2143 | :param postcallback: A callback function called for the next POST requests. 2144 | Arguments are the requested path and a dict of POST 2145 | data as returned by `urlparse.parse_qs()`. The 2146 | callback should return a tuple `(retcode, answer)` 2147 | where `retcode` is the HTTP return code and `answer` 2148 | will be JSON-encoded and sent to the client. Note that 2149 | this callback is run in a different thread, so pay 2150 | attention to race conditions! To disable the previous 2151 | callback, set this to an empty string. 2152 | """ 2153 | md['lock'].acquire() 2154 | if retcode is not None: 2155 | md['retcode'] = retcode 2156 | if contenttype is not None: 2157 | md['content-type'] = contenttype 2158 | if answers is not None: 2159 | md['answers'] = answers.copy() 2160 | if postcallback is not None: 2161 | md['post-callback'] = postcallback 2162 | md['lock'].release() 2163 | 2164 | def apiMockServer(port, mockdata): 2165 | """ 2166 | Thread target for an GitHub API mock server started on the given `port` 2167 | with the given dict of `mockdata`. 2168 | 2169 | :param port: The port to which the server should bind 2170 | :param mockdata: A dictionary with the keys "lock", "retcode", 2171 | "content-type" and "answers" where "lock" is 2172 | a threading.Lock() that will be acquired while the thread 2173 | reads from the dict, "retcode" is the HTTP return code of 2174 | the next requests, "content-type" is the Content-Type HTTP 2175 | header to send with the next requests, and "answers" is 2176 | a dictionary mapping request paths to objects that should 2177 | be JSON-encoded and returned. Use `updateMockData()` to 2178 | update the contents of the mockdata dict. 2179 | """ 2180 | httpd = BaseHTTPServer.HTTPServer(('127.0.0.1', port), GitHubAPIMock) 2181 | # Make mockdata available to server 2182 | httpd.mockdata = mockdata 2183 | httpd.serve_forever() 2184 | 2185 | 2186 | def get_parser(): 2187 | parser = argparse.ArgumentParser("Run the test suite for trac-github") 2188 | parser.add_argument('--with-coverage', action='store_true', help="Enable test coverage") 2189 | parser.add_argument('--with-trac-log', action='store_true', help="Display logs of test trac instances") 2190 | parser.add_argument('--virtualenv', help="Path to the virtualenv where Trac is installed") 2191 | parser.add_argument('--git-default-branch', default="main", help="The default branch used in the test repositories") 2192 | return parser 2193 | 2194 | 2195 | if __name__ == '__main__': 2196 | options, unittest_argv = get_parser().parse_known_args() 2197 | 2198 | COVERAGE = options.with_coverage 2199 | SHOW_LOG = options.with_trac_log 2200 | GIT_DEFAULT_BRANCH = options.git_default_branch 2201 | 2202 | if options.virtualenv: 2203 | TRAC_ADMIN_BIN = os.path.join(options.virtualenv, 'bin', TRAC_ADMIN_BIN) 2204 | TRACD_BIN = os.path.join(options.virtualenv, 'bin', TRACD_BIN) 2205 | COVERAGE_BIN = os.path.join(options.virtualenv, 'bin', COVERAGE_BIN) 2206 | 2207 | TESTDIR = tempfile.mkdtemp(prefix='trac-github-test-') 2208 | print "Starting tests using temporary directory %r" % TESTDIR 2209 | print "Using git version %s" % git_check_output('--version').strip() 2210 | 2211 | try: 2212 | test_program = unittest.main(argv=[sys.argv[0]] + unittest_argv, exit=False) 2213 | finally: 2214 | shutil.rmtree(TESTDIR) 2215 | 2216 | if not test_program.result.wasSuccessful(): 2217 | sys.exit(1) 2218 | else: 2219 | sys.exit(0) 2220 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [egg_info] 2 | tag_build = dev 3 | 4 | [wheel] 5 | universal = 1 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This software is licensed as described in the file LICENSE, which 5 | # you should have received as part of this distribution. 6 | 7 | from setuptools import setup, find_packages 8 | 9 | setup( 10 | name='trac-github', 11 | version='2.4', 12 | author='Aymeric Augustin', 13 | author_email='aymeric.augustin@m4x.org', 14 | url='https://github.com/trac-hacks/trac-github', 15 | description='Trac - GitHub integration', 16 | download_url='https://pypi.python.org/pypi/trac-github', 17 | packages=find_packages(), 18 | namespace_packages=['tracext'], 19 | platforms='all', 20 | license='BSD', 21 | extras_require={'oauth': ['requests_oauthlib >= 0.5']}, 22 | entry_points={'trac.plugins': [ 23 | 'github.browser = tracext.github:GitHubBrowser', 24 | 'github.loginmodule = tracext.github:GitHubLoginModule[oauth]', 25 | 'github.postcommithook = tracext.github:GitHubPostCommitHook', 26 | 'github.groups = tracext.github:GitHubGroupsProvider', 27 | ]}, 28 | test_suite='runtests', 29 | tests_require='lxml', 30 | ) 31 | -------------------------------------------------------------------------------- /tracext/__init__.py: -------------------------------------------------------------------------------- 1 | __import__('pkg_resources').declare_namespace(__name__) 2 | -------------------------------------------------------------------------------- /tracext/github/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # This software is licensed as described in the file LICENSE, which 4 | # you should have received as part of this distribution. 5 | 6 | import collections 7 | import fnmatch 8 | import hashlib 9 | import hmac 10 | import json 11 | import os 12 | import re 13 | import traceback 14 | 15 | from datetime import datetime, timedelta 16 | 17 | from subprocess import Popen, PIPE, STDOUT 18 | 19 | import trac 20 | from trac.cache import cached 21 | from trac.config import ListOption, BoolOption, Option 22 | from trac.core import Component, implements 23 | from trac.perm import IPermissionGroupProvider, IPermissionPolicy 24 | from trac.util.html import html as tag 25 | from trac.util.translation import _ 26 | from trac.versioncontrol.api import is_default, NoSuchChangeset, RepositoryManager 27 | from trac.versioncontrol.web_ui.changeset import ChangesetModule 28 | from trac.web.api import IRequestHandler, RequestDone 29 | from trac.web.auth import LoginModule 30 | from trac.web.chrome import add_warning 31 | 32 | def _config_secret(value): 33 | if re.match(r'[A-Z_]+', value): 34 | return os.environ.get(value, '') 35 | elif value.startswith('/') or value.startswith('./'): 36 | with open(value) as f: 37 | return f.read().strip() 38 | else: 39 | return value 40 | 41 | 42 | class GitHubMixin(object): 43 | username_prefix = Option('github', 'username_prefix', '', 44 | doc="A unique username prefix for all users " 45 | "authenticated via GitHub (as opposed to any " 46 | "other authentication method).") 47 | 48 | webhook_secret = Option('github', 'webhook_secret', '', 49 | doc="""GitHub webhook secret token. 50 | Uppercase environment variable name, filename starting with '/' or './', or plain string.""") 51 | 52 | def _verify_webhook_signature(self, signature, reqdata): 53 | if not self.webhook_secret: 54 | return True 55 | if not signature: 56 | return False 57 | 58 | algorithm, _, expected = signature.partition("=") 59 | supported_algorithms = { 60 | "sha1": hashlib.sha1, 61 | "sha256": hashlib.sha256, 62 | "sha512": hashlib.sha512 63 | } 64 | if algorithm not in supported_algorithms: 65 | return False 66 | 67 | webhook_secret = _config_secret(self.webhook_secret) 68 | 69 | hmac_hash = hmac.new( 70 | webhook_secret.encode('utf-8'), 71 | reqdata, 72 | supported_algorithms[algorithm]) 73 | computed = hmac_hash.hexdigest() 74 | 75 | return hmac.compare_digest(expected, computed) 76 | 77 | def get_gh_repo(self, reponame): 78 | key = 'repository' if is_default(reponame) else '%s.repository' % reponame 79 | return self.config.get('github', key) 80 | 81 | def get_branches(self, reponame): 82 | key = 'branches' if is_default(reponame) else '%s.branches' % reponame 83 | return self.config.getlist('github', key, sep=' ') 84 | 85 | 86 | class GitHubLoginModule(GitHubMixin, LoginModule): 87 | 88 | auth_path_prefix = Option( 89 | 'github', 'auth_path_prefix', '/github', 90 | doc="""Prefix for the login and logout paths. Defaults to `/github`, 91 | which avoids interferring with Trac's !LoginModule. An empty value 92 | should be used if there are no other authentication modules using 93 | the paths `/login` and `/logout`. 94 | """) 95 | 96 | client_id = Option( 97 | 'github', 'client_id', '', 98 | doc="""Client ID for the OAuth Application on GitHub. 99 | Uppercase environment variable name, filename starting with '/' or './', or plain string.""") 100 | 101 | client_secret = Option( 102 | 'github', 'client_secret', '', 103 | doc="""Client secret for the OAuth Application on GitHub. 104 | Uppercase environment variable name, filename starting with '/' or './', or plain string.""") 105 | 106 | request_email = BoolOption( 107 | 'github', 'request_email', 'false', 108 | doc="Request access to the email address of the GitHub user.") 109 | 110 | preferred_email_domain = Option( 111 | 'github', 'preferred_email_domain', '', 112 | doc="Prefer email address under this domain over the primary address.") 113 | 114 | # INavigationContributor methods 115 | 116 | def get_active_navigation_item(self, req): 117 | return 'github_login' 118 | 119 | def get_navigation_items(self, req): 120 | if req.authname and req.authname != 'anonymous': 121 | # Use the same names as LoginModule to avoid duplicates. 122 | yield ('metanav', 'login', _('logged in as %(user)s', 123 | user=req.authname)) 124 | logout_href = req.href('%s/logout' % self.auth_path_prefix) 125 | from pkg_resources import parse_version 126 | if parse_version(trac.__version__) < parse_version('1.0.2'): 127 | yield ('metanav', 'logout', tag.a(_('Logout'), logout_href)) 128 | else: 129 | yield ('metanav', 'logout', 130 | tag.form(tag.div(tag.button(_('Logout'), 131 | name='logout', 132 | type='submit')), 133 | action=logout_href, method='post', id='logout', 134 | class_='trac-logout')) 135 | else: 136 | yield ('metanav', 'github_login', 137 | tag.a(_('GitHub Login'), 138 | href=req.href('%s/login' % self.auth_path_prefix))) 139 | 140 | # IRequestHandler methods 141 | 142 | def match_request(self, req): 143 | return re.match('/github/oauth/?$', req.path_info) or \ 144 | re.match('%s/(login|logout)/?$' 145 | % re.escape(self.auth_path_prefix), req.path_info) 146 | 147 | def process_request(self, req): 148 | if req.path_info.startswith('%s/login' % self.auth_path_prefix): 149 | self._do_login(req) 150 | elif req.path_info.startswith('/github/oauth'): 151 | self._do_oauth(req) 152 | elif req.path_info.startswith('%s/logout' % self.auth_path_prefix): 153 | self._do_logout(req) 154 | self._redirect_back(req) 155 | 156 | # Internal methods 157 | 158 | def _do_login(self, req): 159 | if not self.client_id or not self.client_secret: 160 | add_warning(req, "GitHubLogin configuration incomplete, missing client_id or client_secret") 161 | self._redirect_back(req) 162 | 163 | oauth = self._oauth_session(req) 164 | authorization_url, state = oauth.authorization_url( 165 | 'https://github.com/login/oauth/authorize') 166 | req.session['oauth_state'] = state 167 | req.redirect(authorization_url) 168 | 169 | def _do_oauth(self, req): 170 | try: 171 | state = req.session['oauth_state'] 172 | except KeyError as exc: 173 | self._reject_oauth(req, exc) 174 | 175 | oauth = self._oauth_session(req, state) 176 | 177 | # Inner import to avoid a hard dependency on requests-oauthlib. 178 | import oauthlib 179 | import requests 180 | github_oauth_url = os.environ.get("TRAC_GITHUB_OAUTH_URL", "https://github.com/") 181 | github_api_url = os.environ.get("TRAC_GITHUB_API_URL", "https://api.github.com/") 182 | try: 183 | oauth.fetch_token( 184 | github_oauth_url + 'login/oauth/access_token', 185 | authorization_response=req.abs_href(req.path_info) + '?' + req.query_string, 186 | client_secret=_config_secret(self.client_secret)) 187 | except (oauthlib.oauth2.OAuth2Error, requests.exceptions.ConnectionError) as exc: 188 | self._reject_oauth(req, exc) 189 | 190 | try: 191 | user = oauth.get(github_api_url + 'user').json() 192 | # read all required data here to deal with errors correctly 193 | name = user.get('name') 194 | email = user.get('email') 195 | login = user.get('login') 196 | except Exception as exc: # pylint: disable=broad-except 197 | self._reject_oauth( 198 | req, exc, 199 | reason=_("An error occurred while communicating with the GitHub API")) 200 | if self.request_email: 201 | try: 202 | for item in oauth.get(github_api_url + 'user/emails').json(): 203 | if not item['verified']: 204 | # ignore unverified email addresses 205 | continue 206 | if (self.preferred_email_domain and 207 | item['email'].lower().endswith( 208 | '@' + self.preferred_email_domain.lower())): 209 | email = item['email'] 210 | break 211 | if item['primary']: 212 | email = item['email'] 213 | if not self.preferred_email_domain: 214 | break 215 | except Exception as exc: # pylint: disable=broad-except 216 | self._reject_oauth( 217 | req, exc, 218 | reason=_("An error occurred while retrieving your email address " 219 | "from the GitHub API")) 220 | # Small hack to pass the username to _do_login. 221 | req.environ['REMOTE_USER'] = self.username_prefix + login 222 | # Save other available values in the session. 223 | req.session.setdefault('name', name or '') 224 | req.session.setdefault('email', email or '') 225 | 226 | return super(GitHubLoginModule, self)._do_login(req) 227 | 228 | def _reject_oauth(self, req, exc, reason=None): 229 | self.log.warn("An OAuth authorization attempt was rejected due to an " 230 | "exception: %s\n%s", exc, traceback.format_exc()) 231 | if reason is None: 232 | reason = _("Invalid request. Please try to login again.") 233 | add_warning(req, reason) 234 | self._redirect_back(req) 235 | 236 | def _do_logout(self, req): 237 | req.session.pop('oauth_state', None) 238 | super(GitHubLoginModule, self)._do_logout(req) 239 | 240 | def _oauth_session(self, req, state=None): 241 | client_id = _config_secret(self.client_id) 242 | scope = [''] 243 | if self.request_email: 244 | scope = ['user:email'] 245 | redirect_uri = req.abs_href.github('oauth') 246 | # Inner import to avoid a hard dependency on requests-oauthlib. 247 | from requests_oauthlib import OAuth2Session 248 | return OAuth2Session( 249 | client_id, 250 | scope=scope, 251 | redirect_uri=redirect_uri, 252 | state=state, 253 | ) 254 | 255 | 256 | class GitHubBrowser(GitHubMixin, ChangesetModule): 257 | 258 | repository = Option('github', 'repository', '', 259 | doc="Repository name on GitHub (/)") 260 | 261 | # IRequestHandler methods 262 | 263 | def match_request(self, req): 264 | match = self._request_re.match(req.path_info) 265 | if match: 266 | rev, path = match.groups() 267 | req.args['rev'] = rev 268 | req.args['path'] = path or '/' 269 | return True 270 | 271 | def process_request(self, req): 272 | rev = req.args.get('rev') 273 | path = req.args.get('path') 274 | 275 | rm = RepositoryManager(self.env) 276 | reponame, repos, path = rm.get_repository_by_path(path) 277 | gh_repo = self.get_gh_repo(reponame) 278 | 279 | if not gh_repo: 280 | req.args['new'] = rev 281 | req.args['new_path'] = path 282 | req.args['reponame'] = reponame 283 | return super(GitHubBrowser, self).process_request(req) 284 | 285 | rev = repos.normalize_rev(rev) 286 | 287 | if path and path != '/': 288 | path = path.lstrip('/') 289 | # GitHub will s/blob/tree/ if the path is a directory 290 | url = 'https://github.com/%s/blob/%s/%s' % (gh_repo, rev, path) 291 | else: 292 | url = 'https://github.com/%s/commit/%s' % (gh_repo, rev) 293 | req.redirect(url) 294 | 295 | # ITimelineEventProvider methods 296 | 297 | def get_timeline_events(self, req, start, stop, filters): 298 | for event in super(GitHubBrowser, self).get_timeline_events(req, start, stop, filters): 299 | assert event[0] == 'changeset' 300 | viewable_changesets, show_location, show_files = event[3] 301 | filtered_changesets = [] 302 | for cset, cset_resource, (reponame,) in viewable_changesets: 303 | branches = self.get_branches(reponame) 304 | if rev_in_branches(cset, branches): 305 | filtered_changesets.append((cset, cset_resource, [reponame])) 306 | 307 | if filtered_changesets: 308 | cset = filtered_changesets[-1][0] 309 | yield ('changeset', cset.date, cset.author, 310 | (filtered_changesets, show_location, show_files)) 311 | 312 | class GitHubCachedAPI(object): 313 | """ 314 | Abstract base class for a call of the GitHub API. Implements automatic 315 | caching and deals with errors by re-trying to fetch the data periodically. 316 | """ 317 | 318 | BACKOFF = 60 * 5 319 | """Timeout in seconds until failed API requests are repeated.""" 320 | 321 | def __init__(self, api, env, fullname): 322 | """ 323 | Setup a cached GitHub API call 324 | 325 | :param api: the `GitHubGroupsProvider` providing API access 326 | :param env: the `TracEnvironment` context used to cache results 327 | :param fullname: a unique identifier used to cache this group's 328 | members. 329 | """ 330 | self.api = api 331 | self.env = env 332 | self.name = fullname 333 | # _fullname needs to be a string, not a unicode string, otherwise the 334 | # cache object won't convert it into a hash. 335 | self._fullname = fullname.encode('utf-8') 336 | # next try: immediately 337 | self._next_update = datetime.now() - timedelta(seconds=10) 338 | self._cached_result = self._apiresult_error() 339 | self._first_lookup = True 340 | 341 | def fullname(self): 342 | """ 343 | Return the unique identifier for this cached object. 344 | """ 345 | return self.name 346 | 347 | def _apicall_parameters(self): 348 | """ 349 | Abstract method that returns the API endpoint and all its required 350 | arguments to query in the format of a sequence where the first index is 351 | the format string and all following indices contain the format string 352 | parameters. 353 | """ 354 | raise NotImplementedError( 355 | "_apicall_parameters not implemented in %s" % type(self)) # pragma: no cover 356 | 357 | def _apiresult_postprocess(self, json_obj): 358 | """ 359 | Abstract method that is run on the JSON returned by the GitHub API 360 | after a successful call. Must return a post-processed version of the 361 | results to be cached and returned to the user. 362 | """ 363 | raise NotImplementedError( 364 | "_apiresult_postprocess not implemented in %s" % type(self)) # pragma: no cover 365 | 366 | def _apiresult_error(self): 367 | """ 368 | Abstract method that is run if an API call fails. Should return an empty 369 | 'dummy' result that matches the format returned by 370 | `_apiresult_postprocess`. 371 | """ 372 | raise NotImplementedError( 373 | "_apiresult_error not implemented in %s" % type(self)) # pragma: no cover 374 | 375 | @cached('_fullname') 376 | def _data(self): 377 | """ 378 | Connect to GitHub API endpoint specified by `_apicall_parameters()` if 379 | required. If an error occurs, return the result of `_apiresult_error()` 380 | and set up retrying after a BACKOFF. On success, return the API result 381 | post-processed by `_apiresult_postprocess()`. 382 | 383 | Do not call this method directly. It does not deal with cache 384 | invalidation. Instead, call the `data()` function, which does. 385 | """ 386 | # Disable coverage, because update() will be called during startup, so 387 | # _first_lookup can never be True here; this is just a safegaurd. 388 | if self._first_lookup: # pragma: no cover 389 | self.update() 390 | return self._cached_result 391 | 392 | def update(self): 393 | """ 394 | Connect to GitHub API endpoint specified by `_apicall_parameters()`, 395 | postprocess the result using `_apiresult_postprocess()` and trigger 396 | a cache update if the API call was successful. 397 | 398 | If an error occurs, cache the empty result generated by 399 | `_apiresult_error()`. Additionally, set up retrying after a certain 400 | time. 401 | 402 | Return `True` if the API call was successful, `False` otherwise. 403 | 404 | Call this method directly if you want to invalidate the current cache. 405 | Otherwise, just call `data()`, which will automatically call `update()` 406 | if required. 407 | """ 408 | result = self.api.github_api(*self._apicall_parameters()) 409 | if result is None: 410 | # an error occurred, try again after BACKOFF 411 | self._next_update = datetime.now() + timedelta(seconds=self.BACKOFF) 412 | # assume an empty result until the error disappears 413 | self._cached_result = self._apiresult_error() 414 | else: 415 | # request successful, cache does not expire 416 | self._next_update = None 417 | # Write the new result into self._cached_result to be picked up by 418 | # _data on `del self._data`. 419 | self._cached_result = self._apiresult_postprocess(result) 420 | 421 | # Don't `del self._data` if it has never been cached, that would create 422 | # ugly database entries in the cache table. 423 | if not self._first_lookup: 424 | del self._data 425 | else: 426 | self._first_lookup = False 427 | 428 | # signal success or error 429 | return result is not None 430 | 431 | def data(self): 432 | """ 433 | Get a cached post-processed result of a GitHub API call. Uses Trac cache 434 | to avoid constant querying of the remote API. If a previous API call did 435 | not succeed, automatically retries after a timeout. 436 | """ 437 | if self._next_update and datetime.now() > self._next_update: 438 | self.update() 439 | return self._data 440 | 441 | class GitHubUserCollection(GitHubCachedAPI): 442 | """ 443 | A cached representation of a collection of users at GitHub. use the 444 | `members()` method to get a list of GitHub login names that are part of this 445 | group. To use access the full name of this group, use the `fullname()` 446 | method. 447 | """ 448 | def __init__(self, *args, **kwargs): 449 | super(GitHubUserCollection, self).__init__(*args, **kwargs) 450 | self.members = self.data 451 | 452 | def _apicall_parameters(self): # pragma: no cover 453 | return super(GitHubUserCollection, self)._apicall_parameters() 454 | 455 | def _apiresult_postprocess(self, json_obj): 456 | return [member['login'] for member in json_obj] 457 | 458 | def _apiresult_error(self): 459 | return [] 460 | 461 | class GitHubTeam(GitHubUserCollection): 462 | """ 463 | A cached representation of a team at GitHub. Use the `members()` method to 464 | get a list of GitHub login names that are part of this group. To access the 465 | full name of this team, use the `fullname()` method. 466 | """ 467 | def __init__(self, api, env, org, url, slug): # pylint: disable=too-many-arguments 468 | """ 469 | Create a new team. 470 | 471 | :param api: the `GitHubGroupsProvider` providing API access 472 | :param env: the `TracEnvironment` context used to cache results 473 | :param org: the name of the organization of the team 474 | :param url: the GitHub API URL of the team 475 | :param slug: the GitHub team shortname in URL representation 476 | """ 477 | self._url = url 478 | self._orgid = org 479 | fullname = '-'.join(['github', org, slug]) 480 | super(GitHubTeam, self).__init__(api, env, fullname) 481 | 482 | def _apicall_parameters(self): 483 | return ("{}/members", self._url) 484 | 485 | #class GitHubOrgMembers(GitHubUserCollection): 486 | # """ 487 | # A cached representation of an organization's members at GitHub. Use the 488 | # `members()` method to get a list of GitHub login names that are part of this 489 | # group. To access the full name of this team, use the `fullname()` method. 490 | # """ 491 | # def __init__(self, api, env, org): 492 | # """ 493 | # Create a cached representation of the members of an organization. 494 | # 495 | # :param api: the `GitHubGroupsProvider` providing API access 496 | # :param env: the `TracEnvironment` context used to cache results 497 | # :param org: the name of the organization 498 | # """ 499 | # self._org = org 500 | # fullname = '-'.join(['github', org]) 501 | # super(GitHubOrgMembers, self).__init__(api, env, fullname) 502 | # 503 | # def _apicall_parameters(self): 504 | # return ("orgs/{}/members", self._org) 505 | 506 | class GitHubOrgTeams(GitHubCachedAPI): 507 | """ 508 | A cached representation of an organization's teams at GitHub. Use the 509 | `teams()` method to get a dictionary of teams in this organization where the 510 | key is the slug and the value is the team ID. 511 | """ 512 | 513 | def __init__(self, api, env, org): 514 | """ 515 | Create a cached representation of the teams of an organization. 516 | 517 | :param api: the `GitHubGroupsProvider` providing API access 518 | :param env: the `TracEnvironment` context used to cache results 519 | :param org: the name of the organization 520 | """ 521 | self._org = org 522 | fullname = '-'.join(['githubteams', org]) 523 | super(GitHubOrgTeams, self).__init__(api, env, fullname) 524 | self.teams = self.data 525 | 526 | def _apicall_parameters(self): 527 | return ("orgs/{}/teams", self._org) 528 | 529 | def _apiresult_postprocess(self, json_obj): 530 | github_api_url = os.environ.get("TRAC_GITHUB_API_URL", "https://api.github.com/").rstrip('/') + '/' 531 | return {team['slug']: team['url'][len(github_api_url):] for team in json_obj} 532 | 533 | def _apiresult_error(self): 534 | return {} 535 | 536 | class GitHubOrg(object): 537 | """ 538 | A cached representation of an organization at GitHub. Use the `teams()` and 539 | `members()` methods to get a list of all teams and users in this org, 540 | respectively. 541 | """ 542 | def __init__(self, api, env, org): 543 | self._api = api 544 | self._env = env 545 | self._org = org 546 | self._teamlist = GitHubOrgTeams(api, env, org) 547 | #self._members = GitHubOrgMembers(api, env, org) 548 | self._teamobjects = {} 549 | 550 | def teams(self): 551 | """ 552 | Return a sequence of `GitHubTeam` objects, one for each team in this 553 | org. 554 | """ 555 | teams = self._teamlist.teams() 556 | 557 | # find out which teams have been added or removed since the last sync 558 | current_teams = set(self._teamobjects.keys()) 559 | new_teams = set(teams.keys()) # pylint: disable=no-member 560 | added = new_teams - current_teams 561 | removed = current_teams - new_teams 562 | 563 | for team in removed: 564 | del self._teamobjects[team] 565 | for team in added: 566 | self._teamobjects[team] = GitHubTeam( 567 | self._api, self._env, self._org, teams[team], team) # pylint: disable=unsubscriptable-object 568 | return self._teamobjects.values() 569 | 570 | def has_team(self, slug): 571 | """ 572 | Return `True` iff the given `slug` identifies a team of this org. 573 | 574 | :param slug: The GitHub 'slug' of the team to test for. 575 | """ 576 | return slug in self._teamobjects 577 | 578 | #def fullname(self): 579 | # return self._members.fullname() 580 | 581 | #def members(self): 582 | # return self._members.members() 583 | 584 | def fullname(self): 585 | """ 586 | Return the prefix all of this org's imported groups will get in Trac. 587 | """ 588 | return '-'.join(['github', self._org]) 589 | 590 | def members(self): 591 | """ 592 | Return a list of all users in this organization. Users are identified 593 | by their login name. Note that this is computed from the teams in the 594 | organization, because GitHub does not currently offer a WebHook for 595 | organization membership, so converting org membership would lead to 596 | stale data. 597 | """ 598 | allmembers = set() 599 | for team in self.teams(): 600 | allmembers.update(team.members()) 601 | return sorted(allmembers) 602 | 603 | def update(self): 604 | """ 605 | Trigger an update and cache invalidation for the list of teams in this 606 | organization. Returns `True` on success, `False` otherwise. 607 | """ 608 | success = self._teamlist.update() 609 | #success &= self._members.update() 610 | return success 611 | 612 | def update_team(self, slug): 613 | """ 614 | Trigger an update and cache invalidation for the team identified by the 615 | given `slug`. Returns `True` on success, `False` otherwise. 616 | 617 | :param slug: The GitHub 'slug' that identifies the team in URLs 618 | """ 619 | if slug not in self._teamobjects: 620 | # This case is checked and handled further up, but better be safe 621 | # than sorry. 622 | return False # pragma: no cover 623 | return self._teamobjects[slug].update() 624 | 625 | class GitHubGroupsProvider(GitHubMixin, Component): 626 | """ 627 | Implements the `IPermissionGroupProvider` and `IRequestHandler` extension 628 | points to provide GitHub teams as groups in Trac and an endpoint for GitHub 629 | WebHooks to keep the cached groups up to date. 630 | """ 631 | implements(IPermissionGroupProvider, IRequestHandler) 632 | 633 | organization = Option('github', 'organization', '', 634 | doc="Organization from which to pull teams into groups.") 635 | 636 | username = Option('github', 'username', '', 637 | doc="GitHub user for accessing organization data.") 638 | 639 | access_token = Option('github', 'access_token', '', 640 | doc="""Personal access token for the GitHub user. 641 | Uppercase environment variable name, filename starting with '/' or './', or plain string.""") 642 | 643 | 644 | def __init__(self): 645 | self._org = None 646 | self._orgteams = None 647 | self._teams = {} 648 | self._orgmembers = None 649 | 650 | def github_api(self, url, *args): 651 | """ 652 | Connect to the given GitHub API URL template by replacing all 653 | placeholders with the given parameters and return the decoded JSON 654 | result on success. On error, return `None`. 655 | 656 | :param url: The path to request from the GitHub API. Contains format 657 | string placeholders that will be replaced with all 658 | additional positional arguments. 659 | """ 660 | import requests 661 | import urllib 662 | 663 | github_api_url = os.environ.get("TRAC_GITHUB_API_URL", "https://api.github.com/") 664 | formatted_url = github_api_url + url.format(*(urllib.quote(str(x)) for x in args)) 665 | access_token = _config_secret(self.access_token) 666 | self.log.debug("Hitting GitHub API endpoint %s with user %s", formatted_url, self.username) # pylint: disable=no-member 667 | results = [] 668 | try: 669 | has_next = True 670 | while has_next: 671 | req = requests.get(formatted_url, auth=(self.username, access_token)) 672 | if req.status_code != 200: 673 | try: 674 | message = req.json()['message'] 675 | except Exception: # pylint: disable=broad-except 676 | message = req.text 677 | self.log.error("Error communicating with GitHub API at {}: {}".format( # pylint: disable=no-member 678 | formatted_url, message)) 679 | return None 680 | results.extend(req.json()) 681 | has_next = 'next' in req.links 682 | if has_next: 683 | formatted_url = req.links['next']['url'] 684 | except requests.exceptions.ConnectionError as rce: 685 | self.log.error("Exception while communicating with GitHub API at {}: {}".format( # pylint: disable=no-member 686 | formatted_url, rce)) 687 | return None 688 | return results 689 | 690 | def _fetch_groups(self): 691 | # Fetch teams 692 | if not self._org: 693 | self._org = GitHubOrg(self, self.env, self.organization) # pylint: disable=no-member 694 | # Fetch team members 695 | members = {} 696 | for team in self._org.teams(): 697 | members[team.fullname()] = team.members() 698 | # Fetch organization members 699 | members[self._org.fullname()] = self._org.members() 700 | 701 | # Return data 702 | data = collections.defaultdict(list) 703 | for tname, tmembers in members.iteritems(): 704 | self.log.debug("Team members for group %r: %r", tname, tmembers) # pylint: disable=no-member 705 | for member in tmembers: 706 | data[member].append(tname) 707 | return dict(data) 708 | 709 | def update_organization(self): 710 | """ 711 | Trigger update and cache invalidation for the organization. Returns 712 | `True` if the update was successful, `False` otherwise. 713 | """ 714 | if self._org: 715 | return self._org.update() 716 | # self._org is created during Trac startup, so there should never 717 | # be a case where we try to update an org before it's created; this 718 | # is a sanity check only. 719 | return False # pragma: no cover 720 | 721 | def update_team(self, slug): 722 | """ 723 | Trigger update and cache invalidation for the team identified by the 724 | given `slug`, if any. Returns `True` if the update was successful, 725 | `False` otherwise. 726 | 727 | :param slug: GitHub 'slug' name for the team to be updated. 728 | """ 729 | if self._org: 730 | if not self._org.has_team(slug): 731 | return self._org.update() 732 | return self._org.update_team(slug) 733 | # self._org is created during Trac startup, so there should never 734 | # be a case where we try to update an org before it's created; this 735 | # is a sanity check only. 736 | return False # pragma: no cover 737 | 738 | # IPermissionGroupProvider methods 739 | def get_permission_groups(self, username): 740 | """ 741 | Return a list of names of the groups that the user with the specified 742 | name is a member of. Implements an `IPermissionGroupProvider` API. 743 | 744 | This specific implementation connects to GitHub with a dedicated user, 745 | fetches and caches the teams and their users configured at GitHub and 746 | converts the data into a format usable for easy access by username. 747 | """ 748 | if not self.organization or not self.username or not self.access_token: 749 | return [] 750 | elif (self.username_prefix and 751 | not username.startswith(self.username_prefix)): 752 | return [] 753 | 754 | data = self._fetch_groups() 755 | if not data: 756 | self.log.error("No cached groups from GitHub available") # pylint: disable=no-member 757 | return [] 758 | else: 759 | return data.get(username[len(self.username_prefix):], []) 760 | 761 | # IRequestHandler methods 762 | _request_re = re.compile(r"/github-groups(/.*)?$") 763 | _debug_request_re = re.compile(r"/github-groups-dump/?$") 764 | 765 | def match_request(self, req): 766 | """ 767 | Return whether the handler wants to process the given request. 768 | Implements an `IRequestHandler` API. 769 | """ 770 | match = self._request_re.match(req.path_info) 771 | if match: 772 | return True 773 | if os.environ.get('TRAC_GITHUB_ENABLE_DEBUGGING', None) is not None: 774 | debug_match = self._debug_request_re.match(req.path_info) 775 | if debug_match: 776 | return True 777 | 778 | def process_debug_request(self, req): 779 | """ 780 | Debgging helper used for testing, processes the given request and dumps 781 | the internal state of cached user to group mappings. Note that this is 782 | only callable if TRAC_GITHUB_ENABLE_DEBUGGING is set in the 783 | environment. 784 | """ 785 | req.send(json.dumps(self._fetch_groups()).encode('utf-8'), 'application/json', 200) 786 | 787 | def process_request(self, req): 788 | """ 789 | Process the given request `req`, implements an `IRequestHandler` API. 790 | 791 | Normally, `process_request` would return a tuple, but since none of 792 | these requests will return an HTML page, they will all terminate 793 | without a return value and directly send a response. 794 | """ 795 | if os.environ.get('TRAC_GITHUB_ENABLE_DEBUGGING', None) is not None: 796 | debug_match = self._debug_request_re.match(req.path_info) 797 | if debug_match: 798 | self.process_debug_request(req) 799 | 800 | if req.method != 'POST': 801 | msg = u'Endpoint is ready to accept GitHub Organization membership notifications.\n' 802 | self.log.warning(u'Method not allowed (%s)', req.method) # pylint: disable=no-member 803 | req.send(msg.encode('utf-8'), 'text/plain', 405) 804 | 805 | event = req.get_header('X-GitHub-Event') 806 | supported_events = { 807 | 'ping': self._handle_ping_ev, 808 | 'membership': self._handle_membership_ev 809 | } 810 | 811 | # Check whether this event is supported 812 | if event not in supported_events: 813 | msg = u'Event type %s is not supported\n' % event 814 | self.log.warning(msg.rstrip('\n')) # pylint: disable=no-member 815 | req.send(msg.encode('utf-8'), 'text/plain', 400) 816 | 817 | # Verify the event's signature 818 | reqdata = req.read() 819 | signature = req.get_header('X-Hub-Signature') 820 | if not self._verify_webhook_signature(signature, reqdata): 821 | msg = u'Webhook signature verification failed\n' 822 | self.log.warning(msg.rstrip('\n')) # pylint: disable=no-member 823 | req.send(msg.encode('utf-8'), 'text/plain', 403) 824 | 825 | # Decode JSON and handle errors 826 | try: 827 | payload = json.loads(reqdata) 828 | except (ValueError, KeyError): 829 | msg = u'Invalid payload\n' 830 | self.log.warning(msg.rstrip('\n')) # pylint: disable=no-member 831 | req.send(msg.encode('utf-8'), 'text/plain', 400) 832 | 833 | # Handle the event 834 | try: 835 | supported_events[event](req, payload) 836 | except RequestDone: 837 | # Normal termination, bubble up 838 | raise 839 | except Exception: # pylint: disable=broad-except 840 | msg = (u'Exception occurred while handling payload, ' 841 | 'possible invalid payload\n%s' % traceback.format_exc()) 842 | self.log.warning(msg.rstrip('\n')) # pylint: disable=no-member 843 | req.send(msg.encode('utf-8'), 'text/plain', 500) 844 | 845 | def _handle_ping_ev(self, req, payload): # pylint: disable=no-self-use 846 | req.send(payload['zen'].encode('utf-8'), 'text/plain', 200) 847 | 848 | def _handle_membership_ev(self, req, payload): 849 | # Unfortunately, no events for organization membership, so the idea of 850 | # converting those is pretty much dead :( 851 | 852 | # Deleting teams sends one notification per team member; the "team" object looks like this: 853 | # "team": { 854 | # "id": 2108536, 855 | # "name": "test-team", 856 | # "deleted": true 857 | # } 858 | # Note the absence of the slug field for deletion requests! Also note 859 | # that the "deleted" key does *not* exist on standard add and remove 860 | # updates for teams. 861 | if 'deleted' in payload['team']: 862 | # When a team was deleted, update the organization 863 | success = self.update_organization() 864 | else: 865 | # A team was modified, update only this team; if this is a new 866 | # team, it will automatically trigger an org update. 867 | success = self.update_team(payload['team']['slug']) 868 | # Just in case a new team was added, trigger a re-fetch so the new 869 | # team's members are cached. 870 | self._fetch_groups() 871 | if success: 872 | req.send(u'success'.encode('utf-8'), 'text/plain', 200) 873 | else: 874 | req.send(u'failure'.encode('utf-8'), 'text/plain', 500) 875 | 876 | 877 | class GitHubPolicy(Component): 878 | """Permission policy for trac-github integration 879 | 880 | Grants `FILE_VIEW` and `BROWSER_VIEW` to users that possess 881 | `CHANGESET_VIEW`. `FILE_VIEW` and `BROWSER_VIEW` are needed for the 882 | [[TracIni#timeline-changeset_show_files-option|[timeline] changeset_show_files]] 883 | option to be effective when enabled (value not `0`). With the 884 | `BrowserModule` disabled, `FILE_VIEW` and `BROWSER_VIEW` are only used 885 | for enforcing the visibility of files in the timeline. 886 | 887 | Add the permission policy before `DefaultPermissionsPolicy`. It is 888 | usually correct to make it the first entry in the list. 889 | 890 | The following will be correct for a Trac 1.2 installation that had 891 | the default value for `permission_policies`. 892 | {{{#!ini 893 | [trac] 894 | permission_policies = GitHubPolicy, ReadonlyWikiPolicy, DefaultPermissionPolicy, LegacyAttachmentPolicy 895 | }}} 896 | """ 897 | implements(IPermissionPolicy) 898 | 899 | def check_permission(self, action, username, resource, perm): 900 | if action in ('FILE_VIEW', 'BROWSER_VIEW'): 901 | return 'CHANGESET_VIEW' in perm 902 | 903 | 904 | class GitHubPostCommitHook(GitHubMixin, Component): 905 | implements(IRequestHandler) 906 | 907 | branches = ListOption('github', 'branches', sep=' ', 908 | doc="Notify only commits on these branches to Trac") 909 | 910 | # IRequestHandler methods 911 | 912 | _request_re = re.compile(r"/github(/.*)?$") 913 | 914 | def match_request(self, req): 915 | match = self._request_re.match(req.path_info) 916 | if match: 917 | req.args['path'] = match.group(1) or '/' 918 | return True 919 | 920 | def process_request(self, req): 921 | path = req.args['path'] 922 | 923 | rm = RepositoryManager(self.env) 924 | reponame, repos, path = rm.get_repository_by_path(path) 925 | 926 | if repos is None or path != '/': 927 | msg = u'No such repository (%s)\n' % path 928 | self.log.warning(msg.rstrip('\n')) 929 | req.send(msg.encode('utf-8'), 'text/plain', 400) 930 | 931 | if req.method != 'POST': 932 | msg = u'Endpoint is ready to accept GitHub notifications.\n' 933 | self.log.warning(u'Method not allowed (%s)', req.method) 934 | req.send(msg.encode('utf-8'), 'text/plain', 405) 935 | 936 | # Verify the event's signature 937 | reqdata = req.read() 938 | signature = req.get_header('X-Hub-Signature') 939 | if not self._verify_webhook_signature(signature, reqdata): 940 | msg = u'Webhook signature verification failed\n' 941 | self.log.warning(msg.rstrip('\n')) # pylint: disable=no-member 942 | req.send(msg.encode('utf-8'), 'text/plain', 403) 943 | 944 | event = req.get_header('X-GitHub-Event') 945 | if event == 'ping': 946 | payload = json.loads(reqdata) 947 | req.send(payload['zen'].encode('utf-8'), 'text/plain', 200) 948 | elif event != 'push': 949 | msg = u'Only ping and push are supported\n' 950 | self.log.warning(msg.rstrip('\n')) 951 | req.send(msg.encode('utf-8'), 'text/plain', 400) 952 | 953 | output = u'Running hook on %s\n' % (reponame or '(default)') 954 | 955 | output += u'* Updating clone\n' 956 | try: 957 | git = repos.git.repo # GitRepository 958 | except AttributeError: 959 | git = repos.repos.git.repo # GitCachedRepository 960 | git.remote('update', '--prune') 961 | 962 | # Ensure that repos.get_changeset can find the new changesets. 963 | output += u'* Synchronizing with clone\n' 964 | repos.sync() 965 | 966 | try: 967 | payload = json.loads(reqdata) 968 | revs = [commit['id'] 969 | for commit in payload['commits'] if commit['distinct']] 970 | except (ValueError, KeyError): 971 | msg = u'Invalid payload\n' 972 | self.log.warning(msg.rstrip('\n')) 973 | req.send(msg.encode('utf-8'), 'text/plain', 400) 974 | 975 | branches = self.get_branches(reponame) 976 | added, skipped, unknown = classify_commits(revs, repos, branches) 977 | 978 | if added: 979 | output += u'* Adding %s\n' % describe_commits(added) 980 | # This is where Trac gets notified of the commits in the changeset 981 | rm.notify('changeset_added', reponame, added) 982 | 983 | if skipped: 984 | output += u'* Skipping %s\n' % describe_commits(skipped) 985 | 986 | if unknown: 987 | output += u'* Unknown %s\n' % describe_commits(unknown) 988 | self.log.error(u'Payload contains unknown %s', 989 | describe_commits(unknown)) 990 | 991 | status = 200 992 | 993 | git_dir = git.rev_parse('--git-dir').rstrip('\n') 994 | hook = os.path.join(git_dir, 'hooks', 'trac-github-update') 995 | if os.path.isfile(hook): 996 | output += u'* Running trac-github-update hook\n' 997 | try: 998 | p = Popen(hook, cwd=git_dir, 999 | stdin=PIPE, stdout=PIPE, stderr=STDOUT, 1000 | close_fds=trac.util.compat.close_fds) 1001 | except Exception as e: 1002 | output += u'Error: hook execution failed with exception\n%s' % (traceback.format_exc(),) 1003 | status = 500 1004 | else: 1005 | hookoutput = p.communicate(input=reqdata)[0] 1006 | output += hookoutput.decode('utf-8') 1007 | if p.returncode != 0: 1008 | output += u'Error: hook failed with exit code %d\n' % (p.returncode,) 1009 | status = 500 1010 | 1011 | for line in output.splitlines(): 1012 | self.log.debug(line) 1013 | 1014 | if status == 200 and not output: 1015 | status = 204 1016 | 1017 | req.send(output.encode('utf-8'), 'text/plain', status) 1018 | 1019 | 1020 | def classify_commits(revs, repos, branches): 1021 | added, skipped, unknown = [], [], [] 1022 | for rev in revs: 1023 | try: 1024 | cset = repos.get_changeset(rev) 1025 | except NoSuchChangeset: 1026 | unknown.append(rev) 1027 | else: 1028 | if rev_in_branches(cset, branches): 1029 | added.append(rev) 1030 | else: 1031 | skipped.append(rev) 1032 | return added, skipped, unknown 1033 | 1034 | 1035 | def rev_in_branches(changeset, branches): 1036 | if not branches: # no branches filter configured 1037 | return True 1038 | return any(fnmatch.fnmatchcase(cset_branch, branch) 1039 | for cset_branch, _ in changeset.get_branches() for branch in branches) 1040 | 1041 | 1042 | def describe_commits(revs): 1043 | if len(revs) == 1: 1044 | return u'commit %s' % revs[0] 1045 | else: 1046 | return u'commits %s' % u', '.join(revs) 1047 | --------------------------------------------------------------------------------