├── .gitignore ├── .travis.yml ├── .travis ├── scabot.pem.enc └── ssh_scabot ├── LICENSE ├── NOTICE ├── README.md ├── amazon └── src │ └── main │ └── scala │ └── scabot │ └── amazon │ └── DynamoDb.scala ├── build.sbt ├── cli.conf ├── cli └── src │ └── main │ └── scala │ └── scabot │ └── cli │ └── CLI.scala ├── core └── src │ └── main │ └── scala │ └── scabot │ └── core │ ├── Configuration.scala │ └── Core.scala ├── deploy ├── after_push ├── before_restart └── restart ├── github └── src │ └── main │ └── scala │ └── scabot │ └── github │ ├── GithubApi.scala │ └── GithubService.scala ├── gui ├── app │ └── controllers │ │ └── Scabot.scala └── conf │ ├── application.conf │ ├── prod-logger.xml │ └── routes ├── jenkins └── src │ └── main │ └── scala │ └── scabot │ └── jenkins │ ├── JenkinsApi.scala │ └── JenkinsService.scala ├── log └── .empty ├── project ├── build.properties └── plugins.sbt ├── scripts └── stage ├── server └── src │ └── main │ └── scala │ └── scabot │ └── server │ └── Actors.scala └── tmp └── .empty /.gitignore: -------------------------------------------------------------------------------- 1 | /gui/logs/ 2 | target/ 3 | out/ 4 | .idea 5 | *.iml 6 | scabot.conf -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | script: 3 | - sbt ++$TRAVIS_SCALA_VERSION compile 4 | scala: 5 | - 2.11.12 6 | dist: trusty 7 | jdk: 8 | - openjdk8 9 | notifications: 10 | email: 11 | - seth.tisue@lightbend.com 12 | cache: 13 | directories: 14 | - $HOME/.ivy2/ 15 | - $HOME/.sbt/cache/ 16 | - $HOME/.sbt/boot/ 17 | - $HOME/.sbt/launchers/ 18 | env: 19 | global: 20 | - GIT_SSH=.travis/ssh_scabot 21 | after_success: 22 | - openssl aes-256-cbc -K $encrypted_1e8a2655862c_key -iv $encrypted_1e8a2655862c_iv -in .travis/scabot.pem.enc -out .travis/scabot.pem -d && git push --force scabot@scala-ci.typesafe.com:/home/scabot/scabot main:main 23 | 24 | before_cache: 25 | - find $HOME/.sbt -name "*.lock" | xargs rm 26 | - find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm 27 | -------------------------------------------------------------------------------- /.travis/scabot.pem.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scala/scabot/df337b1031ce07fb68c0808932c2eb95d0086042/.travis/scabot.pem.enc -------------------------------------------------------------------------------- /.travis/ssh_scabot: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | chmod 0600 .travis/scabot.pem 4 | exec ssh -o StrictHostKeychecking=no -o CheckHostIP=no -o UserKnownHostsFile=/dev/null -i .travis/scabot.pem -- "$@" 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2015-2024 Lightbend, Inc. dba Akka 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Scabot helps out on GitHub, manages CI 2 | Scabot (pronounced ska-BOH) helps shepherd pull requests in the [Scala repository](https://github.com/scala/scala) along the path from initial submission to final merge. 3 | 4 | ## Usage 5 | Scabot tries to stay behind the scenes. It speaks to us through actions rather than words, where possible. 6 | You shouldn't have to tell it what to do, unless something goes wrong (usually Jenkins acting up). 7 | 8 | Scabot works by listening to webhook events from GitHub and [our CI server](https://scala-ci.typesafe.com). 9 | 10 | It can be summoned (immediately!) through certain commands, posted as pull request comments (see below). 11 | 12 | ### Automations and Activities 13 | - Trigger CI builds for commits, keeping us informed of their progress. (It will also pick up the results of manual rebuilds done directly on the CI server.) 14 | - Set milestone of a PR based on its target branch. 15 | - Add reviewer request when there's a comment like "review by @authorityfigure" 16 | (obsolete feature, we use GitHub's built-in reviewing features instead now) 17 | - For its ambitions, check out [Scabot's issues](../../issues). 18 | 19 | ### Commands 20 | - `/rebuild`: rebuild failed jobs for all commits 21 | - `/rebuild $sha`: rebuild failed jobs for a given commit 22 | - `/sync`: make sure that commit stati are in sync with the actual builds on the [CI server](https://scala-ci.typesafe.com). 23 | - `/nothingtoseehere`: mark commits green without actually running CI on them 24 | 25 | ### PR Title Modifiers 26 | - `[ci: last-only]`: include anywhere in the PR title to avoid verifying all commits, limiting CI to the last one 27 | 28 | ## Admin 29 | For ssh access to the server running the bot, this assumes you're using our [dev machine setup](https://github.com/scala/scala-jenkins-infra/blob/master/doc/client-setup.md). 30 | 31 | ### Deploy 32 | Scabot runs on the CI server under the `scabot` account. We [push to deploy](../../issues/10). (The deployment process could maybe be redone at some point using sbt-native-packager's JavaServerAppPackaging (see https://github.com/sbt/sbt-native-packager/issues/521) -- or whatever Play folks usually use, now that Scabot is a Play/Akka app, not just Akka anymore.) 33 | 34 | (If push-to-deploy isn't working, [this](https://github.com/scala/scala/pull/10957#issuecomment-2540813129) might help.) 35 | 36 | ### Logs 37 | `ssh jenkins-master`, and `journalctl -u scabot` (add `-f` to see the tail, perhaps with `-n 1000` to see more lines) 38 | 39 | ### Restart (last resort) 40 | `ssh jenkins-master`, and `systemctl restart scabot` 41 | 42 | To check if everything seems okay after restart, `journalctl -u scabot -b -f` to follow (`-f`) the logs since last boot (`-b`) 43 | 44 | ## Command line interface (experimental) 45 | The Scabot code includes a suite of methods corresponding to various 46 | API calls, returning instances of case classes representing the 47 | JSON responses from the API calls. You can use these methods from the 48 | Scala REPL to do your own queries. Here is a sample session. Note 49 | that for API calls that interact with GitHub, you must supply a 50 | [personal GitHub API access token](https://help.github.com/articles/creating-an-access-token-for-command-line-use/) 51 | when starting sbt. 52 | 53 | ```text 54 | % sbt -Dscabot.github.token=... cli/console 55 | Welcome to Scala ... 56 | ... 57 | 58 | scala> val c = new scabot.cli.CLI("scala") 59 | c: scabot.cli.CLI = ... 60 | 61 | scala> val pulls = c.await(c.github.pullRequests) 62 | pulls: List[c.PullRequest] = 63 | List(PullRequest(...), ...) 64 | 65 | scala> pulls.filter(_.user == c.User("retronym")).map(_.number) 66 | res0: List[Int] = List(4535, 4522) 67 | 68 | scala> c.shutdown() 69 | ``` 70 | 71 | ## Implementation notes 72 | It's Akka-based. ("My first Akka app", Adriaan says.) 73 | 74 | It uses Spray to make web API calls. (These days, maybe it should be using akka-http instead, if/when its SSL support is mature.) 75 | 76 | Pull request statuses are updated using GitHub's [Status API](https://developer.github.com/v3/repos/statuses/). 77 | 78 | We listen for webhook events, but just in case we missed some, a 79 | `Synch` event triggers on startup and every 30 minutes thereafter 80 | to make sure that our internal state is still in synch with reality. 81 | 82 | In the SBT build, the root project aggregates five subprojects. 83 | `server` depends on `amazon`, `github`, and `jenkins`; all of 84 | them depend on `core`. (TODO: just make `server` be the root project?) 85 | 86 | The `amazon` subproject uses DynamoDB to persist a `scabot-seen-commands` 87 | table, so at `Synch` time we don't reprocess stuff we already handled 88 | once. 89 | 90 | A good place to start exploring the code is 91 | `server/src/main/scala/Actors.scala`. 92 | 93 | PR status is updated in response to `PullRequestEvent` messages, 94 | which might be real events coming directly from GitHub, or they might 95 | synthetic ones we make when `Synch`ing. 96 | 97 | ## History 98 | Old-timers may fondly remember Scabot's predecessor, the [Scala build kitteh](https://github.com/typesafehub/ghpullrequest-validator). 99 | 100 | ## Contributing 101 | Yes, please! 102 | 103 | -------------------------------------------------------------------------------- /amazon/src/main/scala/scabot/amazon/DynamoDb.scala: -------------------------------------------------------------------------------- 1 | package scabot 2 | package amazon 3 | 4 | 5 | import com.amazonaws.auth._ 6 | import com.amazonaws.regions._ 7 | import com.amazonaws.services.dynamodbv2._ 8 | import com.amazonaws.services.dynamodbv2.util.Tables 9 | import com.amazonaws.services.dynamodbv2.model._ 10 | import com.amazonaws.services.dynamodbv2.document._ 11 | 12 | import scala.collection.JavaConverters._ 13 | 14 | import scala.concurrent.Future 15 | 16 | /** 17 | * Created by adriaan on 1/23/15. 18 | */ 19 | trait DynamoDb extends core.Core { 20 | class DynoDbClient { 21 | private lazy val dynamoDBClient = { 22 | val ipcp = new InstanceProfileCredentialsProvider() 23 | val client = new AmazonDynamoDBClient(ipcp) 24 | client.setRegion(Regions.getCurrentRegion) 25 | client 26 | } 27 | 28 | private lazy val dynamoDB = { 29 | new DynamoDB(dynamoDBClient) 30 | } 31 | 32 | class DynoDbTable(val name: String) extends Table(dynamoDBClient, name) { 33 | private def describeTable = dynamoDBClient.describeTable(new DescribeTableRequest(name)).getTable 34 | private def createTable(request: CreateTableRequest): CreateTableResult = dynamoDBClient.createTable(request) 35 | 36 | def create(schema: List[(String, KeyType)], attribs: List[(String, String)], readCapa: Long = 25, writeCapa: Long = 25): Future[TableDescription] = Future { 37 | def attrib = new AttributeDefinition 38 | def key = new KeySchemaElement 39 | def provi = new ProvisionedThroughput 40 | 41 | val request = (new CreateTableRequest).withTableName(name) 42 | .withKeySchema(schema.map { case (name, tp) => key.withAttributeName(name).withKeyType(tp)}.asJava) 43 | .withAttributeDefinitions(attribs.map { case (name, tp) => attrib.withAttributeName(name).withAttributeType(tp)}.asJava) 44 | .withProvisionedThroughput(provi.withReadCapacityUnits(readCapa).withWriteCapacityUnits(writeCapa)) 45 | 46 | val table = createTable(request) 47 | 48 | Tables.waitForTableToBecomeActive(dynamoDBClient, name) 49 | 50 | table.getTableDescription 51 | } 52 | 53 | def exists: Boolean = 54 | try describeTable.getTableStatus == TableStatus.ACTIVE.toString 55 | catch { case rnfe: ResourceNotFoundException => false } 56 | 57 | def put(item: Item) = Future { putItem(item) } 58 | def get(key: PrimaryKey) = Future { Option(getItem(key)) } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | name := "scabot" 2 | 3 | organization in ThisBuild := "com.lightbend" 4 | version in ThisBuild := "0.1.0" 5 | scalaVersion in ThisBuild := "2.11.12" 6 | 7 | // native packaging settings 8 | maintainer := "Seth Tisue " 9 | packageDescription := "Scala Bot" 10 | packageSummary := "Automates stuff on GitHub" 11 | 12 | scalacOptions in ThisBuild ++= 13 | Seq("-feature", "-deprecation", "-Xfatal-warnings") 14 | 15 | lazy val deps: Seq[sbt.Def.Setting[_]] = Seq( 16 | libraryDependencies ++= Seq( 17 | "com.typesafe.akka" %% "akka-actor" % "2.3.16", 18 | "io.spray" %% "spray-client" % "1.3.4", 19 | "io.spray" %% "spray-json" % "1.3.5" 20 | )) 21 | 22 | lazy val amazonDeps: Seq[sbt.Def.Setting[_]] = Seq( 23 | libraryDependencies += "com.amazonaws" % "aws-java-sdk" % "1.9.13") 24 | 25 | 26 | lazy val guiSettings: Seq[sbt.Def.Setting[_]] = Seq( 27 | assemblyJarName in assembly := "scabot.jar", 28 | mainClass in assembly := Some("play.core.server.ProdServerStart"), 29 | fullClasspath in assembly += Attributed.blank(PlayKeys.playPackageAssets.value), 30 | assemblyExcludedJars in assembly := (fullClasspath in assembly).value filter {_.data.getName.startsWith("commons-logging")}, 31 | routesGenerator := InjectedRoutesGenerator 32 | ) 33 | 34 | lazy val core = project settings (deps: _*) 35 | lazy val github = project dependsOn (core) 36 | lazy val jenkins = project dependsOn (core) 37 | lazy val cli = project dependsOn (github) 38 | lazy val amazon = project dependsOn (core) settings (amazonDeps: _*) 39 | lazy val server = project dependsOn (amazon, github, jenkins) 40 | lazy val gui = project dependsOn (server) enablePlugins(PlayScala) settings (guiSettings: _*) 41 | -------------------------------------------------------------------------------- /cli.conf: -------------------------------------------------------------------------------- 1 | scala: { 2 | jenkins: { 3 | jobSuffix: "" 4 | host: "" 5 | user: "" 6 | token: "" 7 | } 8 | github: { 9 | user: "scala" 10 | repo: "scala" 11 | branches: ["2.11.x"] 12 | lastCommitOnly: false 13 | host: "api.github.com" 14 | token: "" 15 | } 16 | } 17 | 18 | dotty: { 19 | jenkins: { 20 | jobSuffix: "" 21 | host: "" 22 | user: "" 23 | token: "" 24 | } 25 | github: { 26 | user: "lampepfl" 27 | repo: "dotty" 28 | branches: ["master"] 29 | lastCommitOnly: true 30 | host: "api.github.com" 31 | token: "" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /cli/src/main/scala/scabot/cli/CLI.scala: -------------------------------------------------------------------------------- 1 | package scabot 2 | package cli 3 | 4 | import scala.concurrent.{ Future, Await } 5 | import scala.concurrent.duration.Duration 6 | 7 | class CLI(configName: String) 8 | extends github.GithubApi 9 | with core.Configuration with core.HttpClient with core.Core { 10 | 11 | override def configFile = 12 | new java.io.File("./cli.conf") 13 | 14 | override def broadcast( 15 | user: String, repo: String)(msg: ProjectMessage) = 16 | throw new UnsupportedOperationException 17 | 18 | override val system = 19 | akka.actor.ActorSystem() 20 | 21 | lazy val github = 22 | new GithubConnection(configs(configName).github) 23 | 24 | def await[T](f: Future[T]): T = 25 | Await.result(f, Duration.Inf) 26 | 27 | def shutdown(): Unit = system.shutdown() 28 | 29 | } 30 | -------------------------------------------------------------------------------- /core/src/main/scala/scabot/core/Configuration.scala: -------------------------------------------------------------------------------- 1 | package scabot.core 2 | 3 | import com.typesafe.config.{Config => TConfig, _} 4 | 5 | trait Configuration extends Core { 6 | 7 | def configFile: java.io.File 8 | 9 | // TODO generate config using chef 10 | lazy val configs: Map[String, Config] = parseConfig(configFile) 11 | 12 | object Config { 13 | // only PRs targeting a branch in `branches` will be monitored 14 | case class Github(user: String, repo: String, branches: Set[String], lastCommitOnly: Boolean, host: String, token: String) 15 | // the launched job will be BuildHelp.mainValidationJob(pull), 16 | // for example, if `jobSuffix` == "validate-main", we'll launch "scala-2.11.x-validate-main" for repo "scala", PR target "2.11.x" 17 | case class Jenkins(jobSuffix: String, host: String, user: String, token: String) 18 | } 19 | import Config._ 20 | case class Config(github: Github, jenkins: Jenkins) 21 | 22 | // TODO - useful error messages on failure. 23 | def parseConfig(file: java.io.File): Map[String, Config] = { 24 | import scala.collection.JavaConverters._ 25 | val config = ConfigFactory parseFile file 26 | 27 | def configString(c: ConfigValue): Option[String] = 28 | if (c.valueType == ConfigValueType.STRING) Some(c.unwrapped.toString) 29 | else None 30 | 31 | def configBoolean(c: ConfigValue): Option[Boolean] = 32 | if (c.valueType == ConfigValueType.BOOLEAN) Some(c.unwrapped.asInstanceOf[java.lang.Boolean]) 33 | else None 34 | 35 | def configObj(c: ConfigValue): Option[ConfigObject] = 36 | if (c.valueType == ConfigValueType.OBJECT) Some(c.asInstanceOf[ConfigObject]) 37 | else None 38 | 39 | def configList(c: ConfigValue): Option[ConfigList] = 40 | if (c.valueType == ConfigValueType.LIST) 41 | Some(c.asInstanceOf[ConfigList]) 42 | else None 43 | 44 | def configStringList(c: ConfigValue): Option[Seq[String]] = 45 | configList(c) map { x => 46 | x.iterator.asScala.flatMap(configString).toSeq 47 | } 48 | 49 | def jenkins(c: ConfigObject): Option[Jenkins] = 50 | for { 51 | jobSuffix <- configString(c.get("jobSuffix")) 52 | host <- configString(c.get("host")) 53 | user <- configString(c.get("user")) 54 | token <- configString(c.get("token")) 55 | } yield Jenkins(jobSuffix, host, user, token) 56 | 57 | def defaultGitHubToken: Option[String] = 58 | Option(sys.props("scabot.github.token")) 59 | 60 | def github(c: ConfigObject): Option[Github] = 61 | for { 62 | user <- configString(c.get("user")) 63 | repo <- configString(c.get("repo")) 64 | branches <- configStringList(c.get("branches")) 65 | lastCommitOnly <- configBoolean(c.get("lastCommitOnly")) orElse Some(false) // default for scala/scala 66 | host <- configString(c.get("host")) 67 | token <- configString(c.get("token")).filter(_.nonEmpty) orElse defaultGitHubToken 68 | } yield Github(user, repo, branches.toSet, lastCommitOnly, host, token) 69 | 70 | def c2c(c: ConfigObject): Option[Config] = 71 | for { 72 | // Jenkins Config 73 | jenkinsConf <- configObj(c.get("jenkins")) 74 | jenkinsObj <- jenkins(jenkinsConf) 75 | // Github config 76 | githubConf <- configObj(c.get("github")) 77 | githubObj <- github(githubConf) 78 | } yield Config(githubObj, jenkinsObj) 79 | 80 | val configs = 81 | for { 82 | kv <- config.root.entrySet.iterator.asScala 83 | name = kv.getKey.toString 84 | obj <- configObj(kv.getValue) 85 | c <- c2c(obj) 86 | } yield name -> c 87 | 88 | configs.toMap 89 | } 90 | 91 | 92 | } 93 | -------------------------------------------------------------------------------- /core/src/main/scala/scabot/core/Core.scala: -------------------------------------------------------------------------------- 1 | package scabot 2 | package core 3 | 4 | import akka.actor.{ActorSelection, ActorRef, ActorSystem} 5 | import akka.event.Logging 6 | import akka.io.IO 7 | import spray.can.Http 8 | import spray.client.pipelining._ 9 | import spray.http.{StatusCode, HttpCredentials, HttpResponse} 10 | import spray.httpx.PipelineException 11 | import spray.httpx.unmarshalling._ 12 | 13 | import scala.concurrent.{Promise, Future, ExecutionContext} 14 | 15 | class BaseRef(val name: String) extends AnyVal 16 | 17 | trait Core { 18 | 19 | // We need an ActorSystem not only for the actors that make up 20 | // the server, but also in order to use akka.io.IO's HTTP support, 21 | // which is actor-based. (It's not strictly necessary we use 22 | // the *same* actor system in both places, but whatevs.) 23 | implicit def system: ActorSystem 24 | 25 | // needed for marshalling implicits for the json api 26 | implicit def ec: ExecutionContext = system.dispatcher 27 | 28 | def broadcast(user: String, repo: String)(msg: ProjectMessage): Unit 29 | 30 | // marker for messages understood by ProjectActor 31 | trait ProjectMessage 32 | 33 | // marker for messages understood by PullRequestActor 34 | trait PRMessage 35 | 36 | type PullRequest 37 | 38 | // see also scala-jenkins-infra 39 | final val PARAM_REPO_USER = "repo_user" 40 | final val PARAM_REPO_NAME = "repo_name" 41 | final val PARAM_REPO_REF = "repo_ref" 42 | final val PARAM_PR = "_scabot_pr" 43 | final val PARAM_LAST = "_scabot_last" // TODO: temporary until we run real integration on the actual merge commit 44 | 45 | trait JobContextLense { 46 | def contextForJob(job: String, baseRef: BaseRef): Option[String] 47 | def jobForContext(context: String, baseRef: BaseRef): Option[String] 48 | } 49 | } 50 | 51 | trait Util extends Core { 52 | def findFirstSequentially[T](futures: Stream[Future[T]])(p: T => Boolean): Future[T] = { 53 | val resultPromise = Promise[T] 54 | def loop(futures: Stream[Future[T]]): Unit = 55 | futures.headOption match { 56 | case Some(hd) => 57 | val hdF = hd.filter(p) 58 | hdF onFailure { 59 | case filterEx: NoSuchElementException => loop(futures.tail) 60 | case e => resultPromise.failure(e) 61 | } 62 | 63 | hdF onSuccess { case v => resultPromise.success(v) } 64 | 65 | case _ => resultPromise.failure(new NoSuchElementException) 66 | } 67 | loop(futures) 68 | resultPromise.future 69 | } 70 | } 71 | 72 | 73 | trait HttpClient extends Core { 74 | import spray.can.Http.HostConnectorSetup 75 | import spray.client.pipelining._ 76 | import spray.http.{HttpCredentials, HttpRequest} 77 | 78 | // TODO: use spray's url abstraction instead 79 | implicit class SlashyString(_str: String) { def /(o: Any) = _str +"/"+ o.toString } 80 | 81 | val logResponseBody = {response: HttpResponse => system.log.debug(response.entity.asString.take(2000)); response } 82 | 83 | // use this to initialize an implicit of type Future[SendReceive], for use with p (for "pipeline") and px below 84 | def setupConnection(host: String, credentials: Option[HttpCredentials] = None): Future[SendReceive] = { 85 | import akka.pattern.ask 86 | import akka.util.Timeout 87 | import scala.concurrent.duration._ 88 | implicit val timeout = Timeout(15.seconds) 89 | 90 | val noop: RequestTransformer = identity[HttpRequest] 91 | val auth: RequestTransformer = credentials.map(addCredentials).getOrElse(noop) 92 | 93 | for ( 94 | Http.HostConnectorInfo(connector, _) <- IO(Http) ? HostConnectorSetup(host = host, port = 443, sslEncryption = true) 95 | ) yield auth ~> sendReceive(connector) ~> logResponseBody 96 | } 97 | 98 | 99 | def p[T: FromResponseUnmarshaller](req: HttpRequest)(implicit connection: Future[SendReceive]): Future[T] = 100 | connection flatMap { sr => (sr ~> unmarshal[T]).apply(req) } 101 | 102 | def pWithStatus[T: FromResponseUnmarshaller](req: HttpRequest)(implicit connection: Future[SendReceive]): Future[(T, StatusCode)] = 103 | connection flatMap { sr => (sr ~> unmarshalWithStatus[T]).apply(req) } 104 | 105 | def unmarshalWithStatus[T: FromResponseUnmarshaller]: HttpResponse ⇒ (T, StatusCode) = 106 | response ⇒ response.as[T] match { 107 | case Right(value) ⇒ (value, response.status) 108 | case Left(error: MalformedContent) ⇒ throw new PipelineException(error.errorMessage, error.cause.orNull) 109 | case Left(error) ⇒ throw new PipelineException(error.toString) 110 | } 111 | 112 | def px(req: HttpRequest)(implicit connection: Future[SendReceive]): Future[HttpResponse] = 113 | connection flatMap (_.apply(req)) 114 | } 115 | 116 | // for experimenting with the actors logic 117 | trait NOOPHTTPClient extends HttpClient { 118 | override def setupConnection(host: String, credentials: Option[HttpCredentials] = None): Future[SendReceive] = 119 | Future.successful{ x => logRequest(system.log, akka.event.Logging.InfoLevel).apply(x); Future.successful(HttpResponse()) } 120 | } 121 | -------------------------------------------------------------------------------- /deploy/after_push: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | oldrev=$1 4 | newrev=$2 5 | 6 | run() { 7 | [ -x $1 ] && $1 $oldrev $newrev 8 | } 9 | 10 | echo files changed: $(git diff $oldrev $newrev --diff-filter=ACDMR --name-only | wc -l) 11 | 12 | umask 002 13 | 14 | git submodule sync && git submodule update --init --recursive 15 | 16 | run deploy/before_restart 17 | run deploy/restart && run deploy/after_restart 18 | -------------------------------------------------------------------------------- /deploy/before_restart: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | oldrev, newrev = ARGV 3 | 4 | def run(cmd) 5 | exit($?.exitstatus) unless system "umask 002 && #{cmd}" 6 | end 7 | 8 | run "scripts/stage" -------------------------------------------------------------------------------- /deploy/restart: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | killall -1 java ||: # only java process run by scabot -- will be restarted by systemd -------------------------------------------------------------------------------- /github/src/main/scala/scabot/github/GithubApi.scala: -------------------------------------------------------------------------------- 1 | // relevant doc: 2 | // * https://developer.github.com/guides/getting-started/ 3 | // * https://developer.github.com/v3/ 4 | 5 | // A note on terminology: a repository is identified by a combination 6 | // of a user name and a repository name. The user name might be an 7 | // individual's, but is more likely an organization name. 8 | 9 | // A note on pagination: many API calls only return the first n 10 | // results, where n is often 30 but may be 100 or a different 11 | // number depending on the particular call. This can lead to 12 | // random-seeming failage when we happen to exceed the limit. 13 | // For now, we add `per_page` if the limit is found to be a 14 | // problem in practice. per_page cannot be increased past 15 | // 100; if we ever find that's not enough in practice, we'd 16 | // have to add additional code to split the request into 17 | // multiple pages. relevant doc: 18 | // * https://developer.github.com/guides/traversing-with-pagination/ 19 | 20 | package scabot 21 | package github 22 | 23 | import scabot.core.BaseRef 24 | 25 | trait GithubApi extends GithubApiTypes with GithubJsonProtocol with GithubApiActions 26 | 27 | // definitions in topo-order, no cycles in dependencies 28 | trait GithubApiTypes extends core.Core { 29 | // spray json seems to disregard the expected type and won't unmarshall a json number as a String (sometimes github uses longs, sometimes string reps) 30 | type Date = Option[Either[String, Long]] 31 | 32 | object CommitStatusConstants { 33 | final val SUCCESS = "success" 34 | final val PENDING = "pending" 35 | final val FAILURE = "failure" 36 | 37 | // context to enforce that last commit is green only if all prior commits are also green 38 | final val COMBINED = "combined" 39 | final val REVIEWED = "reviewed" 40 | 41 | def jenkinsContext(ctx: String) = ctx match { 42 | case COMBINED | REVIEWED => false 43 | case _ => !ctx.contains("travis-ci") 44 | } 45 | } 46 | import CommitStatusConstants._ 47 | 48 | case class User(login: String) 49 | case class Author(name: String, email: String) {// , username: Option[String] 50 | override def toString = name 51 | } 52 | case class Repository(name: String, full_name: String, git_url: String, 53 | updated_at: Date, created_at: Date, pushed_at: Date) { // owner: Either[User, Author] 54 | override def toString = full_name 55 | } 56 | case class GitRef(sha: String, label: String, ref: String, repo: Repository, user: User) { 57 | override def toString = s"${repo}#${sha.take(7)}" 58 | } 59 | case class PullRequest(number: Int, state: String, title: String, body: Option[String], 60 | created_at: Date, updated_at: Date, closed_at: Date, merged_at: Date, 61 | head: GitRef, base: GitRef, user: User, merged: Option[Boolean], mergeable: Option[Boolean], merged_by: Option[User]) { 62 | override def toString = s"${base.repo}#$number" 63 | } 64 | case class Reviewers(reviewers: List[String]) 65 | //, comments: Int, commits: Int, additions: Int, deletions: Int, changed_files: Int) 66 | 67 | case class Label(name: String, color: Option[String] = None, url: Option[String] = None) { 68 | override def toString = name 69 | } 70 | 71 | object Milestone { 72 | private val MergeBranch = """Merge to (\S+)\b""".r.unanchored 73 | } 74 | case class Milestone(number: Int, state: String, title: String, description: Option[String], creator: User, 75 | created_at: Date, updated_at: Date, closed_at: Option[Date], due_on: Option[Date]) { 76 | override def toString = s"Milestone $title ($state)" 77 | 78 | def mergeBranch = description match { 79 | case Some(Milestone.MergeBranch(branch)) => Some(branch) 80 | case _ => None 81 | } 82 | } 83 | 84 | case class Issue(number: Int, state: String, title: String, body: Option[String], user: User, labels: List[Label], 85 | assignee: Option[User], milestone: Option[Milestone], created_at: Date, updated_at: Date, closed_at: Date) { 86 | override def toString = s"Issue #$number" 87 | } 88 | 89 | case class CommitInfo(id: Option[String], message: String, timestamp: Date, author: Author, committer: Author) 90 | // added: Option[List[String]], removed: Option[List[String]], modified: Option[List[String]] 91 | case class Commit(sha: String, commit: CommitInfo, url: Option[String] = None) 92 | 93 | trait HasState { 94 | def state: String 95 | 96 | def success = state == SUCCESS 97 | def pending = state == PENDING 98 | def failure = state == FAILURE 99 | } 100 | 101 | case class CombiCommitStatus(state: String, sha: String, statuses: List[CommitStatus], total_count: Int) extends HasState { 102 | lazy val byContext = statuses.groupBy(_.context).toMap 103 | def apply(context: String) = byContext.get(Some(context)) 104 | } 105 | 106 | trait HasContext { 107 | def context: Option[String] 108 | 109 | def combined = context == Some(COMBINED) 110 | } 111 | 112 | // TODO: factory method that caps state to 140 chars 113 | case class CommitStatus(state: String, context: Option[String] = None, description: Option[String] = None, target_url: Option[String] = None) extends HasState with HasContext { 114 | def forJob(job: String, baseRef: BaseRef)(implicit lense: JobContextLense): Boolean = lense.contextForJob(job, baseRef) == context 115 | def jobName(baseRef: BaseRef)(implicit lense: JobContextLense): Option[String] = context.flatMap(lense.jobForContext(_, baseRef)) 116 | } 117 | 118 | case class IssueComment(body: String, user: Option[User] = None, created_at: Date = None, updated_at: Date = None, id: Option[Long] = None) extends PRMessage 119 | case class PullRequestComment(body: String, user: Option[User] = None, commit_id: Option[String] = None, path: Option[String] = None, position: Option[Int] = None, 120 | created_at: Date = None, updated_at: Date = None, id: Option[Long] = None) extends PRMessage 121 | // diff_hunk, original_position, original_commit_id 122 | 123 | case class PullRequestEvent(action: String, number: Int, pull_request: PullRequest) extends ProjectMessage with PRMessage 124 | case class PushEvent(ref: String, commits: List[CommitInfo], repository: Repository) extends ProjectMessage 125 | // TODO: https://github.com/scala/scabot/issues/46 --> ref_name is also included, but it comes at the end of the JSON payload, and for some reason we don't see it (chunking?) 126 | 127 | // ref_name: String, before: String, after: String, created: Boolean, deleted: Boolean, forced: Boolean, base_ref: Option[String], commits: List[CommitInfo], head_commit: CommitInfo, repository: Repository, pusher: Author) 128 | case class PullRequestReviewCommentEvent(action: String, pull_request: PullRequest, comment: PullRequestComment, repository: Repository) extends ProjectMessage 129 | case class IssueCommentEvent(action: String, issue: Issue, comment: IssueComment, repository: Repository) extends ProjectMessage 130 | 131 | // case class AuthApp(name: String, url: String) 132 | // case class Authorization(token: String, app: AuthApp, note: Option[String]) 133 | 134 | } 135 | 136 | import spray.http.BasicHttpCredentials 137 | import spray.json.{RootJsonFormat, DefaultJsonProtocol} 138 | 139 | // TODO: can we make this more debuggable? 140 | // TODO: test against https://github.com/github/developer.github.com/tree/master/lib/webhooks 141 | trait GithubJsonProtocol extends GithubApiTypes with DefaultJsonProtocol with core.Configuration { 142 | private type RJF[x] = RootJsonFormat[x] 143 | implicit lazy val _fmtUser : RJF[User] = jsonFormat1(User) 144 | implicit lazy val _fmtAuthor : RJF[Author] = jsonFormat2(Author) 145 | implicit lazy val _fmtRepository : RJF[Repository] = jsonFormat6(Repository) 146 | 147 | implicit lazy val _fmtGitRef : RJF[GitRef] = jsonFormat5(GitRef) 148 | 149 | implicit lazy val _fmtPullRequest : RJF[PullRequest] = jsonFormat14(PullRequest) 150 | implicit lazy val _fmtReviewers : RJF[Reviewers] = jsonFormat1(Reviewers) 151 | 152 | implicit lazy val _fmtLabel : RJF[Label] = jsonFormat3(Label) 153 | implicit lazy val _fmtMilestone : RJF[Milestone] = jsonFormat9(Milestone.apply) 154 | implicit lazy val _fmtIssue : RJF[Issue] = jsonFormat11(Issue) 155 | 156 | implicit lazy val _fmtCommitInfo : RJF[CommitInfo] = jsonFormat5(CommitInfo) 157 | implicit lazy val _fmtCommit : RJF[Commit] = jsonFormat3(Commit) 158 | implicit lazy val _fmtCommitStatus : RJF[CommitStatus] = jsonFormat4(CommitStatus.apply) 159 | implicit lazy val _fmtCombiCommitStatus: RJF[CombiCommitStatus] = jsonFormat(CombiCommitStatus, "state", "sha", "statuses", "total_count") // need to specify field names because we added methods to the case class.. 160 | 161 | implicit lazy val _fmtIssueComment : RJF[IssueComment] = jsonFormat5(IssueComment) 162 | implicit lazy val _fmtPullRequestComment: RJF[PullRequestComment] = jsonFormat8(PullRequestComment) 163 | 164 | implicit lazy val _fmtPullRequestEvent : RJF[PullRequestEvent] = jsonFormat3(PullRequestEvent) 165 | implicit lazy val _fmtPushEvent : RJF[PushEvent] = jsonFormat3(PushEvent) 166 | implicit lazy val _fmtPRCommentEvent : RJF[PullRequestReviewCommentEvent] = jsonFormat4(PullRequestReviewCommentEvent) 167 | implicit lazy val _fmtIssueCommentEvent: RJF[IssueCommentEvent] = jsonFormat4(IssueCommentEvent) 168 | 169 | // implicit lazy val _fmtAuthorization : RJF[Authorization] = jsonFormat3(Authorization) 170 | // implicit lazy val _fmtAuthApp : RJF[AuthApp] = jsonFormat2(AuthApp) 171 | } 172 | 173 | trait GithubApiActions extends GithubJsonProtocol with core.HttpClient { 174 | class GithubConnection(config: Config.Github) { 175 | import spray.http.{GenericHttpCredentials, Uri} 176 | import spray.httpx.SprayJsonSupport._ 177 | import spray.client.pipelining._ 178 | 179 | // NOTE: the token (https://github.com/settings/applications#personal-access-tokens) 180 | // must belong to a collaborator of the repo (https://github.com/$user/$repo/settings/collaboration) 181 | // or we can't set commit statuses 182 | private implicit def connection = setupConnection(config.host, Some(new BasicHttpCredentials(config.token, "x-oauth-basic"))) // https://developer.github.com/v3/auth/#basic-authentication 183 | // addHeader("X-My-Special-Header", "fancy-value") 184 | // "Accept" -> "application/vnd.github.v3+json" 185 | 186 | def api(rest: String) = Uri(s"/repos/${config.user}/${config.repo}" / rest) 187 | import spray.json._ 188 | 189 | def pullRequests = p[List[PullRequest]] (Get(api("pulls"))) 190 | def closedPullRequests = p[List[PullRequest]] (Get(api("pulls") withQuery Map("state" -> "closed"))) 191 | def pullRequest(nb: Int) = p[PullRequest] (Get(api("pulls" / nb))) 192 | def pullRequestCommits(nb: Int) = p[List[Commit]] (Get(api("pulls" / nb / "commits") 193 | withQuery Map("per_page" -> "100"))) 194 | def deletePRComment(id: String) = px (Delete(api("pulls" / "comments" / id))) 195 | def requestReview(nb: Int, reviewers: Reviewers) = px (Post(api("pulls" / nb / "requested_reviewers"), reviewers)~> 196 | /** https://developer.github.com/changes/2016-12-14-reviews-api/ */ addHeader("Accept", "application/vnd.github.black-cat-preview+json")) 197 | 198 | def issueComments(nb: Int) = p[List[IssueComment]](Get(api("issues" / nb / "comments"))) 199 | def postIssueComment(nb: Int, c: IssueComment) = p[IssueComment] (Post(api("issues" / nb / "comments"), c)) 200 | def issue(nb: Int) = p[Issue] (Get(api("issues" / nb))) 201 | def setMilestone(nb: Int, milestone: Int) = px (Patch(api("issues" / nb), JsObject("milestone" -> JsNumber(milestone)))) 202 | def addLabel(nb: Int, labels: List[Label]) = p[Label] (Post(api("issues" / nb / "labels"), labels)) 203 | def deleteLabel(nb: Int, label: String) = px (Delete(api("issues" / nb / "labels" / label))) 204 | def labels(nb: Int) = p[List[Label]] (Get(api("issues" / nb / "labels"))) 205 | 206 | // most recent status comes first in the resulting list! 207 | def commitStatus(sha: String) = p[CombiCommitStatus] (Get(api("commits" / sha / "status"))) 208 | def postStatus(sha: String, status: CommitStatus) = p[CommitStatus] (Post(api("statuses" / sha), status)) 209 | 210 | def allLabels = p[List[Label]] (Get(api("labels"))) 211 | def createLabel(label: Label) = p[List[Label]] (Post(api("labels"), label)) 212 | 213 | def postCommitComment(sha: String, c: PullRequestComment)= p[PullRequestComment] (Post(api("commits" / sha / "comments"), c)) 214 | def commitComments(sha: String) = p[List[PullRequestComment]](Get(api("commits" / sha / "comments"))) 215 | 216 | def deleteCommitComment(id: String): Unit = px (Delete(api("comments" / id))) 217 | 218 | def repoMilestones(state: String = "open") = p[List[Milestone]] (Get(api("milestones") withQuery Map("state" -> state))) 219 | 220 | 221 | // def editPRComment(user: String, repo: String, id: String, comment: IssueComment) = patch[IssueComment](pulls + "/comments/$id") 222 | // // Normalize sha if it's not 40 chars 223 | // // GET /repos/:owner/:repo/commits/:sha 224 | // def normalizeSha(user: String, repo: String, sha: String): String = 225 | // if (sha.length == 40) sha 226 | // else try { 227 | // val url = makeAPIurl(s"/repos/$user/$repo/commits/$sha") 228 | // val action = url >- (x => parseJsonTo[PRCommit](x).sha) 229 | // Http(action) 230 | // } catch { 231 | // case e: Exception => 232 | // println(s"Error: couldn't normalize $sha (for $user/$repo): "+ e) 233 | // sha 234 | // } 235 | } 236 | } 237 | 238 | 239 | //object CommitStatus { 240 | // final val PENDING = "pending" 241 | // final val SUCCESS = "success" 242 | // final val ERROR = "error" 243 | // final val FAILURE = "failure" 244 | // 245 | // // to distinguish PENDING jobs that are done but waiting on other PENDING jobs from truly pending jobs 246 | // // the message of other PENDING jobs should never start with "$job OK" 247 | // final val FAKE_PENDING = "OK" 248 | // 249 | // // TODO: assert(!name.contains(" ")) for all job* methods below 250 | // def jobQueued(name: String) = CommitStatus(PENDING, None, Some(name +" queued.")) 251 | // def jobStarted(name: String, url: String) = CommitStatus(PENDING, Some(url), Some(name +" started.")) 252 | // // assert(!message.startsWith(FAKE_PENDING)) 253 | // def jobEnded(name: String, url: String, ok: Boolean, message: String) = 254 | // CommitStatus(if(ok) SUCCESS else ERROR, Some(url), Some((name +" "+ message).take(140))) 255 | // 256 | // // only used for last commit 257 | // def jobEndedBut(name: String, url: String, message: String)(prev: String) = 258 | // CommitStatus(PENDING, Some(url), Some((name +" "+ FAKE_PENDING +" but waiting for "+ prev).take(140))) 259 | // 260 | // // depends on the invariant maintained by overruleSuccess so that we only have to look at the most recent status 261 | // def jobDoneOk(cs: List[CommitStatus]) = cs.headOption.map(st => st.success || st.fakePending).getOrElse(false) 262 | // 263 | // 264 | // /** Find commit status that's either truly pending (not fake pending) or that found an error, 265 | // * and for which there's no corresponding successful commit status 266 | // */ 267 | // def notDoneOk(commitStati: List[CommitStatus]): Iterable[CommitStatus] = { 268 | // val grouped = commitStati.groupBy(_.job) 269 | // val problems = grouped.flatMap { 270 | // case (Some(jobName), jobAndCommitStati) if !jobAndCommitStati.exists(_.success) => 271 | // jobAndCommitStati.filter(cs => (cs.pending && !cs.fakePending) || cs.error) 272 | // case _ => 273 | // Nil 274 | // } 275 | // // println("notDoneOk grouped: "+ grouped.mkString("\n")) 276 | // // println("problems: "+ problems) 277 | // problems 278 | // } 279 | //} 280 | 281 | //// note: it looks like the buildbot github user needs administrative permission to create labels, 282 | //// but also to set the commit status 283 | //object Authenticate { 284 | // 285 | // private[this] val authorizations = :/("api.github.com").secure / "authorizations" <:< Map("User-Agent" -> USER_AGENT) 286 | // 287 | // val authScopes = """{ 288 | // "scopes": [ 289 | // "user", 290 | // "repo", 291 | // "repo:status" 292 | // ], 293 | // "note": "scabot API Access" 294 | //}""" 295 | // 296 | // /** This method looks for a previous GH authorization for this API and retrieves it, or 297 | // * creates a new one. 298 | // */ 299 | // def authenticate(user: String, pw: String): Authorization = { 300 | // val previousAuth: Option[Authorization] = 301 | // (getAuthentications(user,pw) filter (_.note == Some("scabot API Access"))).headOption 302 | // previousAuth getOrElse makeAuthentication(user, pw) 303 | // } 304 | // 305 | // 306 | // def makeAuthentication(user: String, pw: String): Authorization = 307 | // Http(authorizations.POST.as_!(user, pw) << authScopes >- parseJsonTo[Authorization]) 308 | // 309 | // def getAuthentications(user: String, pw: String): List[Authorization] = 310 | // Http(authorizations.as_!(user, pw) >- parseJsonTo[List[Authorization]]) 311 | // 312 | // def deleteAuthentication(auth: Authorization, user: String, pw: String): Unit = 313 | // Http( (authorizations / auth.id).DELETE.as_!(user,pw) >|) 314 | // 315 | // def deleteAuthentications(user: String, pw: String): Unit = 316 | // getAuthentications(user, pw) foreach { a => 317 | // deleteAuthentication(a, user, pw) 318 | // } 319 | //} 320 | // 321 | 322 | //case class PullRequest( 323 | // number: Int, 324 | // head: GitRef, 325 | // base: GitRef, 326 | // user: User, 327 | // title: String, 328 | // body: String, 329 | // state: String, 330 | // updated_at: String, 331 | // created_at: String, 332 | // mergeable: Option[Boolean], 333 | // milestone: Option[Milestone] // when treating an issue as a pull 334 | // ) extends Ordered[PullRequest] { 335 | // def compare(other: PullRequest): Int = number compare other.number 336 | // def sha10 = head.sha10 337 | // def ref = head.ref 338 | // def branch = head.label.replace(':', '/') 339 | // def date = updated_at takeWhile (_ != 'T') 340 | // def time = updated_at drop (date.length + 1) 341 | // 342 | // override def toString = s"${base.repo.owner.login}/${base.repo.name}#$number" 343 | //} 344 | // 345 | 346 | 347 | 348 | //// { 349 | //// "url": "https://api.github.com/repos/octocat/Hello-World/milestones/1", 350 | //// "number": 1, 351 | //// "state": "open", 352 | //// "title": "v1.0", 353 | //// "description": "", 354 | //// "creator": { 355 | //// "login": "octocat", 356 | //// "id": 1, 357 | //// "avatar_url": "https://github.com/images/error/octocat_happy.gif", 358 | //// "gravatar_id": "somehexcode", 359 | //// "url": "https://api.github.com/users/octocat" 360 | //// }, 361 | //// "open_issues": 4, 362 | //// "closed_issues": 8, 363 | //// "created_at": "2011-04-10T20:09:31Z", 364 | //// "due_on": null 365 | //// } 366 | ////] 367 | -------------------------------------------------------------------------------- /github/src/main/scala/scabot/github/GithubService.scala: -------------------------------------------------------------------------------- 1 | package scabot 2 | package github 3 | 4 | import akka.event.Logging 5 | 6 | import scala.util.{Success, Failure} 7 | 8 | 9 | trait GithubService extends GithubApi { 10 | 11 | private lazy val UserRepo = """([^/]+)/(.+)""".r 12 | def notifyProject(ev: ProjectMessage, repository: Repository): String = { 13 | val UserRepo(user, repo) = repository.full_name 14 | val log = s"Processing $ev for $user/$repo" 15 | system.log.info(log) 16 | broadcast(user, repo)(ev) 17 | log 18 | } 19 | 20 | def pullRequestEvent(ev: PullRequestEvent): String = ev match { 21 | case PullRequestEvent(action, number, pull_request) => 22 | notifyProject(ev, ev.pull_request.base.repo) 23 | } 24 | 25 | def pushEvent(ev: PushEvent): String = ev match { 26 | case PushEvent(_, _, repository) => 27 | notifyProject(ev, repository) 28 | } 29 | 30 | def issueCommentEvent(ev: IssueCommentEvent): String = ev match { 31 | case IssueCommentEvent(action, issue, comment, repository) => 32 | notifyProject(ev, repository) 33 | } 34 | 35 | def pullRequestReviewCommentEvent(ev: PullRequestReviewCommentEvent): String = ev match { 36 | case PullRequestReviewCommentEvent(action, pull_request, comment, repository) => 37 | notifyProject(ev, repository) 38 | } 39 | 40 | } 41 | 42 | -------------------------------------------------------------------------------- /gui/app/controllers/Scabot.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import akka.actor.ActorSystem 4 | import javax.inject.Inject 5 | 6 | import play.api.http.LazyHttpErrorHandler 7 | import play.api.http.Status.REQUEST_ENTITY_TOO_LARGE 8 | import play.api.libs.iteratee.{Cont, Done, Input, Iteratee, Traversable} 9 | import play.api.mvc.{Action => PlayAction, _} 10 | import scabot.github.GithubService 11 | import scabot.jenkins.JenkinsService 12 | import scabot.core 13 | import scabot.server.Actors 14 | import spray.json.ParserInput.ByteArrayBasedParserInput 15 | import spray.json._ 16 | 17 | import scala.concurrent.Future 18 | import scala.util._ 19 | 20 | class Scabot @Inject() (val system: ActorSystem) extends Controller with GithubService with JenkinsService with core.Configuration with core.HttpClient with Actors { 21 | 22 | override def configFile = new java.io.File(sys.props("scabot.config.file")) 23 | 24 | startActors() 25 | 26 | // X-Github-Event: 27 | // commit_comment Any time a Commit is commented on. 28 | // create Any time a Branch or Tag is created. 29 | // delete Any time a Branch or Tag is deleted. 30 | // deployment Any time a Repository has a new deployment created from the API. 31 | // deployment_status Any time a deployment for a Repository has a status update from the API. 32 | // fork Any time a Repository is forked. 33 | // gollum Any time a Wiki page is updated. 34 | // issue_comment Any time an Issue is commented on. 35 | // issues Any time an Issue is assigned, unassigned, labeled, unlabeled, opened, closed, or reopened. 36 | // member Any time a User is added as a collaborator to a non-Organization Repository. 37 | // membership Any time a User is added or removed from a team. Organization hooks only. 38 | // page_build Any time a Pages site is built or results in a failed build. 39 | // public Any time a Repository changes from private to public. 40 | // pull_request_review_comment Any time a Commit is commented on while inside a Pull Request review (the Files Changed tab). 41 | // pull_request Any time a Pull Request is assigned, unassigned, labeled, unlabeled, opened, closed, reopened, or synchronized (updated due to a new push in the branch that the pull request is tracking). 42 | // push Any Git push to a Repository, including editing tags or branches. Commits via API actions that update references are also counted. This is the default event. 43 | // repository Any time a Repository is created. Organization hooks only. 44 | // release Any time a Release is published in a Repository. 45 | // status Any time a Repository has a status update from the API 46 | // team_add Any time a team is added or modified on a Repository. 47 | // watch Any time a User watches a Repository. 48 | 49 | def github() = PlayAction(sprayBodyParser) { implicit request => 50 | request.headers.get("X-GitHub-Event").collect { 51 | case "issue_comment" => handleWith(issueCommentEvent) 52 | case "pull_request_review_comment" => handleWith(pullRequestReviewCommentEvent) 53 | case "pull_request" => handleWith(pullRequestEvent) 54 | case "push" => handleWith(pushEvent) 55 | // case "status" => TODO: use this to propagate combined contexts -- problem: the (payload)[https://developer.github.com/v3/activity/events/types/#statusevent] does not specify the PR 56 | } match { 57 | case Some(Success(message)) => Ok(message) 58 | case Some(Failure(ex)) => InternalServerError(ex.getMessage) 59 | case None => Status(404) 60 | } 61 | } 62 | 63 | def jenkins() = PlayAction(sprayBodyParser) { implicit request => 64 | handleWith(jenkinsEvent) match { 65 | case Success(message) => Ok(message) 66 | case Failure(ex) => 67 | system.log.error(s"Couldn't handle Jenkins event: ${request.body}.\n Fail: $ex") 68 | InternalServerError(ex.getMessage) 69 | } 70 | } 71 | 72 | private def sprayBodyParser: BodyParser[JsValue] = 73 | BodyParsers.parse.raw(memoryThreshold = 640 * 1024 /*should be enough to keep most requests in memory...*/).mapM { 74 | buffer => 75 | val parsed = 76 | Future { JsonParser(new ByteArrayBasedParserInput(buffer.asBytes().get)) } 77 | parsed.onFailure{ case ex => system.log.error(s"Fail in spray body parser: $ex") } 78 | parsed.onSuccess{ case jsv => system.log.debug(s"Received JSON: $jsv") } 79 | parsed 80 | } 81 | 82 | def handleWith[T](handler: T => String)(implicit reader: JsonReader[T], request: Request[JsValue]): Try[String] = 83 | Try(handler(reader.read(request.body))) 84 | 85 | def index = PlayAction { 86 | Ok("ohi scabot") 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /gui/conf/application.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scala/scabot/df337b1031ce07fb68c0808932c2eb95d0086042/gui/conf/application.conf -------------------------------------------------------------------------------- /gui/conf/prod-logger.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %level %logger{15} - %message%n%xException{10} 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /gui/conf/routes: -------------------------------------------------------------------------------- 1 | POST /jenkins controllers.Scabot.jenkins 2 | POST /github controllers.Scabot.github 3 | 4 | GET / controllers.Scabot.index 5 | -------------------------------------------------------------------------------- /jenkins/src/main/scala/scabot/jenkins/JenkinsApi.scala: -------------------------------------------------------------------------------- 1 | package scabot 2 | package jenkins 3 | 4 | import spray.http.BasicHttpCredentials 5 | 6 | import scala.concurrent.Future 7 | import scala.util.Try 8 | 9 | trait JenkinsApi extends JenkinsApiTypes with JenkinsJsonProtocol with JenkinsApiActions 10 | 11 | trait JenkinsApiTypes extends core.Configuration { 12 | object Job { 13 | // avoid humongeous replies that would require more spray sophistication (chunking etc) 14 | val XPath = "?tree=name,description,nextBuildNumber,builds[number,url],queueItem[*],lastBuild[number,url],firstBuild[number,url]" 15 | } 16 | case class Job(name: String, 17 | description: String, 18 | nextBuildNumber: Int, 19 | builds: List[Build], 20 | queueItem: Option[QueueItem], 21 | lastBuild: Build, 22 | firstBuild: Build) 23 | 24 | case class Build(number: Int, url: String) extends Ordered[Build] { 25 | def num: Int = number.toInt 26 | 27 | def compare(that: Build) = that.num - this.num 28 | } 29 | 30 | // value is optional: password-valued parameters hide their value 31 | case class Param(name: String, value: Option[String]) { 32 | override def toString = "%s -> %s" format(name, value) 33 | } 34 | 35 | case class Action(parameters: Option[List[Param]]) { 36 | override def toString = "Parameters(%s)" format (parameters mkString ", ") 37 | } 38 | 39 | object BuildStatus { 40 | // avoid humongeous replies that would require more spray sophistication (chunking etc) 41 | val XPath = "?tree=number,result,building,duration,actions[parameters[*]],url" 42 | } 43 | case class BuildStatus(number: Option[Int], 44 | result: Option[String], 45 | building: Boolean, 46 | duration: Long, 47 | actions: Option[List[Action]], 48 | url: Option[String]) { // TODO url may not need to be optional -- not sure 49 | def friendlyDuration = Try { 50 | val seconds = duration.toInt / 1000 51 | if (seconds == 0) "" 52 | else "Took "+ (if (seconds <= 90) s"$seconds s." else s"${seconds / 60} min.") 53 | } getOrElse "" 54 | 55 | def queued = result.isEmpty 56 | def success = result == Some("SUCCESS") 57 | def failed = !(queued || building || success) 58 | // TODO deal with ABORTED and intermittent failure (should retry these with different strategy from failures?) 59 | 60 | def status = if (building) "Building" else result.getOrElse("Pending") 61 | def parameters = (for { 62 | actions <- actions.toList 63 | action <- actions 64 | parameters <- action.parameters.toList 65 | Param(n, Some(v)) <- parameters 66 | } yield (n, v)).toMap 67 | 68 | def paramsMatch(expectedArgs: Map[String, String]): Boolean = 69 | parameters.filterKeys(expectedArgs.isDefinedAt) == expectedArgs 70 | 71 | override def toString = s"[${number.getOrElse("?")}] $status. $friendlyDuration" 72 | } 73 | 74 | class QueuedBuildStatus(actions: Option[List[Action]], url: Option[String]) extends BuildStatus(None, None, false, 0, actions, url) { 75 | def this(params: Map[String, String], url: Option[String]) { 76 | this(Some(List(Action(Some(params.toList.map { case (k, v) => Param(k, Some(v))})))), url) 77 | } 78 | } 79 | 80 | case class Queue(items: List[QueueItem]) 81 | 82 | case class QueueItem(actions: Option[List[Action]], task: Task, id: Int) { 83 | def jobName = task.name 84 | 85 | // the url is fake but needs to be unique 86 | def toStatus = new QueuedBuildStatus(actions, Some(task.url + "#queued-" + id)) 87 | } 88 | 89 | case class Task(name: String, url: String) 90 | 91 | // for https://wiki.jenkins-ci.org/display/JENKINS/Notification+Plugin 92 | case class JobState(name: String, url: String, build: BuildState) extends ProjectMessage with PRMessage 93 | // phase = STARTED | COMPLETED | FINALIZED 94 | case class BuildState(number: Int, phase: String, parameters: Map[String, String], scm: ScmParams, result: Option[String], full_url: String, log: Option[String]) // TODO result always seems to be None?? 95 | case class ScmParams(url: Option[String], branch: Option[String], commit: Option[String]) // TODO can we lift the `Option`s to `BuildState`'s `scm` arg? 96 | 97 | } 98 | 99 | /* 100 | { 101 | "build": { 102 | "artifacts": {}, 103 | "full_url": "https://scala-ci.typesafe.com/job/scala-2.11.x-validate-main/28/", 104 | "number": 28, 105 | "parameters": { 106 | "prDryRun": "", 107 | "repo_name": "scala", 108 | "repo_ref": "61a16f54b64d1bf020f36c2a6b4baff58c6ec70d", 109 | "repo_user": "adriaanm" 110 | }, 111 | "phase": "FINALIZED", 112 | "scm": {}, 113 | "status": "FAILURE", 114 | "url": "job/scala-2.11.x-validate-main/28/" 115 | }, 116 | "name": "scala-2.11.x-validate-main", 117 | "url": "job/scala-2.11.x-validate-main/" 118 | } 119 | */ 120 | 121 | import spray.json.{RootJsonFormat, DefaultJsonProtocol} 122 | 123 | // TODO: can we make this more debuggable? 124 | trait JenkinsJsonProtocol extends JenkinsApiTypes with DefaultJsonProtocol { 125 | implicit object nonjudgmentalStringJsonFormat extends spray.json.JsonFormat[String] { 126 | import spray.json._ 127 | 128 | def write(x: String) = { 129 | require(x ne null) 130 | JsString(x) 131 | } 132 | def read(value: JsValue) = value match { 133 | case JsString(x) => x 134 | case x => x.toString // whatevs 135 | } 136 | } 137 | 138 | private type RJF[x] = RootJsonFormat[x] 139 | implicit lazy val _fmtJob : RJF[Job ] = jsonFormat7(Job.apply) 140 | implicit lazy val _fmtBuild : RJF[Build ] = jsonFormat2(Build) 141 | implicit lazy val _fmtParam : RJF[Param ] = jsonFormat2(Param) 142 | implicit lazy val _fmtActions : RJF[Action ] = jsonFormat1(Action) 143 | implicit lazy val _fmtBuildStatus : RJF[BuildStatus] = jsonFormat6(BuildStatus.apply) 144 | implicit lazy val _fmtQueue : RJF[Queue ] = jsonFormat1(Queue) 145 | implicit lazy val _fmtQueueItem : RJF[QueueItem ] = jsonFormat3(QueueItem) 146 | implicit lazy val _fmtTask : RJF[Task ] = jsonFormat2(Task) 147 | implicit lazy val _fmtJobState : RJF[JobState ] = jsonFormat3(JobState) 148 | implicit lazy val _fmtBuildState : RJF[BuildState ] = jsonFormat7(BuildState) 149 | implicit lazy val _fmtScmState : RJF[ScmParams ] = jsonFormat3(ScmParams) 150 | } 151 | 152 | trait JenkinsApiActions extends JenkinsJsonProtocol with core.HttpClient { 153 | class JenkinsConnection(config: Config.Jenkins) { 154 | import spray.http.{GenericHttpCredentials, Uri} 155 | import spray.httpx.SprayJsonSupport._ 156 | import spray.client.pipelining._ 157 | 158 | private implicit def connection = setupConnection(config.host, Some(BasicHttpCredentials(config.user, config.token))) 159 | 160 | def api(rest: String) = Uri("/" + rest) 161 | 162 | def buildJob(name: String, params: Map[String, String] = Map.empty) = 163 | p[String](Post( 164 | if (params.isEmpty) api(name / "build") 165 | else api("job" / name / "buildWithParameters") withQuery (params) 166 | )) 167 | 168 | def buildStatus(name: String, buildNumber: Int) = 169 | p[BuildStatus](Get(api("job" / name / buildNumber / "api/json"+ BuildStatus.XPath))) 170 | 171 | 172 | def buildStatus(url: String) = 173 | p[BuildStatus](Get(url / "api/json"+ BuildStatus.XPath)) 174 | 175 | /** A traversable that lazily pulls build status information from jenkins. 176 | * 177 | * Only statuses for the specified job (`job.name`) that have parameters that match all of `expectedArgs` 178 | */ 179 | def buildStatusesForJob(job: String): Future[Stream[Future[BuildStatus]]] = { 180 | def queuedStati(q: Queue) = q.items.toStream.filter(_.jobName == job).map(qs => Future.successful(qs.toStatus)) 181 | def reportedStati(info: Job) = info.builds.sorted.toStream.map(b => buildStatus(job, b.number)) 182 | 183 | // hack: retrieve queued jobs from queue/api/json 184 | // queued items must come first, they have been added more recently or they wouldn't have been queued 185 | for { 186 | queued <- p[Queue](Get(api("queue/api/json"))).map(queuedStati) 187 | reported <- p[Job](Get(api("job" / job / "api/json"+ Job.XPath))).map(reportedStati) 188 | } yield queued ++ reported 189 | } 190 | 191 | def jobInfo(name: String) = 192 | p[Job](Get(api("job" / name / "api/json"))) 193 | 194 | def nextBuildNumber(name: String): Future[Int] = 195 | jobInfo(name).map(_.lastBuild.number.toInt) 196 | 197 | } 198 | } 199 | 200 | -------------------------------------------------------------------------------- /jenkins/src/main/scala/scabot/jenkins/JenkinsService.scala: -------------------------------------------------------------------------------- 1 | package scabot 2 | package jenkins 3 | 4 | trait JenkinsService extends JenkinsApi { 5 | 6 | def jenkinsEvent(jobState: JobState): String = jobState match { 7 | case JobState(name, _, BuildState(number, phase, parameters, scm, result, full_url, log)) => 8 | system.log.info(s"Jenkins event: $name [$number]: $phase ($result) at $full_url.\n Scm: scm\n Params: $parameters\n $log") 9 | 10 | { 11 | for { 12 | user <- parameters.get(PARAM_REPO_USER) 13 | repo <- parameters.get(PARAM_REPO_NAME) 14 | } yield broadcast(user, repo)(jobState) 15 | } getOrElse { 16 | system.log.warning(s"Couldn't identify project for job based on $PARAM_REPO_USER/$PARAM_REPO_NAME in $parameters. Was it started by us?") 17 | } 18 | 19 | "Fascinating, dear Jenkins!" 20 | } 21 | } 22 | 23 | -------------------------------------------------------------------------------- /log/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scala/scabot/df337b1031ce07fb68c0808932c2eb95d0086042/log/.empty -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.18 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.11") 2 | 3 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.9") 4 | -------------------------------------------------------------------------------- /scripts/stage: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # the new jenkins master only has java 8 installed, and no jvm-select script 4 | # source /usr/local/share/jvm/jvm-select 5 | # jvmSelect openjdk 8 6 | 7 | sbt update gui/assembly 8 | -------------------------------------------------------------------------------- /server/src/main/scala/scabot/server/Actors.scala: -------------------------------------------------------------------------------- 1 | package scabot 2 | package server 3 | 4 | import java.util.NoSuchElementException 5 | 6 | import akka.actor._ 7 | import akka.event.LoggingAdapter 8 | import com.amazonaws.services.dynamodbv2.document.{Item, PrimaryKey} 9 | import com.amazonaws.services.dynamodbv2.model.KeyType 10 | import scabot.amazon.DynamoDb 11 | import scabot.core.BaseRef 12 | 13 | import scala.concurrent.{Promise, ExecutionContext, Future} 14 | import scala.concurrent.duration._ 15 | import scala.util.Try 16 | 17 | /** 18 | * Created by adriaan on 1/15/15. 19 | */ 20 | trait Actors extends github.GithubApi with jenkins.JenkinsApi with DynamoDb with core.Util { 21 | def system: ActorSystem 22 | 23 | private lazy val githubActor = system.actorOf(Props(new GithubActor), "github") 24 | 25 | // project actors are supervised by the github actor 26 | // pull request actors are supervised by their project actor 27 | private def projectActorName(user: String, repo: String) = s"$user-$repo" 28 | 29 | def broadcast(user: String, repo: String)(msg: ProjectMessage) = 30 | system.actorSelection(githubActor.path / projectActorName(user, repo)) ! msg 31 | 32 | 33 | def startActors() = { 34 | githubActor ! GithubActor.StartProjectActors(configs) 35 | } 36 | 37 | 38 | object GithubActor { 39 | case class StartProjectActors(configs: Map[String, Config]) 40 | } 41 | 42 | import GithubActor._ 43 | class GithubActor extends Actor with ActorLogging { 44 | override def receive: Receive = { 45 | case StartProjectActors(configs) => 46 | log.info(s"Reading ${configs.size} configs.") 47 | configs map { case (name, config) => 48 | context.actorOf(Props(new ProjectActor(config)), projectActorName(config.github.user, config.github.repo)) 49 | } foreach { _ ! Synch } 50 | } 51 | } 52 | 53 | case object Synch extends ProjectMessage 54 | 55 | case class JenkinsJobResult(name: String, status: BuildStatus) extends PRMessage { 56 | lazy val sha = status.parameters(PARAM_REPO_REF) 57 | } 58 | 59 | // represents a github project at github.com/${config.github.user}/${config.github.repo} 60 | class ProjectActor(val config: Config) extends Actor with ActorLogging with Building { 61 | lazy val githubApi = new GithubConnection(config.github) 62 | lazy val jenkinsApi = new JenkinsConnection(config.jenkins) 63 | private val REFS_HEADS = "refs/heads/" 64 | 65 | import context._ 66 | 67 | // find or create actor responsible for PR #`nb` 68 | def prActor(nb: Int) = child(nb.toString).getOrElse(actorOf(Props(new PullRequestActor(nb, config)), nb.toString)) 69 | 70 | def monitored(pullRequest: PullRequest) = { 71 | val monitor = config.github.branches(pullRequest.base.ref) 72 | if (!monitor) log.warning(s"Not monitoring #${pullRequest.number} because ${pullRequest.base.ref} not in ${config.github.branches}.") 73 | monitor 74 | } 75 | 76 | // supports messages of type ProjectMessage 77 | override def receive: Receive = { 78 | case Synch => 79 | log.info("Synching up! Bleepy-dee-bloop.") 80 | 81 | githubApi.pullRequests.foreach { prs => 82 | prs.filter(monitored).foreach { pr => prActor(pr.number) ! PullRequestEvent("synchronize", pr.number, pr)} 83 | } 84 | // synch every once in a while, just in case we missed a webhook event somehow 85 | // TODO make timeout configurable 86 | context.system.scheduler.scheduleOnce(30.minutes, self, Synch) 87 | 88 | case ev@PullRequestEvent(_, nb, pull_request) if monitored(pull_request) => 89 | prActor(nb) ! ev 90 | 91 | case PullRequestReviewCommentEvent("created", pull_request, comment, _) if monitored(pull_request) => 92 | prActor(pull_request.number) ! comment 93 | 94 | case IssueCommentEvent("created", issue, comment, _) => 95 | prActor(issue.number) ! comment 96 | 97 | case PushEvent(ref, commits, _) if ref startsWith REFS_HEADS => 98 | // only build for master-like branches (we always have milestones that mention this ref for those) 99 | val branch = ref.drop(REFS_HEADS.length) 100 | for(_ <- milestoneForBranch(branch)) 101 | buildPushedCommits(new BaseRef(branch), commits) 102 | 103 | // there are two cases, unfortunately: 104 | // - PARAM_PR's value is an integer ==> assumed to be PR number, 105 | // - otherwise, it's assumed to be the branch name of a job launched for a pushed commit (e.g., a merge or a direct push) 106 | // TODO: report failure 107 | case js@JobState(name, _, BuildState(number, _, _, _, _, _, _)) => 108 | // fetch the state from jenkins -- the webhook doesn't pass in result correctly (???) 109 | for { 110 | bs <- jenkinsApi.buildStatus(name, number) 111 | prParam <- Future { bs.parameters(PARAM_PR) } 112 | } { 113 | log.info(s"Build status for $name #$number: $bs") 114 | val jobRes = JenkinsJobResult(name, bs) 115 | 116 | try prActor(prParam.toInt) ! jobRes 117 | catch { case _: NumberFormatException => 118 | // check prParam is a branch name we recognize 119 | for(_ <- milestoneForBranch(prParam)) 120 | postStatus(new BaseRef(prParam), jobRes) 121 | } 122 | } 123 | } 124 | 125 | // determine jobs needed to be built based on the commit's status, synching github's view with build statuses reported by jenkins 126 | private def buildPushedCommits(baseRef: BaseRef, commits: List[CommitInfo]): Future[List[List[String]]] = { 127 | val lastSha = commits.last.id.getOrElse("???") // the merge commit, typically 128 | Future.sequence(commits map { commit => 129 | for { 130 | combiCs <- fetchCommitStatus(commit.id.get) 131 | buildRes <- { 132 | val params = jobParams(baseRef.name, combiCs.sha, combiCs.sha == lastSha) 133 | Future.sequence(jobsTodo(baseRef, combiCs, rebuild = false).map(launchBuild(combiCs.sha, baseRef, params)(_))) 134 | } 135 | } yield buildRes 136 | }) 137 | } 138 | } 139 | 140 | trait Building { 141 | def config: Config 142 | def githubApi: GithubConnection 143 | def jenkinsApi: JenkinsConnection 144 | def log: LoggingAdapter 145 | 146 | def milestoneForBranch(branch: String): Future[Milestone] = for { 147 | mss <- githubApi.repoMilestones() 148 | ms <- Future { 149 | val msOpt = mss.find(_.mergeBranch == Some(branch)) 150 | log.info(s"Looking for milestone for $branch: $msOpt") 151 | msOpt.get 152 | } 153 | } yield ms 154 | 155 | def fetchCommitStatus(sha: String) = { 156 | val fetcher = githubApi.commitStatus(sha) 157 | fetcher.onFailure { case e => log.warning(s"Couldn't get status for ${sha}: $e")} 158 | fetcher 159 | } 160 | 161 | implicit object jcl extends JobContextLense { 162 | // e.g., scala-2.11.x- for PR targeting 2.11.x of s"$user/scala" (for any user) 163 | def prefix(baseRef: BaseRef) = s"${config.github.repo}-${baseRef.name}-" 164 | 165 | // TODO: as we add more analyses to PR validation, update this predicate to single out jenkins jobs 166 | // NOTE: config.jenkins.job spawns other jobs, which we don't know about here, but still want to retry on /rebuild 167 | def contextForJob(job: String, baseRef: BaseRef): Option[String] = 168 | Some(job.replace(prefix(baseRef), "")) // TODO: should only replace *prefix*, not just anywhere in string 169 | 170 | def jobForContext(context: String, baseRef: BaseRef): Option[String] = 171 | if (CommitStatusConstants.jenkinsContext(context)) Some(prefix(baseRef) + context) 172 | else None 173 | } 174 | 175 | def mainValidationJob(baseRef: BaseRef) = jcl.prefix(baseRef) + config.jenkins.jobSuffix 176 | 177 | // TODO: is this necessary? just to be sure, as it looks like github refuses non-https links 178 | def urlForBuild(bs: BuildStatus) = Some(bs.url.map(_.replace("http://", "https://")).getOrElse("")) 179 | 180 | def stateForBuild(bs: BuildStatus) = 181 | if (bs.building || bs.queued) CommitStatusConstants.PENDING 182 | else if (bs.success) CommitStatusConstants.SUCCESS 183 | else CommitStatusConstants.FAILURE 184 | 185 | def contextForJob(jobName: String, baseRef: BaseRef): Option[String] = implicitly[JobContextLense].contextForJob(jobName, baseRef) 186 | 187 | def commitStatus(jobName: String, bs: BuildStatus, baseRef: BaseRef): CommitStatus = { 188 | val advice = if (bs.failed) " Say /rebuild on PR to retry *spurious* failure." else "" 189 | commitStatusForContext(contextForJob(jobName, baseRef), bs, advice) 190 | } 191 | 192 | def commitStatusForContext(context: Option[String], bs: BuildStatus, advice: String): CommitStatus = { 193 | CommitStatus(stateForBuild(bs), context, 194 | description = Some((bs.toString + advice) take 140), 195 | target_url = urlForBuild(bs)) 196 | } 197 | 198 | def combiStatus(state: String, msg: String): CommitStatus = 199 | CommitStatus(state, Some(CommitStatusConstants.COMBINED), description = Some(msg.take(140))) 200 | 201 | type Parameters = Map[String, String] 202 | def jobParams(pr: String, sha: String, lastCommit: Boolean): Parameters = 203 | Map(PARAM_REPO_USER -> config.github.user, 204 | PARAM_REPO_NAME -> config.github.repo, 205 | PARAM_PR -> pr, // pr number or branch name (for pushed commit) 206 | PARAM_REPO_REF -> sha) ++ ( 207 | if (lastCommit) Map(PARAM_LAST -> "1") 208 | else Map.empty 209 | ) 210 | 211 | // result is a subset of (config.jenkins.job and the contexts found in combiCommitStatus.statuses that are jenkins jobs) 212 | // if not rebuilding or gathering all jobs, this subset is either empty or the main job (if no statuses were found for it) 213 | // unless gatherAllJobs, the result only includes jobs whose most recent status was a failure 214 | def jobsTodo(baseRef: BaseRef, combiCommitStatus: CombiCommitStatus, rebuild: Boolean): List[String] = { 215 | // TODO: filter out aborted stati? 216 | // TODO: for pending jobs, check that they are indeed pending! 217 | def considerStati(stati: List[CommitStatus]): Boolean = 218 | if (rebuild) stati.headOption.forall(_.failure) else stati.isEmpty 219 | 220 | val mainJobForPull = mainValidationJob(baseRef) 221 | val shouldConsider = Map(mainJobForPull -> true) ++: combiCommitStatus.statuses.groupBy(_.jobName(baseRef)).collect { 222 | case (Some(job), stati) => (job, considerStati(stati)) 223 | } 224 | 225 | log.info(s"shouldConsider for ${combiCommitStatus.sha.take(6)} (rebuild=$rebuild, consider main job: ${shouldConsider.get(mainJobForPull)}): $shouldConsider") 226 | 227 | val allToConsider = shouldConsider.collect{case (job, true) => job} 228 | 229 | // We've built this before and we were asked to rebuild. For all jobs that have ended in failure, launch a build. 230 | // TODO: once we support overriding main job with results of its downstream jobs, 231 | // on rebuild, we should yield only the downstream jobs, overriding the main job once they finish 232 | // lazy val nonMainToBuild = allToConsider.toSet - mainValidationJob 233 | 234 | val jobs = 235 | // if (rebuild && nonMainToBuild.nonEmpty) nonMainToBuild.toList 236 | if (shouldConsider(mainJobForPull)) List(mainJobForPull) 237 | else Nil 238 | 239 | val jobMsg = 240 | if (jobs.isEmpty) "No need to build" 241 | else s"Found jobs ${jobs.mkString(", ")} TODO" 242 | 243 | log.info(s"$jobMsg for ${combiCommitStatus.sha.take(6)} (rebuild=$rebuild), based on ${combiCommitStatus.total_count} statuses:\n${combiCommitStatus.statuses.groupBy(_.jobName(baseRef))}") 244 | 245 | jobs 246 | } 247 | 248 | 249 | def launchBuild(sha: String, baseRef: BaseRef, params: Parameters)(job: String = mainValidationJob(baseRef)): Future[String] = { 250 | val status = commitStatus(job, new QueuedBuildStatus(params, None), baseRef) 251 | 252 | val launcher = for { 253 | posting <- githubApi.postStatus(sha, status) 254 | buildRes <- jenkinsApi.buildJob(job, params) 255 | _ <- Future.successful(log.info(s"Launched $job for $sha: $buildRes")) 256 | } yield buildRes 257 | 258 | launcher onFailure { case e => log.warning(s"FAILED launchBuild($job, $baseRef, $sha, $params): $e") } 259 | launcher 260 | } 261 | 262 | 263 | def postStatus(baseRef: BaseRef, jenkinsJobResult: JenkinsJobResult): Future[String] = { 264 | import jenkinsJobResult._ 265 | (for { 266 | currentStatus <- githubApi.commitStatus(sha).map(_.statuses.filter(_.forJob(name, baseRef)).headOption) 267 | newStatus = commitStatus(name, status, baseRef) 268 | _ <- Future.successful(log.info(s"New status (new? ${currentStatus != Some(newStatus)}) for $sha: $newStatus old: $currentStatus")) 269 | if currentStatus != Some(newStatus) 270 | posting <- githubApi.postStatus(sha, newStatus) 271 | _ <- Future.successful(log.info(s"Posted status on $sha for $name $status:\n$posting")) 272 | } yield posting.toString).recover { 273 | case _: NoSuchElementException => s"No need to update status of $sha for context $name" 274 | } 275 | } 276 | } 277 | 278 | // for migration 279 | class NoopPullRequestActor extends Actor with ActorLogging { 280 | override def receive: Actor.Receive = { case _ => log.warning("NOOP ACTOR SAYS HELLO") } 281 | } 282 | 283 | class PullRequestActor(pr: Int, val config: Config) extends Actor with ActorLogging with Building { 284 | lazy val githubApi = new GithubConnection(config.github) 285 | lazy val jenkinsApi = new JenkinsConnection(config.jenkins) 286 | 287 | def baseRef(pull: PullRequest): BaseRef = new BaseRef(pull.base.ref) 288 | 289 | private def pull = githubApi.pullRequest(pr) 290 | private def pullBranch = pull.map(baseRef(_)) 291 | private def pullRequestCommits = githubApi.pullRequestCommits(pr) 292 | private def lastSha = pullRequestCommits map (_.last.sha) 293 | private def issueComments = githubApi.issueComments(pr) 294 | 295 | def jobParams(sha: String, lastCommit: Boolean): Parameters = jobParams(pr.toString, sha, lastCommit) 296 | 297 | private var lastSynchronized: Date = None 298 | 299 | // TODO: distrust input, go back to source to verify 300 | // supports messages of type PRMessage 301 | // PullRequestEvent.action: “assigned”, “unassigned”, “labeled”, “unlabeled”, “opened”, “closed”, or “reopened”, or “synchronize” 302 | override def receive: Actor.Receive = { 303 | // process all commits (need to launch builds?) & PR comments 304 | case PullRequestEvent(a@"synchronize", _, pull_request) if pull_request.updated_at != lastSynchronized => 305 | lastSynchronized = pull_request.updated_at 306 | log.info(s"PR synch --> $pull_request") 307 | handlePR(a, pull_request) 308 | 309 | case PullRequestEvent(a@("opened" | "reopened"), _, pull_request) => 310 | log.info(s"PR open --> $pull_request") 311 | handlePR(a, pull_request) 312 | 313 | case PullRequestEvent("closed", _, _) => 314 | log.info(s"PR closed!") 315 | context.stop(self) 316 | 317 | case comment@IssueComment(body, user, created_at, updated_at, id) => 318 | log.info(s"Comment by ${user.getOrElse("???")}:\n$body") 319 | handleComment(comment) 320 | 321 | // TODO: on CommitStatusEvent, propagateEarlierStati(pull) 322 | 323 | case jenkinsJobResult@JenkinsJobResult(name, bs) => 324 | log.info(s"Job state for $name [${bs.number.getOrElse("?")}] @${jenkinsJobResult.sha.take(6)}: ${bs.status} at ${bs.url}") // result is not passed in correctly? 325 | val poster = 326 | for { 327 | baseRef <- pullBranch 328 | postRes <- postStatus(baseRef, jenkinsJobResult) 329 | pull <- pull 330 | propRes <- propagateEarlierStati(pull, jenkinsJobResult.sha) 331 | // if !(bs.queued || bs.building || bs.success) 332 | // _ <- postFailureComment(pull, bs) 333 | } yield propRes 334 | 335 | poster onFailure { case e => log.warning(s"handleJobState($jenkinsJobResult) failed: $e") } 336 | 337 | case PullRequestComment(body, user, commitId, path, pos, created, update, id) => 338 | log.info(s"Comment by $user on $commitId ($path:$pos):\n$body") 339 | // TODO do something with commit comments? 340 | 341 | } 342 | 343 | // requires pull.number == pr 344 | private def handlePR(action: String, pull: PullRequest, synchOnly: Boolean = false) = { 345 | checkMilestone(pull) 346 | propagateEarlierStati(pull) 347 | // don't exec commands when synching, or we'll keep executing the /sync that triggered this handlePR execution 348 | if (!synchOnly) execCommands(pull) 349 | buildCommitsIfNeeded(baseRef(pull), synchOnly = synchOnly, lastOnly = lastOnly(pull.title)) 350 | } 351 | 352 | 353 | // // not called -- see if we can live with less noise 354 | // def postFailureComment(pull: PullRequest, bs: BuildStatus) = 355 | // (for { 356 | // comments <- githubApi.commitComments(sha) 357 | // header = s"Job $jobName failed for ${sha.take(8)}, ${bs.friendlyDuration} (ping @${pull.user.login}) [(results)](${bs.url}):\n" 358 | // if !comments.exists(_.body.startsWith(header)) 359 | // details = s"If you suspect the failure was spurious, comment `/rebuild $sha` on PR ${pr} to retry.\n"+ 360 | // "NOTE: New commits are rebuilt automatically as they appear. A forced rebuild is only necessary for transient failures.\n"+ 361 | // "`/rebuild` without a sha will force a rebuild for all commits." 362 | // comment <- githubApi.postCommitComment(sha, PullRequestComment(header+details)) 363 | // } yield comment.body).recover { 364 | // case _: NoSuchElementException => s"Avoiding double-commenting on $sha for $jobName" 365 | // } 366 | 367 | 368 | // synch contexts assumed to correspond to jenkins jobs with the most recent result of the corresponding build of the jenkins job specified by the context 369 | private def synchBuildStatuses(expected: Parameters, baseRef: BaseRef, combiCommitStatus: CombiCommitStatus): Future[List[String]] = { 370 | case class GitHubReport(state: String, url: Option[String]) 371 | def toReport(bs: BuildStatus) = GitHubReport(stateForBuild(bs), urlForBuild(bs)) 372 | 373 | def checkLinked(url: String): Future[BuildStatus] = for { 374 | linkedBuild <- jenkinsApi.buildStatus(url) 375 | } yield linkedBuild 376 | 377 | def checkMostRecent(job: String): Future[BuildStatus] = for { 378 | // summarize jenkins's report as the salient parts of a CommitStatus (should match what github reported in combiCommitStatus) 379 | bss <- jenkinsApi.buildStatusesForJob(job) 380 | // don't bombard poor jenkins, find in sequence (usually, the first try is successful) 381 | mostRecentBuild <- findFirstSequentially(bss)(_.paramsMatch(expected)) // first == most recent 382 | } yield mostRecentBuild 383 | 384 | // // the status we found on the PR didn't match what Jenkins told us --> synch 385 | // def updateStatus(job: String, bs: BuildStatus) = for { 386 | // res <- handleJobState(job, combiCommitStatus.sha, bs) 387 | // } yield { 388 | // val msg = s"Updating ${combiCommitStatus.sha} of #$pr from ${combiCommitStatus.statuses.headOption} to $bs." 389 | // log.debug(msg) 390 | // msg 391 | // } 392 | 393 | val githubReports = combiCommitStatus.byContext.collect { 394 | case (Some(context), mostRecentStatus :: _) if CommitStatusConstants.jenkinsContext(context) => 395 | (context, GitHubReport(mostRecentStatus.state, mostRecentStatus.target_url)) 396 | } 397 | 398 | def synchLinked(context: String, report: GitHubReport) = for { 399 | url <- Future { report.url.get } 400 | _ = log.info(s"Checking linked at $url") 401 | bs <- checkLinked(url) 402 | if bs.paramsMatch(expected) 403 | } yield 404 | if (toReport(bs) == report) None 405 | else Some(commitStatusForContext(Some(context), bs, "")) 406 | 407 | def synchMostRecent(context: String, report: GitHubReport) = for { 408 | job <- Future { jcl.jobForContext(context, baseRef).get } 409 | bs <- checkMostRecent(job) 410 | } yield 411 | if (toReport(bs) == report) None 412 | else Some(commitStatus(job, bs, baseRef)) 413 | 414 | 415 | val syncher = Future.sequence(githubReports.toList.map { case (context, report) => 416 | (for { 417 | cs <- 418 | synchMostRecent(context, report) recoverWith { case _: spray.httpx.UnsuccessfulResponseException | _ : NoSuchElementException | _ : spray.httpx.PipelineException => 419 | synchLinked(context, report) recover { case e@(_: spray.httpx.UnsuccessfulResponseException | _ : NoSuchElementException | _ : spray.httpx.PipelineException) => 420 | log.info(s"Synch($context, $report) failed with $e") 421 | Some(CommitStatus(CommitStatusConstants.FAILURE, Some(context), 422 | description = Some("No corresponding job found on Jenkins. Failed to launch? Try /rebuild"), 423 | target_url = report.url)) 424 | }} 425 | res <- githubApi.postStatus(combiCommitStatus.sha, cs.get) 426 | } yield res.toString) recover { case _ : NoSuchElementException => "No need to synch." } 427 | }) 428 | 429 | syncher onFailure { case e => log.error(s"FAILED synchBuildStatuses($combiCommitStatus): $e") } // should never happen with the recover 430 | syncher 431 | } 432 | 433 | // /nothingtoseehere 434 | private def overrideFailures(combiCommitStatus: CombiCommitStatus): Future[List[CommitStatus]] = 435 | Future.sequence(combiCommitStatus.byContext.collect { 436 | case (Some(context), mostRecentStatus :: _) if !mostRecentStatus.success => 437 | CommitStatus(CommitStatusConstants.SUCCESS, Some(context), description = Some("Failure overridden. Nothing to see here."), target_url = mostRecentStatus.target_url) 438 | }.toList.map(githubApi.postStatus(combiCommitStatus.sha, _))) 439 | 440 | 441 | 442 | private def lastOnly(pullTitle: String) = config.github.lastCommitOnly || pullTitle.contains("[ci: last-only]") // only test last commit when requested in PR's title (e.g., for large PRs) 443 | 444 | // determine jobs needed to be built based on the commit's status, synching github's view with build statuses reported by jenkins 445 | private def buildCommitsIfNeeded(baseRef: BaseRef, forceRebuild: Boolean = false, synchOnly: Boolean = false, lastOnly: Boolean = false): Future[List[List[String]]] = { 446 | for { 447 | commits <- pullRequestCommits 448 | _ <- Future { log.info(s"buildCommitsIfNeeded? ${baseRef.name} ${commits.map(_.sha.take(6))} force=$forceRebuild synch=$synchOnly") } 449 | lastSha = commits.last.sha // safe to assume commits of a pr is nonEmpty 450 | results <- Future.sequence(commits map { commit => 451 | log.info(s"Build commit? ${commit.sha.take(6)} ") 452 | for { 453 | combiCs <- fetchCommitStatus(commit.sha) 454 | buildRes <- { 455 | val params = jobParams(combiCs.sha, combiCs.sha == lastSha) 456 | if (synchOnly) synchBuildStatuses(params, baseRef, combiCs) 457 | else if (lastOnly && combiCs.sha != lastSha) Future.successful(List(s"Skipped ${combiCs.sha} on request")) 458 | else Future.sequence(jobsTodo(baseRef, combiCs, rebuild = forceRebuild).map(launchBuild(combiCs.sha, baseRef, params)(_))) 459 | } 460 | } yield buildRes 461 | }) 462 | } yield results 463 | } 464 | 465 | // propagate status of commits before the last one over to the last commit's status, 466 | // so that all statuses are (indirectly) considered by github when coloring the merge button green/red 467 | private def propagateEarlierStati(pull: PullRequest, causeSha: String = ""): Future[List[CommitStatus]] = { 468 | import CommitStatusConstants._ 469 | 470 | def postLast(lastSha: String, desc: String, state: String, lastStss: List[CommitStatus]) = 471 | for { _ <- Future.successful(()) 472 | if ! lastStss.exists(st => st.state == state && st.description == Some(desc)) 473 | res <- githubApi.postStatus(lastSha, combiStatus(state, desc)) } yield res 474 | 475 | def outdated(earlier: List[CombiCommitStatus]) = earlier.filter(_.statuses.exists(st => st.combined && !st.success)) 476 | def nothingToSee(st: CombiCommitStatus) = githubApi.postStatus(st.sha, combiStatus(SUCCESS, "Nothing to see here -- no longer last commit.")) 477 | 478 | def combine(earlierStati: List[CombiCommitStatus], lastSha: String, lastStss: List[CommitStatus]) = { 479 | val failingCommits = earlierStati.filterNot(_.success) 480 | val worst = if (failingCommits.exists(_.failure)) FAILURE else if (failingCommits.isEmpty) SUCCESS else PENDING 481 | 482 | if (worst == SUCCESS) postLast(lastSha, "All previous commits successful.", worst, lastStss).map(List(_)) 483 | else for { 484 | last <- postLast(lastSha, s"Found earlier commit(s) marked $worst: ${failingCommits.map(_.sha.take(6)).mkString(", ")}", worst, lastStss) 485 | earlier <- Future.sequence(outdated(earlierStati) map nothingToSee) // we know this status is not there 486 | } yield last :: earlier 487 | } 488 | 489 | if (lastOnly(pull.title)) Future.successful(Nil) 490 | else (for { 491 | commits <- pullRequestCommits 492 | 493 | lastSha = commits.lastOption.map(_.sha).getOrElse("") 494 | if commits.nonEmpty && commits.tail.nonEmpty && causeSha != lastSha // ignore if caused by an update to the last commit 495 | 496 | earlierStati <- Future.sequence(commits.init.map(c => githubApi.commitStatus(c.sha))) 497 | 498 | lastStss <- githubApi.commitStatus(lastSha).map(_.statuses) 499 | 500 | posting <- combine(earlierStati, lastSha, lastStss) 501 | 502 | } yield posting).recover { case _: NoSuchElementException => Nil } 503 | } 504 | 505 | 506 | // MILESTONE 507 | 508 | // if there's a milestone with description "Merge to ${pull.base.ref}.", set it as the PR's milestone 509 | private def checkMilestone(pull: PullRequest) = 510 | milestoneForBranch(pull.base.ref) foreach { milestone => 511 | for { 512 | issue <- githubApi.issue(pr) 513 | if issue.milestone.isEmpty 514 | } yield { 515 | log.info(s"Setting milestone to ${milestone.title}") 516 | githubApi.setMilestone(pr, milestone.number) 517 | } 518 | } 519 | 520 | // commands 521 | private def execCommands(pullRequest: PullRequest) = for { 522 | comments <- issueComments 523 | commentResults <- Future.sequence(comments.map(handleComment)) 524 | } yield commentResults 525 | 526 | private object seenComments { 527 | private val ddc = new DynoDbClient 528 | private val _table = { 529 | val t = new ddc.DynoDbTable("scabot-seen-commands") 530 | if (!t.exists) t.create(List(("Id", KeyType.HASH)), List(("Id", "N"))) 531 | t 532 | } 533 | 534 | def apply(id: Long): Future[Boolean] = 535 | _table.get(new PrimaryKey("Id", id)).map(_.nonEmpty) 536 | 537 | def +=(id: Long): Future[String] = 538 | _table.put((new Item).withPrimaryKey("Id", id)).map(_.getPutItemResult.toString) 539 | } 540 | 541 | private def handleComment(comment: IssueComment): Future[Any] = { 542 | import Commands._ 543 | 544 | if (isReviewRequest(comment)) { 545 | val REVIEW_BY(reviewer) = comment.body.toLowerCase 546 | log.info(s"Review requested from $reviewer") 547 | requestReview(reviewer) 548 | } 549 | else if (!hasCommand(comment.body)) { 550 | log.debug(s"No command in $comment") 551 | Future.successful("No Command found") 552 | } else { 553 | (for { 554 | id <- Future { 555 | comment.id.get 556 | } // the get will fail the future if somehow id is empty 557 | seen <- seenComments(id) 558 | if !seen 559 | _ <- seenComments += id 560 | res <- { 561 | log.debug(s"Executing command for ${comment.body}") 562 | comment.body match { 563 | case REBUILD_SHA(sha) => rebuildSha(sha) 564 | case REBUILD_ALL() => rebuildAll() 565 | case SYNCH() => synch() 566 | case NOTHINGTOSEEHERE() => nothingToSeeHere() 567 | } 568 | } 569 | } yield res).recover { 570 | case _: NoSuchElementException => 571 | val msg = s"Already handled $comment" 572 | log.info(msg) 573 | msg 574 | case _: MatchError => 575 | val msg = s"Unknown command in $comment" 576 | log.warning(msg) 577 | msg 578 | } 579 | } 580 | } 581 | 582 | object Commands { 583 | 584 | final val REVIEW_BY = """^review (?:by )?@(\w+)""".r.unanchored 585 | def isReviewRequest(comment: IssueComment): Boolean = comment.body.toLowerCase match { 586 | case REVIEW_BY(_) => true 587 | case _ => false 588 | } 589 | def requestReview(user: String) = githubApi.requestReview(pr, Reviewers(List(user))) 590 | 591 | def hasCommand(body: String) = body.startsWith("/") 592 | 593 | final val REBUILD_SHA = """^/rebuild (\w+)""".r.unanchored 594 | def rebuildSha(sha: String) = for { 595 | commits <- pullRequestCommits 596 | baseRef <- pullBranch 597 | build <- launchBuild(sha, baseRef, jobParams(sha, sha == commits.last.sha))() // safe to assume commits of a pr is nonEmpty, so commits.last won't bomb 598 | } yield build 599 | 600 | final val REBUILD_ALL = """^/rebuild""".r.unanchored 601 | def rebuildAll() = 602 | for { 603 | pull <- pull 604 | baseRef <- pullBranch 605 | buildRes <- buildCommitsIfNeeded(baseRef, forceRebuild = true, lastOnly = lastOnly(pull.title)) 606 | } yield buildRes 607 | 608 | final val SYNCH = """^/sync""".r.unanchored 609 | def synch() = 610 | for { 611 | pull <- pull 612 | synchRes <- handlePR("synchronize", pull, synchOnly = true) 613 | } yield synchRes 614 | 615 | final val NOTHINGTOSEEHERE = """^/nothingtoseehere""".r.unanchored 616 | def nothingToSeeHere() = 617 | for { 618 | commits <- pullRequestCommits 619 | results <- Future.sequence(commits map { commit => 620 | for { 621 | combiCs <- fetchCommitStatus(commit.sha) 622 | res <- overrideFailures(combiCs) 623 | } yield res }) 624 | } yield results 625 | 626 | } 627 | } 628 | } 629 | -------------------------------------------------------------------------------- /tmp/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scala/scabot/df337b1031ce07fb68c0808932c2eb95d0086042/tmp/.empty --------------------------------------------------------------------------------