├── AUTHORS ├── Dockerfile ├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── handle_webhook.go ├── internal ├── logging │ └── logging.go └── pubsub │ └── pubsub.go ├── main.go └── server.go /AUTHORS: -------------------------------------------------------------------------------- 1 | Seth Vargo 2 | Google, LLC 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2021 the GitHub Workflow Job to PubSub authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.20 AS builder 16 | 17 | RUN apt-get -qq update && apt-get -yqq install upx 18 | 19 | ENV GO111MODULE=on \ 20 | CGO_ENABLED=0 \ 21 | GOOS=linux \ 22 | GOARCH=amd64 23 | 24 | WORKDIR /src 25 | COPY . . 26 | 27 | RUN go build \ 28 | -a \ 29 | -trimpath \ 30 | -ldflags "-s -w -extldflags '-static'" \ 31 | -tags "osusergo,netgo,static,static_build" \ 32 | -o /bin/app \ 33 | . 34 | 35 | RUN strip -s /bin/app 36 | RUN upx -q -9 /bin/app 37 | 38 | RUN echo "nobody:*:65534:65534:nobody:/:/bin/false" > /tmp/etc-passwd 39 | 40 | 41 | FROM scratch 42 | 43 | COPY --from=builder /tmp/etc-passwd /etc/passwd 44 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 45 | COPY --from=builder /bin/app /bin/app 46 | USER nobody 47 | 48 | ENTRYPOINT ["/bin/app"] 49 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GitHub Workflow Job to Pub/Sub 2 | 3 | The GitHub Workflow Job to Pub/Sub is a small service that fulfills a GitHub 4 | [`workflow_job` webhook][workflow-job-webhook]. 5 | 6 | - When a job is queued, it inserts one message onto a Pub/Sub topic. 7 | 8 | - When a job is finished, it acknowledges one message from a Pub/Sub 9 | subscription. 10 | 11 | This means that, at any time, the number of unacknowledged messages in the 12 | Pub/Sub topic corresponds to the number of queued or active GitHub Actions jobs. 13 | You can use this property as an indicator in an auto-scaling metric or a means 14 | to queue new ephemeral workers. 15 | 16 | **This is not an officially supported Google product, and it is not covered by a 17 | Google Cloud support contract. To report bugs or request features in a Google 18 | Cloud product, please contact [Google Cloud 19 | support](https://cloud.google.com/support).** 20 | 21 | 22 | ## Deployment 23 | 24 | This deployment example uses Google Cloud and [Cloud Run][cloud-run] to deploy 25 | and manage the proxy. 26 | 27 | 1. Create or use an existing Google Cloud project: 28 | 29 | ```sh 30 | export PROJECT_ID="..." 31 | ``` 32 | 33 | 1. Enable required APIs: 34 | 35 | ```sh 36 | gcloud services enable --project="${PROJECT_ID}" \ 37 | artifactregistry.googleapis.com \ 38 | cloudbuild.googleapis.com \ 39 | pubsub.googleapis.com \ 40 | run.googleapis.com \ 41 | secretmanager.googleapis.com 42 | ``` 43 | 44 | 1. Create a service account to run the receiver: 45 | 46 | ```sh 47 | gcloud iam service-accounts create "gh-webhook-receiver" \ 48 | --description="GitHub webhook receiver" \ 49 | --project="${PROJECT_ID}" 50 | ``` 51 | 52 | 1. Create a GitHub Webhook secret and store it in Google Secret Manager. This 53 | secret can be any value. 54 | 55 | ```sh 56 | echo -n "" | gcloud secrets create "gh-webhook-secret" \ 57 | --project="${PROJECT_ID}" \ 58 | --data-file=- 59 | ``` 60 | 61 | If you do not have a secret, you can randomly generate a secret using 62 | `openssl`: 63 | 64 | ```sh 65 | openssl rand -base64 32 66 | ``` 67 | 68 | 1. Grant the service account permissions to access the secret: 69 | 70 | ```sh 71 | gcloud secrets add-iam-policy-binding "gh-webhook-secret" \ 72 | --project="${PROJECT_ID}" \ 73 | --role="roles/secretmanager.secretAccessor" \ 74 | --member="serviceAccount:gh-webhook-receiver@${PROJECT_ID}.iam.gserviceaccount.com" 75 | ``` 76 | 77 | 1. Create a Pub/Sub topic: 78 | 79 | ```sh 80 | gcloud pubsub topics create "gh-topic" \ 81 | --project="${PROJECT_ID}" 82 | ``` 83 | 84 | 1. Grant the service account permissions to publish to the topic: 85 | 86 | ```sh 87 | gcloud pubsub topics add-iam-policy-binding "gh-topic" \ 88 | --project="${PROJECT_ID}" \ 89 | --role="roles/pubsub.publisher" \ 90 | --member="serviceAccount:gh-webhook-receiver@${PROJECT_ID}.iam.gserviceaccount.com" 91 | ``` 92 | 93 | 1. Create a Pub/Sub subscription: 94 | 95 | ```sh 96 | gcloud pubsub subscriptions create "gh-subscription" \ 97 | --project="${PROJECT_ID}" \ 98 | --topic="gh-topic" \ 99 | --ack-deadline="10" \ 100 | --message-retention-duration="1800s" \ 101 | --expiration-period="never" \ 102 | --min-retry-delay="5s" \ 103 | --max-retry-delay="30s" 104 | ``` 105 | 106 | 1. Grant the service account permissions to pull from the subscription: 107 | 108 | ```sh 109 | gcloud pubsub subscriptions add-iam-policy-binding "gh-subscription" \ 110 | --project="${PROJECT_ID}" \ 111 | --role="roles/pubsub.subscriber" \ 112 | --member="serviceAccount:gh-webhook-receiver@${PROJECT_ID}.iam.gserviceaccount.com" 113 | ``` 114 | 115 | 1. Create a repository in Artifact Registry to store the container: 116 | 117 | ```sh 118 | gcloud artifacts repositories create "gh-webhook-receiver" \ 119 | --project="${PROJECT_ID}" \ 120 | --repository-format="docker" \ 121 | --location="us" \ 122 | --description="GitHub webhook receiver" 123 | ``` 124 | 125 | 1. Build and push the container: 126 | 127 | ```sh 128 | gcloud builds submit . \ 129 | --project="${PROJECT_ID}" \ 130 | --tag="us-docker.pkg.dev/${PROJECT_ID}/gh-webhook-receiver/gh-webhook-receiver" 131 | ``` 132 | 133 | 1. Deploy the service and attach the secret (see 134 | [Configuration](#configuration) for more information on available options): 135 | 136 | ```sh 137 | gcloud beta run deploy "gh-webhook-receiver" \ 138 | --quiet \ 139 | --project="${PROJECT_ID}" \ 140 | --region="us-east1" \ 141 | --set-secrets="GITHUB_WEBHOOK_SECRET=gh-webhook-secret:1" \ 142 | --set-env-vars="PUBSUB_TOPIC_NAME=projects/${PROJECT_ID}/topics/gh-topic,PUBSUB_SUBSCRIPTION_NAME=projects/${PROJECT_ID}/subscriptions/gh-subscription" \ 143 | --image="us-docker.pkg.dev/${PROJECT_ID}/gh-webhook-receiver/gh-webhook-receiver" \ 144 | --service-account="gh-webhook-receiver@${PROJECT_ID}.iam.gserviceaccount.com" \ 145 | --allow-unauthenticated 146 | ``` 147 | 148 | Take note of the URL. It is important to note that this is a 149 | **publicly-accessible URL**. 150 | 151 | 1. Create an organization webhook on GitHub: 152 | 153 | - **Payload URL:** URL for the Cloud Run service above. 154 | - **Content type:** application/json 155 | - **Secret:** value from above 156 | - **Events:** select "individual events" and then choose **only** "Workflow jobs" 157 | 158 | 159 | ## Configuration 160 | 161 | - `GITHUB_WEBHOOK_SECRET` - this is the secret key to use for authenticating 162 | the webhook's HMAC. This must match the value given to GitHub when 163 | configuring the webhook. It is very important that you choose a high-entropy 164 | secret, because your service must be publicly accessible. 165 | 166 | - `PUBSUB_TOPIC_NAME` - this is the name of the topic on which to publish. 167 | This must be the full topic name including the project (e.g. 168 | `projects/my-project/topics/my-topic`). 169 | 170 | - `PUBSUB_SUBSCRIPTION_NAME` - this is the name of the subscription on which 171 | to pull and acknowledge. This must be the full subscription name including 172 | the project (e.g. `projects/my-project/subscriptions/my-subscription`). 173 | 174 | 175 | ## FAQ 176 | 177 | **Q: Why is Pub/Sub necessary? Why not just have the service create ephemeral runners or scale a VM pool directly?** 178 |
179 | A: GitHub has a [timeout of 10 seconds][webhook-timeout] for webhook responses, 180 | and strongly recommends asynchronous processing. Modifying an autoscaling group 181 | or spinning up a new virtual machine will almost always exceed this timeout. 182 | 183 | [workflow-job-webhook]: https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job 184 | [webhook-timeout]: https://docs.github.com/en/rest/guides/best-practices-for-integrators#favor-asynchronous-work-over-synchronous 185 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | // Copyright 2021 the GitHub Workflow Job to PubSub authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | module github.com/sethvargo/github-workflow-job-to-pubsub 16 | 17 | go 1.20 18 | 19 | require golang.org/x/oauth2 v0.5.0 20 | 21 | require ( 22 | cloud.google.com/go/compute v1.18.0 // indirect 23 | cloud.google.com/go/compute/metadata v0.2.3 // indirect 24 | github.com/golang/protobuf v1.5.2 // indirect 25 | golang.org/x/net v0.33.0 // indirect 26 | google.golang.org/appengine v1.6.7 // indirect 27 | google.golang.org/protobuf v1.33.0 // indirect 28 | ) 29 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= 2 | cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= 3 | cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= 4 | cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= 5 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 6 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 7 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 8 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 9 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 10 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 11 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 12 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 13 | golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= 14 | golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= 15 | golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= 16 | golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= 17 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 18 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 19 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 20 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 21 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 22 | google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= 23 | google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 24 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 25 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 26 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= 27 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 28 | -------------------------------------------------------------------------------- /handle_webhook.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 the GitHub Workflow Job to PubSub authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "fmt" 21 | "io" 22 | "net/http" 23 | "time" 24 | 25 | "github.com/sethvargo/github-workflow-job-to-pubsub/internal/pubsub" 26 | ) 27 | 28 | type WorkflowJobEvent struct { 29 | Action string `json:"action"` 30 | 31 | WorkflowJob struct { 32 | RunID string `json:"run_id"` 33 | RunURL string `json:"run_url"` 34 | } 35 | 36 | Repository struct { 37 | FullName string `json:"full_name"` 38 | } `json:"repository"` 39 | } 40 | 41 | func (s *Server) handleWebhook() http.Handler { 42 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 43 | ctx := r.Context() 44 | 45 | defer r.Body.Close() 46 | 47 | if !isWorkflowJobEvent(r) { 48 | respondJSON(w, &jsonResponse{ 49 | Error: "invalid event type", 50 | }, http.StatusBadRequest) 51 | return 52 | } 53 | 54 | limitedBody := io.LimitReader(r.Body, 12*1024*1024) // 12 MiB 55 | body, err := io.ReadAll(limitedBody) 56 | if err != nil { 57 | logger.Error("failed to read body", "error", err) 58 | respondJSON(w, &jsonResponse{ 59 | Error: fmt.Sprintf("failed to read body: %s", err), 60 | }, http.StatusBadRequest) 61 | return 62 | } 63 | 64 | givenSignature := r.Header.Get(GitHubSignatureHeaderName) 65 | if !isValidSignature(givenSignature, body) { 66 | respondJSON(w, &jsonResponse{ 67 | Error: "invalid signature", 68 | }, http.StatusBadRequest) 69 | return 70 | } 71 | 72 | // If we got this far, it's safe to try and decode the payload. 73 | var event WorkflowJobEvent 74 | if err := json.Unmarshal(body, &event); err != nil { 75 | logger.Error("failed to unmarshal event", "error", err) 76 | respondJSON(w, &jsonResponse{ 77 | Error: fmt.Sprintf("failed to unmarshal json: %s", err), 78 | }, http.StatusInternalServerError) 79 | return 80 | } 81 | 82 | switch event.Action { 83 | case StatusQueued: 84 | // Increase the size of the pool by adding a message to the queue. 85 | if err := s.pubsubClient.PublishOne(ctx, PubSubTopicName, &pubsub.Message{ 86 | Attributes: map[string]string{ 87 | "run_id": event.WorkflowJob.RunID, 88 | "run_url": event.WorkflowJob.RunURL, 89 | }, 90 | }); err != nil { 91 | logger.Error("failed to publish", "error", err) 92 | respondJSON(w, &jsonResponse{ 93 | Error: fmt.Sprintf("failed to publish message: %s", err), 94 | }, http.StatusInternalServerError) 95 | } 96 | 97 | case StatusInProgress: 98 | // Do nothing, we already queued a worker for the queued job. 99 | respondJSON(w, &jsonResponse{ 100 | Message: "ok", 101 | }, http.StatusOK) 102 | 103 | case StatusCompleted: 104 | // Remove an item from the pool. 105 | ctx, done := context.WithTimeout(context.Background(), 7*time.Second) 106 | defer done() 107 | 108 | if _, err := s.pubsubClient.PullAndAck(ctx, PubSubSubscriptionName); err != nil { 109 | logger.Error("failed to pull and ack", "error", err) 110 | respondJSON(w, &jsonResponse{ 111 | Error: fmt.Sprintf("failed to pull and ack: %s", err), 112 | }, http.StatusInternalServerError) 113 | } 114 | default: 115 | logger.Error("unknown event action", "action", event.Action) 116 | } 117 | }) 118 | } 119 | 120 | // isWorkflowJobEvent verifies the request is of type "workflow_job", which 121 | // indicates it's safe to attempt to unmarshal into the WorkflowJob struct. 122 | func isWorkflowJobEvent(r *http.Request) bool { 123 | return r.Header.Get(GitHubEventHeaderName) == GitHubEventWorkflowJob 124 | } 125 | -------------------------------------------------------------------------------- /internal/logging/logging.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 the GitHub Workflow Job to PubSub authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logging 16 | 17 | import ( 18 | "encoding/json" 19 | "fmt" 20 | "io" 21 | "os" 22 | "time" 23 | ) 24 | 25 | type Severity string 26 | 27 | const ( 28 | SeverityDebug Severity = "DEBUG" 29 | SeverityInfo Severity = "INFO" 30 | SeverityWarn Severity = "WARNING" 31 | SeverityError Severity = "ERROR" 32 | SeverityFatal Severity = "EMERGENCY" 33 | ) 34 | 35 | type Logger struct { 36 | stdout io.Writer 37 | stderr io.Writer 38 | } 39 | 40 | func NewLogger(outw, errw io.Writer) *Logger { 41 | return &Logger{stdout: outw, stderr: errw} 42 | } 43 | 44 | func (l *Logger) Debug(msg string, fields ...interface{}) { 45 | l.log(l.stdout, msg, SeverityDebug, fields...) 46 | } 47 | 48 | func (l *Logger) Info(msg string, fields ...interface{}) { 49 | l.log(l.stdout, msg, SeverityInfo, fields...) 50 | } 51 | 52 | func (l *Logger) Warn(msg string, fields ...interface{}) { 53 | l.log(l.stdout, msg, SeverityWarn, fields...) 54 | } 55 | 56 | func (l *Logger) Error(msg string, fields ...interface{}) { 57 | l.log(l.stderr, msg, SeverityError, fields...) 58 | } 59 | 60 | func (l *Logger) Fatal(msg string, fields ...interface{}) { 61 | l.log(l.stderr, msg, SeverityFatal, fields...) 62 | os.Exit(1) 63 | } 64 | 65 | func (l *Logger) log(w io.Writer, msg string, sev Severity, fields ...interface{}) { 66 | if len(fields)%2 != 0 { 67 | panic("number of fields must be even") 68 | } 69 | 70 | data := make(map[string]interface{}, len(fields)/2) 71 | for i := 0; i < len(fields); i += 2 { 72 | key, ok := fields[i].(string) 73 | if !ok { 74 | panic(fmt.Errorf("field %d is not a string (%T, %q)", i, fields[i], fields[i])) 75 | } 76 | 77 | switch typ := fields[i+1].(type) { 78 | case error: 79 | data[key] = typ.Error() 80 | default: 81 | data[key] = typ 82 | } 83 | } 84 | 85 | jsonPayload, err := json.Marshal(&LogEntry{ 86 | Time: timePtr(time.Now().UTC()), 87 | Severity: sev, 88 | Message: msg, 89 | Data: data, 90 | }) 91 | if err != nil { 92 | panic(fmt.Errorf("failed to marshal log entry: %w", err)) 93 | } 94 | 95 | fmt.Fprintln(w, string(jsonPayload)) 96 | } 97 | 98 | type LogEntry struct { 99 | Time *time.Time 100 | Severity Severity 101 | Message string 102 | Data map[string]interface{} 103 | } 104 | 105 | func (l *LogEntry) MarshalJSON() ([]byte, error) { 106 | d := make(map[string]interface{}, 8) 107 | 108 | if l.Time != nil { 109 | d["time"] = l.Time.Format(time.RFC3339) 110 | } 111 | 112 | d["severity"] = string(l.Severity) 113 | d["message"] = l.Message 114 | 115 | for k, v := range l.Data { 116 | d[k] = v 117 | } 118 | 119 | return json.Marshal(d) 120 | } 121 | 122 | func timePtr(t time.Time) *time.Time { 123 | if t.IsZero() { 124 | return nil 125 | } 126 | return &t 127 | } 128 | -------------------------------------------------------------------------------- /internal/pubsub/pubsub.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 the GitHub Workflow Job to PubSub authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package pubsub 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "encoding/json" 21 | "fmt" 22 | "io" 23 | "io/ioutil" 24 | "net/http" 25 | "strings" 26 | "time" 27 | 28 | "golang.org/x/oauth2/google" 29 | ) 30 | 31 | // Message represents a message on the PubSub topic/subscription. 32 | type Message struct { 33 | // Data is the message data. 34 | Data []byte `json:"data"` 35 | 36 | // Attributes are any message attributes. 37 | Attributes map[string]string `json:"attributes"` 38 | 39 | // MessageID is assigned by the server. 40 | MessageID string `json:"messageId"` 41 | 42 | // PublishTime is the server timestamp for when the message was published. 43 | PublishTime *time.Time `json:"publishTime"` 44 | } 45 | 46 | type Client struct { 47 | httpClient *http.Client 48 | } 49 | 50 | // NewClient creates a new authenticated http client using Google's 51 | // Application Default Credentials. 52 | func NewClient() (*Client, error) { 53 | httpClient, err := google.DefaultClient(context.Background(), "https://www.googleapis.com/auth/cloud-platform") 54 | if err != nil { 55 | return nil, fmt.Errorf("failed to set up credentials: %w", err) 56 | } 57 | 58 | return &Client{ 59 | httpClient: httpClient, 60 | }, nil 61 | } 62 | 63 | type Messages struct { 64 | Messages []*Message `json:"messages"` 65 | } 66 | 67 | // Publish pushes new messages onto the provided topic, returning any errors 68 | // that occur. 69 | func (c *Client) Publish(ctx context.Context, topic string, messages []*Message) error { 70 | pth := fmt.Sprintf("https://pubsub.googleapis.com/v1/%s:publish", topic) 71 | 72 | var body bytes.Buffer 73 | if err := json.NewEncoder(&body).Encode(&Messages{ 74 | Messages: messages, 75 | }); err != nil { 76 | return fmt.Errorf("failed to encode message: %w", err) 77 | } 78 | 79 | req, err := http.NewRequestWithContext(ctx, "POST", pth, &body) 80 | if err != nil { 81 | return fmt.Errorf("failed to build publish request: %w", err) 82 | } 83 | req.Header.Set("Accept", "application/json") 84 | req.Header.Set("Content-Type", "application/json") 85 | 86 | resp, err := c.httpClient.Do(req) 87 | if err != nil { 88 | return fmt.Errorf("failed to make publish request: %w", err) 89 | } 90 | defer resp.Body.Close() 91 | 92 | if resp.StatusCode != 200 { 93 | respBody, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 64*1024)) 94 | return fmt.Errorf("bad response from server on publish (%d): %s", resp.StatusCode, respBody) 95 | } 96 | return nil 97 | } 98 | 99 | // PublishOne pushes a new message onto the provided topic, returning any errors 100 | // that occur. 101 | func (c *Client) PublishOne(ctx context.Context, topic string, message *Message) error { 102 | return c.Publish(ctx, topic, []*Message{message}) 103 | } 104 | 105 | // PullAndAck pulls the message and immediately acks it. 106 | func (c *Client) PullAndAck(ctx context.Context, subscription string) (*Message, error) { 107 | message, err := c.Pull(ctx, subscription) 108 | if err != nil { 109 | return nil, err 110 | } 111 | if message == nil { 112 | return nil, nil 113 | } 114 | 115 | if err := c.Ack(ctx, subscription, message.AckID); err != nil { 116 | return nil, err 117 | } 118 | 119 | return message.Message, err 120 | } 121 | 122 | type ReceivedMessages struct { 123 | ReceivedMessages []*ReceivedMessage `json:"receivedMessages"` 124 | } 125 | 126 | // ReceivedMessage is a message received back from PubSub. 127 | type ReceivedMessage struct { 128 | AckID string `json:"ackId"` 129 | Message *Message `json:"message"` 130 | } 131 | 132 | // Pull gets a message of the given subscription. Use a timeout on the context 133 | // to cancel the pull. The returned message will be nil if there's nothing on 134 | // the subscription. 135 | func (c *Client) Pull(ctx context.Context, subscription string) (*ReceivedMessage, error) { 136 | pth := fmt.Sprintf("https://pubsub.googleapis.com/v1/%s:pull", subscription) 137 | 138 | body := strings.NewReader(`{"maxMessages":1}`) 139 | req, err := http.NewRequestWithContext(ctx, "POST", pth, body) 140 | if err != nil { 141 | return nil, fmt.Errorf("failed to build pull request: %w", err) 142 | } 143 | req.Header.Set("Accept", "application/json") 144 | req.Header.Set("Content-Type", "application/json") 145 | 146 | resp, err := c.httpClient.Do(req) 147 | if err != nil { 148 | return nil, fmt.Errorf("failed to make pull request: %w", err) 149 | } 150 | defer resp.Body.Close() 151 | 152 | if resp.StatusCode != 200 { 153 | respBody, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 64*1024)) 154 | return nil, fmt.Errorf("bad response from server on pull (%d): %s", resp.StatusCode, respBody) 155 | } 156 | 157 | var messages ReceivedMessages 158 | if err := json.NewDecoder(resp.Body).Decode(&messages); err != nil { 159 | return nil, fmt.Errorf("failed to unmarshal pull esponse: %w", err) 160 | } 161 | 162 | if len(messages.ReceivedMessages) == 0 { 163 | return nil, nil 164 | } 165 | 166 | return messages.ReceivedMessages[0], nil 167 | } 168 | 169 | type AckMessage struct { 170 | AckIDs []string `json:"ackIds"` 171 | } 172 | 173 | func (c *Client) Ack(ctx context.Context, subscription, ackID string) error { 174 | pth := fmt.Sprintf("https://pubsub.googleapis.com/v1/%s:acknowledge", subscription) 175 | 176 | message := &AckMessage{ 177 | AckIDs: []string{ackID}, 178 | } 179 | var body bytes.Buffer 180 | if err := json.NewEncoder(&body).Encode(message); err != nil { 181 | return fmt.Errorf("failed to encode ack message: %w", err) 182 | } 183 | 184 | req, err := http.NewRequestWithContext(ctx, "POST", pth, &body) 185 | if err != nil { 186 | return fmt.Errorf("failed to build ack request: %w", err) 187 | } 188 | req.Header.Set("Accept", "application/json") 189 | req.Header.Set("Content-Type", "application/json") 190 | 191 | resp, err := c.httpClient.Do(req) 192 | if err != nil { 193 | return fmt.Errorf("failed to make ack request: %w", err) 194 | } 195 | defer resp.Body.Close() 196 | 197 | if resp.StatusCode != 200 { 198 | respBody, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 64*1024)) 199 | return fmt.Errorf("bad response from server on ack (%d): %s", resp.StatusCode, respBody) 200 | } 201 | return nil 202 | } 203 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 the GitHub Workflow Job to PubSub authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "net/http" 21 | "os" 22 | "os/signal" 23 | "syscall" 24 | "time" 25 | 26 | "github.com/sethvargo/github-workflow-job-to-pubsub/internal/logging" 27 | "github.com/sethvargo/github-workflow-job-to-pubsub/internal/pubsub" 28 | ) 29 | 30 | const ( 31 | // GitHubSignatureHeaderName is the header to search for the GitHub webhook 32 | // payload signature. GitHubSignaturePrefix is the message prefix on the 33 | // signature. 34 | GitHubSignatureHeaderName = "x-hub-signature-256" 35 | GitHubSignaturePrefix = "sha256=" 36 | 37 | // GitHubEventHeaderName is the header for the event name. 38 | // GitHubEventWorkflowJob is the event name for job queueing. 39 | GitHubEventHeaderName = "x-github-event" 40 | GitHubEventWorkflowJob = "workflow_job" 41 | 42 | // Status event types. 43 | StatusQueued = "queued" 44 | StatusInProgress = "in_progress" 45 | StatusCompleted = "completed" 46 | ) 47 | 48 | var ( 49 | // GitHubWebhookSecret is the secret key to use for authenticating the 50 | // webhook's HMAC. This should be injected via Secret Manager or a similar 51 | // process. 52 | GitHubWebhookSecret = os.Getenv("GITHUB_WEBHOOK_SECRET") 53 | 54 | // PubSubTopicName is the name of the topic on which to publish. 55 | // PubSubSubscriptionName is the name of the subscription on which to pull. 56 | // These should be the full topic and subscription including the project (e.g. 57 | // "projects/p/topics/t"). 58 | PubSubTopicName = os.Getenv("PUBSUB_TOPIC_NAME") 59 | PubSubSubscriptionName = os.Getenv("PUBSUB_SUBSCRIPTION_NAME") 60 | 61 | // logger is the logging system. 62 | logger = logging.NewLogger(os.Stdout, os.Stderr) 63 | ) 64 | 65 | func main() { 66 | ctx, done := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) 67 | defer done() 68 | 69 | err := realMain(ctx) 70 | done() 71 | 72 | if err != nil { 73 | logger.Fatal("application failure", "error", err) 74 | } 75 | logger.Info("shutting down") 76 | } 77 | 78 | func realMain(ctx context.Context) error { 79 | pubsubClient, err := pubsub.NewClient() 80 | if err != nil { 81 | return fmt.Errorf("failed to create pubsub client: %w", err) 82 | } 83 | 84 | s := &Server{ 85 | pubsubClient: pubsubClient, 86 | } 87 | 88 | mux := http.NewServeMux() 89 | mux.Handle("/", s.handleWebhook()) 90 | 91 | port := os.Getenv("PORT") 92 | if port == "" { 93 | port = "8080" 94 | } 95 | 96 | srv := &http.Server{ 97 | Addr: ":" + port, 98 | Handler: mux, 99 | } 100 | 101 | go func() { 102 | if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { 103 | logger.Error("failed to listen and serve", "error", err) 104 | } 105 | }() 106 | <-ctx.Done() 107 | 108 | shutdownCtx, done := context.WithTimeout(context.Background(), 5*time.Second) 109 | defer done() 110 | 111 | if err := srv.Shutdown(shutdownCtx); err != nil { 112 | return fmt.Errorf("failed to shutdown server: %w", err) 113 | } 114 | 115 | return nil 116 | } 117 | -------------------------------------------------------------------------------- /server.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 the GitHub Workflow Job to PubSub authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "crypto/hmac" 19 | "crypto/sha256" 20 | "crypto/subtle" 21 | "encoding/hex" 22 | "encoding/json" 23 | "fmt" 24 | "net/http" 25 | 26 | "github.com/sethvargo/github-workflow-job-to-pubsub/internal/pubsub" 27 | ) 28 | 29 | type jsonResponse struct { 30 | Message string `json:"message,omitempty"` 31 | Error string `json:"error,omitempty"` 32 | } 33 | 34 | type Server struct { 35 | pubsubClient *pubsub.Client 36 | } 37 | 38 | func respondJSON(w http.ResponseWriter, i *jsonResponse, status int) { 39 | w.Header().Set("Content-Type", "application/json") 40 | w.WriteHeader(status) 41 | 42 | if i != nil { 43 | b, err := json.Marshal(i) 44 | if err != nil { 45 | panic(err) 46 | } 47 | fmt.Fprint(w, string(b)) 48 | } 49 | } 50 | 51 | // isValidSignature determines if the provided signature matches the expected 52 | // signature. 53 | func isValidSignature(want string, body []byte) bool { 54 | h := hmac.New(sha256.New, []byte(GitHubWebhookSecret)) 55 | h.Write(body) 56 | got := GitHubSignaturePrefix + hex.EncodeToString(h.Sum(nil)) 57 | 58 | return subtle.ConstantTimeCompare([]byte(want), []byte(got)) == 1 59 | } 60 | --------------------------------------------------------------------------------