├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── idempotency ├── .gcloudignore ├── README.md ├── index.js └── package.json ├── order-processing ├── .gcloudignore ├── README.md ├── index.js └── package.json └── retries ├── .gcloudignore ├── README.md ├── caller.js ├── index.js └── package.json /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows 28 | [Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cloud Functions Reliability Samples (Node.js) 2 | 3 | This repository contains code samples that demonstrate how to improve the 4 | reliability of your Cloud Functions: 5 | 6 | * [Retries](retries) - retrying Cloud Functions on failure. 7 | * [Idempotency](idempotency) - making Cloud Functions idempotent. 8 | * [Order processing](order-processing) - a sample restaurant order processing 9 | system built on Cloud Functions, which makes use of retries and idempotency. 10 | -------------------------------------------------------------------------------- /idempotency/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | -------------------------------------------------------------------------------- /idempotency/README.md: -------------------------------------------------------------------------------- 1 | # Idempotency 2 | 3 | This is sample code which demonstrates how to make your Cloud Functions 4 | idempotent. 5 | 6 | ## Instructions 7 | 8 | Create a Cloud Firestore project by following 9 | [these instructions](https://firebase.google.com/docs/firestore/quickstart#create_a_project) 10 | or use a project in which you have already used Cloud Firestore. Set this 11 | project in Cloud SDK: 12 | 13 | ``` 14 | gcloud config set project [PROJECT_ID] 15 | ``` 16 | 17 | Deploy the `flaky` function, which simulates a flaky service on which your other 18 | functions depend: 19 | 20 | ``` 21 | gcloud functions deploy flaky --trigger-http 22 | ``` 23 | 24 | (Note: in this sample code, `package.json` declares dependencies for all 25 | functions from `index.js`. In production scenarios, we recommend isolating 26 | dependencies of your functions to avoid installing unnecessary modules on 27 | function deployment). 28 | 29 | Then, deploy the `nonIdempotentFirestoreFunction`, which is triggered by Cloud 30 | Pub/Sub messages and adds a document to Cloud Firestore before calling the 31 | `flaky` function. Replace `[TOPIC_FOR_FIRESTORE_1]` with the name of the Pub/Sub 32 | topic you want to use and add the `--retry` option to have the function retried 33 | on failure: 34 | 35 | ``` 36 | gcloud functions deploy nonIdempotentFirestoreFunction --trigger-topic [TOPIC_FOR_FIRESTORE_1] --retry 37 | ``` 38 | 39 | To invoke the function, publish some messages on the topic you specified: 40 | 41 | ``` 42 | gcloud pubsub topics publish [TOPIC_FOR_FIRESTORE_1] --message "{ \"value\": \"${RANDOM}\"}" 43 | ``` 44 | 45 | If the function fails and is retried, you will observe duplicate documents in 46 | Cloud Firestore. To prevent this situation, deploy the idempotent version of the 47 | function: 48 | 49 | ``` 50 | gcloud functions deploy idempotentFirestoreFunction --trigger-topic [TOPIC_FOR_FIRESTORE_2] --retry 51 | ``` 52 | 53 | And publish some messages to the new topic: 54 | 55 | ``` 56 | gcloud pubsub topics publish [TOPIC_FOR_FIRESTORE_2] --message "{ \"value\": \"${RANDOM}\"}" 57 | ``` 58 | 59 | You shouldn't observe duplicates anymore. 60 | 61 | Now, deploy the `nonIdempotentEmailFunction`, which is also triggered by Cloud 62 | Pub/Sub messages and simulates sending (or actually sends) an email before 63 | calling the `flaky` function. Replace `[TOPIC_FOR_EMAIL_1]` with the name of the 64 | Pub/Sub topic you want to use and add the `--retry` option to have the function 65 | retried on failure. If you want to only simulate sending an email by logging a 66 | message, deploy the code from the repository unchanged. If you want to actually 67 | send an email from the function, follow the comment in function code, and change 68 | the code accordingly before deploying the function. 69 | 70 | ``` 71 | gcloud functions deploy nonIdempotentEmailFunction --trigger-topic [TOPIC_FOR_EMAIL_1] --retry 72 | ``` 73 | 74 | To invoke the function, publish some messages on the topic you specified. The 75 | function expects the `text` field to be present: 76 | 77 | ``` 78 | gcloud pubsub topics publish [TOPIC_FOR_EMAIL_1] --message "{ \"text\": \"${RANDOM}\"}" 79 | ``` 80 | 81 | If the function fails and is retried, you will observe duplicate entries about 82 | sending an email in Stackdriver Logging (or actual duplicate emails sent if you 83 | had configured Sendgrid and adjusted function code). To get rid of the vast 84 | majority of duplicates, deploy `almostIdempotentEmailFunction`: 85 | 86 | ``` 87 | gcloud functions deploy almostIdempotentEmailFunction --trigger-topic [TOPIC_FOR_EMAIL_2] --retry 88 | ``` 89 | 90 | And publish some messages to the new topic: 91 | 92 | ``` 93 | gcloud pubsub topics publish [TOPIC_FOR_EMAIL_2] --message "{ \"text\": \"${RANDOM}\"}" 94 | ``` 95 | 96 | It is very unlikely that you will observe any duplicates but they can still 97 | occur, occasionally. To practically eliminate them, deploy 98 | `idempotentEmailFunction`: 99 | 100 | ``` 101 | gcloud functions deploy idempotentEmailFunction --trigger-topic [TOPIC_FOR_EMAIL_3] --retry 102 | ``` 103 | 104 | And publish some messages to the new topic: 105 | 106 | ``` 107 | gcloud pubsub topics publish [TOPIC_FOR_EMAIL_3] --message "{ \"text\": \"${RANDOM}\"}" 108 | ``` 109 | 110 | You shouldn't observe duplicates now. 111 | -------------------------------------------------------------------------------- /idempotency/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const admin = require('firebase-admin'); 18 | const functions = require('firebase-functions'); 19 | const request = require('request-promise'); 20 | const sgMail = require('@sendgrid/mail'); 21 | 22 | admin.initializeApp(functions.config().firebase); 23 | 24 | const db = admin.firestore(); 25 | 26 | const project = process.env.GCP_PROJECT; 27 | const region = process.env.FUNCTION_REGION; 28 | 29 | /** 30 | * Non-idempotent Pub/Sub-triggered function which adds a document to Cloud 31 | * Firestore before calling a flaky service. 32 | * 33 | * @param {Object} event The Cloud Pub/Sub event. 34 | */ 35 | exports.nonIdempotentFirestoreFunction = (event) => { 36 | const message = event.data; 37 | const content = 38 | JSON.parse(Buffer.from(message.data || '', 'base64').toString() || '{}'); 39 | return db.collection('contents').add(content).then(() => { 40 | return request({ 41 | method: 'POST', 42 | uri: `https://${region}-${project}.cloudfunctions.net/flaky`, 43 | body: content, 44 | json: true 45 | }); 46 | }); 47 | }; 48 | 49 | /** 50 | * Idempotent Pub/Sub-triggered function which creates or overwrites a document 51 | * in Cloud Firestore before calling a flaky service. 52 | * 53 | * @param {Object} event The Cloud Pub/Sub event. 54 | */ 55 | exports.idempotentFirestoreFunction = (event) => { 56 | const message = event.data; 57 | const content = 58 | JSON.parse(Buffer.from(message.data || '', 'base64').toString() || '{}'); 59 | const eventId = event.context.eventId; 60 | return db.collection('contents').doc(eventId).set(content).then(() => { 61 | return request({ 62 | method: 'POST', 63 | uri: `https://${region}-${project}.cloudfunctions.net/flaky/${eventId}`, 64 | body: content, 65 | json: true 66 | }); 67 | }); 68 | }; 69 | 70 | /** 71 | * Non-idempotent Pub/Sub-triggered function which simulates sending (or 72 | * actually sends) an email before calling a flaky service. 73 | * 74 | * @param {Object} event The Cloud Pub/Sub event. 75 | */ 76 | exports.nonIdempotentEmailFunction = (event) => { 77 | const message = event.data; 78 | const content = 79 | JSON.parse(Buffer.from(message.data || '', 'base64').toString() || '{}'); 80 | 81 | console.log(`Sending email with text ${content.text}`); 82 | // To actually send an email, change the sender and recipient addresses, get 83 | // SendGrid API Key from https://app.sendgrid.com/settings/api_keys, use it as 84 | // setApiKey argument, and uncomment the code below. 85 | // const email = { 86 | // to: 'to@example.com', 87 | // from: 'from@example.com', 88 | // subject: 'Email from Cloud Functions', 89 | // text: content.text, 90 | // }; 91 | // sgMail.setApiKey('SENDGRID_API_KEY'); 92 | // sgMail.send(email); 93 | 94 | // Call another service. 95 | return request({ 96 | method: 'POST', 97 | uri: `https://${region}-${project}.cloudfunctions.net/flaky`, 98 | body: content, 99 | json: true 100 | }); 101 | }; 102 | 103 | /** 104 | * Pub/Sub-triggered function which simulates sending (or actually sends) an 105 | * email before calling a flaky service. In this version, duplicate emails are 106 | * very unlikely. 107 | * 108 | * @param {Object} event The Cloud Pub/Sub event. 109 | */ 110 | exports.almostIdempotentEmailFunction = (event) => { 111 | const message = event.data; 112 | const content = 113 | JSON.parse(Buffer.from(message.data || '', 'base64').toString() || '{}'); 114 | const eventId = event.context.eventId; 115 | const emailRef = db.collection('sentEmails').doc(eventId); 116 | 117 | return shouldSend(emailRef) 118 | .then(send => { 119 | if (send) { 120 | console.log(`Sending email with text ${content.text}`); 121 | // To actually send an email, change the sender and recipient 122 | // addresses, get SendGrid API Key from 123 | // https://app.sendgrid.com/settings/api_keys, use it as setApiKey 124 | // argument, and uncomment the code below. 125 | // const email = { 126 | // to: 'to@example.com', 127 | // from: 'from@example.com', 128 | // subject: 'Email from Cloud Functions', 129 | // text: content.text, 130 | // }; 131 | // sgMail.setApiKey('SENDGRID_API_KEY'); 132 | // sgMail.send(email); 133 | return markSent(emailRef); 134 | } 135 | }) 136 | .then(() => { 137 | // Call another service. 138 | return request({ 139 | method: 'POST', 140 | uri: `https://${region}-${project}.cloudfunctions.net` + 141 | `/flaky/${eventId}`, 142 | body: content, 143 | json: true 144 | }); 145 | }); 146 | }; 147 | 148 | /** 149 | * Returns true if the given email has not yet been recorded as sent in Cloud 150 | * Firestore; otherwise, returns false. 151 | * 152 | * @param {!firebase.firestore.DocumentReference} emailRef Cloud Firestore 153 | * reference to the email. 154 | * @returns {boolean} Whether the email should be sent by the current function 155 | * execution. 156 | */ 157 | function shouldSend(emailRef) { 158 | return emailRef.get().then(emailDoc => { 159 | return !emailDoc.exists || !emailDoc.data().sent; 160 | }); 161 | } 162 | 163 | /** 164 | * Records the given email as sent in Cloud Firestore. 165 | * 166 | * @param {!firebase.firestore.DocumentReference} emailRef Cloud Firestore 167 | * reference to the email. 168 | * @returns {!Promise} Promise which indicates that the data has successfully 169 | * been recorded in Cloud Firestore. 170 | */ 171 | function markSent(emailRef) { 172 | return emailRef.set({sent: true}); 173 | } 174 | 175 | /** 176 | * Pub/Sub-triggered function which simulates sending (or actually sends) an 177 | * email before calling a flaky service. In this version, duplicate emails are 178 | * practically eliminated. 179 | * 180 | * @param {Object} event The Cloud Pub/Sub event. 181 | */ 182 | exports.idempotentEmailFunction = (event) => { 183 | const message = event.data; 184 | const content = 185 | JSON.parse(Buffer.from(message.data || '', 'base64').toString() || '{}'); 186 | const eventId = event.context.eventId; 187 | const emailRef = db.collection('sentEmails').doc(eventId); 188 | 189 | return shouldSendWithLease(emailRef) 190 | .then(send => { 191 | if (send) { 192 | console.log(`Sending email with text ${content.text}`); 193 | // To actually send an email, change the sender and recipient 194 | // addresses, get SendGrid API Key from 195 | // https://app.sendgrid.com/settings/api_keys, use it as setApiKey 196 | // argument, and uncomment the code below. 197 | // const email = { 198 | // to: 'to@example.com', 199 | // from: 'from@example.com', 200 | // subject: 'Email from Cloud Functions', 201 | // text: content.text, 202 | // }; 203 | // sgMail.setApiKey('SENDGRID_API_KEY'); 204 | // sgMail.send(email); 205 | return markSent(emailRef); 206 | } 207 | }) 208 | .then(() => { 209 | // Call another service. 210 | return request({ 211 | method: 'POST', 212 | uri: `https://${region}-${project}.cloudfunctions.net` + 213 | `/flaky/${eventId}`, 214 | body: content, 215 | json: true 216 | }); 217 | }); 218 | }; 219 | 220 | const leaseTime = 60 * 1000; // 60s, equals function timeout. 221 | 222 | /** 223 | * Returns true if the given email has not yet been recorded as sent in Cloud 224 | * Firestore and the current execution took the lease; returns a rejected 225 | * Promise if the email has not been recorded as sent but the lease is already 226 | * taken by a concurrent function execution; otherwise, returns false. 227 | * 228 | * @param {!firebase.firestore.DocumentReference} emailRef Cloud Firestore 229 | * reference to the email. 230 | * @returns {boolean|!Promise} Whether the email should be sent by the current 231 | * function execution, or rejected Promise if the lease is already taken. 232 | */ 233 | function shouldSendWithLease(emailRef) { 234 | return db.runTransaction(transaction => { 235 | return transaction.get(emailRef).then(emailDoc => { 236 | if (emailDoc.exists && emailDoc.data().sent) { 237 | return false; 238 | } 239 | if (emailDoc.exists && new Date() < emailDoc.data().lease) { 240 | return Promise.reject('Lease already taken, try later.'); 241 | } 242 | transaction.set( 243 | emailRef, {lease: new Date(new Date().getTime() + leaseTime)}); 244 | return true; 245 | }); 246 | }); 247 | } 248 | 249 | const flakySuccessRatio = 0.5; 250 | 251 | /** 252 | * Simulates a flaky service. 253 | * 254 | * @param {Object} req The HTTP request. 255 | * @param {Object} res The HTTP response. 256 | */ 257 | exports.flaky = (req, res) => { 258 | if (req.path && req.path.length > 1) { 259 | console.log(`Received idempotency key ${req.path.substring(1)}`); 260 | } 261 | if (Math.random() < flakySuccessRatio) { 262 | res.status(200).send('Flaky service succeeded!'); 263 | } else { 264 | res.status(500).send('Flaky service failed.'); 265 | } 266 | }; 267 | -------------------------------------------------------------------------------- /idempotency/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": 3 | { 4 | "request" : "2.88.0", 5 | "request-promise" : "4.2.2", 6 | "firebase-admin" : "5.12.1", 7 | "firebase-functions" : "1.1.0", 8 | "@sendgrid/mail": "6.3.1" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /order-processing/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | -------------------------------------------------------------------------------- /order-processing/README.md: -------------------------------------------------------------------------------- 1 | # Order processing 2 | 3 | This is an example which demonstrates the usefulness of retries and idempotency 4 | when building a solution based on Cloud Functions. Here, it is a sample 5 | restaurant order processing pipeline. 6 | 7 | ## Instructions 8 | 9 | Create a Cloud Firestore project by following 10 | [these instructions](https://firebase.google.com/docs/firestore/quickstart#create_a_project) 11 | or use a project in which you have already used Cloud Firestore. Configure the 12 | Cloud SDK to use this project: 13 | 14 | ``` 15 | gcloud config set project [PROJECT_ID] 16 | ``` 17 | 18 | Deploy the `publish` function, which generates an order by publishing a Cloud 19 | Pub/Sub message: 20 | 21 | ``` 22 | gcloud functions deploy publish --trigger-http 23 | ``` 24 | 25 | (Note: in this sample code, `package.json` declares dependencies for all 26 | functions from `index.js`. In production scenarios, we recommend isolating 27 | dependencies of your functions to avoid installing unnecessary modules on 28 | function deployment). 29 | 30 | Then, deploy the `processOrder` function, which is triggered by Cloud Pub/Sub 31 | messages and does three things sequentially: calls the simulated third-party 32 | `chooseCook` service to choose the cook who will handle the order; stores the 33 | order in Cloud Firestore; and calls another simulated third-party service, 34 | `prepareMeal`, which notifies the cook about the order. 35 | 36 | ``` 37 | gcloud functions deploy processOrder --trigger-topic outgoing 38 | ``` 39 | 40 | (Note: if `chooseCook` and `prepareMeal` services took awhile to respond and 41 | supported callback URLs, you could introduce a tail-call optimization by 42 | chaining the calls and eliminating the `processOrder` function). 43 | 44 | Finally, deploy the `chooseCook` and `prepareMeal` functions, which simulate 45 | flaky third-party services on which the `processOrder` function depends: 46 | 47 | ``` 48 | gcloud functions deploy chooseCook --trigger-http 49 | gcloud functions deploy prepareMeal --trigger-http 50 | ``` 51 | 52 | To test the pipeline you just created, generate some load by invoking the 53 | `publish` function multiple times. Replace `[REGION]` and `[PROJECT]` with the 54 | name and ID of the region and project to which you deployed the functions: 55 | 56 | ``` 57 | for i in {1..30} 58 | do 59 | curl https://[REGION]-[PROJECT].cloudfunctions.net/publish \ 60 | --header "Content-Type: application/json" \ 61 | --data "{ \"topic\": \"outgoing\", \"data\": \"Order $i\" }" 62 | done 63 | ``` 64 | 65 | Wait a minute, and go to the Database viewer for Cloud Firestore in the 66 | [Firebase Console](https://console.firebase.google.com/). You will likely 67 | observe less orders in the `incoming` collection than you generated. This means 68 | that some of the orders got lost in the process. To prevent this situation, 69 | deploy `processOrderRetry` function with the 'retry on failure' option enabled. 70 | The code for `processOrderRetry` remains unchanged compared to `processOrder`, 71 | it just stores the orders in a different Cloud Firestore collection. 72 | 73 | ``` 74 | gcloud functions deploy processOrderRetry --trigger-topic outgoingRetry --retry 75 | ``` 76 | 77 | To test the new version of the pipeline, generate load by invoking the `publish` 78 | function multiple times again, this time pointing to the new topic: 79 | 80 | ``` 81 | for i in {1..30} 82 | do 83 | curl https://[REGION]-[PROJECT].cloudfunctions.net/publish \ 84 | --header "Content-Type: application/json" \ 85 | --data "{ \"topic\": \"outgoingRetry\", \"data\": \"Order $i\" }" 86 | done 87 | ``` 88 | 89 | Wait a minute, then open the Cloud Firestore data viewer and take a look at the 90 | `incomingRetry` collection. You will likely observe more orders than you 91 | generated. This means that some of the orders got duplicated in the process. To 92 | prevent this situation, deploy `processOrderRetryIdempotent` function, which 93 | uses a Cloud Firestore transaction to avoid duplicates among orders stored in 94 | another Cloud Firestore collection. Keep the 'retry on failure' option enabled: 95 | 96 | ``` 97 | gcloud functions deploy processOrderRetryIdempotent --trigger-topic outgoingRetryIdempotent --retry 98 | ``` 99 | 100 | To test this version of the pipeline, generate load by invoking the `publish` 101 | function multiple times again, pointing to the topic you just used: 102 | 103 | ``` 104 | for i in {1..30} 105 | do 106 | curl https://[REGION]-[PROJECT].cloudfunctions.net/publish \ 107 | --header "Content-Type: application/json" \ 108 | --data "{ \"topic\": \"outgoingRetryIdempotent\", \"data\": \"Order $i\" }" 109 | done 110 | ``` 111 | 112 | Wait a minute, and open the Cloud Firestore data viewer again. Take a look at 113 | the `incomingRetryIdempotent` collection. You shouldn't observe any lost or 114 | duplicate orders. 115 | -------------------------------------------------------------------------------- /order-processing/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const PubSub = require('@google-cloud/pubsub'); 18 | const admin = require('firebase-admin'); 19 | const functions = require('firebase-functions'); 20 | const request = require('request-promise'); 21 | const randomItem = require('random-item'); 22 | 23 | admin.initializeApp(functions.config().firebase); 24 | 25 | const pubsubClient = new PubSub(); 26 | const db = admin.firestore(); 27 | 28 | const project = process.env.GCP_PROJECT; 29 | const region = process.env.FUNCTION_REGION; 30 | 31 | /** 32 | * Publishes the given data to the given Cloud Pub/Sub topic. 33 | * 34 | * @param {Object} req The HTTP request. 35 | * @param {Object} res The HTTP response. 36 | */ 37 | exports.publish = (req, res) => { 38 | console.log( 39 | `Publishing data '${req.body.data}' to topic '${req.body.topic}'.`); 40 | const topic = req.body.topic; 41 | const data = Buffer.from(req.body.data); 42 | pubsubClient.topic(topic) 43 | .publisher() 44 | .publish(data) 45 | .then(messageId => { 46 | console.log(`Message ${messageId} published to topic ${topic}.`); 47 | res.status(200).send(`'${data}' published to '${topic}'.\n`); 48 | }) 49 | .catch(err => { 50 | console.error(`Publish error: ${err}`); 51 | res.status(500).send('Failed.\n'); 52 | }); 53 | }; 54 | 55 | const successRatio = 0.9; 56 | const cooks = [ 57 | 'John', 58 | 'Patricia', 59 | 'Mike', 60 | 'Linda', 61 | 'Steve', 62 | 'Katie', 63 | ]; 64 | 65 | /** 66 | * Simulates a service which selects a random cook, or fails occasionally. 67 | * 68 | * @param {Object} req The HTTP request. 69 | * @param {Object} res The HTTP response. 70 | */ 71 | exports.chooseCook = (req, res) => { 72 | if (Math.random() < successRatio) { 73 | res.status(200).send({cook: randomItem(cooks)}); 74 | } else { 75 | res.status(500).send('Transient failure from chooseCook.'); 76 | } 77 | }; 78 | 79 | /** 80 | * Simulates a service which notifies a cook about an order, or fails 81 | * occasionally. 82 | * 83 | * @param {Object} req The HTTP request. 84 | * @param {Object} res The HTTP response. 85 | */ 86 | exports.prepareMeal = (req, res) => { 87 | if (Math.random() < successRatio) { 88 | res.status(200).send('Cook successfully notified to prepare a meal.'); 89 | } else { 90 | res.status(500).send('Transient failure from prepareMeal.'); 91 | } 92 | }; 93 | 94 | /** 95 | * Non-idempotent Pub/Sub-triggered function which handles an order, using 96 | * 'incoming' collection in Cloud Firestore. 97 | * 98 | * @param {Object} event The Cloud Pub/Sub event. 99 | */ 100 | exports.processOrder = (event) => { 101 | return nonIdempotentProcessOrder('incoming', event); 102 | }; 103 | 104 | /** 105 | * Non-idempotent Pub/Sub-triggered function which handles an order, using 106 | * 'incomingRetry' collection in Cloud Firestore. 107 | * 108 | * @param {Object} event The Cloud Pub/Sub event. 109 | */ 110 | exports.processOrderRetry = (event) => { 111 | return nonIdempotentProcessOrder('incomingRetry', event); 112 | }; 113 | 114 | /** 115 | * Non-idempotent function which does three things sequentially: calls 116 | * chooseCook service to choose the cook who will handle the order, then stores 117 | * the order in the given collection in Cloud Firestore, and then calls 118 | * prepareMeal service to start meal preparation. 119 | * 120 | * @param {String} collection The name of the Cloud Firestore collection. 121 | * @param {Object} event The Cloud Pub/Sub event. 122 | */ 123 | function nonIdempotentProcessOrder(collection, event) { 124 | const context = event.context; 125 | const message = event.data; 126 | const order = { 127 | id: context.eventId, 128 | timestamp: context.timestamp, 129 | meal: message.data ? Buffer.from(message.data, 'base64').toString() : '', 130 | }; 131 | console.log(`Received an order for meal ${order.meal}`); 132 | return request({ 133 | method: 'GET', 134 | uri: `https://${region}-${project}.cloudfunctions.net/chooseCook/`, 135 | json: true 136 | }) 137 | // The code below is not executed if the call to chooseCook failed. 138 | .then(res => { 139 | order.cook = res.cook; 140 | console.log(`Assigning cook ${order.cook} and storing order`); 141 | return db.collection(collection).add(order); 142 | }) 143 | .then(() => { 144 | return request({ 145 | method: 'POST', 146 | uri: `https://${region}-${project}.cloudfunctions.net/prepareMeal/`, 147 | body: order, 148 | json: true 149 | }); 150 | }); 151 | } 152 | 153 | /** 154 | * Idempotent Pub/Sub-triggered function which does three things sequentially: 155 | * calls chooseCook service to choose the cook who will handle the order, then 156 | * stores the order in the 'incomingRetryIdempotent' collection in Cloud 157 | * Firestore, and then calls prepareMeal service to start meal preparation. 158 | * 159 | * @param {Object} event The Cloud Pub/Sub event. 160 | */ 161 | exports.processOrderRetryIdempotent = (event) => { 162 | const context = event.context; 163 | const message = event.data; 164 | const order = { 165 | id: context.eventId, 166 | timestamp: context.timestamp, 167 | meal: message.data ? Buffer.from(message.data, 'base64').toString() : '', 168 | }; 169 | console.log(`Received an order for meal ${order.meal}`); 170 | return request({ 171 | method: 'GET', 172 | uri: `https://${region}-${project}.cloudfunctions.net/chooseCook/`, 173 | json: true 174 | }) 175 | .then(res => { 176 | return db.runTransaction(transaction => { 177 | const doc = 178 | db.collection('incomingRetryIdempotent').doc(context.eventId); 179 | return transaction.get(doc).then(snapshot => { 180 | if (!snapshot.exists) { 181 | order.cook = res.cook; 182 | console.log(`Assigning cook ${order.cook} and storing order`); 183 | transaction.set(doc, order); 184 | } 185 | }); 186 | }); 187 | }) 188 | .then(() => { 189 | return request({ 190 | method: 'POST', 191 | uri: `https://${region}-${project}.cloudfunctions.net/prepareMeal/`, 192 | body: {id: order.id}, 193 | json: true 194 | }); 195 | }); 196 | }; 197 | -------------------------------------------------------------------------------- /order-processing/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": 3 | { 4 | "random-item" : "1.0.0", 5 | "request" : "2.88.0", 6 | "request-promise" : "4.2.2", 7 | "firebase-admin" : "5.12.1", 8 | "firebase-functions" : "1.1.0", 9 | "@google-cloud/pubsub": "0.20.1" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /retries/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | -------------------------------------------------------------------------------- /retries/README.md: -------------------------------------------------------------------------------- 1 | # Retries 2 | 3 | This is sample code which demonstrates retrying Cloud Functions on failure. 4 | 5 | ## Instructions 6 | 7 | Deploy the `flaky` function, which simulates a flaky service on which your other 8 | functions depend: 9 | 10 | ``` 11 | gcloud functions deploy flaky --trigger-http 12 | ``` 13 | 14 | Deploy the `httpFunction`, which is triggered by HTTP requests and calls the 15 | `flaky` function internally: 16 | 17 | ``` 18 | gcloud functions deploy httpFunction --trigger-http 19 | ``` 20 | 21 | To have the `httpFunction` function retried on failure, use retries at the call 22 | site. For example, call the function from your machine using the `caller.js` 23 | script: 24 | 25 | ``` 26 | npm install 27 | node caller.js --project [PROJECT] --region [REGION] 28 | ``` 29 | 30 | (Note: in this sample code, `package.json` declares dependencies for both 31 | `index.js` and `caller.js`. In production scenarios, we recommend isolating 32 | dependencies of your functions to avoid installing unnecessary modules on 33 | function deployment). 34 | 35 | Deploy the `pubSubFunction`, which is triggered by Cloud Pub/Sub messages and 36 | calls the `flaky` function internally. Use any topic name for `[TOPIC]` To have 37 | the function retried on failure, add the `--retry` option: 38 | 39 | ``` 40 | gcloud functions deploy pubSubFunction --trigger-topic [TOPIC] --retry 41 | ``` 42 | 43 | To invoke the function, publish a message on the topic you specified: 44 | 45 | ``` 46 | gcloud pubsub topics publish [TOPIC] --message {} 47 | ``` 48 | -------------------------------------------------------------------------------- /retries/caller.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const request = require('request-promise'); 18 | const promiseRetry = require('promise-retry'); 19 | 20 | const argv = require('yargs').demandOption(['project', 'region']).argv; 21 | const project = argv.project; 22 | const region = argv.region; 23 | 24 | // Call an HTTP endpoint with retries using exponential backoff with 25 | // randomization (see https://en.wikipedia.org/wiki/Exponential_backoff). 26 | console.log(`Calling httpFunction in project ${project} region ${region}`); 27 | promiseRetry( 28 | {retries: 10, factor: 2, randomize: true}, 29 | (retry, number) => { 30 | console.log(`Attempt number ${number}.`); 31 | return request({ 32 | method: 'POST', 33 | uri: `https://${region}-${project}.cloudfunctions.net` + 34 | '/httpFunction', 35 | body: {foo: 'bar'}, 36 | json: true 37 | }) 38 | .catch(retry); 39 | }) 40 | .then(res => { 41 | console.log(`Success! ${res}`); 42 | }) 43 | .catch(err => { 44 | console.log(`Failure. ${err}`); 45 | }); 46 | 47 | -------------------------------------------------------------------------------- /retries/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const request = require('request-promise'); 18 | 19 | const project = process.env.GCP_PROJECT; 20 | const region = process.env.FUNCTION_REGION; 21 | 22 | /** 23 | * HTTP-triggered function which calls a flaky service. 24 | * 25 | * @param {Object} req The HTTP request. 26 | * @param {Object} res The HTTP response. 27 | */ 28 | exports.httpFunction = (req, res) => { 29 | request({ 30 | method: 'POST', 31 | uri: `https://${region}-${project}.cloudfunctions.net/flaky`, 32 | body: req.body, 33 | json: true 34 | }) 35 | .then(apiRes => { 36 | res.status(200).send('HTTP function succeeded!'); 37 | }) 38 | .catch(err => { 39 | res.status(500).send('HTTP function failed.'); 40 | }); 41 | }; 42 | 43 | 44 | /** 45 | * Pub/Sub-triggered function which calls a flaky service. 46 | * 47 | * @param {Object} event The Cloud Pub/Sub event. 48 | */ 49 | exports.pubSubFunction = (event) => { 50 | const message = event.data; 51 | const content = 52 | JSON.parse(Buffer.from(message.data || '', 'base64').toString() || '{}'); 53 | return request({ 54 | method: 'POST', 55 | uri: `https://${region}-${project}.cloudfunctions.net/flaky`, 56 | body: content, 57 | json: true 58 | }); 59 | }; 60 | 61 | const flakySuccessRatio = 0.5; 62 | 63 | /** 64 | * Simulates a flaky service. 65 | * 66 | * @param {Object} req The HTTP request. 67 | * @param {Object} res The HTTP response. 68 | */ 69 | exports.flaky = (req, res) => { 70 | if (Math.random() < flakySuccessRatio) { 71 | res.status(200).send('Flaky service succeeded!'); 72 | } else { 73 | res.status(500).send('Flaky service failed.'); 74 | } 75 | }; 76 | 77 | -------------------------------------------------------------------------------- /retries/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies" : 3 | { 4 | "request": "2.88.0", 5 | "request-promise" : "4.2.2", 6 | "promise-retry" : "1.1.1", 7 | "yargs" : "12.0.2" 8 | } 9 | } 10 | --------------------------------------------------------------------------------