├── .gitignore ├── README.md ├── config.json ├── googleapi.js ├── handler.js ├── package-lock.json ├── package.json ├── pgClient.js └── serverless.yml /.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | credentials.json 5 | token.json 6 | 7 | # Serverless directories 8 | .serverless -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lambda-google-sheets 2 | Lambda function to pull data from Postgres database and dump to Google Spreadsheet 3 | 4 | More context in this [blog post](https://blog.bitsrc.io/serverless-function-to-sync-data-from-a-database-to-google-spreadsheet-c71af04a1a34) 5 | 6 | ### Installation 7 | 8 | Clone the repository and run 9 | 10 | ```bash 11 | $ npm install 12 | $ npm install -g serverless 13 | ``` 14 | 15 | Make sure the environment variables as present in `pgClient.js` are populated correctly. 16 | 17 | ### Configuration 18 | 19 | Review the contents of `config.json` to add the source of importing data. 20 | 21 | ### Testing Locally 22 | 23 | Make sure you have the following setup in place on ur machine before executing the script: 24 | 25 | * have node 8.10 runtime or higher 26 | * you have a local tunnel to DWH available on port 5439 27 | * you have the AWS KEY and SECRET available in ~/.aws/credentials 28 | 29 | ```bash 30 | $ SLS_DEBUG=* serverless invoke local --function lambda_sheets 31 | ``` 32 | 33 | 34 | ### Deployment 35 | 36 | To deploy the function to AWS, make sure aws_access_key_id, aws_secret_access_key and region are passed as environment vars to deploy command or configured in ~/.aws/config 37 | 38 | ```bash 39 | $ serverless deploy 40 | ``` 41 | 42 | The deployment script should exit with status code 0. 43 | -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Customers", 3 | "description": "All customer related details sync", 4 | "spreadsheetUrl": "https://docs.google.com/spreadsheets/d/1d4QXXx8qFRhcajpVcEbwVM8TlEXAvrclWpLGmqvw-Pc/edit#gid=0", 5 | "spreadsheetId": "1d4QXXx8qFRhcajpVcEbwVM8TlEXAvrclWpLGmqvw-Pc", 6 | "tables": [ 7 | { 8 | "tableName": "customer_details", 9 | "sheetName": "Sheet1" 10 | } 11 | ] 12 | } -------------------------------------------------------------------------------- /googleapi.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const fs = require('fs') 4 | const readline = require('readline') 5 | const { google } = require('googleapis') 6 | const { promisify } = require('es6-promisify'); 7 | const config = require('./config.json') 8 | 9 | // If modifying these scopes, delete token.json. 10 | const SCOPES = ['https://www.googleapis.com/auth/spreadsheets.readonly'] 11 | // The file token.json stores the user's access and refresh tokens, and is 12 | // created automatically when the authorization flow completes for the first 13 | // time. 14 | const TOKEN_PATH = 'token.json' 15 | 16 | /** 17 | * Create an OAuth2 client with the given credentials, and then execute the 18 | * given callback function. 19 | * @param {Object} credentials The authorization client credentials. 20 | * @param {function} callback The callback to call with the authorized client. 21 | */ 22 | function authorize(credentials, callback) { 23 | const { client_secret, client_id, redirect_uris } = credentials.installed 24 | const oAuth2Client = new google.auth.OAuth2( 25 | client_id, 26 | client_secret, 27 | redirect_uris[0] 28 | ) 29 | 30 | // Check if we have previously stored a token. 31 | fs.readFile(TOKEN_PATH, (err, token) => { 32 | if (err) return getNewToken(oAuth2Client, callback) 33 | oAuth2Client.setCredentials(JSON.parse(token)) 34 | callback(oAuth2Client) 35 | }) 36 | } 37 | 38 | /** 39 | * Get and store new token after prompting for user authorization, and then 40 | * execute the given callback with the authorized OAuth2 client. 41 | * @param {google.auth.OAuth2} oAuth2Client The OAuth2 client to get token for. 42 | * @param {getEventsCallback} callback The callback for the authorized client. 43 | */ 44 | function getNewToken(oAuth2Client, callback) { 45 | const authUrl = oAuth2Client.generateAuthUrl({ 46 | access_type: 'offline', 47 | scope: SCOPES 48 | }) 49 | console.log('Authorize this app by visiting this url:', authUrl) 50 | const rl = readline.createInterface({ 51 | input: process.stdin, 52 | output: process.stdout 53 | }) 54 | rl.question('Enter the code from that page here: ', code => { 55 | rl.close() 56 | oAuth2Client.getToken(code, (err, token) => { 57 | if (err) 58 | return console.error('Error while trying to retrieve access token', err) 59 | oAuth2Client.setCredentials(token) 60 | // Store the token to disk for later program executions 61 | fs.writeFile(TOKEN_PATH, JSON.stringify(token), err => { 62 | if (err) console.error(err) 63 | console.log('Token stored to', TOKEN_PATH) 64 | }) 65 | callback(oAuth2Client) 66 | }) 67 | }) 68 | } 69 | 70 | async function pingSheet(auth) { 71 | const sheets = google.sheets({ version: 'v4', auth }) 72 | const updateSheet = promisify(sheets.spreadsheets.values.update) 73 | 74 | const resource = { 75 | values: [['Greeting', 'Subject'],['Hello', 'World']] 76 | } 77 | 78 | const res = await updateSheet({ 79 | spreadsheetId: config.spreadsheetId, 80 | resource, 81 | range: 'Sheet1', 82 | valueInputOption: 'RAW', 83 | }) 84 | } 85 | 86 | fs.readFile('credentials.json', (err, content) => { 87 | if (err) return console.log('Error loading client secret file:', err) 88 | // Authorize a client with credentials, then call the Google Sheets API. 89 | authorize(JSON.parse(content), pingSheet) 90 | }) -------------------------------------------------------------------------------- /handler.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const pgClient = require('./pgClient') 3 | const readline = require('readline'); 4 | const {google} = require('googleapis'); 5 | const config = require('./config.json') 6 | const { promisify } = require('es6-promisify'); 7 | 8 | // If modifying these scopes, delete token.json. 9 | const SCOPES = ['https://www.googleapis.com/auth/spreadsheets']; 10 | const TOKEN_PATH = 'token.json'; 11 | 12 | let dbClient; 13 | 14 | /** 15 | * Create an OAuth2 client with the given credentials, and then execute the 16 | * given callback function. 17 | * @param {Object} credentials The authorization client credentials. 18 | * @param {function} callback The callback to call with the authorized client. 19 | */ 20 | async function authorize(credentials, callback) { 21 | const {client_secret, client_id, redirect_uris} = credentials.installed; 22 | const oAuth2Client = new google.auth.OAuth2( 23 | client_id, client_secret, redirect_uris[0]); 24 | 25 | // Check if we have previously stored a token. 26 | const token = fs.readFileSync(TOKEN_PATH) 27 | if (!token) return getNewToken(oAuth2Client, callback) 28 | oAuth2Client.setCredentials(JSON.parse(token)) 29 | 30 | console.log('OAuth Authentication successful!') 31 | await callback(oAuth2Client) 32 | } 33 | 34 | /** 35 | * Get and store new token after prompting for user authorization, and then 36 | * execute the given callback with the authorized OAuth2 client. 37 | * @param {google.auth.OAuth2} oAuth2Client The OAuth2 client to get token for. 38 | * @param {getEventsCallback} callback The callback for the authorized client. 39 | */ 40 | async function getNewToken(oAuth2Client, callback) { 41 | const authUrl = oAuth2Client.generateAuthUrl({ 42 | access_type: 'offline', 43 | scope: SCOPES, 44 | }); 45 | console.log('Authorize this app by visiting this url:', authUrl); 46 | const rl = readline.createInterface({ 47 | input: process.stdin, 48 | output: process.stdout, 49 | }); 50 | rl.question('Enter the code from that page here: ', (code) => { 51 | rl.close(); 52 | oAuth2Client.getToken(code, (err, token) => { 53 | if (err) return console.error('Error while trying to retrieve access token', err); 54 | oAuth2Client.setCredentials(token); 55 | // Store the token to disk for later program executions 56 | fs.writeFile(TOKEN_PATH, JSON.stringify(token), (err) => { 57 | if (err) console.error(err); 58 | console.log('Token stored to', TOKEN_PATH); 59 | }); 60 | callback(oAuth2Client); 61 | }); 62 | }); 63 | } 64 | 65 | async function processTables(auth) { 66 | console.log(`processDataconfig started!`); 67 | dbClient = await pgClient.init(); 68 | console.log('Initialised Postgress Client'); 69 | for (const task of config.tables) { 70 | console.log(`Processing table and sheet: ${task.tableName} ${task.sheetName}`) 71 | await queryDatabase(auth, config.spreadsheetId, task.tableName, task.sheetName) 72 | } 73 | } 74 | 75 | async function lambdaCheckIn(auth, spreadsheetId) { 76 | const resource = { 77 | values: [['LastTriggeredAt:', new Date().toLocaleString('en-US', {timeZone: 'America/Los_Angeles'})]] 78 | } 79 | await upsertSheet(auth, spreadsheetId, resource, 'RunLog') 80 | } 81 | 82 | async function queryDatabase(auth, spreadsheetId, tableName, sheetName) { 83 | let sData = []; 84 | const data = await dbClient.query(`select * from ${tableName}`); 85 | 86 | sData.push(Object.keys(data.rows[0])); 87 | for (let i = 0; i < data.rows.length; i++) { 88 | sData.push(Object.values(data.rows[i])); 89 | } 90 | const resource = { 91 | values: sData 92 | }; 93 | 94 | console.log(`[${sheetName}] Rows from Redshift: ${sData.length}`); 95 | await upsertSheet(auth, spreadsheetId, resource, sheetName); 96 | } 97 | 98 | async function upsertSheet(auth, spreadsheetId, resource, sheetName) { 99 | const sheets = google.sheets({ version: "v4", auth }); 100 | const range = sheetName; 101 | 102 | const getValues = promisify(sheets.spreadsheets.values.get); 103 | 104 | const res = await getValues({ 105 | spreadsheetId: spreadsheetId, 106 | range: range 107 | }) 108 | 109 | if (res) { 110 | await writeToSheet(sheets, range, spreadsheetId, resource, sheetName); 111 | } else { 112 | console.log(`Sheet: ${sheetName} not found. Creating sheet!`); 113 | const batchUpdate = promisify(sheets.spreadsheets.batchUpdate); 114 | await batchUpdate( 115 | { 116 | spreadsheetId, 117 | resource: { requests: [{ addSheet: { properties: { title: sheetName } } }] } 118 | } 119 | ) 120 | } 121 | } 122 | 123 | async function writeToSheet(sheets, range, spreadsheetId, resource, sheetName, retryCount = 0) { 124 | const valueInputOption = "RAW"; 125 | 126 | const spreadSheetClear = promisify(sheets.spreadsheets.values.clear); 127 | 128 | await spreadSheetClear({ 129 | spreadsheetId, 130 | range 131 | }); 132 | 133 | const updateSheet = promisify(sheets.spreadsheets.values.update) 134 | 135 | const res = await updateSheet({ 136 | spreadsheetId, 137 | range, 138 | valueInputOption, 139 | resource 140 | }) 141 | 142 | console.log(`[${sheetName}] Rows updated to sheet: ${res.data.updatedRows}`); 143 | } 144 | 145 | module.exports.run = async (event, context) => { 146 | console.log('Google Lambda-Sheets handler triggered') 147 | 148 | try { 149 | // Load client secrets from a local file. 150 | const content = fs.readFileSync('credentials.json') 151 | if (!content) return console.log('Error loading client secret file') 152 | // Authorize a client with credentials, then call the Google Sheets API. 153 | await authorize(JSON.parse(content), processTables) 154 | } catch (e) { 155 | console.log(e) 156 | } 157 | 158 | console.log("Google Lambda-Sheets handler ended"); 159 | }; -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lambda-google-sheets", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "axios": { 8 | "version": "0.19.2", 9 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", 10 | "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==", 11 | "requires": { 12 | "follow-redirects": "1.5.10" 13 | } 14 | }, 15 | "buffer-equal-constant-time": { 16 | "version": "1.0.1", 17 | "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", 18 | "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" 19 | }, 20 | "buffer-writer": { 21 | "version": "2.0.0", 22 | "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", 23 | "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" 24 | }, 25 | "debug": { 26 | "version": "3.1.0", 27 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", 28 | "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", 29 | "requires": { 30 | "ms": "2.0.0" 31 | } 32 | }, 33 | "ecdsa-sig-formatter": { 34 | "version": "1.0.10", 35 | "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.10.tgz", 36 | "integrity": "sha1-HFlQAPBKiJffuFAAiSoPTDOvhsM=", 37 | "requires": { 38 | "safe-buffer": "^5.0.1" 39 | } 40 | }, 41 | "es6-promisify": { 42 | "version": "6.0.1", 43 | "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-6.0.1.tgz", 44 | "integrity": "sha512-J3ZkwbEnnO+fGAKrjVpeUAnZshAdfZvbhQpqfIH9kSAspReRC4nJnu8ewm55b4y9ElyeuhCTzJD0XiH8Tsbhlw==" 45 | }, 46 | "extend": { 47 | "version": "3.0.2", 48 | "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", 49 | "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" 50 | }, 51 | "follow-redirects": { 52 | "version": "1.5.10", 53 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", 54 | "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", 55 | "requires": { 56 | "debug": "=3.1.0" 57 | } 58 | }, 59 | "gcp-metadata": { 60 | "version": "0.6.3", 61 | "resolved": "http://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.6.3.tgz", 62 | "integrity": "sha512-MSmczZctbz91AxCvqp9GHBoZOSbJKAICV7Ow/AIWSJZRrRchUd5NL1b2P4OfP+4m490BEUPhhARfpHdqCxuCvg==", 63 | "requires": { 64 | "axios": "^0.18.0", 65 | "extend": "^3.0.1", 66 | "retry-axios": "0.3.2" 67 | }, 68 | "dependencies": { 69 | "axios": { 70 | "version": "0.18.1", 71 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz", 72 | "integrity": "sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g==", 73 | "requires": { 74 | "follow-redirects": "1.5.10", 75 | "is-buffer": "^2.0.2" 76 | } 77 | } 78 | } 79 | }, 80 | "google-auth-library": { 81 | "version": "1.6.1", 82 | "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-1.6.1.tgz", 83 | "integrity": "sha512-jYiWC8NA9n9OtQM7ANn0Tk464do9yhKEtaJ72pKcaBiEwn4LwcGYIYOfwtfsSm3aur/ed3tlSxbmg24IAT6gAg==", 84 | "requires": { 85 | "axios": "^0.18.0", 86 | "gcp-metadata": "^0.6.3", 87 | "gtoken": "^2.3.0", 88 | "jws": "^3.1.5", 89 | "lodash.isstring": "^4.0.1", 90 | "lru-cache": "^4.1.3", 91 | "retry-axios": "^0.3.2" 92 | }, 93 | "dependencies": { 94 | "axios": { 95 | "version": "0.18.1", 96 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz", 97 | "integrity": "sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g==", 98 | "requires": { 99 | "follow-redirects": "1.5.10", 100 | "is-buffer": "^2.0.2" 101 | } 102 | } 103 | } 104 | }, 105 | "google-p12-pem": { 106 | "version": "1.0.3", 107 | "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.3.tgz", 108 | "integrity": "sha512-KGnAiMMWaJp4j4tYVvAjfP3wCKZRLv9M1Nir2wRRNWUYO7j1aX8O9Qgz+a8/EQ5rAvuo4SIu79n6SIdkNl7Msg==", 109 | "requires": { 110 | "node-forge": "^0.7.5", 111 | "pify": "^4.0.0" 112 | }, 113 | "dependencies": { 114 | "pify": { 115 | "version": "4.0.1", 116 | "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", 117 | "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" 118 | } 119 | } 120 | }, 121 | "googleapis": { 122 | "version": "27.0.0", 123 | "resolved": "http://registry.npmjs.org/googleapis/-/googleapis-27.0.0.tgz", 124 | "integrity": "sha512-Cz0BRsZmewc21N50x5nAUW5cqaGhJ9ETQKZMGqGL4BxmCV7ETELazSqmNi4oCDeRwM4Iub/fIJWAWZk2i6XLCg==", 125 | "requires": { 126 | "google-auth-library": "^1.3.1", 127 | "pify": "^3.0.0", 128 | "qs": "^6.5.1", 129 | "string-template": "1.0.0", 130 | "uuid": "^3.2.1" 131 | } 132 | }, 133 | "gtoken": { 134 | "version": "2.3.0", 135 | "resolved": "http://registry.npmjs.org/gtoken/-/gtoken-2.3.0.tgz", 136 | "integrity": "sha512-Jc9/8mV630cZE9FC5tIlJCZNdUjwunvlwOtCz6IDlaiB4Sz68ki29a1+q97sWTnTYroiuF9B135rod9zrQdHLw==", 137 | "requires": { 138 | "axios": "^0.18.0", 139 | "google-p12-pem": "^1.0.0", 140 | "jws": "^3.1.4", 141 | "mime": "^2.2.0", 142 | "pify": "^3.0.0" 143 | }, 144 | "dependencies": { 145 | "axios": { 146 | "version": "0.18.1", 147 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz", 148 | "integrity": "sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g==", 149 | "requires": { 150 | "follow-redirects": "1.5.10", 151 | "is-buffer": "^2.0.2" 152 | } 153 | } 154 | } 155 | }, 156 | "is-buffer": { 157 | "version": "2.0.4", 158 | "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz", 159 | "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==" 160 | }, 161 | "jwa": { 162 | "version": "1.1.6", 163 | "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.1.6.tgz", 164 | "integrity": "sha512-tBO/cf++BUsJkYql/kBbJroKOgHWEigTKBAjjBEmrMGYd1QMBC74Hr4Wo2zCZw6ZrVhlJPvoMrkcOnlWR/DJfw==", 165 | "requires": { 166 | "buffer-equal-constant-time": "1.0.1", 167 | "ecdsa-sig-formatter": "1.0.10", 168 | "safe-buffer": "^5.0.1" 169 | } 170 | }, 171 | "jws": { 172 | "version": "3.1.5", 173 | "resolved": "https://registry.npmjs.org/jws/-/jws-3.1.5.tgz", 174 | "integrity": "sha512-GsCSexFADNQUr8T5HPJvayTjvPIfoyJPtLQBwn5a4WZQchcrPMPMAWcC1AzJVRDKyD6ZPROPAxgv6rfHViO4uQ==", 175 | "requires": { 176 | "jwa": "^1.1.5", 177 | "safe-buffer": "^5.0.1" 178 | } 179 | }, 180 | "lodash.isstring": { 181 | "version": "4.0.1", 182 | "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", 183 | "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=" 184 | }, 185 | "lru-cache": { 186 | "version": "4.1.5", 187 | "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", 188 | "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", 189 | "requires": { 190 | "pseudomap": "^1.0.2", 191 | "yallist": "^2.1.2" 192 | } 193 | }, 194 | "mime": { 195 | "version": "2.4.0", 196 | "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.0.tgz", 197 | "integrity": "sha512-ikBcWwyqXQSHKtciCcctu9YfPbFYZ4+gbHEmE0Q8jzcTYQg5dHCr3g2wwAZjPoJfQVXZq6KXAjpXOTf5/cjT7w==" 198 | }, 199 | "ms": { 200 | "version": "2.0.0", 201 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 202 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" 203 | }, 204 | "node-forge": { 205 | "version": "0.7.6", 206 | "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz", 207 | "integrity": "sha512-sol30LUpz1jQFBjOKwbjxijiE3b6pjd74YwfD0fJOKPjF+fONKb2Yg8rYgS6+bK6VDl+/wfr4IYpC7jDzLUIfw==" 208 | }, 209 | "packet-reader": { 210 | "version": "0.3.1", 211 | "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-0.3.1.tgz", 212 | "integrity": "sha1-zWLmCvjX/qinBexP+ZCHHEaHHyc=" 213 | }, 214 | "pg": { 215 | "version": "7.7.1", 216 | "resolved": "https://registry.npmjs.org/pg/-/pg-7.7.1.tgz", 217 | "integrity": "sha512-p3I0mXOmUvCoVlCMFW6iYSrnguPol6q8He15NGgSIdM3sPGjFc+8JGCeKclw8ZR4ETd+Jxy2KNiaPUcocHZeMw==", 218 | "requires": { 219 | "buffer-writer": "2.0.0", 220 | "packet-reader": "0.3.1", 221 | "pg-connection-string": "0.1.3", 222 | "pg-pool": "^2.0.4", 223 | "pg-types": "~1.12.1", 224 | "pgpass": "1.x", 225 | "semver": "4.3.2" 226 | } 227 | }, 228 | "pg-connection-string": { 229 | "version": "0.1.3", 230 | "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", 231 | "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" 232 | }, 233 | "pg-pool": { 234 | "version": "2.0.5", 235 | "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.5.tgz", 236 | "integrity": "sha512-T4W9qzP2LjItXuXbW6jgAF2AY0jHp6IoTxRhM3GB7yzfBxzDnA3GCm0sAduzmmiCybMqD0+V1HiqIG5X2YWqlQ==" 237 | }, 238 | "pg-types": { 239 | "version": "1.12.1", 240 | "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-1.12.1.tgz", 241 | "integrity": "sha1-1kCH45A7WP+q0nnnWVxSIIoUw9I=", 242 | "requires": { 243 | "postgres-array": "~1.0.0", 244 | "postgres-bytea": "~1.0.0", 245 | "postgres-date": "~1.0.0", 246 | "postgres-interval": "^1.1.0" 247 | } 248 | }, 249 | "pgpass": { 250 | "version": "1.0.2", 251 | "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", 252 | "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", 253 | "requires": { 254 | "split": "^1.0.0" 255 | } 256 | }, 257 | "pify": { 258 | "version": "3.0.0", 259 | "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", 260 | "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=" 261 | }, 262 | "postgres-array": { 263 | "version": "1.0.3", 264 | "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-1.0.3.tgz", 265 | "integrity": "sha512-5wClXrAP0+78mcsNX3/ithQ5exKvCyK5lr5NEEEeGwwM6NJdQgzIJBVxLvRW+huFpX92F2QnZ5CcokH0VhK2qQ==" 266 | }, 267 | "postgres-bytea": { 268 | "version": "1.0.0", 269 | "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", 270 | "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" 271 | }, 272 | "postgres-date": { 273 | "version": "1.0.3", 274 | "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.3.tgz", 275 | "integrity": "sha1-4tiXAu/bJY/52c7g/pG9BpdSV6g=" 276 | }, 277 | "postgres-interval": { 278 | "version": "1.1.2", 279 | "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.1.2.tgz", 280 | "integrity": "sha512-fC3xNHeTskCxL1dC8KOtxXt7YeFmlbTYtn7ul8MkVERuTmf7pI4DrkAxcw3kh1fQ9uz4wQmd03a1mRiXUZChfQ==", 281 | "requires": { 282 | "xtend": "^4.0.0" 283 | } 284 | }, 285 | "pseudomap": { 286 | "version": "1.0.2", 287 | "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", 288 | "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" 289 | }, 290 | "qs": { 291 | "version": "6.6.0", 292 | "resolved": "https://registry.npmjs.org/qs/-/qs-6.6.0.tgz", 293 | "integrity": "sha512-KIJqT9jQJDQx5h5uAVPimw6yVg2SekOKu959OCtktD3FjzbpvaPr8i4zzg07DOMz+igA4W/aNM7OV8H37pFYfA==" 294 | }, 295 | "retry-axios": { 296 | "version": "0.3.2", 297 | "resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-0.3.2.tgz", 298 | "integrity": "sha512-jp4YlI0qyDFfXiXGhkCOliBN1G7fRH03Nqy8YdShzGqbY5/9S2x/IR6C88ls2DFkbWuL3ASkP7QD3pVrNpPgwQ==" 299 | }, 300 | "safe-buffer": { 301 | "version": "5.1.2", 302 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", 303 | "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" 304 | }, 305 | "semver": { 306 | "version": "4.3.2", 307 | "resolved": "http://registry.npmjs.org/semver/-/semver-4.3.2.tgz", 308 | "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" 309 | }, 310 | "split": { 311 | "version": "1.0.1", 312 | "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", 313 | "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", 314 | "requires": { 315 | "through": "2" 316 | } 317 | }, 318 | "string-template": { 319 | "version": "1.0.0", 320 | "resolved": "https://registry.npmjs.org/string-template/-/string-template-1.0.0.tgz", 321 | "integrity": "sha1-np8iM9wA8hhxjsN5oopWc+zKi5Y=" 322 | }, 323 | "through": { 324 | "version": "2.3.8", 325 | "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", 326 | "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" 327 | }, 328 | "uuid": { 329 | "version": "3.3.2", 330 | "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", 331 | "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" 332 | }, 333 | "xtend": { 334 | "version": "4.0.1", 335 | "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", 336 | "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" 337 | }, 338 | "yallist": { 339 | "version": "2.1.2", 340 | "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", 341 | "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" 342 | } 343 | } 344 | } 345 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lambda-google-sheets", 3 | "version": "1.0.0", 4 | "description": "Lambda function to pull data from Redshift DWH and dump to Google Spreadsheet", 5 | "main": "handler.js", 6 | "author": "RC", 7 | "license": "MIT", 8 | "dependencies": { 9 | "es6-promisify": "^6.0.1", 10 | "googleapis": "^27.0.0", 11 | "pg": "^7.7.1", 12 | "axios": "0.19.2" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /pgClient.js: -------------------------------------------------------------------------------- 1 | const { Client } = require('pg') 2 | 3 | const pg = { 4 | init: async function() { 5 | const client = new Client({ 6 | user: process.env['DB_USER'] || 'postgres', 7 | host: process.env['DB_HOST'] || 'localhost', 8 | database: process.env['DB_NAME'] || 'customers', 9 | password: process.env['DB_PASSWORD'] || '', 10 | port: 5432 11 | }) 12 | 13 | await client.connect(); 14 | return client; 15 | } 16 | } 17 | 18 | module.exports = pg 19 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | # Welcome to Serverless! 2 | # 3 | # This file is the main config file for your service. 4 | # It's very minimal at this point and uses default values. 5 | # You can always add more config options for more control. 6 | # We've included some commented out config examples here. 7 | # Just uncomment any of them to get that config option. 8 | # 9 | # For full config options, check the docs: 10 | # docs.serverless.com 11 | # 12 | # Happy Coding! 13 | 14 | service: SheetImporter # NOTE: update this with your service name 15 | 16 | # You can pin your service to only deploy with a specific Serverless version 17 | # Check out our docs for more details 18 | # frameworkVersion: "=X.X.X" 19 | 20 | provider: 21 | name: aws 22 | runtime: nodejs8.10 23 | 24 | # you can overwrite defaults here 25 | # stage: dev 26 | # region: us-east-1 27 | 28 | # you can add statements to the Lambda function's IAM Role here 29 | # iamRoleStatements: 30 | # - Effect: "Allow" 31 | # Action: 32 | # - "s3:ListBucket" 33 | # Resource: { "Fn::Join" : ["", ["arn:aws:s3:::", { "Ref" : "ServerlessDeploymentBucket" } ] ] } 34 | # - Effect: "Allow" 35 | # Action: 36 | # - "s3:PutObject" 37 | # Resource: 38 | # Fn::Join: 39 | # - "" 40 | # - - "arn:aws:s3:::" 41 | # - "Ref" : "ServerlessDeploymentBucket" 42 | # - "/*" 43 | 44 | # you can define service wide environment variables here 45 | # environment: 46 | # variable1: value1 47 | 48 | # you can add packaging information here 49 | #package: 50 | # include: 51 | # - include-me.js 52 | # - include-me-dir/** 53 | # exclude: 54 | # - exclude-me.js 55 | # - exclude-me-dir/** 56 | 57 | functions: 58 | lambda_sheets: 59 | handler: handler.runs 60 | description: Lambda function to pull data from Postgres and dump to Google Spreadsheet 61 | timeout: 900 62 | events: 63 | - schedule: 64 | rate: rate(1 hour) 65 | enabled: true 66 | 67 | # The following are a few example events you can configure 68 | # NOTE: Please make sure to change your handler code to work with those events 69 | # Check the event documentation for details 70 | # events: 71 | # - http: 72 | # path: users/create 73 | # method: get 74 | # - s3: ${env:BUCKET} 75 | # - schedule: rate(10 minutes) 76 | # - sns: greeter-topic 77 | # - stream: arn:aws:dynamodb:region:XXXXXX:table/foo/stream/1970-01-01T00:00:00.000 78 | # - alexaSkill: amzn1.ask.skill.xx-xx-xx-xx 79 | # - alexaSmartHome: amzn1.ask.skill.xx-xx-xx-xx 80 | # - iot: 81 | # sql: "SELECT * FROM 'some_topic'" 82 | # - cloudwatchEvent: 83 | # event: 84 | # source: 85 | # - "aws.ec2" 86 | # detail-type: 87 | # - "EC2 Instance State-change Notification" 88 | # detail: 89 | # state: 90 | # - pending 91 | # - cloudwatchLog: '/aws/lambda/hello' 92 | # - cognitoUserPool: 93 | # pool: MyUserPool 94 | # trigger: PreSignUp 95 | 96 | # Define function environment variables here 97 | # environment: 98 | # variable2: value2 99 | 100 | # you can add CloudFormation resource templates here 101 | #resources: 102 | # Resources: 103 | # NewResource: 104 | # Type: AWS::S3::Bucket 105 | # Properties: 106 | # BucketName: my-new-bucket 107 | # Outputs: 108 | # NewOutput: 109 | # Description: "Description for the output" 110 | # Value: "Some output value" 111 | --------------------------------------------------------------------------------