├── LICENSE ├── README.md ├── coupons.js ├── demoAPIKeys.js ├── errors.js ├── gstore ├── .gcloudignore ├── .gitignore ├── deploy ├── ds.js ├── expressToGCF.js ├── gcp-ds-key.json ├── hardcodedCredentials.js ├── index.html ├── index.js ├── package-lock.json ├── package.json ├── public │ ├── console.html │ └── index.html └── server.js ├── index.js ├── package-lock.json ├── package.json ├── plans.js ├── pocketwatch-cache ├── .gitignore ├── package-lock.json ├── package.json └── redis.js ├── pocketwatch ├── .ebextensions │ ├── 00_postdep.config │ └── 10_link_node.config ├── .gitignore ├── .gitmodules ├── allowedKeys.js ├── api.js ├── cost.js ├── errors.js ├── gstore │ ├── .gcloudignore │ ├── .gitignore │ ├── deploy │ ├── ds.js │ ├── expressToGCF.js │ ├── gcp-ds-key.json │ ├── hardcodedCredentials.js │ ├── index.html │ ├── index.js │ ├── package-lock.json │ ├── package.json │ ├── public │ │ ├── console.html │ │ └── index.html │ └── server.js ├── index.js ├── package-lock.json ├── package.json ├── pocketwatch-mechanism │ ├── .ebextensions │ │ ├── 00_postdep.config │ │ ├── 00_rootaccess.config │ │ └── 10_link_node.config │ ├── .gitignore │ ├── .gitmodules │ ├── QU.js │ ├── enqueueMessage.js │ ├── errors.js │ ├── index.js │ ├── package-lock.json │ ├── package.json │ ├── performTask.js │ ├── pocketwatch-cache │ │ ├── .gitignore │ │ ├── package-lock.json │ │ ├── package.json │ │ └── redis.js │ ├── promisify.js │ ├── public │ │ └── console.html │ ├── receiveMessage.js │ ├── redis.js │ ├── sm.js │ └── wrapAsync.js ├── pocketwatch-supervisor │ ├── .ebextensions │ │ ├── 00_postdep.config │ │ ├── 00_rootaccess.config │ │ └── 10_link_node.config │ ├── .gitignore │ ├── .gitmodules │ ├── QU.js │ ├── TODO │ ├── cron.yaml │ ├── enqueueSupervisorMessage.js │ ├── errors.js │ ├── functions.js │ ├── gstore │ │ ├── .gcloudignore │ │ ├── .gitignore │ │ ├── deploy │ │ ├── ds.js │ │ ├── expressToGCF.js │ │ ├── gcp-ds-key.json │ │ ├── hardcodedCredentials.js │ │ ├── index.html │ │ ├── index.js │ │ ├── package-lock.json │ │ ├── package.json │ │ ├── public │ │ │ ├── console.html │ │ │ └── index.html │ │ └── server.js │ ├── index.js │ ├── package-lock.json │ ├── package.json │ ├── pocketwatch-cache │ │ ├── .gitignore │ │ ├── package-lock.json │ │ ├── package.json │ │ └── redis.js │ ├── pocketwatch-mechanism │ │ ├── .ebextensions │ │ │ ├── 00_postdep.config │ │ │ ├── 00_rootaccess.config │ │ │ └── 10_link_node.config │ │ ├── .gitignore │ │ ├── .gitmodules │ │ ├── QU.js │ │ ├── enqueueMessage.js │ │ ├── errors.js │ │ ├── index.js │ │ ├── package-lock.json │ │ ├── package.json │ │ ├── performTask.js │ │ ├── pocketwatch-cache │ │ │ ├── .gitignore │ │ │ ├── package-lock.json │ │ │ ├── package.json │ │ │ └── redis.js │ │ ├── promisify.js │ │ ├── public │ │ │ └── console.html │ │ ├── receiveMessage.js │ │ ├── sm.js │ │ └── wrapAsync.js │ ├── promisify.js │ ├── public │ │ └── console.html │ ├── receiveMessage.js │ ├── sm.js │ ├── supervision.js │ ├── test.js │ └── wrapAsync.js ├── promisify.js ├── public │ ├── cost-calculated-modal.html │ ├── fancy.html │ ├── favicon.ico │ ├── index.html │ ├── modal.css │ ├── old-style.css │ ├── ph-free-demo.html │ ├── ph.html │ ├── style.css │ ├── success.html │ ├── terms.html │ └── transparent.html ├── sm.js ├── stripeKeysPublic.js ├── stripeKeysSecret.js ├── views.js └── wrapAsync.js ├── sm.js ├── stripeKeysPublic.js ├── stripeKeysSecret.js ├── subscribe ├── console.html ├── index.html ├── modal.css ├── old-style.css ├── style.css ├── subscription-confirm-modal.html ├── success.html ├── terms.html └── transparent.html ├── views.js └── wrapAsync.js /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 dosyago 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CronStorm 2 | 3 | CronStorm (aka Pocketwatch, aka Kairoi) is a SaaS I created at the beginning of 2018. 4 | 5 | I've opensourced the repositories that I serve it with on AWS and GCP. 6 | For simplicity I have merged the separate services into a monorepo for this open-source release. 7 | If you want to mirror my configuration on AWS or GCP you will have to break each subdirectory module into 8 | its own repo, install its dependencies and deploy it to the appropriate service. As well as setting up the 9 | external services (such as redis and datastore). 10 | 11 | ## System Overview 12 | 13 | There's 6 services: 14 | 15 | - redis cache (ElastiCache) (pocketwatch-cache) 16 | - GCP Datastore (gstore) 17 | - Timekeeper (An EB SQS worker that runs the cron jobs and manages message delays) (pocketwatch-mechanism) 18 | - Supervisor (An EB SQS worker that manages the Timekeeper and makes sure all jobs that are supposed to be running are running and restarts any that have missed their calls) 19 | - An API service (the root directory, cronstorm-services aka pocketwatch-api) combined with a buy a subscription flow 20 | - A web UI to purcash a one-off time (pocketwatch) 21 | 22 | -------------------------------------------------------------------------------- /coupons.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const coupons = { 4 | "TOTALRECALL": "super_duper_coupon_1" 5 | }; 6 | 7 | module.exports = coupons; 8 | } 9 | -------------------------------------------------------------------------------- /demoAPIKeys.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const demoKeys = new Set([ 4 | "i_am_hn_and_proud", 5 | "i_am_ph_and_proud" 6 | ]); 7 | 8 | module.exports = demoKeys; 9 | } 10 | -------------------------------------------------------------------------------- /errors.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const service = 'api'; 4 | const errors = { 5 | log, errorView 6 | }; 7 | 8 | module.exports = errors; 9 | 10 | function log(error, req, res, next) { 11 | console.log("\n"); 12 | console.log(error); 13 | res.status(error.code || error.statusCode).end(JSON.stringify({error})); 14 | } 15 | 16 | function safe( data ) { 17 | const dataString = JSON.stringify(data); 18 | const safe = dataString.replace(/&/g, '&').replace(//g, '>'); 19 | return JSON.parse(safe); 20 | } 21 | 22 | function errorView( data ) { 23 | data = safe(data); 24 | return ` 25 |
26 |

An error occurred

27 |
28 |

29 | ${data.message} 30 |

31 |
32 | 33 |
34 | 42 |
43 | `; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /gstore/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | #!include:.gitignore 18 | -------------------------------------------------------------------------------- /gstore/.gitignore: -------------------------------------------------------------------------------- 1 | *.zip 2 | 3 | node_modules 4 | 5 | .*.swp 6 | 7 | -------------------------------------------------------------------------------- /gstore/deploy: -------------------------------------------------------------------------------- 1 | gcloud beta functions deploy gstore --trigger-http 2 | -------------------------------------------------------------------------------- /gstore/ds.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const crypto = require('crypto'); 4 | const path = require('path'); 5 | const Datastore = require('@google-cloud/datastore'); 6 | const projectId = ''; 7 | const keyFilename = path.join(__dirname, 'gcp-ds-key.json'); 8 | const datastore = new Datastore({projectId,keyFilename}); 9 | const ds = { 10 | test, save, query, keyFromName, newRandom, update, 11 | "delete": del 12 | }; 13 | 14 | module.exports = ds; 15 | 16 | if ( require.main === module ) { 17 | (function() { 18 | test().then( r => console.log(r)); 19 | }()); 20 | } 21 | 22 | function newRandom() { 23 | return crypto.randomBytes(20).toString('hex'); 24 | } 25 | 26 | function keyFromName( {namespace,kind,name} = {} ) { 27 | const key = datastore.key({namespace,path:[kind,name]}); 28 | return key; 29 | } 30 | 31 | function newKey( {namespace,kind} = {} ) { 32 | if ( ! kind || ! namespace ) { 33 | throw new TypeError("Supply kind and namespace"); 34 | } 35 | const randName = newRandom(); 36 | const key = datastore.key({namespace, path:[kind, randName]}); 37 | return key; 38 | } 39 | 40 | async function query( {namespace, kind, q} = {} ) { 41 | const { lines, cursor } = typeof q == "string" ? JSON.parse(q) : q; 42 | let query = datastore.createQuery( namespace, kind ); 43 | while( lines.length ) { 44 | const nextLine = lines.shift(); 45 | switch( nextLine.type ) { 46 | case "order": 47 | query = query.order(nextLine.prop, nextLine.dir); 48 | break; 49 | case "filter": 50 | query = query.filter(nextLine.prop, nextLine.op, nextLine.val); 51 | break; 52 | case "groupBy": 53 | query = query.groupBy(nextLine.prop); 54 | break; 55 | case "select": 56 | query = query.select(nextLine.prop || nextLine.propArray); 57 | break; 58 | case "limit": 59 | query = query.limit(nextLine.limit); 60 | break; 61 | case "default": 62 | throw new TypeError(`Invalid query line type ${nextLine}`); 63 | break; 64 | } 65 | } 66 | if ( !! cursor ) { 67 | query = query.start(cursor); 68 | } 69 | const result = await datastore.runQuery(query); 70 | return result; 71 | } 72 | 73 | async function save( {namespace, kind, excludeFromIndexes: excludeFromIndexes = [], data} = {} ) { 74 | const key = newKey({namespace,kind}); 75 | const keyName = key.name; 76 | data.keyName = keyName; 77 | const entity = { key, data, excludeFromIndexes }; 78 | const resp = await datastore.insert(entity); 79 | return {resp,keyName}; 80 | } 81 | 82 | async function del( {namespace, kind, keyName} = {} ) { 83 | const key = datastore.key({namespace,path:[kind,keyName]}); 84 | const result = await datastore.delete(key); 85 | return result; 86 | } 87 | 88 | 89 | async function update( {namespace,kind,data} = {} ) { 90 | const key = datastore.key({namespace,path:[kind,data.keyName]}); 91 | remove_undefined_keys(data); 92 | const entity = { key, data }; 93 | const resp = await datastore.upsert(entity); 94 | return {resp,keyName:data.keyName}; 95 | } 96 | 97 | function test() { 98 | // The kind for the new entity 99 | const kind = 'Task'; 100 | const namespace = 'test1'; 101 | const data = { 102 | description: 'Buy milk', 103 | }; 104 | 105 | // Saves the entity 106 | return save({namespace,kind,data}); 107 | } 108 | 109 | function remove_undefined_keys(o) { 110 | const keys = Object.keys(o); 111 | const to_remove = []; 112 | for( const k of keys ) { 113 | if ( o[k] == undefined || o[k] == null || o[k] == '' ) { 114 | to_remove.push(k); 115 | } 116 | } 117 | to_remove.forEach( k => o[k] = ' ' ); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /gstore/expressToGCF.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const app = require('./server.js'); 4 | 5 | module.exports = ( req, res ) => { 6 | if ( ! req.path ) { 7 | req.url = '/'; 8 | } 9 | return app( req, res ); 10 | }; 11 | } 12 | -------------------------------------------------------------------------------- /gstore/gcp-ds-key.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | } 4 | 5 | -------------------------------------------------------------------------------- /gstore/hardcodedCredentials.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const hardcodedCredentials = new Set([ 5 | "api key for special access to gstore from pocketwatch" 6 | ]); 7 | 8 | module.exports = hardcodedCredentials; 9 | } 10 | -------------------------------------------------------------------------------- /gstore/index.html: -------------------------------------------------------------------------------- 1 | GSTORE TEST 2 | -------------------------------------------------------------------------------- /gstore/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const expressToGCF = require('./expressToGCF.js'); 4 | 5 | exports.gstore = expressToGCF; 6 | } 7 | 8 | -------------------------------------------------------------------------------- /gstore/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gstore", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "watch": { 7 | "start": { 8 | "patterns": [ 9 | "*" 10 | ], 11 | "extensions": "js,html,css", 12 | "quiet": false 13 | }, 14 | "dev": { 15 | "patterns": [ 16 | "*" 17 | ], 18 | "extensions": "js,html,css", 19 | "quiet": false 20 | } 21 | }, 22 | "scripts": { 23 | "start": "node index.js", 24 | "endless": "node ./node_modules/pm2/bin/pm2 start index.js --name gstore", 25 | "postendless": "node ./node_modules/pm2/bin/pm2 logs", 26 | "dev": "node server.js", 27 | "watch": "npm-watch", 28 | "test": "npm run watch dev" 29 | }, 30 | "repository": { 31 | "type": "git", 32 | "url": "git@git.dosaygo.com:/home/git/gstore" 33 | }, 34 | "author": "@dosy", 35 | "private": true, 36 | "license": "MIT", 37 | "dependencies": { 38 | "@google-cloud/datastore": "^1.4.0", 39 | "@google-cloud/storage": "^1.7.0", 40 | "body-parser": "^1.18.3", 41 | "crypto": "^1.0.1", 42 | "express": "^4.16.3", 43 | "npm-watch": "^0.3.0" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /gstore/public/console.html: -------------------------------------------------------------------------------- 1 |
2 |

3 | 4 | 5 | 6 | 7 | 8 |

9 | 10 |

11 |
12 | 17 |

18 | 19 | 20 | 21 |

22 | 23 |

24 | -------------------------------------------------------------------------------- /gstore/public/index.html: -------------------------------------------------------------------------------- 1 | GSTORE 2 | 7 | CONSOLE 8 | -------------------------------------------------------------------------------- /gstore/server.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const credentials = require('./hardcodedCredentials.js'); 4 | const exp = require('express'); 5 | const ds = require('./ds.js'); 6 | 7 | const app = exp(); 8 | 9 | if ( require.main == module ) { 10 | const bodyParser = require('body-parser'); 11 | app.use(bodyParser.urlencoded({extended:true})); 12 | } 13 | 14 | app.use("/",exp.static("public")); 15 | 16 | app.post("/save/:namespace/:kind/", (req,res,next) => { 17 | const {namespace,kind} = req.params; 18 | const data = Object.assign({},req.body); 19 | console.log(data); 20 | if ( credentials.has(data.apikey) ) { 21 | ds.save({namespace,kind,data}).then( result => res.end("saved")); 22 | } else { 23 | res.end("forbidden"); 24 | } 25 | }); 26 | 27 | app.get("/query/:namespace/:kind/", (req,res,next) => { 28 | const {namespace,kind} = req.params; 29 | const { q, apikey } = req.query; 30 | if ( credentials.has(apikey) ) { 31 | ds.query({namespace,kind,q}).then( result => res.end(JSON.stringify(result)) ); 32 | } else { 33 | res.end("forbidden"); 34 | } 35 | }); 36 | 37 | module.exports = app; 38 | 39 | if ( require.main == module ) { 40 | const port = process.env.PORT || 8080; 41 | const server = app.listen(port, () => console.log(`Server up at ${new Date()} on port ${port}`)); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pw-api", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "watch": { 6 | "start": { 7 | "patterns": [ 8 | "*" 9 | ], 10 | "extensions": "js,html,css", 11 | "quiet": false 12 | }, 13 | "dev": { 14 | "patterns": [ 15 | "*" 16 | ], 17 | "extensions": "js,html,css", 18 | "quiet": false 19 | } 20 | }, 21 | "scripts": { 22 | "prestart": "node sm.js", 23 | "start": "node index.js", 24 | "watch": "npm-watch", 25 | "dev": "node index.js", 26 | "test": "npm run watch dev" 27 | }, 28 | "repository": { 29 | "type": "git", 30 | "url": "git@git.dosaygo.com:/home/git/pa-dataentry-client-ui" 31 | }, 32 | "author": "@dosy", 33 | "license": "MIT", 34 | "dependencies": { 35 | "body-parser": "^1.18.3", 36 | "express": "^4.16.3", 37 | "npm-watch": "^0.3.0", 38 | "path": "^0.12.7", 39 | "stripe": "^6.1.1", 40 | "url": "^0.11.0" 41 | }, 42 | "description": "" 43 | } 44 | -------------------------------------------------------------------------------- /plans.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const economy_saver = { 4 | plan_name: "Economy Saver", 5 | plan_monthly_price: "69.00", 6 | usdCostCents: 6900, 7 | max_timers: 100, 8 | max_hits: 5000000 9 | }; 10 | const medium_enterprise = { 11 | plan_name: "Medium Enterprise", 12 | plan_monthly_price: "185.00", 13 | usdCostCents: 18500, 14 | max_timers: 1000, 15 | max_hits: 25000000 16 | }; 17 | const major_player = { 18 | plan_name: "Major Player", 19 | plan_monthly_price: "555.00", 20 | usdCostCents: 55500, 21 | max_timers: 10000, 22 | max_hits: 125000000 23 | }; 24 | const stripeIdLive = { 25 | economy_saver: "plan_D2jQoArlu2zKBG", 26 | medium_enterprise: "plan_D2jSxGrlu2YKOl", 27 | major_player: "plan_D2jT5Ux5FoN0uG" 28 | }; 29 | const stripeIdTest = { 30 | economy_saver: "plan_D2k1QjY36KfVAO", 31 | medium_enterprise: "plan_D2k1WqKUZ1HCUL", 32 | major_player: "plan_D2k11Ld2dkQH1C" 33 | } 34 | const stripeId = process.env.PAYMENTS == 'live' ? stripeIdLive : stripeIdTest; 35 | const plans = { 36 | economy_saver, medium_enterprise, major_player, 37 | stripeId 38 | }; 39 | 40 | module.exports = plans; 41 | } 42 | -------------------------------------------------------------------------------- /pocketwatch-cache/.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | 3 | node_modules 4 | 5 | # Elastic Beanstalk Files 6 | .elasticbeanstalk/* 7 | !.elasticbeanstalk/*.cfg.yml 8 | !.elasticbeanstalk/*.global.yml 9 | -------------------------------------------------------------------------------- /pocketwatch-cache/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketwatch-cache", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "bluebird": { 8 | "version": "3.5.1", 9 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", 10 | "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" 11 | }, 12 | "double-ended-queue": { 13 | "version": "2.1.0-0", 14 | "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz", 15 | "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=" 16 | }, 17 | "redis": { 18 | "version": "2.8.0", 19 | "resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz", 20 | "integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==", 21 | "requires": { 22 | "double-ended-queue": "^2.1.0-0", 23 | "redis-commands": "^1.2.0", 24 | "redis-parser": "^2.6.0" 25 | } 26 | }, 27 | "redis-commands": { 28 | "version": "1.3.5", 29 | "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.3.5.tgz", 30 | "integrity": "sha512-foGF8u6MXGFF++1TZVC6icGXuMYPftKXt1FBT2vrfU9ZATNtZJ8duRC5d1lEfE8hyVe3jhelHGB91oB7I6qLsA==" 31 | }, 32 | "redis-parser": { 33 | "version": "2.6.0", 34 | "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz", 35 | "integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs=" 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pocketwatch-cache/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketwatch-cache", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "npm test" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git@git.dosaygo.com:/home/git/pocketwatch-cache" 12 | }, 13 | "author": "@dosy", 14 | "license": "MIT", 15 | "dependencies": { 16 | "bluebird": "^3.5.1", 17 | "redis": "^2.8.0" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /pocketwatch-cache/redis.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const EC = { 4 | name: process.env.NODE_ENV !== 'dev' ? 5 | 'localhost' : 6 | '', 7 | port: 6379 8 | }; 9 | 10 | const Redis = require('redis'); 11 | const bluebird = require('bluebird'); 12 | bluebird.promisifyAll(Redis.RedisClient.prototype); 13 | bluebird.promisifyAll(Redis.Multi.prototype); 14 | 15 | let client; 16 | 17 | connect(); 18 | 19 | const redis = client; 20 | 21 | const api = { 22 | see_interval, add_interval, remove_interval, get_missing_intervals, 23 | is_message_duplicate, 24 | add_origin, 25 | mark_for_deletion, 26 | is_marked_for_deletion, 27 | clear, 28 | client 29 | }; 30 | 31 | Object.assign( redis, api ); 32 | 33 | module.exports = redis; 34 | 35 | function connect() { 36 | if ( !! client ) { 37 | client.end(true); 38 | } 39 | client = Redis.createClient(EC.port, EC.name, {retry_strategy}); 40 | } 41 | 42 | function clear() { 43 | return redis.flushallAsync(); 44 | } 45 | 46 | function is_marked_for_deletion( id ) { 47 | return redis.client.getAsync(`deleting:${id}`); 48 | } 49 | 50 | function mark_for_deletion( id, timer_delay = 1 ) { 51 | // notes on deletion 52 | // we set the key long enough for either our whole system to reboot 53 | // 10 minutes is an overestimate 54 | // or for 3 intervals to pass 55 | // the idea is if we haven't see the timer for 3 intervals 56 | // then it is not coming back 57 | // but if the delay is short 58 | // and our system goes down 59 | // we might miss it and 60 | // we need to wait at least until our system goes back to give us a chance to see 61 | // the timer 62 | // the weakness in this system is that we will not delete it if we purge redis 63 | // but we can fix that by writing to datastore as above 64 | // and then doing a deleteReconcile task 65 | // looking for intervals that are marked for deletion but not yet deleted 66 | // or some other such mechansim 67 | // we want to keep any heavy lifting ( datastore writes ) out of the critical path 68 | // in timekeeper receive message 69 | return redis.client.setAsync(`deleting:${id}`, 70 | "OK", "EX", Math.max(600,2*timer_delay)); 71 | } 72 | 73 | function is_message_duplicate( id, msg ) { 74 | return redis.getAsync( `messages:${id}` ).then( result => { 75 | if ( !! result ) { 76 | return true; 77 | } else { 78 | return redis.setAsync( `messages:${id}`, "OK", "EX", Math.max(30,msg.delay_seconds*3) ).then( result => false ); 79 | } 80 | }); 81 | } 82 | 83 | function see_interval( id ) { 84 | return remove_interval( id ); 85 | } 86 | 87 | function has_origin( i ) { 88 | return redis.getAsync( `inserted:${i.keyName}` ); 89 | } 90 | 91 | function add_origin( i ) { 92 | const originKey = `inserted:${i.keyName}`; 93 | const expire_time = Math.ceil(1.5*i.delay_seconds); 94 | return redis.setAsync( originKey, "OK", "EX", expire_time); 95 | } 96 | 97 | async function add_interval( i ) { 98 | const key = `interval:${i.keyName}`; 99 | const ji = JSON.stringify(i); 100 | const missing_timeout = 2*i.delay_seconds; 101 | const expire_timeout = Math.max(120,missing_timeout); 102 | const already_set = !! ( await redis.getAsync(key) ); 103 | if ( ! already_set ) { 104 | await redis.setAsync( key, ji, "EX", expire_timeout ); 105 | const currentTime = +new Date; 106 | const score = currentTime + missing_timeout*1000; 107 | await redis.zaddAsync('intervals',score,key); 108 | return `added:${key}`; 109 | } else { 110 | return `already_present_not_added:${key}`; 111 | } 112 | } 113 | 114 | function remove_interval( id ) { 115 | const key = `interval:${id}`; 116 | const originKey = `inserted:${id}`; 117 | return redis.delAsync( key ).then( 118 | () => redis.zremAsync( 'intervals', key ) 119 | ).then( 120 | () => redis.delAsync( originKey ) 121 | ); 122 | } 123 | 124 | async function get_missing_intervals() { 125 | const currentTime = +new Date; 126 | const keys = await redis.zrangebyscoreAsync( 'intervals', 0, currentTime ); 127 | if ( process.env.DEBUG == 'full' ) { 128 | console.log("\n"); 129 | console.log(JSON.stringify({missingIntervalKeys:keys})); 130 | } 131 | let intervals = []; 132 | if ( keys.length ) { 133 | intervals = await redis 134 | .mgetAsync(...keys) 135 | .then(json_intervals => json_intervals 136 | .filter( ji => !! ji ) 137 | .map( ji => JSON.parse(ji) ) 138 | ); 139 | } 140 | const intervals_to_insert = []; 141 | for ( const i of intervals ) { 142 | const has_o = await has_origin(i); 143 | const inserted = has_o == "OK"; 144 | if ( ! inserted ) { 145 | intervals_to_insert.push( i ); 146 | } 147 | } 148 | if ( process.env.DEBUG == 'full' ) { 149 | console.log("\n"); 150 | console.log(JSON.stringify({intervals_to_insert})); 151 | } 152 | return intervals_to_insert; 153 | } 154 | 155 | function retry_strategy(options) { 156 | if (options.error && options.error.code === 'ECONNREFUSED') { 157 | // End reconnecting on a specific error and flush all commands with 158 | // a individual error 159 | return new Error('The server refused the connection'); 160 | } 161 | if (options.total_retry_time > 1000 * 60 * 60) { 162 | // End reconnecting after a specific timeout and flush all commands 163 | // with a individual error 164 | return new Error('Retry time exhausted'); 165 | } 166 | if (options.attempt > 10) { 167 | // End reconnecting with built in error 168 | return undefined; 169 | } 170 | // reconnect after 171 | return Math.min(options.attempt * 100, 3000); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /pocketwatch/.ebextensions/00_postdep.config: -------------------------------------------------------------------------------- 1 | files: 2 | "/opt/elasticbeanstalk/hooks/appdeploy/post/99_fix_node_permissions.sh": 3 | mode: "000755" 4 | owner: root 5 | group: root 6 | content: | 7 | #!/usr/bin/env bash 8 | chown -R nodejs:nodejs /tmp/.npm/ 9 | -------------------------------------------------------------------------------- /pocketwatch/.ebextensions/10_link_node.config: -------------------------------------------------------------------------------- 1 | files: 2 | "/opt/elasticbeanstalk/hooks/appdeploy/pre/some_job.sh": 3 | mode: "000755" 4 | owner: root 5 | group: root 6 | content: | 7 | #!/usr/bin/env bash 8 | ln -sf `ls -td /opt/elasticbeanstalk/node-install/node-* | head -1`/bin/node /bin/node 9 | ln -sf `ls -td /opt/elasticbeanstalk/node-install/node-* | head -1`/bin/npm /bin/npm 10 | 11 | -------------------------------------------------------------------------------- /pocketwatch/.gitignore: -------------------------------------------------------------------------------- 1 | # vim swap 2 | .*.swp 3 | 4 | *.zip 5 | node_modules 6 | 7 | # Elastic Beanstalk Files 8 | .elasticbeanstalk/* 9 | !.elasticbeanstalk/*.cfg.yml 10 | !.elasticbeanstalk/*.global.yml 11 | -------------------------------------------------------------------------------- /pocketwatch/.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "pocketwatch-mechanism"] 2 | path = pocketwatch-mechanism 3 | url = git@git.dosaygo.com:/home/git/pocketwatch-mechanism 4 | [submodule "gstore"] 5 | path = gstore 6 | url = git@git.dosaygo.com:/home/git/gstore 7 | [submodule "pocketwatch-supervisor"] 8 | path = pocketwatch-supervisor 9 | url = git@git.dosaygo.com:/home/git/pocketwatch-supervisor 10 | -------------------------------------------------------------------------------- /pocketwatch/allowedKeys.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const timerInitial = new Set([ 4 | 'name', 'url', 'method', 5 | 'interval_unit_count', 'interval_unit_type', 6 | 'duration_unit_count', 'duration_unit_type', 7 | 'body', 'contentType', 8 | 'request_source', 'apiKey', 'action', 9 | 'created_at', 'will_end_after' 10 | ]); 11 | const timerFinal = new Set([ 12 | ...timerInitial.values(), 13 | 'intervalDescription', 14 | 'customerId', 'subscriptionId', 15 | 'stripeEmail', 'stripeToken', 'stripeTokenType', 16 | 'name', 'plan', 'plan_choice', 17 | 'interval_count', 'interval_seconds', 18 | 'first_inject', 'delay_seconds', 19 | 'duration_seconds', 20 | 'cost', 'usdCostCents' 21 | ]) 22 | const allowedKeys = { 23 | timerInitial, is_valid, timerFinal 24 | }; 25 | 26 | module.exports = allowedKeys; 27 | 28 | function is_valid( obj, defName ) { 29 | if ( ! allowedKeys[defName] ) { 30 | throw {code:500,message:'that definition name is not known'}; 31 | } 32 | const objKeys = Object.keys(obj); 33 | const allowed = allowedKeys[defName]; 34 | for( const k of objKeys ) { 35 | if ( ! allowed.has(k) ) { 36 | throw {code:400,message:`key ${k} is not allowed in this request.`}; 37 | } 38 | } 39 | return true; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /pocketwatch/api.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const ds = require('./gstore/ds.js'); 4 | const enqueueMessage = require('./pocketwatch-mechanism/enqueueMessage.js'); 5 | 6 | const namespace = 'pocketwatch-gen-1'; 7 | const kind = 'Interval'; 8 | 9 | const excludeFromIndexes = [ 10 | 'action', 11 | 'body', 12 | 'contentType', 13 | 'cost', 14 | 'delay_seconds', 15 | 'duration_seconds', 16 | 'duration_unit_count', 17 | 'duration_unit_type', 18 | 'first_inject', 19 | 'interval_count', 20 | 'interval_seconds', 21 | 'interval_unit_count', 22 | 'interval_unit_type', 23 | 'stripeTokenType', 24 | 'usdCostCents' 25 | ]; 26 | 27 | const api = { 28 | create_timer 29 | }; 30 | 31 | module.exports = api; 32 | 33 | async function create_timer(data) { 34 | const ct = + new Date; 35 | data.created_at = ct; 36 | data.will_end_after = ct + data.duration_seconds * 1000; 37 | data.marked_for_deletion = false; 38 | data.first_inject = true; 39 | const store_result = await ds.save({namespace,kind,data,excludeFromIndexes}); 40 | data.keyName = store_result.keyName; 41 | const result = await enqueueMessage(data); 42 | data.trackingCode = data.keyName; 43 | return result; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /pocketwatch/cost.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const seconds = { 4 | "second": 1, 5 | "minute": 60, 6 | "hour": 3600, 7 | "day": 86400, 8 | "week": 604800, 9 | "month": 2629800 10 | }; 11 | const cost = { 12 | calculate, seconds, calculateApproximateHitsRemaining 13 | }; 14 | 15 | const {URL} = require('url'); 16 | const METHODS = new Set([ 17 | 'GET', 18 | 'POST', 19 | 'PATCH', 20 | 'PUT', 21 | 'DELETE', 22 | 'HEAD' 23 | ]); 24 | const TIMEUNITS = new Set([ 25 | 'second', 26 | 'minute', 27 | 'hour', 28 | 'day', 29 | 'week', 30 | 'month' 31 | ]); 32 | const EMAIL_EX = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; 33 | 34 | module.exports = cost; 35 | 36 | function calculate(body) { 37 | const { 38 | interval_unit_count, interval_unit_type, 39 | duration_unit_count, duration_unit_type 40 | } = body; 41 | const data = Object.assign({}, body); 42 | guardValidate(data); 43 | const iuc = parseInt(interval_unit_count); 44 | const duc = parseInt(duration_unit_count); 45 | const interval_seconds = iuc*seconds[interval_unit_type]; 46 | const delay_seconds = interval_seconds; 47 | const duration_seconds = duc*seconds[duration_unit_type]; 48 | const interval_count = Math.floor(duration_seconds/interval_seconds); 49 | // 1 right now and 1 every interval seconds until duration seconds elapsed from first invocation 50 | const cost = Math.max( 51 | 85, 52 | (0.001*interval_count/duration_seconds)+(duration_seconds*0.000001)+(interval_count*0.001) 53 | ); 54 | data.intervalDescription = data.intervalDescription || `${data.name}: every ${iuc} ${interval_unit_type} for ${duc} ${duration_unit_type}s.` 55 | data.usdCostCents = cost.toFixed(0); 56 | data.cost = (cost/100).toFixed(2); 57 | Object.assign(data, {interval_count, interval_seconds, delay_seconds, duration_seconds }); 58 | guardFinal(data); 59 | return data; 60 | } 61 | 62 | function calculateApproximateHitsRemaining(timer) { 63 | const ct = +new Date; 64 | const {created_at, interval_count,will_end_after} = timer; 65 | const runningDuration = ct - created_at; 66 | const totalDuration = Math.max(1,will_end_after - created_at); 67 | const completeRatio = runningDuration/totalDuration; 68 | const remainingRatio = 1 - Math.abs(completeRatio); 69 | const approximateHitsRemaining = Math.floor(remainingRatio*interval_count); 70 | return parseInt(Math.abs(approximateHitsRemaining)); 71 | } 72 | 73 | function guardValidate(data) { 74 | guardInteger(data.interval_unit_count); 75 | guardInteger(data.duration_unit_count); 76 | guardUrl(data.url); 77 | guardMember(data.method,METHODS); 78 | guardString(data.contentType,{optional:true}); 79 | guardString(data.body,{optional:true}); 80 | guardEmail(data.stripeEmail,{optional:true}); 81 | guardMember(data.interval_unit_type,TIMEUNITS); 82 | guardMember(data.duration_unit_type,TIMEUNITS); 83 | } 84 | 85 | function guardString(i, {optional:optional=false,message:message=null}={}) { 86 | const type = typeof i; 87 | if ( type === "string" ) { 88 | return true; 89 | } else if ( ( i == null || i == undefined ) && optional ) { 90 | return true; 91 | } else { 92 | throw {code:400,message:`${ 93 | optional? 'optional' : 'required' 94 | } value ${i} was asked to be of type String. it was not. ${message||''}`}; 95 | } 96 | } 97 | 98 | function guardFinal(data) { 99 | guardFloat(data.cost); 100 | guardInteger(data.usdCostCents); 101 | guardInteger(data.interval_seconds); 102 | guardInteger(data.delay_seconds); 103 | guardInteger(data.duration_seconds); 104 | } 105 | 106 | function guardInteger(i, {optional:optional=false, message:message=null} = {}) { 107 | const int = parseInt(i); 108 | if ( Number.isInteger(int) ) { 109 | return true; 110 | } else { 111 | throw {code:400,message:`${i} was asked to be integer. it was not. ${message||''}` }; 112 | } 113 | } 114 | 115 | function guardFloat(i, {optional:optional=false, message:message=null} = {}) { 116 | const float = parseFloat(i); 117 | if ( Number.isFinite(float) && ! Number.isNaN(float)) { 118 | return true; 119 | } else { 120 | throw {code:400,message:`${i} was asked to be floating point number. it was not. ${message||''}` }; 121 | } 122 | } 123 | 124 | function guardMember(i, set, {optional: optional=false, message: message=null}={}) { 125 | const test = !! i && set.has(i); 126 | if ( test ) { 127 | return true; 128 | } else { 129 | throw {code:400,message:`${i} was asked to be one of ${[...set.values()].join(',')}, it was not. ${message||''}`}; 130 | } 131 | } 132 | 133 | function guardUrl(i, {optional: optional=false, message: message=null}={}) { 134 | try { 135 | return !!(new URL(i)); 136 | } catch(e) { 137 | throw {code:400, message:`${i} was required to be a valid URL. It was not.${message||''}`}; 138 | } 139 | } 140 | 141 | function guardEmail(i, {optional: optional=false, message: message=null}={}) { 142 | const exists = !! i; 143 | if ( optional && ! exists ) { 144 | return true; 145 | } else { 146 | const test = !! i && EMAIL_EX.test(i); 147 | if ( test ) { 148 | return true; 149 | } else { 150 | throw {code:400,message:`${i} is required to be an email. it is not. ${message||''}`}; 151 | } 152 | } 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /pocketwatch/errors.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const service = 'web'; 4 | const errors = { 5 | log, xhr, html, errorView 6 | }; 7 | 8 | module.exports = errors; 9 | 10 | function log(error, req, res, next) { 11 | console.log("\n"); 12 | console.log(error); 13 | next(error); 14 | } 15 | 16 | function xhr(error, req, res, next) { 17 | res.status(error.code || error.statusCode).end(JSON.stringify({error})); 18 | } 19 | 20 | function html(err, req, res, next) { 21 | res.type('html'); 22 | res.end('unspecified error'); 23 | } 24 | 25 | function safe( data ) { 26 | const dataString = JSON.stringify(data); 27 | const safe = dataString.replace(/&/g, '&').replace(//g, '>'); 28 | return JSON.parse(safe); 29 | } 30 | 31 | function errorView( data ) { 32 | data = safe(data); 33 | return ` 34 |
35 |

An error occurred

36 |
37 |

38 | ${data.message} 39 |

40 |
41 | 42 |
43 | 51 |
52 | `; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /pocketwatch/gstore/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | #!include:.gitignore 18 | -------------------------------------------------------------------------------- /pocketwatch/gstore/.gitignore: -------------------------------------------------------------------------------- 1 | *.zip 2 | 3 | node_modules 4 | 5 | .*.swp 6 | 7 | -------------------------------------------------------------------------------- /pocketwatch/gstore/deploy: -------------------------------------------------------------------------------- 1 | gcloud beta functions deploy gstore --trigger-http 2 | -------------------------------------------------------------------------------- /pocketwatch/gstore/ds.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const crypto = require('crypto'); 4 | const path = require('path'); 5 | const Datastore = require('@google-cloud/datastore'); 6 | const projectId = ''; 7 | const keyFilename = path.join(__dirname, 'gcp-ds-key.json'); 8 | const datastore = new Datastore({projectId,keyFilename}); 9 | const ds = { 10 | test, save, query, keyFromName, newRandom, update, 11 | "delete": del 12 | }; 13 | 14 | module.exports = ds; 15 | 16 | if ( require.main === module ) { 17 | (function() { 18 | test().then( r => console.log(r)); 19 | }()); 20 | } 21 | 22 | function newRandom() { 23 | return crypto.randomBytes(20).toString('hex'); 24 | } 25 | 26 | function keyFromName( {namespace,kind,name} = {} ) { 27 | const key = datastore.key({namespace,path:[kind,name]}); 28 | return key; 29 | } 30 | 31 | function newKey( {namespace,kind} = {} ) { 32 | if ( ! kind || ! namespace ) { 33 | throw new TypeError("Supply kind and namespace"); 34 | } 35 | const randName = newRandom(); 36 | const key = datastore.key({namespace, path:[kind, randName]}); 37 | return key; 38 | } 39 | 40 | async function query( {namespace, kind, q} = {} ) { 41 | const { lines, cursor } = typeof q == "string" ? JSON.parse(q) : q; 42 | let query = datastore.createQuery( namespace, kind ); 43 | while( lines.length ) { 44 | const nextLine = lines.shift(); 45 | switch( nextLine.type ) { 46 | case "order": 47 | query = query.order(nextLine.prop, nextLine.dir); 48 | break; 49 | case "filter": 50 | query = query.filter(nextLine.prop, nextLine.op, nextLine.val); 51 | break; 52 | case "groupBy": 53 | query = query.groupBy(nextLine.prop); 54 | break; 55 | case "select": 56 | query = query.select(nextLine.prop || nextLine.propArray); 57 | break; 58 | case "limit": 59 | query = query.limit(nextLine.limit); 60 | break; 61 | case "default": 62 | throw new TypeError(`Invalid query line type ${nextLine}`); 63 | break; 64 | } 65 | } 66 | if ( !! cursor ) { 67 | query = query.start(cursor); 68 | } 69 | const result = await datastore.runQuery(query); 70 | return result; 71 | } 72 | 73 | async function save( {namespace, kind, excludeFromIndexes: excludeFromIndexes = [], data} = {} ) { 74 | const key = newKey({namespace,kind}); 75 | const keyName = key.name; 76 | data.keyName = keyName; 77 | const entity = { key, data, excludeFromIndexes }; 78 | const resp = await datastore.insert(entity); 79 | return {resp,keyName}; 80 | } 81 | 82 | async function del( {namespace, kind, keyName} = {} ) { 83 | const key = datastore.key({namespace,path:[kind,keyName]}); 84 | const result = await datastore.delete(key); 85 | return result; 86 | } 87 | 88 | 89 | async function update( {namespace,kind,data} = {} ) { 90 | const key = datastore.key({namespace,path:[kind,data.keyName]}); 91 | remove_undefined_keys(data); 92 | const entity = { key, data }; 93 | const resp = await datastore.upsert(entity); 94 | return {resp,keyName:data.keyName}; 95 | } 96 | 97 | function test() { 98 | // The kind for the new entity 99 | const kind = 'Task'; 100 | const namespace = 'test1'; 101 | const data = { 102 | description: 'Buy milk', 103 | }; 104 | 105 | // Saves the entity 106 | return save({namespace,kind,data}); 107 | } 108 | 109 | function remove_undefined_keys(o) { 110 | const keys = Object.keys(o); 111 | const to_remove = []; 112 | for( const k of keys ) { 113 | if ( o[k] == undefined || o[k] == null || o[k] == '' ) { 114 | to_remove.push(k); 115 | } 116 | } 117 | to_remove.forEach( k => o[k] = ' ' ); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /pocketwatch/gstore/expressToGCF.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const app = require('./server.js'); 4 | 5 | module.exports = ( req, res ) => { 6 | if ( ! req.path ) { 7 | req.url = '/'; 8 | } 9 | return app( req, res ); 10 | }; 11 | } 12 | -------------------------------------------------------------------------------- /pocketwatch/gstore/gcp-ds-key.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | } 4 | 5 | -------------------------------------------------------------------------------- /pocketwatch/gstore/hardcodedCredentials.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const hardcodedCredentials = new Set([ 5 | "api key for special access to gstore from pocketwatch" 6 | ]); 7 | 8 | module.exports = hardcodedCredentials; 9 | } 10 | -------------------------------------------------------------------------------- /pocketwatch/gstore/index.html: -------------------------------------------------------------------------------- 1 | GSTORE TEST 2 | -------------------------------------------------------------------------------- /pocketwatch/gstore/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const expressToGCF = require('./expressToGCF.js'); 4 | 5 | exports.gstore = expressToGCF; 6 | } 7 | 8 | -------------------------------------------------------------------------------- /pocketwatch/gstore/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gstore", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "watch": { 7 | "start": { 8 | "patterns": [ 9 | "*" 10 | ], 11 | "extensions": "js,html,css", 12 | "quiet": false 13 | }, 14 | "dev": { 15 | "patterns": [ 16 | "*" 17 | ], 18 | "extensions": "js,html,css", 19 | "quiet": false 20 | } 21 | }, 22 | "scripts": { 23 | "start": "node index.js", 24 | "endless": "node ./node_modules/pm2/bin/pm2 start index.js --name gstore", 25 | "postendless": "node ./node_modules/pm2/bin/pm2 logs", 26 | "dev": "node server.js", 27 | "watch": "npm-watch", 28 | "test": "npm run watch dev" 29 | }, 30 | "repository": { 31 | "type": "git", 32 | "url": "git@git.dosaygo.com:/home/git/gstore" 33 | }, 34 | "author": "@dosy", 35 | "private": true, 36 | "license": "MIT", 37 | "dependencies": { 38 | "@google-cloud/datastore": "^1.4.0", 39 | "@google-cloud/storage": "^1.7.0", 40 | "body-parser": "^1.18.3", 41 | "crypto": "^1.0.1", 42 | "express": "^4.16.3", 43 | "npm-watch": "^0.3.0" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /pocketwatch/gstore/public/console.html: -------------------------------------------------------------------------------- 1 |
2 |

3 | 4 | 5 | 6 | 7 | 8 |

9 | 10 |

11 |
12 | 17 |

18 | 19 | 20 | 21 |

22 | 23 |

24 | -------------------------------------------------------------------------------- /pocketwatch/gstore/public/index.html: -------------------------------------------------------------------------------- 1 | GSTORE 2 | 7 | CONSOLE 8 | -------------------------------------------------------------------------------- /pocketwatch/gstore/server.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const credentials = require('./hardcodedCredentials.js'); 4 | const exp = require('express'); 5 | const ds = require('./ds.js'); 6 | 7 | const app = exp(); 8 | 9 | if ( require.main == module ) { 10 | const bodyParser = require('body-parser'); 11 | app.use(bodyParser.urlencoded({extended:true})); 12 | } 13 | 14 | app.use("/",exp.static("public")); 15 | 16 | app.post("/save/:namespace/:kind/", (req,res,next) => { 17 | const {namespace,kind} = req.params; 18 | const data = Object.assign({},req.body); 19 | console.log(data); 20 | if ( credentials.has(data.apikey) ) { 21 | ds.save({namespace,kind,data}).then( result => res.end("saved")); 22 | } else { 23 | res.end("forbidden"); 24 | } 25 | }); 26 | 27 | app.get("/query/:namespace/:kind/", (req,res,next) => { 28 | const {namespace,kind} = req.params; 29 | const { q, apikey } = req.query; 30 | if ( credentials.has(apikey) ) { 31 | ds.query({namespace,kind,q}).then( result => res.end(JSON.stringify(result)) ); 32 | } else { 33 | res.end("forbidden"); 34 | } 35 | }); 36 | 37 | module.exports = app; 38 | 39 | if ( require.main == module ) { 40 | const port = process.env.PORT || 8080; 41 | const server = app.listen(port, () => console.log(`Server up at ${new Date()} on port ${port}`)); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /pocketwatch/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const service = 'pw-web:'; 4 | 5 | process.on('unhandledRejection', error => { 6 | const event = "Received unhandled promise rejection"; 7 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 8 | console.log("\n"); 9 | console.log(JSON.stringify({service,event,error})); 10 | }); 11 | process.on('uncaughtException', error => { 12 | const event = "Received uncaught exception"; 13 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 14 | console.log("\n"); 15 | console.log(JSON.stringify({service,event,error})); 16 | }); 17 | 18 | const keys = require('./stripeKeysSecret.js')[process.env.PAYMENTS == 'live' ? 'live' : 'test']; 19 | const redis = require('./pocketwatch-mechanism/pocketwatch-cache/redis.js'); 20 | const ds = require('./pocketwatch-supervisor/gstore/ds.js'); 21 | const stripe = require('stripe')(keys.secret); 22 | const path = require('path'); 23 | const exp = require('express'); 24 | const ba = require('express-basic-auth'); 25 | const bodyParser = require('body-parser'); 26 | const wrapAsync = require('./wrapAsync.js'); 27 | const errors = require('./errors.js'); 28 | const allowedKeys = require('./allowedKeys.js'); 29 | const api = require('./api.js'); 30 | const views = require('./views.js'); 31 | const cost = require('./cost.js'); 32 | 33 | stripe.setTimeout(20000); 34 | stripe.on('request', req => { 35 | //console.log(`Stripe request: ${JSON.stringify(req)}`); 36 | }); 37 | stripe.on('response', res => { 38 | //console.log(`Stripe response: ${JSON.stringify(res)}`); 39 | }) 40 | 41 | const app = exp(); 42 | const port = process.env.PORT || 8080; 43 | 44 | app.use(bodyParser.urlencoded({extended:true})); 45 | app.use(bodyParser.json({extended:true})); 46 | 47 | app.get("/health", (req,res,next) => { 48 | res.end("OK"); 49 | }); 50 | 51 | app.get("/clear-pw-redis-cache-ALPHAGARDEN", wrapAsync(async (req,res,next) => { 52 | res.end(await redis.clear()); 53 | })); 54 | 55 | app.get("/demo-functions/purge-timers", wrapAsync(async (req,res,next) => { 56 | const namespace = 'pocketwatch-gen-1'; 57 | const kind = 'Interval'; 58 | const {apiKey, maxDurationUnit, maxDurationCount} = req.query; 59 | if ( ! apiKey || ! maxDurationUnit || ! maxDurationCount ) { 60 | throw {code:400,message:`request for timer purge has incorrect parameters. #1`}; 61 | } 62 | if ( ! cost.seconds[maxDurationUnit] ) { 63 | throw {code:400,message:`request for timer purge has incorrect parameters. #2`}; 64 | } 65 | // get timers made with that key that were created more than 66 | // max duration time ago 67 | const maxDurationUnitSeconds = cost.seconds[maxDurationUnit]; 68 | const currentTime = +new Date; 69 | const maxTimeAgo = currentTime - (maxDurationUnitSeconds * maxDurationCount * 1000); 70 | let timers_to_purge; 71 | let args = {namespace,kind,q: { 72 | lines: [ 73 | { 74 | type: 'filter', 75 | prop: 'apiKey', 76 | op: '=', 77 | val: apiKey 78 | }, 79 | { 80 | type: 'filter', 81 | prop: 'created_at', 82 | op: '<', 83 | val: maxTimeAgo 84 | } 85 | ] 86 | }}; 87 | try { 88 | [timers_to_purge] = await ds.query(args); 89 | } catch(error) { 90 | const event = "Received error from ds.query"; 91 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 92 | console.log("\n"); 93 | console.log(JSON.stringify({service,event,error,args})); 94 | throw {code:400,message:'there was an error querying for timers to purge'}; 95 | } 96 | // for each such timer, call redis mark for deletion 97 | // on its keyName 98 | const mark_for_deletion_results = await Promise.all( 99 | timers_to_purge.map(t => redis.mark_for_deletion(t.keyName, t.delay_seconds))); 100 | // and delete it from the database so it is not revived later by supervisor 101 | const delete_from_db_results = await Promise.all( 102 | timers_to_purge.map(t => ds.delete({namespace,kind,keyName:t.keyName}))); 103 | res.end("OK"); 104 | })); 105 | 106 | app.use(errors.log); 107 | app.use(errors.xhr); 108 | 109 | /** 110 | app.use(ba({ 111 | challenge: true, 112 | realm: 'pocketwatch-999', 113 | users: { 114 | "dominate": "all situations" 115 | } 116 | })); 117 | **/ 118 | 119 | app.use("/",exp.static(path.join(__dirname, "public"))); 120 | 121 | app.post("/interval/purchase_flow", wrapAsync(async (req,res,next) => { 122 | switch( req.body.action ) { 123 | case "calculate_cost": { 124 | allowedKeys.is_valid(req.body,"timerInitial"); 125 | const data = cost.calculate(req.body); 126 | res.type('html'); 127 | res.end(views.costCalculated(data)); 128 | break; 129 | } 130 | case "proceed_to_stripe_payment": { 131 | const data = cost.calculate(req.body); 132 | res.type('html'); 133 | res.end(views.stripePayment(data)); 134 | break; 135 | } 136 | case "verify_stripe_charge": { 137 | const data = cost.calculate(req.body); 138 | const charge = await stripe.charges.create({ 139 | amount: data.usdCostCents, 140 | source: data.stripeToken, 141 | currency: 'usd', 142 | description: data.intervalDescription, 143 | statement_descriptor: '1 pocketwatch interval' 144 | }); 145 | res.type('html'); 146 | //console.log('charge', charge); 147 | if ( charge.status == 'succeeded' ) { 148 | allowedKeys.is_valid(data,"timerFinal"); 149 | const result = await api.create_timer(data); 150 | console.log("Message data", data); 151 | //console.log("First enqueue result", result); 152 | res.end(views.successPayment(data)); 153 | } else { 154 | res.end(views.error({message:charge.failure_message})); 155 | } 156 | break; 157 | } 158 | default: 159 | next(); 160 | } 161 | })); 162 | 163 | app.use(errors.log); 164 | app.use(errors.html); 165 | app.use((req,res,next) => { 166 | res.status(404).send(views.errorView({message: 'Page not found'})); 167 | }); 168 | 169 | const server = app.listen(port, () => console.log(`Server up at ${new Date()} on port ${port}`)); 170 | 171 | server.on('clientError', (err,socket) => { 172 | socket.end('HTTP/1.1 400 Bad Request\r\n\r\n'); 173 | }); 174 | } 175 | -------------------------------------------------------------------------------- /pocketwatch/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketwatch", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "watch": { 6 | "start": { 7 | "patterns": [ 8 | "*" 9 | ], 10 | "extensions": "js,html,css", 11 | "quiet": false 12 | }, 13 | "dev": { 14 | "patterns": [ 15 | "*" 16 | ], 17 | "extensions": "js,html,css", 18 | "quiet": false 19 | } 20 | }, 21 | "scripts": { 22 | "prestart": "node sm.js", 23 | "start": "node index.js", 24 | "dev": "node index.js", 25 | "watch": "npm-watch", 26 | "test": "npm run watch dev" 27 | }, 28 | "repository": { 29 | "type": "git", 30 | "url": "git@git.dosaygo.com:/home/git/hnsakura" 31 | }, 32 | "author": "@dosy", 33 | "license": "MIT", 34 | "description": "", 35 | "dependencies": { 36 | "aws-sdk": "^2.224.1", 37 | "body-parser": "^1.18.2", 38 | "express": "^4.16.3", 39 | "express-basic-auth": "^1.1.5", 40 | "hn-api": "^0.1.5", 41 | "node-fetch": "^2.1.2", 42 | "npm-watch": "^0.3.0", 43 | "pm2": "^2.10.2", 44 | "request": "^2.85.0", 45 | "stripe": "^5.8.0", 46 | "url": "^0.11.0" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/.ebextensions/00_postdep.config: -------------------------------------------------------------------------------- 1 | files: 2 | "/opt/elasticbeanstalk/hooks/appdeploy/post/99_fix_node_permissions.sh": 3 | mode: "000755" 4 | owner: root 5 | group: root 6 | content: | 7 | #!/usr/bin/env bash 8 | chown -R nodejs:nodejs /tmp/.npm/ 9 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/.ebextensions/00_rootaccess.config: -------------------------------------------------------------------------------- 1 | container_commands: 2 | 01_enable_rootaccess: 3 | command: echo Defaults:root \!requiretty >> /etc/sudoers 4 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/.ebextensions/10_link_node.config: -------------------------------------------------------------------------------- 1 | files: 2 | "/opt/elasticbeanstalk/hooks/appdeploy/pre/some_job.sh": 3 | mode: "000755" 4 | owner: root 5 | group: root 6 | content: | 7 | #!/usr/bin/env bash 8 | ln -sf `ls -td /opt/elasticbeanstalk/node-install/node-* | head -1`/bin/node /bin/node 9 | ln -sf `ls -td /opt/elasticbeanstalk/node-install/node-* | head -1`/bin/npm /bin/npm 10 | 11 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/.gitignore: -------------------------------------------------------------------------------- 1 | *.rdb 2 | 3 | node_modules 4 | 5 | # Elastic Beanstalk Files 6 | .elasticbeanstalk/* 7 | !.elasticbeanstalk/*.cfg.yml 8 | !.elasticbeanstalk/*.global.yml 9 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "pocketwatch-cache"] 2 | path = pocketwatch-cache 3 | url = git@git.dosaygo.com:/home/git/pocketwatch-cache 4 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/QU.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const QU = 'mechanism SQS endpoint'; 5 | 6 | module.exports = QU; 7 | } 8 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/enqueueMessage.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const QU = require('./QU.js'); 4 | const redis = require('./pocketwatch-cache/redis.js'); 5 | const promisify = require('./promisify.js'); 6 | const crypto = require('crypto'); 7 | const AWS = require('aws-sdk'); 8 | 9 | AWS.config.update({region:'us-west-2'}); 10 | const sqs = new AWS.SQS({apiVersion:'2012-11-05'}); 11 | 12 | const sendMsg = promisify((...args) => sqs.sendMessage(...args)); 13 | 14 | module.exports = enqueueMessage; 15 | 16 | function newRandom() { 17 | return crypto.randomBytes(20).toString('hex'); 18 | } 19 | 20 | async function enqueueMessage( msg ) { 21 | if ( !! msg.first_inject ) { 22 | delete msg.first_inject; 23 | const ct = +new Date; 24 | msg.will_next_execute_at = ct + msg.interval_seconds * 1000; 25 | } 26 | const packagedMessage = createMessage(msg); 27 | await redis.add_origin(msg); 28 | const result = await sendMsg(packagedMessage); 29 | const resultStatus = makeStatus( result ); 30 | const msgDeduplicatorKey = `${msg.origin}/${msg.msgName}`; 31 | //console.log(JSON.stringify({QU,msgDeduplicatorKey,resultStatus})); 32 | return resultStatus; 33 | } 34 | 35 | function createMessage( obj ) { 36 | obj.msgName = ( obj.msgName || 0 ) + 1; 37 | obj.origin = obj.origin || newRandom(); 38 | const MessageBody = JSON.stringify(obj); 39 | const QueueUrl = QU; 40 | const DelaySeconds = obj.delay_seconds || 0; 41 | const params = { 42 | MessageBody, QueueUrl, DelaySeconds 43 | } 44 | return params; 45 | } 46 | 47 | function makeStatus( [ err, data ] = [] ) { 48 | if ( !! err ) { 49 | return `fail`; 50 | } else if ( !! data ) { 51 | return `success`; 52 | } else { 53 | return `unknown`; 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/errors.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const service = 'timekeeper'; 4 | const errors = { 5 | log, xhr, html, errorView 6 | }; 7 | 8 | module.exports = errors; 9 | 10 | function log(error, req, res, next) { 11 | const event = "received error"; 12 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 13 | console.log("\n"); 14 | console.error(JSON.stringify({service,event,error})); 15 | next(error); 16 | } 17 | 18 | function xhr(err, req, res, next) { 19 | if ( req.xhr ) { 20 | res.status(500).end({ error: 'Something failed!' }); 21 | } else { 22 | next(err); 23 | } 24 | } 25 | 26 | function html(err, req, res, next) { 27 | res.type('html'); 28 | res.end('unspecified error'); 29 | } 30 | 31 | function safe( data ) { 32 | const dataString = JSON.stringify(data); 33 | const safe = dataString.replace(/&/g, '&').replace(//g, '>'); 34 | return JSON.parse(safe); 35 | } 36 | 37 | function errorView( data ) { 38 | data = safe(data); 39 | return ` 40 |
41 |

An error occurred

42 |
43 |

44 | ${data.message} 45 |

46 |
47 | 48 |
49 | 57 |
58 | `; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const service = 'timekeeper'; 4 | 5 | process.on('unhandledRejection', error => { 6 | const event = "Received unhandled promise rejection"; 7 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 8 | console.log("\n"); 9 | console.error(JSON.stringify({service,event,error})); 10 | }); 11 | process.on('uncaughtException', error => { 12 | const event = "Received uncaught exception"; 13 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 14 | console.log("\n"); 15 | console.error(JSON.stringify({service,event,error})); 16 | }); 17 | 18 | const receiveMessage = require('./receiveMessage.js'); 19 | const redis = require('./pocketwatch-cache/redis.js'); 20 | const exp = require('express'); 21 | const bodyParser = require('body-parser'); 22 | const path = require('path'); 23 | const wrapAsync = require('./wrapAsync.js'); 24 | const errors = require('./errors.js'); 25 | 26 | const app = exp(); 27 | const port = process.env.PORT || 8080; 28 | 29 | app.use("/", exp.static(path.join(__dirname, "public"))); 30 | app.use(bodyParser.urlencoded({extended:true})); 31 | app.use(bodyParser.json({extended:true})); 32 | 33 | app.get("/", async (req,res,next) => { 34 | if ( process.env.NODE_ENV !== 'dev' ) { 35 | const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress; 36 | const time = new Date(); 37 | const path = req.originalUrl; 38 | const method = req.method; 39 | const body = req.body; 40 | const q = req.query; 41 | console.log("\n"); 42 | console.log(JSON.stringify({request:{service,method,path,body,q,ip,time}})); 43 | } 44 | res.type('text').status(200).end("OK"); 45 | }); 46 | 47 | app.post("/inbox", wrapAsync(async (req,res,next) => { 48 | const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress; 49 | const time = new Date(); 50 | // init 51 | console.log("\n"); 52 | const message = !! req.body.MessageBody ? JSON.parse(req.body.MessageBody) : req.body; 53 | const {keyName,msgName,origin} = message; 54 | console.log( 55 | JSON.stringify({service,ip,time,path:'/inbox',message:{keyName,msgName,origin}})); 56 | // receive it 57 | const receiveResult = await receiveMessage({message}); 58 | // response ( always OK ) 59 | return res.type("text").status(200).end("OK"); 60 | })); 61 | 62 | app.use(errors.log); 63 | app.use(errors.xhr); 64 | app.use(errors.html); 65 | app.use((req,res,next) => { 66 | res.status(404).send(errors.errorView({message: 'Page not found'})); 67 | }); 68 | 69 | const server = app.listen(port, () => { 70 | const currentTime = new Date; 71 | const serverLive = `Server up at ${currentTime} on port ${port}`; 72 | console.log(JSON.stringify({serverLive,port,currentTime})); 73 | return true; 74 | }); 75 | 76 | module.exports = server; 77 | } 78 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketwatch-mechanism", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "watch": { 7 | "start": { 8 | "patterns": [ 9 | "*" 10 | ], 11 | "extensions": "js,html,css", 12 | "quiet": false 13 | }, 14 | "dev": { 15 | "patterns": [ 16 | "*" 17 | ], 18 | "extensions": "js,html,css", 19 | "quiet": false 20 | } 21 | }, 22 | "scripts": { 23 | "prestart": "node sm.js", 24 | "start": "node index.js", 25 | "dev": "node index.js", 26 | "watch": "npm-watch", 27 | "test": "npm run watch dev" 28 | }, 29 | "repository": { 30 | "type": "git", 31 | "url": "git@git.dosaygo.com:/home/git/pocketwatch-mechanism" 32 | }, 33 | "author": "@dosy", 34 | "license": "MIT", 35 | "dependencies": { 36 | "aws-sdk": "^2.224.1", 37 | "express": "^4.16.3", 38 | "node-fetch": "^2.1.2", 39 | "npm-watch": "^0.3.0", 40 | "pm2": "^2.10.2" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/performTask.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const fetch = require('node-fetch'); 4 | const HEADER = { 5 | 'User-Agent': 'Pocketwatch Timer Fetch; Report Misuse to: criscanbereached+pw.bad.actors@gmail.com; Subscribe for your own timers at https://api.pocketwatch.xyz' 6 | }; 7 | 8 | module.exports = performTask; 9 | 10 | async function performTask(msg) { 11 | const { method, url } = msg; 12 | const { keyName, msgName, origin } = msg; 13 | let taskPromise; 14 | try { 15 | switch( method ) { 16 | case 'DELETE': 17 | case 'PUT': 18 | case 'PATCH': 19 | case 'POST': { 20 | let { body, contentType: contentType = 'application/x-www-form-urlencoded' } = msg; 21 | if ( !! body ) { 22 | if ( process.env.DEBUG == 'full' ) { 23 | console.log("\n"); 24 | console.log( 25 | JSON.stringify({message:"Body request",where:"performTask",body})); 26 | } 27 | } 28 | const headers = Object.assign({ 29 | 'Content-Type': contentType 30 | }, HEADER); 31 | taskPromise = await fetch(url, {method, headers, body}); 32 | break; 33 | } 34 | case 'HEAD': 35 | case 'GET': { 36 | taskPromise = await fetch(url, {method, headers: HEADER}); 37 | break; 38 | } 39 | } 40 | } catch(error) { 41 | if ( process.env.DEBUG == 'full' ) { 42 | const event = "performTask: error when making HTTP fetch"; 43 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 44 | console.log("\n"); 45 | console.warn( 46 | JSON.stringify({event,error, 47 | message:{keyName,msgName,origin}})); 48 | } 49 | return; 50 | } 51 | if ( process.env.DEBUG == 'full' ) { 52 | const event = "performTask: reached end of function block"; 53 | console.log("\n"); 54 | console.log( 55 | JSON.stringify({event, 56 | message:{keyName,msgName,origin}})); 57 | } 58 | return taskPromise; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/pocketwatch-cache/.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | 3 | node_modules 4 | 5 | # Elastic Beanstalk Files 6 | .elasticbeanstalk/* 7 | !.elasticbeanstalk/*.cfg.yml 8 | !.elasticbeanstalk/*.global.yml 9 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/pocketwatch-cache/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketwatch-cache", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "bluebird": { 8 | "version": "3.5.1", 9 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", 10 | "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" 11 | }, 12 | "double-ended-queue": { 13 | "version": "2.1.0-0", 14 | "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz", 15 | "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=" 16 | }, 17 | "redis": { 18 | "version": "2.8.0", 19 | "resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz", 20 | "integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==", 21 | "requires": { 22 | "double-ended-queue": "^2.1.0-0", 23 | "redis-commands": "^1.2.0", 24 | "redis-parser": "^2.6.0" 25 | } 26 | }, 27 | "redis-commands": { 28 | "version": "1.3.5", 29 | "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.3.5.tgz", 30 | "integrity": "sha512-foGF8u6MXGFF++1TZVC6icGXuMYPftKXt1FBT2vrfU9ZATNtZJ8duRC5d1lEfE8hyVe3jhelHGB91oB7I6qLsA==" 31 | }, 32 | "redis-parser": { 33 | "version": "2.6.0", 34 | "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz", 35 | "integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs=" 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/pocketwatch-cache/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketwatch-cache", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "npm test" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git@git.dosaygo.com:/home/git/pocketwatch-cache" 12 | }, 13 | "author": "@dosy", 14 | "license": "MIT", 15 | "dependencies": { 16 | "bluebird": "^3.5.1", 17 | "redis": "^2.8.0" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/pocketwatch-cache/redis.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const EC = { 4 | name: process.env.NODE_ENV !== 'dev' ? 5 | 'localhost' : 6 | 'redis endpoint', 7 | port: 6379 8 | }; 9 | 10 | const Redis = require('redis'); 11 | const bluebird = require('bluebird'); 12 | bluebird.promisifyAll(Redis.RedisClient.prototype); 13 | bluebird.promisifyAll(Redis.Multi.prototype); 14 | 15 | let client; 16 | 17 | connect(); 18 | 19 | const redis = client; 20 | 21 | const api = { 22 | see_interval, add_interval, remove_interval, get_missing_intervals, 23 | is_message_duplicate, 24 | add_origin, 25 | mark_for_deletion, 26 | is_marked_for_deletion, 27 | clear, 28 | client 29 | }; 30 | 31 | Object.assign( redis, api ); 32 | 33 | module.exports = redis; 34 | 35 | function connect() { 36 | if ( !! client ) { 37 | client.end(true); 38 | } 39 | client = Redis.createClient(EC.port, EC.name, {retry_strategy}); 40 | } 41 | 42 | function clear() { 43 | return redis.flushallAsync(); 44 | } 45 | 46 | function is_marked_for_deletion( id ) { 47 | return redis.client.getAsync(`deleting:${id}`); 48 | } 49 | 50 | function mark_for_deletion( id, timer_delay = 1 ) { 51 | // notes on deletion 52 | // we set the key long enough for either our whole system to reboot 53 | // 10 minutes is an overestimate 54 | // or for 3 intervals to pass 55 | // the idea is if we haven't see the timer for 3 intervals 56 | // then it is not coming back 57 | // but if the delay is short 58 | // and our system goes down 59 | // we might miss it and 60 | // we need to wait at least until our system goes back to give us a chance to see 61 | // the timer 62 | // the weakness in this system is that we will not delete it if we purge redis 63 | // but we can fix that by writing to datastore as above 64 | // and then doing a deleteReconcile task 65 | // looking for intervals that are marked for deletion but not yet deleted 66 | // or some other such mechansim 67 | // we want to keep any heavy lifting ( datastore writes ) out of the critical path 68 | // in timekeeper receive message 69 | return redis.client.setAsync(`deleting:${id}`, 70 | "OK", "EX", Math.max(600,2*timer_delay)); 71 | } 72 | 73 | function is_message_duplicate( id, msg ) { 74 | return redis.getAsync( `messages:${id}` ).then( result => { 75 | if ( !! result ) { 76 | return true; 77 | } else { 78 | return redis.setAsync( `messages:${id}`, "OK", "EX", Math.max(30,msg.delay_seconds*3) ).then( result => false ); 79 | } 80 | }); 81 | } 82 | 83 | function see_interval( id ) { 84 | return remove_interval( id ); 85 | } 86 | 87 | function has_origin( i ) { 88 | return redis.getAsync( `inserted:${i.keyName}` ); 89 | } 90 | 91 | function add_origin( i ) { 92 | const originKey = `inserted:${i.keyName}`; 93 | const expire_time = Math.ceil(1.5*i.delay_seconds); 94 | return redis.setAsync( originKey, "OK", "EX", expire_time); 95 | } 96 | 97 | async function add_interval( i ) { 98 | const key = `interval:${i.keyName}`; 99 | const ji = JSON.stringify(i); 100 | const missing_timeout = 2*i.delay_seconds; 101 | const expire_timeout = Math.max(120,missing_timeout); 102 | const already_set = !! ( await redis.getAsync(key) ); 103 | if ( ! already_set ) { 104 | await redis.setAsync( key, ji, "EX", expire_timeout ); 105 | const currentTime = +new Date; 106 | const score = currentTime + missing_timeout*1000; 107 | await redis.zaddAsync('intervals',score,key); 108 | return `added:${key}`; 109 | } else { 110 | return `already_present_not_added:${key}`; 111 | } 112 | } 113 | 114 | function remove_interval( id ) { 115 | const key = `interval:${id}`; 116 | const originKey = `inserted:${id}`; 117 | return redis.delAsync( key ).then( 118 | () => redis.zremAsync( 'intervals', key ) 119 | ).then( 120 | () => redis.delAsync( originKey ) 121 | ); 122 | } 123 | 124 | async function get_missing_intervals() { 125 | const currentTime = +new Date; 126 | const keys = await redis.zrangebyscoreAsync( 'intervals', 0, currentTime ); 127 | if ( process.env.DEBUG == 'full' ) { 128 | console.log("\n"); 129 | console.log(JSON.stringify({missingIntervalKeys:keys})); 130 | } 131 | let intervals = []; 132 | if ( keys.length ) { 133 | intervals = await redis 134 | .mgetAsync(...keys) 135 | .then(json_intervals => json_intervals 136 | .filter( ji => !! ji ) 137 | .map( ji => JSON.parse(ji) ) 138 | ); 139 | } 140 | const intervals_to_insert = []; 141 | for ( const i of intervals ) { 142 | const has_o = await has_origin(i); 143 | const inserted = has_o == "OK"; 144 | if ( ! inserted ) { 145 | intervals_to_insert.push( i ); 146 | } 147 | } 148 | if ( process.env.DEBUG == 'full' ) { 149 | console.log("\n"); 150 | console.log(JSON.stringify({intervals_to_insert})); 151 | } 152 | return intervals_to_insert; 153 | } 154 | 155 | function retry_strategy(options) { 156 | if (options.error && options.error.code === 'ECONNREFUSED') { 157 | // End reconnecting on a specific error and flush all commands with 158 | // a individual error 159 | return new Error('The server refused the connection'); 160 | } 161 | if (options.total_retry_time > 1000 * 60 * 60) { 162 | // End reconnecting after a specific timeout and flush all commands 163 | // with a individual error 164 | return new Error('Retry time exhausted'); 165 | } 166 | if (options.attempt > 10) { 167 | // End reconnecting with built in error 168 | return undefined; 169 | } 170 | // reconnect after 171 | return Math.min(options.attempt * 100, 3000); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/promisify.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | // aim is to promisify all extension apis that use callbacks 4 | 5 | module.exports = promisify; 6 | 7 | function promisify(func) { 8 | return async function(...args) { 9 | return new Promise((res,rej) => { 10 | try { 11 | func(...args, (...cb_args) => res(cb_args)); 12 | } catch(e) { 13 | rej(e); 14 | } 15 | }); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/public/console.html: -------------------------------------------------------------------------------- 1 |
2 |

3 | 4 |

5 | 6 |

7 | 8 |

9 | 10 |

11 | 12 |

13 | 14 | 33 | 34 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/receiveMessage.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const redis = require('./pocketwatch-cache/redis.js'); 4 | const performTask = require('./performTask.js'); 5 | const enqueueMessage = require('./enqueueMessage.js'); 6 | const MAX = 660; // 11 * 60 ( 11 minutes in seconds ) 7 | const warning = "TIMEKEEPER RECEIVE MESSAGE WARNING"; 8 | 9 | module.exports = receiveMessage; 10 | 11 | async function receiveMessage({message}) { 12 | // initialization 13 | const ct = + new Date; 14 | const { 15 | will_next_execute_at, will_end_after 16 | } = message; 17 | // see the message 18 | const msgDeduplicatorKey = `${message.origin}/${message.msgName}`; 19 | const isDuplicate = await redis.is_message_duplicate(msgDeduplicatorKey, message); 20 | if ( isDuplicate ) { 21 | console.info("\n"); 22 | console.warn(JSON.stringify({warning,isDuplicate,msgDeduplicatorKey})); 23 | return false; 24 | } 25 | if ( message.being_reinserted ) { 26 | // when a message is reinserted we do not see only delete it from insert table 27 | delete message.being_reinserted; 28 | } else { 29 | // if it is not reinserted we do delete all table entries 30 | await redis.see_interval( message.keyName ); 31 | } 32 | const is_marked_for_deletion = await redis.is_marked_for_deletion( message.keyName ); 33 | if ( is_marked_for_deletion ) { 34 | return 'not reinserted as marked for deletion'; 35 | } 36 | // task performance branching 37 | let taskPerformed = false; 38 | if ( ct >= will_next_execute_at ) { // be prompt with executing tasks ( execute on OR after ) 39 | performTask(message); 40 | taskPerformed = true; 41 | } 42 | // message enqueuing branching 43 | if ( ct > will_end_after ) { // be generous with people's intevals ( only end *after* ) 44 | // don't enqueue another message 45 | } else { 46 | if ( taskPerformed ) { 47 | message.last_executed_at = ct; 48 | message.will_next_execute_at = ct + message.interval_seconds*1000; 49 | } else { 50 | // don't update last and next execute 51 | } 52 | if ( message.will_next_execute_at > ct + (MAX*1000) ) { 53 | message.delay_seconds = MAX; 54 | } else { 55 | message.delay_seconds = Math.max(1, 56 | Math.ceil((message.will_next_execute_at - ct)/1000)); 57 | } 58 | return enqueueMessage(message); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/redis.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const EC = { 4 | name: process.env.NODE_ENV !== 'dev' ? 5 | 'localhost' : 6 | '', 7 | port: 6379 8 | }; 9 | 10 | const Redis = require('redis'); 11 | const bluebird = require('bluebird'); 12 | bluebird.promisifyAll(Redis.RedisClient.prototype); 13 | bluebird.promisifyAll(Redis.Multi.prototype); 14 | 15 | let client; 16 | 17 | connect(); 18 | 19 | const redis = client; 20 | 21 | const api = { 22 | see_interval, add_interval, remove_interval, get_missing_intervals, 23 | is_message_duplicate, 24 | add_origin, 25 | mark_for_deletion, 26 | is_marked_for_deletion, 27 | clear, 28 | client 29 | }; 30 | 31 | Object.assign( redis, api ); 32 | 33 | module.exports = redis; 34 | 35 | function connect() { 36 | if ( !! client ) { 37 | client.end(true); 38 | } 39 | client = Redis.createClient(EC.port, EC.name, {retry_strategy}); 40 | } 41 | 42 | function clear() { 43 | return redis.flushallAsync(); 44 | } 45 | 46 | function is_marked_for_deletion( id ) { 47 | return redis.client.getAsync(`deleting:${id}`); 48 | } 49 | 50 | function mark_for_deletion( id, timer_delay = 1 ) { 51 | // notes on deletion 52 | // we set the key long enough for either our whole system to reboot 53 | // 10 minutes is an overestimate 54 | // or for 3 intervals to pass 55 | // the idea is if we haven't see the timer for 3 intervals 56 | // then it is not coming back 57 | // but if the delay is short 58 | // and our system goes down 59 | // we might miss it and 60 | // we need to wait at least until our system goes back to give us a chance to see 61 | // the timer 62 | // the weakness in this system is that we will not delete it if we purge redis 63 | // but we can fix that by writing to datastore as above 64 | // and then doing a deleteReconcile task 65 | // looking for intervals that are marked for deletion but not yet deleted 66 | // or some other such mechansim 67 | // we want to keep any heavy lifting ( datastore writes ) out of the critical path 68 | // in timekeeper receive message 69 | return redis.client.setAsync(`deleting:${id}`, 70 | "OK", "EX", Math.max(600,2*timer_delay)); 71 | } 72 | 73 | function is_message_duplicate( id, msg ) { 74 | return redis.getAsync( `messages:${id}` ).then( result => { 75 | if ( !! result ) { 76 | return true; 77 | } else { 78 | return redis.setAsync( `messages:${id}`, "OK", "EX", Math.max(30,msg.delay_seconds*3) ).then( result => false ); 79 | } 80 | }); 81 | } 82 | 83 | function see_interval( id ) { 84 | return remove_interval( id ); 85 | } 86 | 87 | function has_origin( i ) { 88 | return redis.getAsync( `inserted:${i.keyName}` ); 89 | } 90 | 91 | function add_origin( i ) { 92 | const originKey = `inserted:${i.keyName}`; 93 | const expire_time = Math.ceil(1.5*i.delay_seconds); 94 | return redis.setAsync( originKey, "OK", "EX", expire_time); 95 | } 96 | 97 | async function add_interval( i ) { 98 | const key = `interval:${i.keyName}`; 99 | const ji = JSON.stringify(i); 100 | const missing_timeout = 2*i.delay_seconds; 101 | const expire_timeout = Math.max(120,missing_timeout); 102 | const already_set = !! ( await redis.getAsync(key) ); 103 | if ( ! already_set ) { 104 | await redis.setAsync( key, ji, "EX", expire_timeout ); 105 | const currentTime = +new Date; 106 | const score = currentTime + missing_timeout*1000; 107 | await redis.zaddAsync('intervals',score,key); 108 | return `added:${key}`; 109 | } else { 110 | return `already_present_not_added:${key}`; 111 | } 112 | } 113 | 114 | function remove_interval( id ) { 115 | const key = `interval:${id}`; 116 | const originKey = `inserted:${id}`; 117 | return redis.delAsync( key ).then( 118 | () => redis.zremAsync( 'intervals', key ) 119 | ).then( 120 | () => redis.delAsync( originKey ) 121 | ); 122 | } 123 | 124 | async function get_missing_intervals() { 125 | const currentTime = +new Date; 126 | const keys = await redis.zrangebyscoreAsync( 'intervals', 0, currentTime ); 127 | if ( process.env.DEBUG == 'full' ) { 128 | console.log("\n"); 129 | console.log(JSON.stringify({missingIntervalKeys:keys})); 130 | } 131 | let intervals = []; 132 | if ( keys.length ) { 133 | intervals = await redis 134 | .mgetAsync(...keys) 135 | .then(json_intervals => json_intervals 136 | .filter( ji => !! ji ) 137 | .map( ji => JSON.parse(ji) ) 138 | ); 139 | } 140 | const intervals_to_insert = []; 141 | for ( const i of intervals ) { 142 | const has_o = await has_origin(i); 143 | const inserted = has_o == "OK"; 144 | if ( ! inserted ) { 145 | intervals_to_insert.push( i ); 146 | } 147 | } 148 | if ( process.env.DEBUG == 'full' ) { 149 | console.log("\n"); 150 | console.log(JSON.stringify({intervals_to_insert})); 151 | } 152 | return intervals_to_insert; 153 | } 154 | 155 | function retry_strategy(options) { 156 | if (options.error && options.error.code === 'ECONNREFUSED') { 157 | // End reconnecting on a specific error and flush all commands with 158 | // a individual error 159 | return new Error('The server refused the connection'); 160 | } 161 | if (options.total_retry_time > 1000 * 60 * 60) { 162 | // End reconnecting after a specific timeout and flush all commands 163 | // with a individual error 164 | return new Error('Retry time exhausted'); 165 | } 166 | if (options.attempt > 10) { 167 | // End reconnecting with built in error 168 | return undefined; 169 | } 170 | // reconnect after 171 | return Math.min(options.attempt * 100, 3000); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/sm.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const fs = require('fs'); 5 | const cp = require('child_process'); 6 | const path = require('path'); 7 | const {promisify} = require('util'); 8 | 9 | const readdir = promisify(fs.readdir); 10 | const stat = promisify(fs.stat); 11 | const access = promisify(fs.access); 12 | const exec = promisify(cp.exec); 13 | 14 | const delay = ms => new Promise(res => setTimeout(res, ms)); 15 | 16 | perform(); 17 | 18 | async function perform() { 19 | const thisDir = path.join(__dirname); 20 | await exec('npm i; npm rebuild;'); 21 | await exec('npm set progress=false'); 22 | //await exec('npm i -g pnpm'); 23 | await recurser( thisDir ); 24 | await delay(1000); 25 | } 26 | 27 | async function recurser( dir ) { 28 | const message = {status:`installed in ${dir}`,dir}; 29 | console.log("\n"); 30 | console.log(JSON.stringify(message)); 31 | await delay(500); 32 | const files = await readdir( dir ); 33 | for( const f of files ) { 34 | try { 35 | const isDir = (await stat(path.join(dir, f))).isDirectory(); 36 | if ( isDir ) { 37 | const isSubmodule = await stat(path.join(dir,f,'package.json')); 38 | if ( isDir && isSubmodule ) { 39 | await exec(`cd ${path.join(dir,f)}; npm i; npm rebuild;`); 40 | await recurser( path.join(dir,f) ); 41 | } 42 | } 43 | } catch(error) { 44 | if ( process.env.DEBUG == 'full' ) { 45 | const event = "Received error in npm install script (sm.js)"; 46 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 47 | console.log("\n"); 48 | console.error(JSON.stringify({event,error})); 49 | } 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-mechanism/wrapAsync.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const wrapAsync = fn => 4 | (req, res, next) => { 5 | return Promise.resolve(fn(req,res,next)).catch(next); 6 | }; 7 | 8 | module.exports = wrapAsync; 9 | } 10 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/.ebextensions/00_postdep.config: -------------------------------------------------------------------------------- 1 | files: 2 | "/opt/elasticbeanstalk/hooks/appdeploy/post/99_fix_node_permissions.sh": 3 | mode: "000755" 4 | owner: root 5 | group: root 6 | content: | 7 | #!/usr/bin/env bash 8 | chown -R nodejs:nodejs /tmp/.npm/ 9 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/.ebextensions/00_rootaccess.config: -------------------------------------------------------------------------------- 1 | container_commands: 2 | 01_enable_rootaccess: 3 | command: echo Defaults:root \!requiretty >> /etc/sudoers 4 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/.ebextensions/10_link_node.config: -------------------------------------------------------------------------------- 1 | files: 2 | "/opt/elasticbeanstalk/hooks/appdeploy/pre/some_job.sh": 3 | mode: "000755" 4 | owner: root 5 | group: root 6 | content: | 7 | #!/usr/bin/env bash 8 | ln -sf `ls -td /opt/elasticbeanstalk/node-install/node-* | head -1`/bin/node /bin/node 9 | ln -sf `ls -td /opt/elasticbeanstalk/node-install/node-* | head -1`/bin/npm /bin/npm 10 | 11 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/.gitignore: -------------------------------------------------------------------------------- 1 | *.rdb 2 | 3 | .*.swp 4 | 5 | node_modules 6 | 7 | # Elastic Beanstalk Files 8 | .elasticbeanstalk/* 9 | !.elasticbeanstalk/*.cfg.yml 10 | !.elasticbeanstalk/*.global.yml 11 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "gstore"] 2 | path = gstore 3 | url = git@git.dosaygo.com:/home/git/gstore/ 4 | [submodule "pocketwatch-cache"] 5 | path = pocketwatch-cache 6 | url = git@git.dosaygo.com:/home/git/pocketwatch-cache 7 | [submodule "pocketwatch-mechanism"] 8 | path = pocketwatch-mechanism 9 | url = git@git.dosaygo.com:/home/git/pocketwatch-mechanism 10 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/QU.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const QU = 'supervisor SQS endpoint'; 5 | 6 | module.exports = QU; 7 | } 8 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/TODO: -------------------------------------------------------------------------------- 1 | - we are adding intervals to supervision task queue even if they are already present. 2 | - this is not good 3 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/cron.yaml: -------------------------------------------------------------------------------- 1 | version: 1 2 | cron: 3 | - name: "check-supervision" 4 | url: "/check-supervision" 5 | schedule: "* * * * *" 6 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/enqueueSupervisorMessage.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const QU = require('./QU.js'); 4 | const promisify = require('./promisify.js'); 5 | const redis = require('./pocketwatch-cache/redis.js'); 6 | const crypto = require('crypto'); 7 | const AWS = require('aws-sdk'); 8 | 9 | AWS.config.update({region:'us-west-2'}); 10 | const sqs = new AWS.SQS({apiVersion:'2012-11-05'}); 11 | 12 | const sendMsg = promisify((...args) => sqs.sendMessage(...args)); 13 | 14 | module.exports = enqueueMessage; 15 | 16 | function newRandom() { 17 | return crypto.randomBytes(20).toString('hex'); 18 | } 19 | 20 | async function enqueueMessage( msg ) { 21 | const packagedMessage = createMessage(msg); 22 | const already_in_queue = await redis.client.getAsync(`s_inserted:${msg.keyName}`); 23 | if ( !! already_in_queue ) { 24 | return 'not enqueued: already in queue'; 25 | } 26 | const result = await sendMsg(packagedMessage); 27 | const resultStatus = makeStatus( result ); 28 | if ( process.env.DEBUG == 'full' ) { 29 | console.log("\n"); 30 | const {origin,msgName,keyName} = msg; 31 | console.log(JSON.stringify({resultStatus,message:{origin,msgName,keyName},QU})); 32 | } 33 | if ( resultStatus == 'success' ) { 34 | await redis.client.setAsync(`s_inserted:${msg.keyName}`,"OK","EX",msg.delay_seconds*2); 35 | } 36 | return resultStatus; 37 | } 38 | 39 | function createMessage( obj ) { 40 | obj.msgName = ( obj.msgName || 0 ) + 1; 41 | obj.origin = obj.origin || newRandom(); 42 | const MessageBody = JSON.stringify(obj); 43 | const QueueUrl = QU; 44 | const DelaySeconds = obj.delay_seconds || 0; 45 | const params = { 46 | MessageBody, QueueUrl, DelaySeconds 47 | } 48 | return params; 49 | } 50 | 51 | function makeStatus( [ err, data ] = [] ) { 52 | if ( !! err ) { 53 | return `fail`; 54 | } else if ( !! data ) { 55 | return `success`; 56 | } else { 57 | return `unknown`; 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/errors.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const service = 'supervisor'; 4 | const errors = { 5 | log, xhr, html, errorView 6 | }; 7 | 8 | module.exports = errors; 9 | 10 | function log(error, req, res, next) { 11 | const event = "received error"; 12 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 13 | console.log("\n"); 14 | console.error(JSON.stringify({service,event,error})); 15 | next(error); 16 | } 17 | 18 | function xhr(err, req, res, next) { 19 | if ( req.xhr ) { 20 | res.status(500).end({ error: 'Something failed!' }); 21 | } else { 22 | next(err); 23 | } 24 | } 25 | 26 | function html(err, req, res, next) { 27 | res.type('html'); 28 | res.end( 29 | errorView({message: err.message || 'unspecified error'}) 30 | ); 31 | } 32 | 33 | function safe( data ) { 34 | const dataString = JSON.stringify(data); 35 | const safe = dataString.replace(/&/g, '&').replace(//g, '>'); 36 | return JSON.parse(safe); 37 | } 38 | 39 | function errorView( data ) { 40 | data = safe(data); 41 | return ` 42 |
43 |

An error occurred

44 |
45 |

46 | ${data.message} 47 |

48 |
49 | 50 |
51 | 59 |
60 | `; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/functions.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const enqueueMechanismMessage = require('./pocketwatch-mechanism/enqueueMessage.js'); 5 | const ds = require('./gstore/ds.js'); 6 | const redis = require('./pocketwatch-cache/redis.js'); 7 | const supervision = require('./supervision.js'); 8 | 9 | const namespace = 'pocketwatch-gen-1'; 10 | const kind = 'Interval'; 11 | 12 | const functions = { 13 | load_live_intervals, reinsert_missing_intervals 14 | }; 15 | 16 | module.exports = functions; 17 | 18 | async function load_live_intervals() { 19 | const currentTime = +new Date; 20 | await redis.client.setAsync(supervision.tasks.loadIntervals.type, currentTime); 21 | const q = { 22 | lines: [ 23 | { 24 | type: 'filter', 25 | prop: 'will_end_after', 26 | op: '>', 27 | val: currentTime 28 | }, 29 | { 30 | type: 'filter', 31 | prop: 'marked_for_deletion', 32 | op: '=', 33 | val: false 34 | } 35 | ] 36 | } 37 | const load_results = await ds.query({namespace,kind,q}).then( ([results,info]) => { 38 | if ( process.env.DEBUG == 'full' ) { 39 | console.log("\n"); 40 | console.log( 41 | JSON.stringify({ 42 | count:results.length, 43 | currentTime, 44 | liveIntervals:results.map(i => i.keyName) 45 | })); 46 | } 47 | return Promise.all( results.map( i => redis.add_interval( i ) ) ); 48 | }); 49 | if ( process.env.DEBUG == 'full' ) { 50 | console.log("\n"); 51 | console.log({load_results,currentTime}); 52 | } 53 | return load_results; 54 | } 55 | 56 | async function reinsert_missing_intervals() { 57 | const currentTime = +new Date; 58 | const task = supervision.tasks.reconcileIntervals; 59 | return redis.get_missing_intervals().then( async intervals => { 60 | await redis.client.setAsync(task.type, currentTime); 61 | if ( process.env.DEBUG == 'full' ) { 62 | console.log("\n"); 63 | console.log({missing_intervals:intervals.map( i => i.keyName )}); 64 | } 65 | return Promise.all(intervals.map(message => { 66 | message.being_reinserted = true; 67 | if ( process.env.DEBUG == 'full' ) { 68 | const {keyName} = message; 69 | console.log("\n"); 70 | console.log( 71 | JSON.stringify({reinserting:{keyName}})); 72 | } 73 | return enqueueMechanismMessage(message); 74 | })); 75 | }); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | #!include:.gitignore 18 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/.gitignore: -------------------------------------------------------------------------------- 1 | *.zip 2 | 3 | node_modules 4 | 5 | .*.swp 6 | 7 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/deploy: -------------------------------------------------------------------------------- 1 | gcloud beta functions deploy gstore --trigger-http 2 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/ds.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const crypto = require('crypto'); 4 | const path = require('path'); 5 | const Datastore = require('@google-cloud/datastore'); 6 | const projectId = ''; 7 | const keyFilename = path.join(__dirname, 'gcp-ds-key.json'); 8 | const datastore = new Datastore({projectId,keyFilename}); 9 | const ds = { 10 | test, save, query, keyFromName, newRandom, update, 11 | "delete": del 12 | }; 13 | 14 | module.exports = ds; 15 | 16 | if ( require.main === module ) { 17 | (function() { 18 | test().then( r => console.log(r)); 19 | }()); 20 | } 21 | 22 | function newRandom() { 23 | return crypto.randomBytes(20).toString('hex'); 24 | } 25 | 26 | function keyFromName( {namespace,kind,name} = {} ) { 27 | const key = datastore.key({namespace,path:[kind,name]}); 28 | return key; 29 | } 30 | 31 | function newKey( {namespace,kind} = {} ) { 32 | if ( ! kind || ! namespace ) { 33 | throw new TypeError("Supply kind and namespace"); 34 | } 35 | const randName = newRandom(); 36 | const key = datastore.key({namespace, path:[kind, randName]}); 37 | return key; 38 | } 39 | 40 | async function query( {namespace, kind, q} = {} ) { 41 | const { lines, cursor } = typeof q == "string" ? JSON.parse(q) : q; 42 | let query = datastore.createQuery( namespace, kind ); 43 | while( lines.length ) { 44 | const nextLine = lines.shift(); 45 | switch( nextLine.type ) { 46 | case "order": 47 | query = query.order(nextLine.prop, nextLine.dir); 48 | break; 49 | case "filter": 50 | query = query.filter(nextLine.prop, nextLine.op, nextLine.val); 51 | break; 52 | case "groupBy": 53 | query = query.groupBy(nextLine.prop); 54 | break; 55 | case "select": 56 | query = query.select(nextLine.prop || nextLine.propArray); 57 | break; 58 | case "limit": 59 | query = query.limit(nextLine.limit); 60 | break; 61 | case "default": 62 | throw new TypeError(`Invalid query line type ${nextLine}`); 63 | break; 64 | } 65 | } 66 | if ( !! cursor ) { 67 | query = query.start(cursor); 68 | } 69 | const result = await datastore.runQuery(query); 70 | return result; 71 | } 72 | 73 | async function save( {namespace, kind, excludeFromIndexes: excludeFromIndexes = [], data} = {} ) { 74 | const key = newKey({namespace,kind}); 75 | const keyName = key.name; 76 | data.keyName = keyName; 77 | const entity = { key, data, excludeFromIndexes }; 78 | const resp = await datastore.insert(entity); 79 | return {resp,keyName}; 80 | } 81 | 82 | async function del( {namespace, kind, keyName} = {} ) { 83 | const key = datastore.key({namespace,path:[kind,keyName]}); 84 | const result = await datastore.delete(key); 85 | return result; 86 | } 87 | 88 | 89 | async function update( {namespace,kind,data} = {} ) { 90 | const key = datastore.key({namespace,path:[kind,data.keyName]}); 91 | remove_undefined_keys(data); 92 | const entity = { key, data }; 93 | const resp = await datastore.upsert(entity); 94 | return {resp,keyName:data.keyName}; 95 | } 96 | 97 | function test() { 98 | // The kind for the new entity 99 | const kind = 'Task'; 100 | const namespace = 'test1'; 101 | const data = { 102 | description: 'Buy milk', 103 | }; 104 | 105 | // Saves the entity 106 | return save({namespace,kind,data}); 107 | } 108 | 109 | function remove_undefined_keys(o) { 110 | const keys = Object.keys(o); 111 | const to_remove = []; 112 | for( const k of keys ) { 113 | if ( o[k] == undefined || o[k] == null || o[k] == '' ) { 114 | to_remove.push(k); 115 | } 116 | } 117 | to_remove.forEach( k => o[k] = ' ' ); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/expressToGCF.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const app = require('./server.js'); 4 | 5 | module.exports = ( req, res ) => { 6 | if ( ! req.path ) { 7 | req.url = '/'; 8 | } 9 | return app( req, res ); 10 | }; 11 | } 12 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/gcp-ds-key.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | } 4 | 5 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/hardcodedCredentials.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const hardcodedCredentials = new Set([ 5 | "api key for special access to gstore from pocketwatch" 6 | ]); 7 | 8 | module.exports = hardcodedCredentials; 9 | } 10 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/index.html: -------------------------------------------------------------------------------- 1 | GSTORE TEST 2 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const expressToGCF = require('./expressToGCF.js'); 4 | 5 | exports.gstore = expressToGCF; 6 | } 7 | 8 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gstore", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "watch": { 7 | "start": { 8 | "patterns": [ 9 | "*" 10 | ], 11 | "extensions": "js,html,css", 12 | "quiet": false 13 | }, 14 | "dev": { 15 | "patterns": [ 16 | "*" 17 | ], 18 | "extensions": "js,html,css", 19 | "quiet": false 20 | } 21 | }, 22 | "scripts": { 23 | "start": "node index.js", 24 | "endless": "node ./node_modules/pm2/bin/pm2 start index.js --name gstore", 25 | "postendless": "node ./node_modules/pm2/bin/pm2 logs", 26 | "dev": "node server.js", 27 | "watch": "npm-watch", 28 | "test": "npm run watch dev" 29 | }, 30 | "repository": { 31 | "type": "git", 32 | "url": "git@git.dosaygo.com:/home/git/gstore" 33 | }, 34 | "author": "@dosy", 35 | "private": true, 36 | "license": "MIT", 37 | "dependencies": { 38 | "@google-cloud/datastore": "^1.4.0", 39 | "@google-cloud/storage": "^1.7.0", 40 | "body-parser": "^1.18.3", 41 | "crypto": "^1.0.1", 42 | "express": "^4.16.3", 43 | "npm-watch": "^0.3.0" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/public/console.html: -------------------------------------------------------------------------------- 1 |
2 |

3 | 4 | 5 | 6 | 7 | 8 |

9 | 10 |

11 |
12 | 17 |

18 | 19 | 20 | 21 |

22 | 23 |

24 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/public/index.html: -------------------------------------------------------------------------------- 1 | GSTORE 2 | 7 | CONSOLE 8 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/gstore/server.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const credentials = require('./hardcodedCredentials.js'); 4 | const exp = require('express'); 5 | const ds = require('./ds.js'); 6 | 7 | const app = exp(); 8 | 9 | if ( require.main == module ) { 10 | const bodyParser = require('body-parser'); 11 | app.use(bodyParser.urlencoded({extended:true})); 12 | } 13 | 14 | app.use("/",exp.static("public")); 15 | 16 | app.post("/save/:namespace/:kind/", (req,res,next) => { 17 | const {namespace,kind} = req.params; 18 | const data = Object.assign({},req.body); 19 | console.log(data); 20 | if ( credentials.has(data.apikey) ) { 21 | ds.save({namespace,kind,data}).then( result => res.end("saved")); 22 | } else { 23 | res.end("forbidden"); 24 | } 25 | }); 26 | 27 | app.get("/query/:namespace/:kind/", (req,res,next) => { 28 | const {namespace,kind} = req.params; 29 | const { q, apikey } = req.query; 30 | if ( credentials.has(apikey) ) { 31 | ds.query({namespace,kind,q}).then( result => res.end(JSON.stringify(result)) ); 32 | } else { 33 | res.end("forbidden"); 34 | } 35 | }); 36 | 37 | module.exports = app; 38 | 39 | if ( require.main == module ) { 40 | const port = process.env.PORT || 8080; 41 | const server = app.listen(port, () => console.log(`Server up at ${new Date()} on port ${port}`)); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const service = 'supervisor'; 4 | 5 | process.on('unhandledRejection', error => { 6 | const event = "Received unhandled promise rejection"; 7 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 8 | console.log("\n"); 9 | console.error(JSON.stringify({service,event,error})); 10 | }); 11 | process.on('uncaughtException', error => { 12 | const event = "Received uncaught exception"; 13 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 14 | console.log("\n"); 15 | console.error(JSON.stringify({service,event,error})); 16 | }); 17 | 18 | const supervision = require('./supervision.js'); 19 | const receiveMessage = require('./receiveMessage.js'); 20 | const ds = require('./gstore/ds.js'); 21 | const exp = require('express'); 22 | const bodyParser = require('body-parser'); 23 | const path = require('path'); 24 | const wrapAsync = require('./wrapAsync.js'); 25 | const errors = require('./errors.js'); 26 | 27 | const app = exp(); 28 | const port = process.env.PORT || 8080; 29 | 30 | app.use("/", exp.static(path.join(__dirname, "public"))); 31 | app.use(bodyParser.urlencoded({extended:true})); 32 | app.use(bodyParser.json({extended:true})); 33 | 34 | app.get("/", async (req,res,next) => { 35 | res.type("text").status(200).end("OK"); 36 | }); 37 | 38 | app.post("/inbox", wrapAsync(async (req,res,next) => { 39 | const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress; 40 | const time = new Date(); 41 | // init 42 | const message = !! req.body.MessageBody ? JSON.parse(req.body.MessageBody) : req.body; 43 | const {keyName,msgName,origin} = message; 44 | console.log("\n"); 45 | console.log( 46 | JSON.stringify({service,ip,time,path:'/inbox',message:{keyName,msgName,origin}})); 47 | // receive it 48 | const receiveResult = await receiveMessage({message}); 49 | // response ( always OK ) 50 | res.type("text").status(200).end("OK"); 51 | })); 52 | 53 | app.post("/check-supervision", async (req,res,next) => { 54 | const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress; 55 | console.log("\n"); 56 | console.log( 57 | JSON.stringify({service,ip,time: new Date(),path:req.originalUrl})); 58 | await supervision.check(); 59 | res.end("OK"); 60 | }); 61 | 62 | app.use(errors.log); 63 | app.use(errors.xhr); 64 | app.use(errors.html); 65 | app.use((req,res,next) => { 66 | res.status(404).send(errors.errorView({message: 'Page not found'})); 67 | }); 68 | 69 | const server = app.listen(port, () => { 70 | const currentTime = new Date; 71 | const serverLive = `Server up at ${currentTime} on port ${port}`; 72 | console.log("\n"); 73 | console.log(JSON.stringify({serverLive,port,currentTime})); 74 | if ( process.env.NODE_ENV !== 'dev' ) { 75 | setInterval(() => { 76 | const currentTime = new Date; 77 | console.log("\n"); 78 | console.log(JSON.stringify({supervisionCheck:{currentTime}})); 79 | supervision.check(); 80 | }, 81 | 5*1000 82 | ); 83 | } 84 | return true; 85 | }); 86 | 87 | module.exports = server; 88 | } 89 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketwatch-supervisor", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "watch": { 7 | "start": { 8 | "patterns": [ 9 | "*" 10 | ], 11 | "extensions": "js,html,css", 12 | "quiet": false 13 | }, 14 | "dev": { 15 | "patterns": [ 16 | "*" 17 | ], 18 | "extensions": "js,html,css", 19 | "quiet": false 20 | } 21 | }, 22 | "scripts": { 23 | "prestart": "node sm.js", 24 | "start": "node index.js", 25 | "dev": "node index.js", 26 | "watch": "npm-watch", 27 | "test": "npm run watch dev" 28 | }, 29 | "repository": { 30 | "type": "git", 31 | "url": "git@git.dosaygo.com:/home/git/pocketwatch-supervisor" 32 | }, 33 | "author": "@dosy", 34 | "license": "MIT", 35 | "dependencies": { 36 | "aws-sdk": "^2.224.1", 37 | "express": "^4.16.3", 38 | "node-fetch": "^2.1.2", 39 | "npm-watch": "^0.3.0", 40 | "pm2": "^2.10.2", 41 | "redis": "^2.8.0" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-cache/.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | 3 | node_modules 4 | 5 | # Elastic Beanstalk Files 6 | .elasticbeanstalk/* 7 | !.elasticbeanstalk/*.cfg.yml 8 | !.elasticbeanstalk/*.global.yml 9 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-cache/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketwatch-cache", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "bluebird": { 8 | "version": "3.5.1", 9 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", 10 | "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" 11 | }, 12 | "double-ended-queue": { 13 | "version": "2.1.0-0", 14 | "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz", 15 | "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=" 16 | }, 17 | "redis": { 18 | "version": "2.8.0", 19 | "resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz", 20 | "integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==", 21 | "requires": { 22 | "double-ended-queue": "^2.1.0-0", 23 | "redis-commands": "^1.2.0", 24 | "redis-parser": "^2.6.0" 25 | } 26 | }, 27 | "redis-commands": { 28 | "version": "1.3.5", 29 | "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.3.5.tgz", 30 | "integrity": "sha512-foGF8u6MXGFF++1TZVC6icGXuMYPftKXt1FBT2vrfU9ZATNtZJ8duRC5d1lEfE8hyVe3jhelHGB91oB7I6qLsA==" 31 | }, 32 | "redis-parser": { 33 | "version": "2.6.0", 34 | "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz", 35 | "integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs=" 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-cache/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketwatch-cache", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "npm test" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git@git.dosaygo.com:/home/git/pocketwatch-cache" 12 | }, 13 | "author": "@dosy", 14 | "license": "MIT", 15 | "dependencies": { 16 | "bluebird": "^3.5.1", 17 | "redis": "^2.8.0" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-cache/redis.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const EC = { 4 | name: process.env.NODE_ENV !== 'dev' ? 5 | 'localhost' : 6 | '', 7 | port: 6379 8 | }; 9 | 10 | const Redis = require('redis'); 11 | const bluebird = require('bluebird'); 12 | bluebird.promisifyAll(Redis.RedisClient.prototype); 13 | bluebird.promisifyAll(Redis.Multi.prototype); 14 | 15 | let client; 16 | 17 | connect(); 18 | 19 | const redis = client; 20 | 21 | const api = { 22 | see_interval, add_interval, remove_interval, get_missing_intervals, 23 | is_message_duplicate, 24 | add_origin, 25 | mark_for_deletion, 26 | is_marked_for_deletion, 27 | clear, 28 | client 29 | }; 30 | 31 | Object.assign( redis, api ); 32 | 33 | module.exports = redis; 34 | 35 | function connect() { 36 | if ( !! client ) { 37 | client.end(true); 38 | } 39 | client = Redis.createClient(EC.port, EC.name, {retry_strategy}); 40 | } 41 | 42 | function clear() { 43 | return redis.flushallAsync(); 44 | } 45 | 46 | function is_marked_for_deletion( id ) { 47 | return redis.client.getAsync(`deleting:${id}`); 48 | } 49 | 50 | function mark_for_deletion( id, timer_delay = 1 ) { 51 | // notes on deletion 52 | // we set the key long enough for either our whole system to reboot 53 | // 10 minutes is an overestimate 54 | // or for 3 intervals to pass 55 | // the idea is if we haven't see the timer for 3 intervals 56 | // then it is not coming back 57 | // but if the delay is short 58 | // and our system goes down 59 | // we might miss it and 60 | // we need to wait at least until our system goes back to give us a chance to see 61 | // the timer 62 | // the weakness in this system is that we will not delete it if we purge redis 63 | // but we can fix that by writing to datastore as above 64 | // and then doing a deleteReconcile task 65 | // looking for intervals that are marked for deletion but not yet deleted 66 | // or some other such mechansim 67 | // we want to keep any heavy lifting ( datastore writes ) out of the critical path 68 | // in timekeeper receive message 69 | return redis.client.setAsync(`deleting:${id}`, 70 | "OK", "EX", Math.max(600,2*timer_delay)); 71 | } 72 | 73 | function is_message_duplicate( id, msg ) { 74 | return redis.getAsync( `messages:${id}` ).then( result => { 75 | if ( !! result ) { 76 | return true; 77 | } else { 78 | return redis.setAsync( `messages:${id}`, "OK", "EX", Math.max(30,msg.delay_seconds*3) ).then( result => false ); 79 | } 80 | }); 81 | } 82 | 83 | function see_interval( id ) { 84 | return remove_interval( id ); 85 | } 86 | 87 | function has_origin( i ) { 88 | return redis.getAsync( `inserted:${i.keyName}` ); 89 | } 90 | 91 | function add_origin( i ) { 92 | const originKey = `inserted:${i.keyName}`; 93 | const expire_time = Math.ceil(1.5*i.delay_seconds); 94 | return redis.setAsync( originKey, "OK", "EX", expire_time); 95 | } 96 | 97 | async function add_interval( i ) { 98 | const key = `interval:${i.keyName}`; 99 | const ji = JSON.stringify(i); 100 | const missing_timeout = 2*i.delay_seconds; 101 | const expire_timeout = Math.max(120,missing_timeout); 102 | const already_set = !! ( await redis.getAsync(key) ); 103 | if ( ! already_set ) { 104 | await redis.setAsync( key, ji, "EX", expire_timeout ); 105 | const currentTime = +new Date; 106 | const score = currentTime + missing_timeout*1000; 107 | await redis.zaddAsync('intervals',score,key); 108 | return `added:${key}`; 109 | } else { 110 | return `already_present_not_added:${key}`; 111 | } 112 | } 113 | 114 | function remove_interval( id ) { 115 | const key = `interval:${id}`; 116 | const originKey = `inserted:${id}`; 117 | return redis.delAsync( key ).then( 118 | () => redis.zremAsync( 'intervals', key ) 119 | ).then( 120 | () => redis.delAsync( originKey ) 121 | ); 122 | } 123 | 124 | async function get_missing_intervals() { 125 | const currentTime = +new Date; 126 | const keys = await redis.zrangebyscoreAsync( 'intervals', 0, currentTime ); 127 | if ( process.env.DEBUG == 'full' ) { 128 | console.log("\n"); 129 | console.log(JSON.stringify({missingIntervalKeys:keys})); 130 | } 131 | let intervals = []; 132 | if ( keys.length ) { 133 | intervals = await redis 134 | .mgetAsync(...keys) 135 | .then(json_intervals => json_intervals 136 | .filter( ji => !! ji ) 137 | .map( ji => JSON.parse(ji) ) 138 | ); 139 | } 140 | const intervals_to_insert = []; 141 | for ( const i of intervals ) { 142 | const has_o = await has_origin(i); 143 | const inserted = has_o == "OK"; 144 | if ( ! inserted ) { 145 | intervals_to_insert.push( i ); 146 | } 147 | } 148 | if ( process.env.DEBUG == 'full' ) { 149 | console.log("\n"); 150 | console.log(JSON.stringify({intervals_to_insert})); 151 | } 152 | return intervals_to_insert; 153 | } 154 | 155 | function retry_strategy(options) { 156 | if (options.error && options.error.code === 'ECONNREFUSED') { 157 | // End reconnecting on a specific error and flush all commands with 158 | // a individual error 159 | return new Error('The server refused the connection'); 160 | } 161 | if (options.total_retry_time > 1000 * 60 * 60) { 162 | // End reconnecting after a specific timeout and flush all commands 163 | // with a individual error 164 | return new Error('Retry time exhausted'); 165 | } 166 | if (options.attempt > 10) { 167 | // End reconnecting with built in error 168 | return undefined; 169 | } 170 | // reconnect after 171 | return Math.min(options.attempt * 100, 3000); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/.ebextensions/00_postdep.config: -------------------------------------------------------------------------------- 1 | files: 2 | "/opt/elasticbeanstalk/hooks/appdeploy/post/99_fix_node_permissions.sh": 3 | mode: "000755" 4 | owner: root 5 | group: root 6 | content: | 7 | #!/usr/bin/env bash 8 | chown -R nodejs:nodejs /tmp/.npm/ 9 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/.ebextensions/00_rootaccess.config: -------------------------------------------------------------------------------- 1 | container_commands: 2 | 01_enable_rootaccess: 3 | command: echo Defaults:root \!requiretty >> /etc/sudoers 4 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/.ebextensions/10_link_node.config: -------------------------------------------------------------------------------- 1 | files: 2 | "/opt/elasticbeanstalk/hooks/appdeploy/pre/some_job.sh": 3 | mode: "000755" 4 | owner: root 5 | group: root 6 | content: | 7 | #!/usr/bin/env bash 8 | ln -sf `ls -td /opt/elasticbeanstalk/node-install/node-* | head -1`/bin/node /bin/node 9 | ln -sf `ls -td /opt/elasticbeanstalk/node-install/node-* | head -1`/bin/npm /bin/npm 10 | 11 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/.gitignore: -------------------------------------------------------------------------------- 1 | *.rdb 2 | 3 | node_modules 4 | 5 | # Elastic Beanstalk Files 6 | .elasticbeanstalk/* 7 | !.elasticbeanstalk/*.cfg.yml 8 | !.elasticbeanstalk/*.global.yml 9 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "pocketwatch-cache"] 2 | path = pocketwatch-cache 3 | url = git@git.dosaygo.com:/home/git/pocketwatch-cache 4 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/QU.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const QU = 'mechanism SQS endpoint'; 5 | 6 | module.exports = QU; 7 | } 8 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/enqueueMessage.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const QU = require('./QU.js'); 4 | const redis = require('./pocketwatch-cache/redis.js'); 5 | const promisify = require('./promisify.js'); 6 | const crypto = require('crypto'); 7 | const AWS = require('aws-sdk'); 8 | 9 | AWS.config.update({region:'us-west-2'}); 10 | const sqs = new AWS.SQS({apiVersion:'2012-11-05'}); 11 | 12 | const sendMsg = promisify((...args) => sqs.sendMessage(...args)); 13 | 14 | module.exports = enqueueMessage; 15 | 16 | function newRandom() { 17 | return crypto.randomBytes(20).toString('hex'); 18 | } 19 | 20 | async function enqueueMessage( msg ) { 21 | if ( !! msg.first_inject ) { 22 | delete msg.first_inject; 23 | const ct = +new Date; 24 | msg.will_next_execute_at = ct + msg.interval_seconds * 1000; 25 | } 26 | const packagedMessage = createMessage(msg); 27 | await redis.add_origin(msg); 28 | const result = await sendMsg(packagedMessage); 29 | const resultStatus = makeStatus( result ); 30 | const msgDeduplicatorKey = `${msg.origin}/${msg.msgName}`; 31 | //console.log(JSON.stringify({QU,msgDeduplicatorKey,resultStatus})); 32 | return resultStatus; 33 | } 34 | 35 | function createMessage( obj ) { 36 | obj.msgName = ( obj.msgName || 0 ) + 1; 37 | obj.origin = obj.origin || newRandom(); 38 | const MessageBody = JSON.stringify(obj); 39 | const QueueUrl = QU; 40 | const DelaySeconds = obj.delay_seconds || 0; 41 | const params = { 42 | MessageBody, QueueUrl, DelaySeconds 43 | } 44 | return params; 45 | } 46 | 47 | function makeStatus( [ err, data ] = [] ) { 48 | if ( !! err ) { 49 | return `fail`; 50 | } else if ( !! data ) { 51 | return `success`; 52 | } else { 53 | return `unknown`; 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/errors.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const service = 'timekeeper'; 4 | const errors = { 5 | log, xhr, html, errorView 6 | }; 7 | 8 | module.exports = errors; 9 | 10 | function log(error, req, res, next) { 11 | const event = "received error"; 12 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 13 | console.log("\n"); 14 | console.error(JSON.stringify({service,event,error})); 15 | next(error); 16 | } 17 | 18 | function xhr(err, req, res, next) { 19 | if ( req.xhr ) { 20 | res.status(500).end({ error: 'Something failed!' }); 21 | } else { 22 | next(err); 23 | } 24 | } 25 | 26 | function html(err, req, res, next) { 27 | res.type('html'); 28 | res.end('unspecified error'); 29 | } 30 | 31 | function safe( data ) { 32 | const dataString = JSON.stringify(data); 33 | const safe = dataString.replace(/&/g, '&').replace(//g, '>'); 34 | return JSON.parse(safe); 35 | } 36 | 37 | function errorView( data ) { 38 | data = safe(data); 39 | return ` 40 |
41 |

An error occurred

42 |
43 |

44 | ${data.message} 45 |

46 |
47 | 48 |
49 | 57 |
58 | `; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const service = 'timekeeper'; 4 | 5 | process.on('unhandledRejection', error => { 6 | const event = "Received unhandled promise rejection"; 7 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 8 | console.log("\n"); 9 | console.error(JSON.stringify({service,event,error})); 10 | }); 11 | process.on('uncaughtException', error => { 12 | const event = "Received uncaught exception"; 13 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 14 | console.log("\n"); 15 | console.error(JSON.stringify({service,event,error})); 16 | }); 17 | 18 | const receiveMessage = require('./receiveMessage.js'); 19 | const redis = require('./pocketwatch-cache/redis.js'); 20 | const exp = require('express'); 21 | const bodyParser = require('body-parser'); 22 | const path = require('path'); 23 | const wrapAsync = require('./wrapAsync.js'); 24 | const errors = require('./errors.js'); 25 | 26 | const app = exp(); 27 | const port = process.env.PORT || 8080; 28 | 29 | app.use("/", exp.static(path.join(__dirname, "public"))); 30 | app.use(bodyParser.urlencoded({extended:true})); 31 | app.use(bodyParser.json({extended:true})); 32 | 33 | app.get("/", async (req,res,next) => { 34 | if ( process.env.NODE_ENV !== 'dev' ) { 35 | const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress; 36 | const time = new Date(); 37 | const path = req.originalUrl; 38 | const method = req.method; 39 | const body = req.body; 40 | const q = req.query; 41 | console.log("\n"); 42 | console.log(JSON.stringify({request:{service,method,path,body,q,ip,time}})); 43 | } 44 | res.type('text').status(200).end("OK"); 45 | }); 46 | 47 | app.post("/inbox", wrapAsync(async (req,res,next) => { 48 | const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress; 49 | const time = new Date(); 50 | // init 51 | console.log("\n"); 52 | const message = !! req.body.MessageBody ? JSON.parse(req.body.MessageBody) : req.body; 53 | const {keyName,msgName,origin} = message; 54 | console.log( 55 | JSON.stringify({service,ip,time,path:'/inbox',message:{keyName,msgName,origin}})); 56 | // receive it 57 | const receiveResult = await receiveMessage({message}); 58 | // response ( always OK ) 59 | return res.type("text").status(200).end("OK"); 60 | })); 61 | 62 | app.use(errors.log); 63 | app.use(errors.xhr); 64 | app.use(errors.html); 65 | app.use((req,res,next) => { 66 | res.status(404).send(errors.errorView({message: 'Page not found'})); 67 | }); 68 | 69 | const server = app.listen(port, () => { 70 | const currentTime = new Date; 71 | const serverLive = `Server up at ${currentTime} on port ${port}`; 72 | console.log(JSON.stringify({serverLive,port,currentTime})); 73 | return true; 74 | }); 75 | 76 | module.exports = server; 77 | } 78 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketwatch-mechanism", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "watch": { 7 | "start": { 8 | "patterns": [ 9 | "*" 10 | ], 11 | "extensions": "js,html,css", 12 | "quiet": false 13 | }, 14 | "dev": { 15 | "patterns": [ 16 | "*" 17 | ], 18 | "extensions": "js,html,css", 19 | "quiet": false 20 | } 21 | }, 22 | "scripts": { 23 | "prestart": "node sm.js", 24 | "start": "node index.js", 25 | "dev": "node index.js", 26 | "watch": "npm-watch", 27 | "test": "npm run watch dev" 28 | }, 29 | "repository": { 30 | "type": "git", 31 | "url": "git@git.dosaygo.com:/home/git/pocketwatch-mechanism" 32 | }, 33 | "author": "@dosy", 34 | "license": "MIT", 35 | "dependencies": { 36 | "aws-sdk": "^2.224.1", 37 | "express": "^4.16.3", 38 | "node-fetch": "^2.1.2", 39 | "npm-watch": "^0.3.0", 40 | "pm2": "^2.10.2" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/performTask.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const fetch = require('node-fetch'); 4 | const HEADER = { 5 | 'User-Agent': 'Pocketwatch Timer Fetch; Report Misuse to: criscanbereached+pw.bad.actors@gmail.com; Subscribe for your own timers at https://api.pocketwatch.xyz' 6 | }; 7 | 8 | module.exports = performTask; 9 | 10 | async function performTask(msg) { 11 | const { method, url } = msg; 12 | const { keyName, msgName, origin } = msg; 13 | let taskPromise; 14 | try { 15 | switch( method ) { 16 | case 'DELETE': 17 | case 'PUT': 18 | case 'PATCH': 19 | case 'POST': { 20 | let { body, contentType: contentType = 'application/x-www-form-urlencoded' } = msg; 21 | if ( !! body ) { 22 | if ( process.env.DEBUG == 'full' ) { 23 | console.log("\n"); 24 | console.log( 25 | JSON.stringify({message:"Body request",where:"performTask",body})); 26 | } 27 | } 28 | const headers = Object.assign({ 29 | 'Content-Type': contentType 30 | }, HEADER); 31 | taskPromise = await fetch(url, {method, headers, body}); 32 | break; 33 | } 34 | case 'HEAD': 35 | case 'GET': { 36 | taskPromise = await fetch(url, {method, headers: HEADER}); 37 | break; 38 | } 39 | } 40 | } catch(error) { 41 | if ( process.env.DEBUG == 'full' ) { 42 | const event = "performTask: error when making HTTP fetch"; 43 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 44 | console.log("\n"); 45 | console.warn( 46 | JSON.stringify({event,error, 47 | message:{keyName,msgName,origin}})); 48 | } 49 | return; 50 | } 51 | if ( process.env.DEBUG == 'full' ) { 52 | const event = "performTask: reached end of function block"; 53 | console.log("\n"); 54 | console.log( 55 | JSON.stringify({event, 56 | message:{keyName,msgName,origin}})); 57 | } 58 | return taskPromise; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/pocketwatch-cache/.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | 3 | node_modules 4 | 5 | # Elastic Beanstalk Files 6 | .elasticbeanstalk/* 7 | !.elasticbeanstalk/*.cfg.yml 8 | !.elasticbeanstalk/*.global.yml 9 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/pocketwatch-cache/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketwatch-cache", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "bluebird": { 8 | "version": "3.5.1", 9 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", 10 | "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" 11 | }, 12 | "double-ended-queue": { 13 | "version": "2.1.0-0", 14 | "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz", 15 | "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=" 16 | }, 17 | "redis": { 18 | "version": "2.8.0", 19 | "resolved": "https://registry.npmjs.org/redis/-/redis-2.8.0.tgz", 20 | "integrity": "sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A==", 21 | "requires": { 22 | "double-ended-queue": "^2.1.0-0", 23 | "redis-commands": "^1.2.0", 24 | "redis-parser": "^2.6.0" 25 | } 26 | }, 27 | "redis-commands": { 28 | "version": "1.3.5", 29 | "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.3.5.tgz", 30 | "integrity": "sha512-foGF8u6MXGFF++1TZVC6icGXuMYPftKXt1FBT2vrfU9ZATNtZJ8duRC5d1lEfE8hyVe3jhelHGB91oB7I6qLsA==" 31 | }, 32 | "redis-parser": { 33 | "version": "2.6.0", 34 | "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz", 35 | "integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs=" 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/pocketwatch-cache/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketwatch-cache", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "npm test" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git@git.dosaygo.com:/home/git/pocketwatch-cache" 12 | }, 13 | "author": "@dosy", 14 | "license": "MIT", 15 | "dependencies": { 16 | "bluebird": "^3.5.1", 17 | "redis": "^2.8.0" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/pocketwatch-cache/redis.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const EC = { 4 | name: process.env.NODE_ENV !== 'dev' ? 5 | 'localhost' : 6 | '', 7 | port: 6379 8 | }; 9 | 10 | const Redis = require('redis'); 11 | const bluebird = require('bluebird'); 12 | bluebird.promisifyAll(Redis.RedisClient.prototype); 13 | bluebird.promisifyAll(Redis.Multi.prototype); 14 | 15 | let client; 16 | 17 | connect(); 18 | 19 | const redis = client; 20 | 21 | const api = { 22 | see_interval, add_interval, remove_interval, get_missing_intervals, 23 | is_message_duplicate, 24 | add_origin, 25 | mark_for_deletion, 26 | is_marked_for_deletion, 27 | clear, 28 | client 29 | }; 30 | 31 | Object.assign( redis, api ); 32 | 33 | module.exports = redis; 34 | 35 | function connect() { 36 | if ( !! client ) { 37 | client.end(true); 38 | } 39 | client = Redis.createClient(EC.port, EC.name, {retry_strategy}); 40 | } 41 | 42 | function clear() { 43 | return redis.flushallAsync(); 44 | } 45 | 46 | function is_marked_for_deletion( id ) { 47 | return redis.client.getAsync(`deleting:${id}`); 48 | } 49 | 50 | function mark_for_deletion( id, timer_delay = 1 ) { 51 | // notes on deletion 52 | // we set the key long enough for either our whole system to reboot 53 | // 10 minutes is an overestimate 54 | // or for 3 intervals to pass 55 | // the idea is if we haven't see the timer for 3 intervals 56 | // then it is not coming back 57 | // but if the delay is short 58 | // and our system goes down 59 | // we might miss it and 60 | // we need to wait at least until our system goes back to give us a chance to see 61 | // the timer 62 | // the weakness in this system is that we will not delete it if we purge redis 63 | // but we can fix that by writing to datastore as above 64 | // and then doing a deleteReconcile task 65 | // looking for intervals that are marked for deletion but not yet deleted 66 | // or some other such mechansim 67 | // we want to keep any heavy lifting ( datastore writes ) out of the critical path 68 | // in timekeeper receive message 69 | return redis.client.setAsync(`deleting:${id}`, 70 | "OK", "EX", Math.max(600,2*timer_delay)); 71 | } 72 | 73 | function is_message_duplicate( id, msg ) { 74 | return redis.getAsync( `messages:${id}` ).then( result => { 75 | if ( !! result ) { 76 | return true; 77 | } else { 78 | return redis.setAsync( `messages:${id}`, "OK", "EX", Math.max(30,msg.delay_seconds*3) ).then( result => false ); 79 | } 80 | }); 81 | } 82 | 83 | function see_interval( id ) { 84 | return remove_interval( id ); 85 | } 86 | 87 | function has_origin( i ) { 88 | return redis.getAsync( `inserted:${i.keyName}` ); 89 | } 90 | 91 | function add_origin( i ) { 92 | const originKey = `inserted:${i.keyName}`; 93 | const expire_time = Math.ceil(1.5*i.delay_seconds); 94 | return redis.setAsync( originKey, "OK", "EX", expire_time); 95 | } 96 | 97 | async function add_interval( i ) { 98 | const key = `interval:${i.keyName}`; 99 | const ji = JSON.stringify(i); 100 | const missing_timeout = 2*i.delay_seconds; 101 | const expire_timeout = Math.max(120,missing_timeout); 102 | const already_set = !! ( await redis.getAsync(key) ); 103 | if ( ! already_set ) { 104 | await redis.setAsync( key, ji, "EX", expire_timeout ); 105 | const currentTime = +new Date; 106 | const score = currentTime + missing_timeout*1000; 107 | await redis.zaddAsync('intervals',score,key); 108 | return `added:${key}`; 109 | } else { 110 | return `already_present_not_added:${key}`; 111 | } 112 | } 113 | 114 | function remove_interval( id ) { 115 | const key = `interval:${id}`; 116 | const originKey = `inserted:${id}`; 117 | return redis.delAsync( key ).then( 118 | () => redis.zremAsync( 'intervals', key ) 119 | ).then( 120 | () => redis.delAsync( originKey ) 121 | ); 122 | } 123 | 124 | async function get_missing_intervals() { 125 | const currentTime = +new Date; 126 | const keys = await redis.zrangebyscoreAsync( 'intervals', 0, currentTime ); 127 | if ( process.env.DEBUG == 'full' ) { 128 | console.log("\n"); 129 | console.log(JSON.stringify({missingIntervalKeys:keys})); 130 | } 131 | let intervals = []; 132 | if ( keys.length ) { 133 | intervals = await redis 134 | .mgetAsync(...keys) 135 | .then(json_intervals => json_intervals 136 | .filter( ji => !! ji ) 137 | .map( ji => JSON.parse(ji) ) 138 | ); 139 | } 140 | const intervals_to_insert = []; 141 | for ( const i of intervals ) { 142 | const has_o = await has_origin(i); 143 | const inserted = has_o == "OK"; 144 | if ( ! inserted ) { 145 | intervals_to_insert.push( i ); 146 | } 147 | } 148 | if ( process.env.DEBUG == 'full' ) { 149 | console.log("\n"); 150 | console.log(JSON.stringify({intervals_to_insert})); 151 | } 152 | return intervals_to_insert; 153 | } 154 | 155 | function retry_strategy(options) { 156 | if (options.error && options.error.code === 'ECONNREFUSED') { 157 | // End reconnecting on a specific error and flush all commands with 158 | // a individual error 159 | return new Error('The server refused the connection'); 160 | } 161 | if (options.total_retry_time > 1000 * 60 * 60) { 162 | // End reconnecting after a specific timeout and flush all commands 163 | // with a individual error 164 | return new Error('Retry time exhausted'); 165 | } 166 | if (options.attempt > 10) { 167 | // End reconnecting with built in error 168 | return undefined; 169 | } 170 | // reconnect after 171 | return Math.min(options.attempt * 100, 3000); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/promisify.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | // aim is to promisify all extension apis that use callbacks 4 | 5 | module.exports = promisify; 6 | 7 | function promisify(func) { 8 | return async function(...args) { 9 | return new Promise((res,rej) => { 10 | try { 11 | func(...args, (...cb_args) => res(cb_args)); 12 | } catch(e) { 13 | rej(e); 14 | } 15 | }); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/public/console.html: -------------------------------------------------------------------------------- 1 |
2 |

3 | 4 |

5 | 6 |

7 | 8 |

9 | 10 |

11 | 12 |

13 | 14 | 33 | 34 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/receiveMessage.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const redis = require('./pocketwatch-cache/redis.js'); 4 | const performTask = require('./performTask.js'); 5 | const enqueueMessage = require('./enqueueMessage.js'); 6 | const MAX = 660; // 11 * 60 ( 11 minutes in seconds ) 7 | const warning = "TIMEKEEPER RECEIVE MESSAGE WARNING"; 8 | 9 | module.exports = receiveMessage; 10 | 11 | async function receiveMessage({message}) { 12 | // initialization 13 | const ct = + new Date; 14 | const { 15 | will_next_execute_at, will_end_after 16 | } = message; 17 | // see the message 18 | const msgDeduplicatorKey = `${message.origin}/${message.msgName}`; 19 | const isDuplicate = await redis.is_message_duplicate(msgDeduplicatorKey, message); 20 | if ( isDuplicate ) { 21 | console.info("\n"); 22 | console.warn(JSON.stringify({warning,isDuplicate,msgDeduplicatorKey})); 23 | return false; 24 | } 25 | if ( message.being_reinserted ) { 26 | // when a message is reinserted we do not see only delete it from insert table 27 | delete message.being_reinserted; 28 | } else { 29 | // if it is not reinserted we do delete all table entries 30 | await redis.see_interval( message.keyName ); 31 | } 32 | const is_marked_for_deletion = await redis.is_marked_for_deletion( message.keyName ); 33 | if ( is_marked_for_deletion ) { 34 | return 'not reinserted as marked for deletion'; 35 | } 36 | // task performance branching 37 | let taskPerformed = false; 38 | if ( ct >= will_next_execute_at ) { // be prompt with executing tasks ( execute on OR after ) 39 | performTask(message); 40 | taskPerformed = true; 41 | } 42 | // message enqueuing branching 43 | if ( ct > will_end_after ) { // be generous with people's intevals ( only end *after* ) 44 | // don't enqueue another message 45 | } else { 46 | if ( taskPerformed ) { 47 | message.last_executed_at = ct; 48 | message.will_next_execute_at = ct + message.interval_seconds*1000; 49 | } else { 50 | // don't update last and next execute 51 | } 52 | if ( message.will_next_execute_at > ct + (MAX*1000) ) { 53 | message.delay_seconds = MAX; 54 | } else { 55 | message.delay_seconds = Math.max(1, 56 | Math.ceil((message.will_next_execute_at - ct)/1000)); 57 | } 58 | return enqueueMessage(message); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/sm.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const fs = require('fs'); 5 | const cp = require('child_process'); 6 | const path = require('path'); 7 | const {promisify} = require('util'); 8 | 9 | const readdir = promisify(fs.readdir); 10 | const stat = promisify(fs.stat); 11 | const access = promisify(fs.access); 12 | const exec = promisify(cp.exec); 13 | 14 | const delay = ms => new Promise(res => setTimeout(res, ms)); 15 | 16 | perform(); 17 | 18 | async function perform() { 19 | const thisDir = path.join(__dirname); 20 | await exec('npm i; npm rebuild;'); 21 | await exec('npm set progress=false'); 22 | //await exec('npm i -g pnpm'); 23 | await recurser( thisDir ); 24 | await delay(1000); 25 | } 26 | 27 | async function recurser( dir ) { 28 | const message = {status:`installed in ${dir}`,dir}; 29 | console.log("\n"); 30 | console.log(JSON.stringify(message)); 31 | await delay(500); 32 | const files = await readdir( dir ); 33 | for( const f of files ) { 34 | try { 35 | const isDir = (await stat(path.join(dir, f))).isDirectory(); 36 | if ( isDir ) { 37 | const isSubmodule = await stat(path.join(dir,f,'package.json')); 38 | if ( isDir && isSubmodule ) { 39 | await exec(`cd ${path.join(dir,f)}; npm i; npm rebuild;`); 40 | await recurser( path.join(dir,f) ); 41 | } 42 | } 43 | } catch(error) { 44 | if ( process.env.DEBUG == 'full' ) { 45 | const event = "Received error in npm install script (sm.js)"; 46 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 47 | console.log("\n"); 48 | console.error(JSON.stringify({event,error})); 49 | } 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/pocketwatch-mechanism/wrapAsync.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const wrapAsync = fn => 4 | (req, res, next) => { 5 | return Promise.resolve(fn(req,res,next)).catch(next); 6 | }; 7 | 8 | module.exports = wrapAsync; 9 | } 10 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/promisify.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | // aim is to promisify all extension apis that use callbacks 4 | 5 | module.exports = promisify; 6 | 7 | function promisify(func) { 8 | return async function(...args) { 9 | return new Promise((res,rej) => { 10 | try { 11 | func(...args, (...cb_args) => res(cb_args)); 12 | } catch(e) { 13 | rej(e); 14 | } 15 | }); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/public/console.html: -------------------------------------------------------------------------------- 1 | CONSOLE 2 |
3 |

4 | 5 | 6 | 7 |

8 | 9 |

10 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/receiveMessage.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const enqueueSupervisorMessage = require('./enqueueSupervisorMessage.js'); 5 | const functions = require('./functions.js'); 6 | const supervision = require('./supervision.js'); 7 | const redis = require('./pocketwatch-cache/redis.js'); 8 | const warning = 'SUPERVISOR RECEIVE MESSAGE WARNING'; 9 | 10 | module.exports = receiveMessage; 11 | 12 | async function receiveMessage({message} = {}) { 13 | await redis.client.delAsync(`s_inserted:${message.keyName}`); 14 | const msgDeduplicatorKey = `${message.origin}/${message.msgName}`; 15 | const isDuplicate = await redis.is_message_duplicate(msgDeduplicatorKey, message); 16 | if ( isDuplicate ) { 17 | console.warn("\n"); 18 | console.warn(JSON.stringify({warning,isDuplicate,msgDeduplicatorKey})); 19 | return false; 20 | } 21 | if ( await supervision.is_time_to_go_again( message.type ) ) { 22 | switch( message.type ) { 23 | case 'loadIntervals': 24 | await functions.load_live_intervals(); 25 | return enqueueSupervisorMessage(message); 26 | break; 27 | case 'reconcileIntervals': 28 | await functions.reinsert_missing_intervals(); 29 | return enqueueSupervisorMessage(message); 30 | break; 31 | default: 32 | console.warn("\n"); 33 | console.warn(JSON.stringify({warning,unknown_message:message})); 34 | break; 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/sm.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const fs = require('fs'); 5 | const cp = require('child_process'); 6 | const path = require('path'); 7 | const {promisify} = require('util'); 8 | 9 | const readdir = promisify(fs.readdir); 10 | const stat = promisify(fs.stat); 11 | const access = promisify(fs.access); 12 | const exec = promisify(cp.exec); 13 | 14 | const delay = ms => new Promise(res => setTimeout(res, ms)); 15 | 16 | perform(); 17 | 18 | async function perform() { 19 | const thisDir = path.join(__dirname); 20 | await exec('npm i; npm rebuild;'); 21 | await exec('npm set progress=false'); 22 | //await exec('npm i -g pnpm'); 23 | await recurser( thisDir ); 24 | await delay(1000); 25 | } 26 | 27 | async function recurser( dir ) { 28 | const message = {status:`installed in ${dir}`,dir}; 29 | console.log("\n"); 30 | console.log(JSON.stringify(message)); 31 | await delay(500); 32 | const files = await readdir( dir ); 33 | for( const f of files ) { 34 | try { 35 | const isDir = (await stat(path.join(dir, f))).isDirectory(); 36 | if ( isDir ) { 37 | const isSubmodule = await stat(path.join(dir,f,'package.json')); 38 | if ( isDir && isSubmodule ) { 39 | await exec(`cd ${path.join(dir,f)}; npm i; npm rebuild;`); 40 | await recurser( path.join(dir,f) ); 41 | } 42 | } 43 | } catch(error) { 44 | if ( process.env.DEBUG == 'full' ) { 45 | const event = "Received error in npm install script (sm.js)"; 46 | error = {error,message:error+'',stack:error.stack.split('\n').map(l => l.trim())}; 47 | console.log("\n"); 48 | console.error(JSON.stringify({event,error})); 49 | } 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/supervision.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const enqueueSupervisorMessage = require('./enqueueSupervisorMessage.js'); 5 | const redis = require('./pocketwatch-cache/redis.js'); 6 | 7 | const tasks = { 8 | reconcileIntervals: { 9 | type: 'reconcileIntervals', 10 | delay_seconds: 7, 11 | keyName: 'reconcile_missing_intervals' 12 | }, 13 | loadIntervals: { 14 | type: 'loadIntervals', 15 | delay_seconds: 11, 16 | keyName: 'load_live_intervals' 17 | } 18 | }; 19 | 20 | const supervision = { 21 | check, is_time_to_go_again, tasks 22 | }; 23 | 24 | module.exports = supervision; 25 | 26 | 27 | async function is_time_to_go_again(type) { 28 | const task = tasks[type]; 29 | const lastTime = (await redis.client.getAsync(type)) || 0; 30 | const currentTime = +new Date; 31 | const secondsSinceLastTime = (currentTime - lastTime)/1000; 32 | if ( secondsSinceLastTime >= task.delay_seconds ) { 33 | return true; 34 | } 35 | return false; 36 | } 37 | 38 | async function check() { 39 | if ( await is_time_to_go_again('reconcileIntervals') ) { 40 | enqueueSupervisorMessage(tasks.reconcileIntervals); 41 | } 42 | if ( await is_time_to_go_again('loadIntervals') ) { 43 | enqueueSupervisorMessage(tasks.loadIntervals); 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const functions = require('./functions.js'); 4 | 5 | test(); 6 | 7 | async function test() { 8 | let r = await functions.load_live_intervals(); 9 | console.log(r); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /pocketwatch/pocketwatch-supervisor/wrapAsync.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const wrapAsync = fn => 4 | (req, res, next) => { 5 | return Promise.resolve(fn(req,res,next)).catch(next); 6 | }; 7 | 8 | module.exports = wrapAsync; 9 | } 10 | -------------------------------------------------------------------------------- /pocketwatch/promisify.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | // aim is to promisify all extension apis that use callbacks 4 | 5 | module.exports = promisify; 6 | 7 | function promisify(func) { 8 | return async function(...args) { 9 | return new Promise((res,rej) => { 10 | try { 11 | func(...args, (...cb_args) => res(cb_args)); 12 | } catch(e) { 13 | rej(e); 14 | } 15 | }); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /pocketwatch/public/cost-calculated-modal.html: -------------------------------------------------------------------------------- 1 | 2 |
3 |
4 |

5 | Your interval will place a ${data.method} request to ${data.url} 6 | every ${data.interval_unit_count} ${data.invertal_unit_type}s, for 7 | a total of ${data.duration_unit_count} ${data.duration_unit_type}s, 8 | and a total of ${data.invocations_count} invocations, with 1 occurring 9 | immediately after purchase. 10 |

11 | Do you wish to proceed to purchase, and start your interval? 12 |

13 |
14 |
15 |

16 | 21 |

22 | 23 |

24 | 25 |
26 | 40 |
41 | -------------------------------------------------------------------------------- /pocketwatch/public/fancy.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | Pocketwatch 8 | 48 | 49 | 50 |
51 |
52 |

53 | Pocketwatch Free Demo 54 |
55 | for Hacker News Readers 56 |

57 |
58 |
59 |

What is Pocketwatch?

60 |
61 |
62 |

63 | Recurring job scheduling as a service 64 |

65 |
66 |
67 |

68 | The system is robust, fault-tolerant and scalable. Normally you can purchase either a subscription, or a one-off timer. But this is a free demo API key for HN readers to try it out without paying anything. 69 |

70 | Pocketwatch provides Pocketwatch Timers which are your schedulers for recurring tasks that can issue HTTP requests at intervals as fine as 1 second, or longer. 71 |

72 | You can use them to: 73 |

    74 |
  • schedule recurring jobs
    e.g. POST /purge-cache every 15 minutes 75 |
  • schedule background tasks
    e.g. POST /check-status every 10 seconds 76 |
  • run jobs
    e.g. GET /tweet-web-crawl every 1 minute 77 |
  • trigger recurrent workloads
    e.g. PATCH /run-chromeless-tests every 5 minutes 78 |
79 |

all the things you might use cron/crontab for (or systemd timers for), but when you want it more scalable, more fault-tolerant than one machine running cron — you use Pocketwatch Timers. 80 |

Just fire and forget. 81 |

Just pick an interval (say, 3 seconds), and a duration (say 5 weeks), set your URL and HTTP method, and you're good to go. 82 |

83 |
84 |
85 |
86 |
87 |
Form D-5: free demo for Hacker News Readers 88 | 89 | 90 |

91 | 97 | 101 |

102 | 105 | 113 | 116 | 124 |

125 |

126 | 127 |

128 |

129 | 135 |

136 |

137 | If you liked this demo, take a look at our developer portal to get your own API key by purchasing a Monthly Subscription. You can also read the docs, and get the client library. 138 |

139 | You can also simply purchase timers on a one-off basis, from this page here, with no subscription. 140 |

141 |
142 |
143 |
144 |
145 |
Limitations of this demo 146 |
147 |

148 | This demo uses an API key I set up specifically for HN readers. The key is i_am_hn_and_proud. To save resources, any timers created with this free demo key will only last roughly 2 weeks regardless of the maximum duration you put, and the key itself also has a maximum number of timers and hits. You can also try other HTTP methods, and include a body and Content-Type for the requests by using this key with the API directly. If you'd like to use the API directly, see the docs, and if you use Node.js, get the Node.js client library. By using this demo you also agree to the terms. If you have any issues, open one here to let me know. 149 |

150 |
151 |
152 |
153 | 160 |
161 | 170 | 171 | -------------------------------------------------------------------------------- /pocketwatch/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DOSYCORPS/cronstorm-opensource/3f007df10787e318524b3c2271fd3e124df96ef5/pocketwatch/public/favicon.ico -------------------------------------------------------------------------------- /pocketwatch/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | Pocketwatch 8 | 45 | 46 | 47 |
48 |
49 |

50 | 🕜 51 | Buy 1 Pocketwatch Timer 52 |

53 |
54 |
55 |

What is an On-Demand Pocketwatch Timer?

56 |
57 |

Flexible, fine-resolution task scheduling

58 |
59 |

60 | On-demand Pocketwatch Timers are schedulers that can issue HTTP requests at intervals as fine as 1 second, or longer. 61 |

62 | You can use them to: 63 |

    64 |
  • schedule background tasks, 65 |
  • run jobs, 66 |
  • trigger recurrent workloads, 67 |
68 |

all the things you might use cron/crontab for, but when you want it more scalable, more fault-tolerant than one machine running cron — you use Pocketwatch Timers. 69 |

Just fire and forget. 70 |

71 |
72 |
73 |
74 |
75 |
Form P-9: purchase of 1 Timer 76 | 77 |

78 | Describe your timer 79 |

80 | 83 |

84 | 90 | request to 91 | 92 |

93 | 96 | 104 | 107 | 115 |

116 |

117 | 118 |

119 |

120 | or 121 | Purchase a Monthly Subscription to get many more Timers 122 |

123 |
124 |
125 | 132 |
133 | 142 | 143 | -------------------------------------------------------------------------------- /pocketwatch/public/modal.css: -------------------------------------------------------------------------------- 1 | :root { 2 | position: relative; 3 | background: repeating-linear-gradient( 4 | 135deg, 5 | rgba(192,192,192,0.5), 6 | rgba(192,192,192,0.5) 20px, 7 | transparent 20px, 8 | transparent 30px 9 | ); 10 | height: 100%; 11 | } 12 | body { 13 | margin: 0; 14 | height: 100%; 15 | } 16 | .modal { 17 | word-break: break-word; 18 | font-family: Times, serif; 19 | display: table; 20 | width: 80%; 21 | max-width: 555px; 22 | padding: 1rem; 23 | position: relative; 24 | border: 1px solid grey; 25 | border-radius: 16px; 26 | background-color: white; 27 | z-index:100000000; 28 | min-width: 100px; 29 | min-height: 100px; 30 | margin: 5rem auto; 31 | overflow: hidden; 32 | } 33 | .long { 34 | word-break: break-all; 35 | display: inline-block; 36 | } 37 | .important { 38 | font-weight: bold; 39 | } 40 | .transparent { 41 | opacity: 0; 42 | } 43 | .result { 44 | color: green; 45 | font-weight: bold; 46 | } 47 | .fail { 48 | color: red; 49 | } 50 | -------------------------------------------------------------------------------- /pocketwatch/public/old-style.css: -------------------------------------------------------------------------------- 1 | main { 2 | background: darkkhaki; 3 | margin: 0 auto; 4 | width: 80%; 5 | min-width: 320px; 6 | margin-bottom: 3rem; 7 | } 8 | form { 9 | display: table; 10 | margin:2rem auto; 11 | } 12 | main header { 13 | background: khaki; 14 | color: forestgreen; 15 | font-family: sans-serif; 16 | } 17 | main > * { 18 | padding: 0 0.25rem; 19 | } 20 | main header h1 { 21 | margin: 0; 22 | } 23 | h1 a { 24 | color: inherit; 25 | } 26 | main header h1 a { 27 | text-decoration: none; 28 | } 29 | section.stories { 30 | padding: 1rem 2rem; 31 | background: white; 32 | } 33 | footer { 34 | text-align: center; 35 | padding-bottom: 0.5rem; 36 | } 37 | .txcent { 38 | text-align: center; 39 | } 40 | .classy { 41 | font-family: serif; 42 | } 43 | .byline { 44 | color: lemonchiffon; 45 | } 46 | fieldset { 47 | margin: 0 auto; 48 | } 49 | form, fieldset { 50 | display: table; 51 | position: relative; 52 | max-width: 90vw !important; 53 | } 54 | @media screen and (min-width: 777px) { 55 | main { 56 | min-width: 650px; 57 | } 58 | form { 59 | min-width: 630px; 60 | } 61 | form p { 62 | position: relative; 63 | } 64 | label { 65 | width: 180px; 66 | display: inline-block; 67 | text-align: right; 68 | position: relative; 69 | padding: 5px 0; 70 | } 71 | label > *, label + * { 72 | position: absolute; 73 | left: 200px; 74 | top: 50%; 75 | transform: translate(0, -50%); 76 | width: auto; 77 | } 78 | label:not(:only-child) > * { 79 | width: 80px; 80 | } 81 | label + * { 82 | left: 300px; 83 | } 84 | } 85 | @media screen and (max-width: 777px) { 86 | header h1 { 87 | text-align: center; 88 | } 89 | label, input, select{ 90 | position: relative; 91 | display: inline-block; 92 | max-width: 80vw; 93 | width: 100%; 94 | box-sizing: border-box; 95 | margin: 5px 0; 96 | } 97 | } 98 | span.icon { 99 | display: inline-block; 100 | vertical-align: baseline; 101 | margin-right: 0.2rem; 102 | font-size: 90%; 103 | } 104 | .light { 105 | color: silver; 106 | font-size: 80%; 107 | font-style: italic; 108 | } 109 | a.no-link-style:link, a.no-link-style:visited { 110 | text-decoration: none; 111 | color: inherit; 112 | cursor: default; 113 | } 114 | -------------------------------------------------------------------------------- /pocketwatch/public/ph.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | Pocketwatch 8 | 48 | 49 | 50 |
51 |
52 |

53 | Pocketwatch Free Demo 54 |
55 | for Product Hunt Readers 56 |

57 |
58 |
59 |

What is Pocketwatch?

60 |
61 |
62 |

63 | Recurring job scheduling as a service 64 |

65 |
66 |
67 |

68 | The system is robust, fault-tolerant and scalable. Normally you can purchase either a subscription, or a one-off timer. But this is a free demo API key for Product Hunt readers to try it out without paying anything. 69 |

70 | Pocketwatch provides Pocketwatch Timers which are your schedulers for recurring tasks that can issue HTTP requests at intervals as fine as 1 second, or longer. 71 |

72 | You can use them to: 73 |

    74 |
  • schedule recurring jobs
    e.g. POST /purge-cache every 15 minutes 75 |
  • schedule background tasks
    e.g. POST /check-status every 10 seconds 76 |
  • run jobs
    e.g. GET /tweet-web-crawl every 1 minute 77 |
  • trigger recurrent workloads
    e.g. PATCH /run-chromeless-tests every 5 minutes 78 |
79 |

all the things you might use cron/crontab for (or systemd timers for), but when you want it more scalable, more fault-tolerant than one machine running cron — you use Pocketwatch Timers. 80 |

Just fire and forget. 81 |

Just pick an interval (say, 3 seconds), and a duration (say 5 weeks), set your URL and HTTP method, and you're good to go. 82 |

83 |
84 |
85 |
86 |
87 |
Form D-5: free demo for Product Hunt Readers 88 | 89 | 90 |

91 | 97 | 101 |

102 | 105 | 113 | 116 | 124 |

125 |

126 | 127 |

128 |

129 | 135 |

136 |

137 | If you liked this demo, take a look at our developer portal to get your own API key by purchasing a Monthly Subscription. You can also read the docs, and get the client library. 138 |

139 | You can also simply purchase timers on a one-off basis, from this page here, with no subscription. 140 |

141 |
142 |
143 |
144 |
145 |
Limitations of this demo 146 |
147 |

148 | This demo uses an API key I set up specifically for Product Hunt readers. The key is i_am_ph_and_proud. To save resources, any timers created with this free demo key will only last roughly 2 weeks regardless of the maximum duration you put, and the key itself also has a maximum number of timers and hits. You can also try other HTTP methods, and include a body and Content-Type for the requests by using this key with the API directly. If you'd like to use the API directly, see the docs, and if you use Node.js, get the Node.js client library. By using this demo you also agree to the terms. If you have any issues, open one here to let me know. 149 |

150 |
151 |
152 |
153 | 160 |
161 | 170 | 171 | -------------------------------------------------------------------------------- /pocketwatch/public/style.css: -------------------------------------------------------------------------------- 1 | :root { 2 | font-family: Times, serif; 3 | } 4 | body { 5 | background: linear-gradient(aquamarine, dodgerblue); 6 | } 7 | header { 8 | text-align: center; 9 | } 10 | main { 11 | display: table; 12 | margin: 0 auto; 13 | max-width: 85vw; 14 | min-width: 280px; 15 | background: white; 16 | margin-bottom: 30vh; 17 | padding: 0 1rem; 18 | box-shadow: 0 1px 0 1px grey; 19 | } 20 | table { 21 | margin: 0 auto; 22 | } 23 | .txcent { 24 | text-align: center; 25 | } 26 | .hidden { 27 | display: none; 28 | } 29 | section { 30 | margin-top: 2rem; 31 | } 32 | input[type=radio]:nth-of-type(1):checked ~ table.plans th:nth-of-type(1) { 33 | background: dodgerblue; 34 | } 35 | input[type=radio]:nth-of-type(2):checked ~ table.plans th:nth-of-type(2) { 36 | background: dodgerblue; 37 | } 38 | input[type=radio]:nth-of-type(3):checked ~ table.plans th:nth-of-type(3) { 39 | background: dodgerblue; 40 | } 41 | button { 42 | font-family: serif; 43 | } 44 | button.cta { 45 | padding: 0; 46 | } 47 | button.cta label { 48 | padding: 1px 6px; 49 | } 50 | button.main-cta { 51 | background: linear-gradient(aquamarine, dodgerblue); 52 | } 53 | header { 54 | font-family: Georgia; 55 | } 56 | section.info header, 57 | section.sell header { 58 | font-family: "Times New Roman"; 59 | } 60 | footer { 61 | margin-bottom: 3rem; 62 | } 63 | section.sell { 64 | font-family: Palatino, Times, serif; 65 | text-align: center; 66 | } 67 | dl { 68 | vertical-align: top; 69 | margin: 0 auto; 70 | min-width: 250px; 71 | display: inline-block; 72 | max-width: 400px; 73 | width: 80%; 74 | } 75 | dl dd { 76 | padding-right: 1rem; 77 | text-align: left; 78 | } 79 | ul.flat { 80 | margin-left:1rem; 81 | padding-left:0; 82 | } 83 | @media screen and (max-width:320px) { 84 | main { 85 | padding: 0 3px; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /pocketwatch/public/success.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 8 | Thank you for purchasing a Timer 9 | 45 | 46 |
47 |
48 |

49 | Thank you for purchasing a timer! 50 |

51 |
52 |
53 |
54 |

Where would you like to go now?

55 |
56 | 65 |
66 | 70 |
71 | 80 | 96 | -------------------------------------------------------------------------------- /pocketwatch/public/terms.html: -------------------------------------------------------------------------------- 1 |

2 | Simple terms for Pocketwatch.xyz 3 |

4 |
    5 |
  1. 6 | When you buy an interval you purchase a number of invocations. But actually that number is only the maximum number of invocations. The actualy invocations effected are always less than that because each request always takes slightly longer than the interval duration ( for example, a 1 seconds request typically takes 1.1 seconds ), so that means that over a 5 minute ( 300 invocation ) duration, you will typically only see about 270 invocations. This is normal. What you are in effect paying for is utilizing of our system for the specificied duration of time, which is always exact. But the number of invocations we provide at time of sale is only a guide and you agree to take it as such. If you have a need for a specific number of invocations ( rather than a specific exact duration over which they occur ) please contact us via email or raise an issue on our github issue tracker. 7 |
  2. 8 | This is the beta release of pocketwatch. Currently limitations include (but not limited to): GET requests only, 9 | cannot delete an interval once it starts without knowing it's tracking number (long, secret code which you need to save 10 | if you want to prove you created the interval and want to request its deletion, currently request is via email). 11 |
  3. 12 | If you do request an interval deletion we do not currently offer any refunds. So please begin your intervals carefully. We can delete it for you if you request it (and have the tracking number) but you will not get any money back. 13 |
  4. 14 | Use this tool at your own discretion. We are not responsible for how you use it. 15 |
  5. 16 | We reserve the right to delete intervals with no refund if it appears they are harmful in any way. 17 |
  6. 18 | In general we will not offer refunds. But in case we do, they will be determined on a case by case basis. 19 |
  7. 20 | Don't use the names The Dosaygo Corporation, DOSY, Dosycorp, or Pocketwatch.xyz in anyway that suggests partnership or endoresment or other relationship when none in fact exists. 21 |
  8. 22 | This product is offered by the Dosyago Corporation, any legal action will be decided with respect to the laws of the United States, and through the courts in that jurisdiction. 23 |
  9. 24 | Unless otherwise made explicit, no content or other corpyrightable or protected work may be reused or incorporated without prior written permission from us. 25 |
  10. 26 | If you need to contact us open an issue at: https://github.com/dosyago-corp/service-issues/issues 27 |
  11. 28 | We do our best to place and fulfill your orders as requested, and to achieve the specified rates of requests for the specified duration as ordered in your interval, however we do not guarantee we shall achieve this, and we are not responsible for issues related to external network latency, routing or other such issues, and in the case your interval suffers degraded performance on account of such or other issues we have limited control over, we certainly offer no refunds. This service does not currently have any SLA ( service level agreement ). 29 |
  12. 30 | If you need to contact us privately join our Discord channel using this invite and DM us: https://discord.gg/VWAudDX 31 |
  13. 32 | These terms are a living document. We may update them at any time, with or without notice. 33 |
34 | -------------------------------------------------------------------------------- /pocketwatch/public/transparent.html: -------------------------------------------------------------------------------- 1 | 7 | -------------------------------------------------------------------------------- /pocketwatch/sm.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const fs = require('fs'); 5 | const cp = require('child_process'); 6 | const path = require('path'); 7 | const {promisify} = require('util'); 8 | 9 | const readdir = promisify(fs.readdir); 10 | const stat = promisify(fs.stat); 11 | const access = promisify(fs.access); 12 | const exec = promisify(cp.exec); 13 | 14 | const delay = ms => new Promise(res => setTimeout(res, ms)); 15 | 16 | perform(); 17 | 18 | async function perform() { 19 | const thisDir = path.join(__dirname); 20 | await exec('npm i; npm rebuild;'); 21 | await exec('npm set progress=false'); 22 | //await exec('npm i -g pnpm'); 23 | await recurser( thisDir ); 24 | await delay(1000); 25 | } 26 | 27 | async function recurser( dir ) { 28 | console.log("installed in ", dir); 29 | await delay(500); 30 | const files = await readdir( dir ); 31 | for( const f of files ) { 32 | try { 33 | const isDir = (await stat(path.join(dir, f))).isDirectory(); 34 | if ( isDir ) { 35 | const isSubmodule = await stat(path.join(dir,f,'package.json')); 36 | if ( isDir && isSubmodule ) { 37 | await exec(`cd ${path.join(dir,f)}; npm i; npm rebuild;`); 38 | await recurser( path.join(dir,f) ); 39 | } 40 | } 41 | } catch(e) {/*console.warn(e)*/} 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /pocketwatch/stripeKeysPublic.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const stripeKeysPublic = { 4 | test: { 5 | public: 'pk_test_', 6 | }, 7 | live: { 8 | public: 'pk_live_', 9 | } 10 | }; 11 | 12 | module.exports = stripeKeysPublic; 13 | } 14 | -------------------------------------------------------------------------------- /pocketwatch/stripeKeysSecret.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const stripeKeysSecret = { 4 | test: { 5 | secret: 'sk_test_', 6 | }, 7 | live: { 8 | secret: 'sk_live_', 9 | } 10 | }; 11 | 12 | module.exports = stripeKeysSecret; 13 | } 14 | -------------------------------------------------------------------------------- /pocketwatch/wrapAsync.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const wrapAsync = fn => 4 | (req, res, next) => { 5 | return Promise.resolve(fn(req,res,next)).catch(next); 6 | }; 7 | 8 | module.exports = wrapAsync; 9 | } 10 | -------------------------------------------------------------------------------- /sm.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | { 4 | const fs = require('fs'); 5 | const cp = require('child_process'); 6 | const path = require('path'); 7 | const {promisify} = require('util'); 8 | 9 | const readdir = promisify(fs.readdir); 10 | const stat = promisify(fs.stat); 11 | const access = promisify(fs.access); 12 | const exec = promisify(cp.exec); 13 | 14 | const delay = ms => new Promise(res => setTimeout(res, ms)); 15 | 16 | perform(); 17 | 18 | async function perform() { 19 | const thisDir = path.join(__dirname); 20 | await exec('npm i; npm rebuild;'); 21 | await exec('npm set progress=false'); 22 | //await exec('npm i -g pnpm'); 23 | await recurser( thisDir ); 24 | await delay(1000); 25 | } 26 | 27 | async function recurser( dir ) { 28 | console.log("installed in ", dir); 29 | await delay(500); 30 | const files = await readdir( dir ); 31 | for( const f of files ) { 32 | try { 33 | const isDir = (await stat(path.join(dir, f))).isDirectory(); 34 | if ( isDir ) { 35 | const isSubmodule = await stat(path.join(dir,f,'package.json')); 36 | if ( isDir && isSubmodule ) { 37 | await exec(`cd ${path.join(dir,f)}; npm i; npm rebuild;`); 38 | await recurser( path.join(dir,f) ); 39 | } 40 | } 41 | } catch(e) {/*console.warn(e)*/} 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /stripeKeysPublic.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const stripeKeysPublic = { 4 | test: { 5 | public: 'pk_test_', 6 | }, 7 | live: { 8 | public: 'pk_live_', 9 | } 10 | }; 11 | 12 | module.exports = stripeKeysPublic; 13 | } 14 | -------------------------------------------------------------------------------- /stripeKeysSecret.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const stripeKeysSecret = { 4 | test: { 5 | secret: 'sk_test_', 6 | }, 7 | live: { 8 | secret: 'sk_live_', 9 | } 10 | }; 11 | 12 | module.exports = stripeKeysSecret; 13 | } 14 | -------------------------------------------------------------------------------- /subscribe/console.html: -------------------------------------------------------------------------------- 1 |
2 |
Form P-8: creation of 1 interval 3 |

4 | 5 |

6 | 9 |

10 | 16 | 17 |

18 | 21 | 29 |

30 | 33 | 41 |

42 |

43 | 44 |

45 |
46 |
47 |
Form P-15: deletion of 1 interval 48 |

49 | 50 |

51 | 52 |

53 | 54 |

55 |
56 | 57 | -------------------------------------------------------------------------------- /subscribe/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CronStorm — Experience the Power of Scheduled Jobs 6 | 7 |
8 |

9 | CronStorm. 10 | Scalable Job Scheduling as a Service. 11 |

12 |

13 | 14 |
15 | 18 | 26 |
27 | 28 |

29 |
30 |
31 |
32 |

33 | Fault-tolerant, fine-resolution task scheduling. 34 | At scale. 35 |

36 |
37 |
38 |
39 |
    40 |
  • 41 |
    42 |

    Managed and scalable.

    43 |

    CronStorm manages the infrastructure required for to run millions of cron jobs on time. CronStorm's FailureGuard technology keeps executing your jobs even under partial system failure.

    44 |
    • 45 |
  • 46 |
    47 |

    Every 1 second.

    48 |

    Is how often you can schedule your jobs, at no extra cost. Most competitors only get to 20 seconds, and charge extra, making CronStorm the choice for triggering collection of fast-changing data. 49 |

    50 |
    • 51 |
  • 52 |
    53 |

    Ideal for automation.

    54 |

    CronStorm is great for recurring actions such as periodically gathering data from the web into a feed. It’s also ideal for cleaning up logs, kicking off routine backups, and integrating into your app with CronStorm API. 55 |

    56 |
    • 57 |
58 |
59 |
60 |
61 |

Choose a Plan.

62 |
63 |
64 |
65 | 66 | 67 | 68 | 73 | 78 | 83 | 84 | 85 | 86 | 87 | 92 | 97 | 102 | 103 | 104 | 109 | 114 | 119 | 120 | 121 | 126 | 131 | 136 | 137 | 138 | 143 | 148 | 153 | 154 | 155 |
69 |

70 | Starter. 71 |

72 | 		74 |

75 | Business. 76 |

77 | 		79 |

80 | Enterprise. 81 |

82 |
88 |

89 | Side project. 90 |

91 | 		93 |

94 | Recommended. 95 |

96 | 		98 |

99 | Web scale. 100 |

101 |
105 |

106 | 5,000,000 requests a month. 107 |

108 | 		110 |

111 | 25,000,000 requests a month. 112 |

113 | 		115 |

116 | 125,000,000 requests a month. 117 |

118 |
122 |

123 | 100 cron jobs. 124 |

125 | 		127 |

128 | 1,000 cron jobs. 129 |

130 | 		132 |

133 | 10,000 cron jobs. 134 |

135 |
139 |

140 | Email support in 3 days. 141 |

142 | 		144 |

145 | Email support in 1 day. 146 |

147 | 		149 |

150 | Phone support in half a day. 151 |

152 |
156 |
157 |
158 |
159 |
160 |
161 |

162 | 165 | 173 |

174 |
175 |
176 |
177 |
178 |
179 |

180 | CronStorm API and CLI. 181 |

182 |
183 |
184 |

185 | The CronStorm API lets you create cron jobs that will issue web requests at intervals, right from your application. 186 | Read the CronStorm API Documentation. 187 | The CronStorm CLI lets you easily access the full power of CronStorm from the command-line. 188 | Get the CronStorm CLI. 189 | When you want a more scalable, fault-tolerant and powerful scheduled task than cron provides — you use CronStorm. 190 |

191 |
192 |
193 |
194 | 211 | 212 | 213 | -------------------------------------------------------------------------------- /subscribe/modal.css: -------------------------------------------------------------------------------- 1 | :root { 2 | position: relative; 3 | background: rgba(192,192,192,0.5); 4 | } 5 | body { 6 | margin: 0; 7 | height: 100%; 8 | } 9 | .modal, a, a:link, a:visited { 10 | color: #37bbe4; 11 | } 12 | .modal { 13 | word-break: break-word; 14 | display: table; 15 | width: 80%; 16 | font-family: Helvetica, Arial, sans-serif; 17 | max-width: 555px; 18 | padding: 1rem; 19 | position: relative; 20 | border: 3px solid; 21 | background: #35342f; 22 | z-index:100000000; 23 | min-width: 100px; 24 | min-height: 100px; 25 | margin: 5rem auto; 26 | overflow: hidden; 27 | } 28 | .long { 29 | word-break: break-all; 30 | display: inline-block; 31 | } 32 | .important { 33 | font-weight: bold; 34 | } 35 | .transparent { 36 | opacity: 0; 37 | } 38 | .result { 39 | color: green; 40 | font-weight: bold; 41 | } 42 | .fail { 43 | color: red; 44 | } 45 | .bonus strong { 46 | color: green; 47 | } 48 | -------------------------------------------------------------------------------- /subscribe/old-style.css: -------------------------------------------------------------------------------- 1 | :root { 2 | font-family: Times, serif; 3 | } 4 | body { 5 | background: linear-gradient(aquamarine, dodgerblue); 6 | } 7 | header { 8 | text-align: center; 9 | } 10 | main { 11 | display: table; 12 | margin: 0 auto; 13 | max-width: 55vw; 14 | min-width: 280px; 15 | background: white; 16 | margin-bottom: 30vh; 17 | padding: 0 1rem; 18 | box-shadow: 0 1px 0 1px grey; 19 | } 20 | table { 21 | margin: 0 auto; 22 | } 23 | .txcent { 24 | text-align: center; 25 | } 26 | .hidden { 27 | display: none; 28 | } 29 | section { 30 | margin-top: 2rem; 31 | } 32 | input[type=radio]:nth-of-type(1):checked ~ table.plans td:nth-of-type(1) button.cta { 33 | background: linear-gradient(silver, lime); 34 | color: navy; 35 | } 36 | input[type=radio]:nth-of-type(2):checked ~ table.plans td:nth-of-type(2) button.cta { 37 | background: linear-gradient(aquamarine, dodgerblue); 38 | } 39 | input[type=radio]:nth-of-type(3):checked ~ table.plans td:nth-of-type(3) button.cta { 40 | background: linear-gradient(purple, hotpink); 41 | color: white; 42 | } 43 | input[type=radio]:nth-of-type(1):checked ~ table.plans th:nth-of-type(1) { 44 | color: navy; 45 | background: lime; 46 | } 47 | input[type=radio]:nth-of-type(2):checked ~ table.plans th:nth-of-type(2) { 48 | color: white; 49 | background: dodgerblue; 50 | } 51 | input[type=radio]:nth-of-type(3):checked ~ table.plans th:nth-of-type(3) { 52 | color: white; 53 | background: hotpink; 54 | } 55 | button { 56 | font-family: serif; 57 | } 58 | button.cta { 59 | padding: 0; 60 | } 61 | button.cta label { 62 | padding: 1px 6px; 63 | } 64 | input[type=radio]:nth-of-type(1):checked ~ p button.main-cta { 65 | background: linear-gradient(silver, lime); 66 | color: navy; 67 | } 68 | input[type=radio]:nth-of-type(2):checked ~ p button.main-cta { 69 | background: linear-gradient(aquamarine, dodgerblue); 70 | } 71 | input[type=radio]:nth-of-type(3):checked ~ p button.main-cta { 72 | background: linear-gradient(purple, hotpink); 73 | color: white; 74 | } 75 | header { 76 | font-family: Georgia; 77 | } 78 | section.info header, 79 | section.sell header { 80 | font-family: "Times New Roman"; 81 | } 82 | footer { 83 | margin-bottom: 3rem; 84 | } 85 | section.sell { 86 | font-family: Palatino, Times, serif; 87 | text-align: center; 88 | } 89 | dl { 90 | vertical-align: top; 91 | margin: 0 auto; 92 | min-width: 250px; 93 | display: inline-block; 94 | max-width: 400px; 95 | width: 80%; 96 | } 97 | dl dd { 98 | padding-right: 1rem; 99 | text-align: left; 100 | } 101 | ul { 102 | margin-left:1rem; 103 | padding-left:0; 104 | } 105 | @media screen and (max-width:320px) { 106 | main { 107 | padding: 0 1px; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /subscribe/style.css: -------------------------------------------------------------------------------- 1 | :root { 2 | position: relative; 3 | min-height: 100%; 4 | font-size: 16px; 5 | font-family: Helvetica, Arial, sans-serif; 6 | background: #35342f; 7 | line-height: 1.15; 8 | } 9 | :root, a, a:link, a:visited { 10 | color: #37bbe4; 11 | } 12 | footer a, footer a:link, footer a:visited { 13 | color: #e1e0dd; 14 | } 15 | body { 16 | min-height: 100%; 17 | margin: 0; 18 | } 19 | iframe#modals_frame { 20 | z-index:20000000; 21 | background-color: transparent; 22 | display: none; 23 | position: fixed; 24 | top: 0; 25 | left: 0; 26 | bottom: 0; 27 | right: 0; 28 | border: 0; 29 | width:100%; 30 | height:100vh; 31 | overflow: scroll; 32 | } 33 | iframe { 34 | border: 0; 35 | display: block; 36 | width: 100%; 37 | -webkit-overflow-scrolling: touch; 38 | } 39 | @media screen and (min-width: 60em) { 40 | header { 41 | position: sticky; 42 | top: 0; 43 | } 44 | } 45 | header { 46 | background: #e1e0dd; 47 | font-weight: bold; 48 | } 49 | section.invert-colors { 50 | color: #35342f; 51 | background: #37bbe4; 52 | } 53 | section.invert-colors h1 { 54 | font-weight: 300; 55 | } 56 | header h1 { 57 | display: inline; 58 | } 59 | h1 { 60 | margin: 0 0 0.25rem; 61 | } 62 | h1 small { 63 | font-weight: 300; 64 | } 65 | dd { 66 | margin-left: 0; 67 | } 68 | section.table-inside { 69 | overflow: auto; 70 | } 71 | table { 72 | border-spacing: 0 0.25rem; 73 | table-layout: fixed; 74 | width: 100%; 75 | max-width: 48em; 76 | min-width: 24em; 77 | margin: 0 auto; 78 | margin-bottom: 0.5rem; 79 | } 80 | th, td { 81 | vertical-align: top; 82 | text-align: left; 83 | } 84 | ul { 85 | list-style-type: circle; 86 | } 87 | .cta p, .cta select, .cta button { 88 | font-size: 1.5rem; 89 | } 90 | footer ul, 91 | main section ul { 92 | text-align: center; 93 | padding: 0; 94 | margin: 0; 95 | display: table; 96 | margin: 0 auto; 97 | } 98 | main section li, 99 | footer li { 100 | display: inline-block; 101 | } 102 | footer li { 103 | margin: 0.5rem; 104 | } 105 | footer { 106 | padding-bottom: 2rem; 107 | } 108 | li h1 { 109 | height: 2rem; 110 | text-align: left; 111 | line-height: 2; 112 | } 113 | main section h1 { 114 | clear: both; 115 | } 116 | main section li { 117 | max-width: 16em; 118 | width: 16em; 119 | vertical-align: top; 120 | } 121 | label { 122 | white-space: nowrap; 123 | } 124 | main section li p, 125 | main section td p { 126 | max-width: 12em; 127 | margin: 0; 128 | } 129 | article dt h1 { 130 | margin: 1rem 0; 131 | } 132 | p { 133 | text-align: left; 134 | line-height: 1.4; 135 | } 136 | section { 137 | margin: 0 0 0.5rem; 138 | } 139 | small form, 140 | section form { 141 | display: inline; 142 | } 143 | small form { 144 | float: right; 145 | margin-top: 0.25rem; 146 | } 147 | small form, small form select, small form button, 148 | section form, section form select, section form button { 149 | display: inline-block; 150 | vertical-align: middle; 151 | } 152 | -------------------------------------------------------------------------------- /subscribe/subscription-confirm-modal.html: -------------------------------------------------------------------------------- 1 | 2 |
3 |
4 |

5 | Your interval will place a ${data.method} request to ${data.url} 6 | every ${data.interval_unit_count} ${data.invertal_unit_type}s, for 7 | a total of ${data.duration_unit_count} ${data.duration_unit_type}s, 8 | and a total of ${data.invocations_count} invocations, with 1 occurring 9 | immediately after purchase. 10 |

11 | Do you wish to proceed to purchase, and start your interval? 12 |

13 |
14 |
15 |

16 | 21 |

22 | 23 |

24 | 25 |
26 | 40 |
41 | -------------------------------------------------------------------------------- /subscribe/success.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | CronStorm — Access Granted. 5 | 6 |
7 |

8 | CronsStorm. 9 | Access granted. 10 |

11 |
12 |
13 |
14 |

What would you like to do now?

15 |
16 |
17 | Read the API Documentation. 18 |
19 | Get the Node.js client library. 20 |
21 | Open an Issue. 22 |
23 | Email us for help. 24 |
25 | The Dosyago Corporation on GitHub. 26 |
27 | The Dosyago Corporation Website. 28 |
29 |
30 |
31 | 32 | -------------------------------------------------------------------------------- /subscribe/terms.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | CronStorm — Access Granted. 5 | 6 |
7 |

8 | CronsStorm. 9 |

10 |
11 |
12 |
13 |

14 | Simple terms for CronStorm 15 |

16 |

17 | These terms are between you and CronStorm ("we", "us"). 18 |

19 | When you subscribe to the CronStorm API service to get a developer API key, you agree to these terms. If you cancel your subscription (which you can do at any time) we do not provide a refund for partial months, but instead your subscription (your API key) will remain operational until the end of that billing cycle. Billing cycles begin each month on the same date that you started your subscription, or the closest day to that. 20 |

21 | If you provided an email address when you paid (you may have had to do so, since Stripe (a payment provider we use, not affiliated with us) may require it), therefore you agree that you consent to us sending you some promotion mails about your subscription, reminders about subscription, overdue payments, and support emails about your service, or other notifications over email. If you wish to not receive these mails then you can email us at this support address to opt out and we will no longer contact you, except as we are required to do so by law or to resolve or clarify any issues about payment that, if we were not to contact you, would possibly have adverse legal consequences. 22 |

23 | When you buy an interval you purchase a number of invocations. But actually that number is only the maximum number of invocations. The actualy invocations effected are always less than that because each request always takes slightly longer than the interval duration ( for example, a 1 seconds request typically takes 1.1 seconds ), so that means that over a 5 minute ( 300 invocation ) duration, you will typically only see about 270 invocations. This is normal. What you are in effect paying for is utilizing of our system for the specificied duration of time, which is always exact. But the number of invocations we provide at time of sale is only a guide and you agree to take it as such. If you have a need for a specific number of invocations ( rather than a specific exact duration over which they occur ) please contact us via email or raise an issue on our github issue tracker. 24 |

25 | This is the beta release of pocketwatch. Currently limitations include (but not limited to): GET requests only, 26 | cannot delete an interval once it starts without knowing it's tracking number (long, secret code which you need to save 27 | if you want to prove you created the interval and want to request its deletion, currently request is via email). 28 |

29 | If you do request an interval deletion we do not currently offer any refunds. So please begin your intervals carefully. We can delete it for you if you request it (and have the tracking number) but you will not get any money back. 30 |

31 | Use this tool at your own discretion. We are not responsible for how you use it. 32 |

33 | We reserve the right to delete intervals with no refund if it appears they are harmful in any way. 34 |

35 | In general we will not offer refunds. But in case we do, they will be determined on a case by case basis. 36 |

37 | Don't use the names The Dosaygo Corporation, DOSY, Dosycorp, or CronStorm.com in anyway that suggests partnership or endoresment or other relationship when none in fact exists. 38 |

39 | This product is offered by the Dosyago Corporation, any legal action will be decided with respect to the laws of the United States, and through the courts in that jurisdiction. 40 |

41 | Unless otherwise made explicit, no content or other corpyrightable or protected work may be reused or incorporated without prior written permission from us. 42 |

43 | If you need to contact us open an issue at: https://github.com/dosyago-corp/service-issues/issues 44 |

45 | We do our best to place and fulfill your orders as requested, and to achieve the specified rates of requests for the specified duration as ordered in your interval, however we do not guarantee we shall achieve this, and we are not responsible for issues related to external network latency, routing or other such issues, and in the case your interval suffers degraded performance on account of such or other issues we have limited control over, we certainly offer no refunds. This service does not currently have any SLA ( service level agreement ). 46 |

47 | If you need to contact CronStorm privately please email CronStorm for help. 48 |

49 | These terms are a living document. We may update them at any time, with or without notice. 50 |

51 |
52 | 69 | 70 | -------------------------------------------------------------------------------- /subscribe/transparent.html: -------------------------------------------------------------------------------- 1 | 7 | -------------------------------------------------------------------------------- /wrapAsync.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | { 3 | const wrapAsync = fn => 4 | (req, res, next) => { 5 | return Promise.resolve(fn(req,res,next)).catch(next); 6 | }; 7 | 8 | module.exports = wrapAsync; 9 | } 10 | --------------------------------------------------------------------------------