├── .eslintignore ├── .eslintrc.json ├── .gitignore ├── .prettierignore ├── Procfile ├── README.md ├── TODO.md ├── app.json ├── bin ├── addtopics └── getfixture ├── clientLogger.js ├── docs ├── heroku-connect │ ├── image1.png │ ├── image10.png │ ├── image11.png │ ├── image12.png │ ├── image13.png │ ├── image14.png │ ├── image15.png │ ├── image16.png │ ├── image17.png │ ├── image2.png │ ├── image3.png │ ├── image4.png │ ├── image5.png │ ├── image6.png │ ├── image7.png │ ├── image8.png │ ├── image9.png │ └── mappings │ │ ├── account.jpg │ │ ├── contract.jpg │ │ ├── order-item.jpg │ │ ├── order.jpg │ │ ├── pricebook-entry.jpg │ │ ├── pricebook2.jpg │ │ ├── product2.jpg │ │ └── record-type.jpg └── kafka-stream-viz-architecture.gif ├── generate_data ├── README.md ├── fixture.json ├── generate_products.js ├── index.js ├── kafka.js ├── package-lock.json ├── package.json ├── products.json └── shoppingfeed.js ├── generate_orders ├── README.md ├── config │ └── default.json ├── index.js ├── package-lock.json ├── package.json └── runner.js ├── logger.js ├── mq_broker ├── constants.js ├── index.js ├── package-lock.json └── package.json ├── mq_worker ├── api.js ├── constants.js ├── index.js ├── package-lock.json ├── package.json └── processMessage.js ├── package-lock.json ├── package.json ├── redshift_batch ├── README.md ├── config │ └── default.js ├── index.js ├── package-lock.json └── package.json ├── sfdx └── order-fulfillment │ ├── .eslintignore │ ├── .forceignore │ ├── .gitignore │ ├── .prettierignore │ ├── .prettierrc │ ├── README.md │ ├── config │ └── project-scratch-def.json │ ├── force-app │ └── main │ │ └── default │ │ ├── applications │ │ └── Product_Orders.app-meta.xml │ │ ├── contentassets │ │ ├── Box1320568095448898951.asset │ │ ├── Box1320568095448898951.asset-meta.xml │ │ ├── Box13205680954488989511.asset │ │ └── Box13205680954488989511.asset-meta.xml │ │ ├── dataSources │ │ └── Heroku_Connect.dataSource-meta.xml │ │ ├── flexipages │ │ ├── Fulfillment_Orders.flexipage-meta.xml │ │ └── Supply_Demand_UtilityBar.flexipage-meta.xml │ │ ├── layouts │ │ └── public_orders__x-Fulfillment Orders Layout.layout-meta.xml │ │ ├── lwc │ │ └── .eslintrc.json │ │ ├── objects │ │ └── public_orders__x │ │ │ ├── fields │ │ │ ├── amount__c.field-meta.xml │ │ │ ├── approved__c.field-meta.xml │ │ │ ├── category__c.field-meta.xml │ │ │ ├── createdat__c.field-meta.xml │ │ │ ├── id__c.field-meta.xml │ │ │ └── updatedat__c.field-meta.xml │ │ │ ├── listViews │ │ │ ├── All.listView-meta.xml │ │ │ └── Approved_Orders.listView-meta.xml │ │ │ └── public_orders__x.object-meta.xml │ │ ├── profilePasswordPolicies │ │ ├── PT1_profilePasswordPolicy1585938656989.profilePasswordPolicy-meta.xml │ │ └── PT2_profilePasswordPolicy1585938581099.profilePasswordPolicy-meta.xml │ │ ├── profileSessionSettings │ │ ├── PT1_profileSessionSetting1585938657131.profileSessionSetting-meta.xml │ │ └── PT2_profileSessionSetting1585938581288.profileSessionSetting-meta.xml │ │ ├── profiles │ │ └── Admin.profile-meta.xml │ │ ├── quickActions │ │ └── public_orders__x.Approve.quickAction-meta.xml │ │ ├── reports │ │ └── unfiled$public │ │ │ └── Product_Demand_40o.report-meta.xml │ │ └── tabs │ │ ├── Product_Demand.tab-meta.xml │ │ └── public_orders__x.tab-meta.xml │ ├── package.json │ ├── scripts │ ├── apex │ │ └── hello.apex │ └── soql │ │ └── account.soql │ └── sfdx-project.json └── viz ├── .editorconfig ├── .prettierignore ├── README.md ├── app.js ├── bin ├── env └── start ├── config └── default.json ├── consumer ├── constants.js └── index.js ├── package-lock.json ├── package.json ├── public ├── diagram │ └── heroku-connect-diagram.html ├── heroku-connect-diagram │ ├── heroku-connect.html │ └── kafka-salesforce-data-demo-animation.hyperesources │ │ ├── 1F13B8-restorable.plist │ │ ├── HYPE-654.full.min.js │ │ ├── HYPE-654.thin.min.js │ │ ├── PIE.htc │ │ ├── arrow-one.svg │ │ ├── arrow-two.svg │ │ ├── blank.gif │ │ ├── heroku-app.svg │ │ ├── heroku-connect.svg │ │ ├── heroku-postgres.svg │ │ ├── kafkasalesforcedatademoanimation_hype_generated_script.js │ │ ├── mobile-arrow-1.svg │ │ ├── mobile-arrow-2.svg │ │ ├── mobile-heroku-app.svg │ │ ├── mobile-heroku-connect.svg │ │ ├── mobile-heroku-postgres.svg │ │ ├── mobile-outline.svg │ │ ├── mobile-speech-1.svg │ │ ├── mobile-speech-2.svg │ │ ├── mobile-speech-3.svg │ │ ├── mobile-speech-4.svg │ │ ├── outline.svg │ │ ├── salesforce.svg │ │ ├── speech-1.svg │ │ ├── speech-2.svg │ │ ├── speech-3.svg │ │ └── speech-4.svg ├── images │ ├── add-mark-28.svg │ ├── layout-list-28.svg │ ├── loading.gif │ ├── metrics-28.svg │ ├── remove-mark-28.svg │ ├── salesforce-heroku-connect.png │ └── usage-28.svg ├── kafka-diagram │ ├── kafka-diagram-v2.html │ └── kafka-diagram-v2.hyperesources │ │ ├── 9EF7D6-restorable.plist │ │ ├── Asset 14_2x.png │ │ ├── HYPE-600.full.min.js │ │ ├── HYPE-600.thin.min.js │ │ ├── PIE.htc │ │ ├── amazon-arrow-mobile-1.svg │ │ ├── amazon-arrow-mobile.svg │ │ ├── amazon-arrow-new.svg │ │ ├── amazon-arrow.svg │ │ ├── amazon-container-mobile.svg │ │ ├── amazon-container-new.svg │ │ ├── amazon-container.svg │ │ ├── blank.gif │ │ ├── data-arrow-mobile.svg │ │ ├── data-pro-tooltip-sm.svg │ │ ├── data-prod-arrow.svg │ │ ├── data-prod-container-mobile.svg │ │ ├── data-prod-container.svg │ │ ├── data-producers-tooltip-med-1.svg │ │ ├── data-producers-tooltip.svg │ │ ├── data-producers.svg │ │ ├── data-viz-arrow.svg │ │ ├── data-viz-tooltip-1.svg │ │ ├── data-viz-tooltip-sm.svg │ │ ├── data-viz.svg │ │ ├── data-writer.svg │ │ ├── heroku-postgres.svg │ │ ├── kafka-container-mobile.svg │ │ ├── kafka-container.svg │ │ ├── kafka-icon.svg │ │ ├── kafkadiagramv2_hype_generated_script.js │ │ ├── metabase-app.svg │ │ ├── postgres-tooltip-1.svg │ │ ├── postgres-tooltip-sm.svg │ │ ├── redshift-icon.svg │ │ ├── viz-arrow-mobile.svg │ │ ├── viz-container-mobile.svg │ │ └── viz-container.svg ├── logo-heroku-white.svg └── logo-salesforce.svg ├── sql ├── create_pg.sql ├── load.sql ├── load_pg.sql └── truncate.sql ├── src ├── demand │ ├── DemandChart.js │ ├── DemandControls.js │ ├── DemandFulfillmentForm.js │ └── order-control.js ├── index.js └── lib │ ├── audienceControls.js │ ├── boothControls.js │ ├── nav.js │ ├── queue.js │ ├── sizedArray.js │ └── stream.js ├── styles ├── components │ ├── audience.css │ ├── booth.css │ ├── chart-colors.css │ ├── demand.css │ ├── main.css │ ├── nav.css │ └── typography.css └── style.css ├── supplyDemand.js ├── views ├── audience.pug ├── booth.pug ├── connect.pug ├── includes │ ├── architecture.pug │ └── charts.pug ├── index.pug ├── layout.pug ├── nav-home.pug ├── ordercontrol.pug └── presentation.pug └── webpack.config.js /.eslintignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | public/ 3 | node_modules/ 4 | config/ -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["eslint:recommended", "plugin:prettier/recommended"], 3 | "parserOptions": { 4 | "ecmaVersion": 10, 5 | "sourceType": "module" 6 | }, 7 | "env": { 8 | "browser": true, 9 | "node": true, 10 | "es6": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | local.json 2 | # Logs 3 | logs 4 | *.log 5 | npm-debug.log* 6 | yarn-debug.log* 7 | yarn-error.log* 8 | 9 | # Runtime data 10 | pids 11 | *.pid 12 | *.seed 13 | *.pid.lock 14 | 15 | # Directory for instrumented libs generated by jscoverage/JSCover 16 | lib-cov 17 | 18 | # Coverage directory used by tools like istanbul 19 | coverage 20 | 21 | # nyc test coverage 22 | .nyc_output 23 | 24 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 25 | .grunt 26 | 27 | # Bower dependency directory (https://bower.io/) 28 | bower_components 29 | 30 | # node-waf configuration 31 | .lock-wscript 32 | 33 | # Compiled binary addons (https://nodejs.org/api/addons.html) 34 | build/Release 35 | 36 | # Dependency directories 37 | node_modules/ 38 | jspm_packages/ 39 | 40 | # TypeScript v1 declaration files 41 | typings/ 42 | 43 | # Optional npm cache directory 44 | .npm 45 | 46 | # Optional eslint cache 47 | .eslintcache 48 | 49 | # Optional REPL history 50 | .node_repl_history 51 | 52 | # Output of 'npm pack' 53 | *.tgz 54 | 55 | # Yarn Integrity file 56 | .yarn-integrity 57 | 58 | # dotenv environment variables file 59 | .env 60 | 61 | # local config file 62 | local.json 63 | 64 | # parcel-bundler cache (https://parceljs.org/) 65 | .cache 66 | 67 | # next.js build output 68 | .next 69 | 70 | # nuxt.js build output 71 | .nuxt 72 | 73 | # vuepress build output 74 | .vuepress/dist 75 | 76 | # Serverless directories 77 | .serverless 78 | 79 | # FuseBox cache 80 | .fusebox/ 81 | 82 | #dist/ 83 | client.crt 84 | client.key 85 | .DS_Store 86 | viz/sql/fixture.csv 87 | viz/dist 88 | 89 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | package.json 2 | package-lock.json 3 | dist/ 4 | public/ 5 | viz/styles/components/demand.css 6 | viz/config/ 7 | generate_orders/config/ 8 | sfdx/ -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | redshift_batch: cd redshift_batch && node index.js 2 | web: cd viz && npm run start 3 | dataworker: cd generate_data && node index.js -c kafka.js 4 | mq_broker: cd mq_broker && node index.js 5 | mq_worker: cd mq_worker && node index.js 6 | order_maker: cd generate_orders && node index.js 7 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODOs 2 | 3 | - [x] Finish top-level README docs 4 | - [x] Edit `viz` README to remove duplicate info in top-level README 5 | - [x] Try out demo script 6 | - [ ] Edit demo script 7 | - [ ] Create mockup image of possible data producers 8 | - [x] Write SQL query to delete all data 9 | - [x] Write SQL query to load fixture data (from S3? or through Kafka?) 10 | - [x] Design way for demoer to trigger delete all data, load fixture data, and start live stream 11 | - [ ] Create Metabase charts and add to a dashboard 12 | - [ ] Record demo script 13 | - [ ] Replace all hard-coded configuration references with env vars 14 | - [ ] Consider adding Metabase deploy to the demo script. Will require saving Metabase configuration from Postgres so that it can be injected quickly during demo. 15 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Example Product/User Analytics System Using Apache Kafka, AWS RedShift, and Metabase", 3 | "description": "This is an example of a system that captures a large stream of product usage data, or events, and provides both real-time data visualization and SQL-based data analytics.", 4 | "keywords": [ 5 | "heroku", 6 | "aws-redshift", 7 | "metabase", 8 | "kafka", 9 | "data analytics", 10 | "data visualization" 11 | ], 12 | "repository": "https://github.com/heroku-examples/analytics-with-kafka-redshift-metabase", 13 | "success_url": "/", 14 | "env": { 15 | "USE_DB": { 16 | "description": "Use AWS Redshift or Heroku Postres. Can be 'redshift' or 'heroku'.", 17 | "value": "redshift" 18 | }, 19 | "AWS_ACCESS_KEY_ID": { 20 | "description": "AWS access key used by RedShift to access fixture data in S3" 21 | }, 22 | "AWS_SECRET_ACCESS_KEY": { 23 | "description": "AWS secret access key used by RedShift to access fixture data in S3" 24 | }, 25 | "ADMIN_PASSWORD": { 26 | "description": "Password to use Demo Admin functionality. Default is 'supersecret'." 27 | }, 28 | "AWS_DATABASE_URL": { 29 | "description": "Connection string to RedShift database (e.g. postgres://username:password@my-redshift-cluster.abcdef123456.us-east-1.redshift.amazonaws.com:5439/db_name)" 30 | }, 31 | "FIXTURE_DATA_S3": { 32 | "description": "S3 path to CSV of fixture data to load into Redshift before starting data stream through Kafka. Leave the default unless you have a reason to change it.", 33 | "value": "s3://aws-heroku-integration-demo/fixture.csv" 34 | }, 35 | "KAFKA_PREFIX": { 36 | "description": "Prefix for Kafka topics. Include a trailing seperator character if you expect one.", 37 | "required": false 38 | }, 39 | "KAFKA_TOPIC": { 40 | "description": "Kafka topic name. Leave the default unless you have a reason to change it.", 41 | "value": "ecommerce-logs", 42 | "required": true 43 | }, 44 | "KAFKA_CMD_TOPIC": { 45 | "description": "Kafka topic name for audience cmds. Leave the default unless you have a reason to change it.", 46 | "value": "audience-cmds", 47 | "required": true 48 | }, 49 | "KAFKA_WEIGHT_TOPIC": { 50 | "description": "Kafka topic name for category weight updates. Leave the default unless you have a reason to change it.", 51 | "value": "weight-updates", 52 | "required": true 53 | }, 54 | "KAFKA_QUEUE_TOPIC": { 55 | "description": "Kafka topic name for queue length updates. Leave the default unless you have a reason to change it.", 56 | "value": "queue-length", 57 | "required": true 58 | }, 59 | "KAFKA_QUEUE_WORKER": { 60 | "description": "Kafka topic name for queue worker updates. Leave the default unless you have a reason to change it.", 61 | "value": "queue-worker", 62 | "required": true 63 | }, 64 | "KAFKA_CONSUMER_GROUP": { 65 | "description": "Kafka consumer group name that is used by `redshift_batch` process type. Leave the default unless you have a reason to change it.", 66 | "value": "redshift-batch", 67 | "required": true 68 | } 69 | }, 70 | "formation": { 71 | "web": { 72 | "quantity": 1 73 | }, 74 | "redshift_batch": { 75 | "quantity": 1 76 | } 77 | }, 78 | "addons": ["heroku-kafka:basic-0"] 79 | } 80 | -------------------------------------------------------------------------------- /bin/addtopics: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | heroku kafka:topics:create $KAFKA_PREFIX$KAFKA_TOPIC 4 | heroku kafka:topics:create $KAFKA_PREFIX$KAFKA_CMD_TOPIC 5 | heroku kafka:topics:create $KAFKA_PREFIX$KAFKA_WEIGHT_TOPIC 6 | heroku kafka:topics:create $KAFKA_PREFIX$KAFKA_QUEUE_TOPIC 7 | heroku kafka:topics:create $KAFKA_PREFIX$KAFKA_QUEUE_WORKER 8 | -------------------------------------------------------------------------------- /bin/getfixture: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | wget http://s3.amazonaws.com/aws-heroku-integration-demo/fixture.csv -O ./viz/sql/fixture.csv 4 | -------------------------------------------------------------------------------- /clientLogger.js: -------------------------------------------------------------------------------- 1 | const winstonMethods = { 2 | error: 'error', 3 | warn: 'log', 4 | info: 'log', 5 | http: 'log', 6 | verbose: 'log', 7 | debug: 'log', 8 | silly: 'log' 9 | } 10 | 11 | module.exports = () => 12 | Object.keys(winstonMethods).reduce((acc, winstonMethod) => { 13 | const consoleMethod = winstonMethods[winstonMethod] 14 | acc[winstonMethod] = (...message) => 15 | console[consoleMethod](winstonMethod, ...message) 16 | return acc 17 | }, {}) 18 | -------------------------------------------------------------------------------- /docs/heroku-connect/image1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image1.png -------------------------------------------------------------------------------- /docs/heroku-connect/image10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image10.png -------------------------------------------------------------------------------- /docs/heroku-connect/image11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image11.png -------------------------------------------------------------------------------- /docs/heroku-connect/image12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image12.png -------------------------------------------------------------------------------- /docs/heroku-connect/image13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image13.png -------------------------------------------------------------------------------- /docs/heroku-connect/image14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image14.png -------------------------------------------------------------------------------- /docs/heroku-connect/image15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image15.png -------------------------------------------------------------------------------- /docs/heroku-connect/image16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image16.png -------------------------------------------------------------------------------- /docs/heroku-connect/image17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image17.png -------------------------------------------------------------------------------- /docs/heroku-connect/image2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image2.png -------------------------------------------------------------------------------- /docs/heroku-connect/image3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image3.png -------------------------------------------------------------------------------- /docs/heroku-connect/image4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image4.png -------------------------------------------------------------------------------- /docs/heroku-connect/image5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image5.png -------------------------------------------------------------------------------- /docs/heroku-connect/image6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image6.png -------------------------------------------------------------------------------- /docs/heroku-connect/image7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image7.png -------------------------------------------------------------------------------- /docs/heroku-connect/image8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image8.png -------------------------------------------------------------------------------- /docs/heroku-connect/image9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/image9.png -------------------------------------------------------------------------------- /docs/heroku-connect/mappings/account.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/mappings/account.jpg -------------------------------------------------------------------------------- /docs/heroku-connect/mappings/contract.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/mappings/contract.jpg -------------------------------------------------------------------------------- /docs/heroku-connect/mappings/order-item.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/mappings/order-item.jpg -------------------------------------------------------------------------------- /docs/heroku-connect/mappings/order.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/mappings/order.jpg -------------------------------------------------------------------------------- /docs/heroku-connect/mappings/pricebook-entry.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/mappings/pricebook-entry.jpg -------------------------------------------------------------------------------- /docs/heroku-connect/mappings/pricebook2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/mappings/pricebook2.jpg -------------------------------------------------------------------------------- /docs/heroku-connect/mappings/product2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/mappings/product2.jpg -------------------------------------------------------------------------------- /docs/heroku-connect/mappings/record-type.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/heroku-connect/mappings/record-type.jpg -------------------------------------------------------------------------------- /docs/kafka-stream-viz-architecture.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/docs/kafka-stream-viz-architecture.gif -------------------------------------------------------------------------------- /generate_data/README.md: -------------------------------------------------------------------------------- 1 | # Heroku Kafka AWS Redshift MetaBase -- Data Generator 2 | 3 | ## First Time 4 | 5 | Install the modules. 6 | `npm i` 7 | 8 | Create or using an existing config. 9 | 10 | `node index.js -c [file.json]` 11 | 12 | ## Startup 13 | 14 | `node index.js -c fixture.json` 15 | 16 | `node index.js -c kafka.json` 17 | 18 | ## JSON config file fields 19 | 20 | ```js 21 | { 22 | "maxWait": 10, // float, max number of seconds to wait between log entries if the volume is 0 23 | "campaignVolumeMult": 1.5, // float, factor to increase the volume, following the campaign start 24 | "maxSessions": 500, // max number of sessions to simulate at a time (affected by current volume) 25 | "campaign": "ae271515-3b71-4c02-88b1-a009fe34279e", // campaign unique id 26 | "startTime": "2018-09-02T07:00:00-07:00", // simulation start time 27 | "endTime": "2018-09-03T00:00:00-07:00", // simulation end time 28 | "campaignTime": "2018-09-02T06:10:00-07:00", // simulation time that the campaign factor starts (and wishlisting) 29 | "primeUntil": "2018-09-02T09:00:00-07:00", // for a kafka output, simulate as fast as you can until this time to prime the sessions 30 | "campaignPercentage": 0.33, // float 0.0-1.0 what percentage of sessions should be campaign session after the session start time 31 | "skipBatchEvents": true, // don't send kafka events during the prime 32 | "mode": "", // no longer used 33 | "output": { 34 | "type": "kafka", // kafka or csv 35 | "topic": "ecommerce-logs", // kafka specific topic 36 | "kafka": { // config that initializes no-kafka client 37 | "connectionString": "kafka://localhost:9092" 38 | "ssl": { 39 | "cert": "", 40 | "key": "", 41 | } 42 | } 43 | }, 44 | "badCategory": "EKUX", // category of products where the wishlist feature is broken 45 | "products": { // any number of products 46 | "sku": { // unique sku for product 47 | "category": "sub-sku", // category of product 48 | "weight": 49 // 0-100 integer wait 49 | } 50 | }, 51 | volume: [ // array of 24 floats 0.0 - 1.0 to indicate the relative volume percentage 52 | ] 53 | } 54 | ``` 55 | -------------------------------------------------------------------------------- /generate_data/generate_products.js: -------------------------------------------------------------------------------- 1 | const skuChars = 'ABCDEFGHJKMNPQRTUVWXYZ2346789' 2 | const prefixCompany = 'ACME' 3 | const numCategories = 8 4 | const numProducts = 78 5 | const maxRev = 6 6 | 7 | const categories = new Set() 8 | const categoryChars = 4 9 | 10 | const products = new Set() 11 | const productChars = 6 12 | 13 | const categoryMap = {} 14 | const productMap = {} 15 | 16 | const generateSkuStr = (num) => { 17 | const strchars = [] 18 | for (let i = 0; i < num; i++) { 19 | strchars.push(skuChars.charAt(Math.floor(Math.random() * skuChars.length))) 20 | } 21 | return strchars.join('') 22 | } 23 | 24 | while (categories.size < numCategories) { 25 | const category = generateSkuStr(categoryChars) 26 | categories.add(category) 27 | categoryMap[category] = [] 28 | } 29 | 30 | const catArr = [...categories] 31 | 32 | while (products.size < numProducts) { 33 | const category = catArr[Math.floor(Math.random() * catArr.length)] 34 | const product = `${prefixCompany}-${category}-${generateSkuStr( 35 | productChars 36 | )}-${Math.ceil(Math.random() * maxRev)}` 37 | 38 | const startSize = products.size 39 | products.add(product) 40 | 41 | if (products.size > startSize) { 42 | categoryMap[category].push(product) 43 | productMap[product] = { 44 | category: category, 45 | weight: Math.ceil(Math.random() * 90 + 10) 46 | } 47 | } 48 | } 49 | 50 | console.log(JSON.stringify(productMap, null, 4)) 51 | -------------------------------------------------------------------------------- /generate_data/index.js: -------------------------------------------------------------------------------- 1 | const Moment = require('moment') 2 | const ShoppingFeed = require('./shoppingfeed') 3 | const ProgressBar = require('progress') 4 | const CsvStringify = require('csv-stringify') 5 | const Kafka = require('no-kafka') 6 | const fs = require('fs') 7 | const path = require('path') 8 | const argv = require('minimist')(process.argv) 9 | const logger = require('../logger')('generate_data') 10 | 11 | const configFilePath = path.resolve(argv.c) 12 | const config = require(configFilePath) 13 | 14 | if (config.output.type === 'csv') { 15 | const start = Moment(config.startTime) 16 | const end = Moment(config.endTime) 17 | const diff = end.diff(start, 'seconds') 18 | 19 | const bar = new ProgressBar(':bar ', { total: diff }) 20 | let last = start.clone() 21 | const csvStream = CsvStringify({ 22 | header: true, 23 | 24 | columns: { 25 | time: 'time', 26 | session: 'session', 27 | action: 'action', 28 | product: 'product', 29 | category: 'category', 30 | campaign: 'campaign' 31 | } 32 | }) 33 | const writeStream = fs.createWriteStream(config.output.path) 34 | 35 | csvStream.pipe(writeStream) 36 | 37 | const csvOutput = (event) => { 38 | csvStream.write(event) 39 | } 40 | 41 | const updateProgress = (time) => { 42 | const amount = time.diff(last, 'ms') 43 | bar.tick(amount / 1000) 44 | last = time.clone() 45 | } 46 | 47 | const endCallback = () => { 48 | csvStream.end() 49 | } 50 | 51 | const sf = new ShoppingFeed( 52 | config, 53 | csvOutput, 54 | () => {}, 55 | updateProgress, 56 | endCallback 57 | ) 58 | //sf.updateBatch(Moment(config.primeUntil)); 59 | //sf.updateLive(); 60 | sf.updateBatch() 61 | } else if (config.output.type === 'kafka') { 62 | const start = Moment(config.startTime) 63 | const end = Moment(config.endTime) 64 | const diff = end.diff(start, 'seconds') 65 | 66 | const bar = new ProgressBar(':bar ', { total: diff }) 67 | let last = start.clone() 68 | 69 | const producer = new Kafka.Producer(config.output.kafka) 70 | const consumer = new Kafka.SimpleConsumer({ 71 | idleTimeout: 1000, 72 | connectionTimeout: 10 * 1000, 73 | clientId: config.output.cmd_topic, 74 | consumer: config.output.kafka 75 | }) 76 | 77 | let ended = 0 78 | let sf = null 79 | 80 | const handleCmdTopic = (messageSet) => { 81 | const items = messageSet.map((m) => 82 | JSON.parse(m.message.value.toString('utf8')) 83 | ) 84 | for (const cmd of items) { 85 | sf.handleCmd(cmd) 86 | } 87 | } 88 | 89 | const handleOutput = (event) => { 90 | producer 91 | .send({ 92 | topic: config.output.topic, 93 | message: { value: JSON.stringify(event) }, 94 | partition: 0 95 | }) 96 | .catch((e) => { 97 | throw e 98 | }) 99 | } 100 | 101 | const handleWeight = (event) => { 102 | const output = { 103 | type: 'weights', 104 | weights: event 105 | } 106 | producer 107 | .send({ 108 | topic: config.output.weight_topic, 109 | message: { value: JSON.stringify(output) }, 110 | partition: 0 111 | }) 112 | .catch((e) => { 113 | throw e 114 | }) 115 | } 116 | 117 | const handleProgress = (time) => { 118 | const amount = time.diff(last, 'ms') 119 | bar.tick(amount / 1000) 120 | if (ended > 0) { 121 | logger.info({ total: sf.counts.TOTAL }) 122 | } 123 | last = time.clone() 124 | } 125 | 126 | const handleEnd = () => { 127 | ended++ 128 | if (ended === 1) { 129 | sf.updateLive() 130 | } 131 | } 132 | 133 | sf = new ShoppingFeed( 134 | config, 135 | handleOutput, 136 | handleWeight, 137 | handleProgress, 138 | handleEnd 139 | ) 140 | 141 | producer 142 | .init() 143 | .then(() => { 144 | return consumer.init() 145 | }) 146 | .then(() => { 147 | return consumer.subscribe(config.output.cmd_topic, handleCmdTopic) 148 | }) 149 | .then(() => { 150 | sf.updateBatch(Moment(config.primeUntil)) 151 | }) 152 | } 153 | -------------------------------------------------------------------------------- /generate_data/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "generate_data", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "bunyan": "^1.8.12", 13 | "csv-stringify": "^5.3.3", 14 | "minimist": "^1.2.3", 15 | "moment": "^2.24.0", 16 | "no-kafka": "^3.4.3", 17 | "progress": "^2.0.3", 18 | "uuid": "^3.3.3", 19 | "winston": "^3.2.1" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /generate_orders/README.md: -------------------------------------------------------------------------------- 1 | 2 | # generate_orders 3 | 4 | This worker creates both fulfillment and purchase orders periodically. 5 | 6 | ## How often and how many 7 | 8 | It produces about 150 orders total every minute and it creates more puchase orders than fulfillment orders overtime. 9 | Sometimes it creates more fulfillment orders in one period. 10 | 11 | ## Development Setup 12 | 13 | ### Requirements 14 | 15 | - Heroku connect 16 | - Heroku Postgres add-on 17 | - Redis add-on 18 | 19 | ### Nodejs setup 20 | 21 | ```shell 22 | npm install 23 | ``` 24 | 25 | ### Environment variables 26 | 27 | You can find the most of these variables on Salesforce 28 | 29 | - `REDIS_URL`: Redis' endpoint url with credentials. [https://devcenter.heroku.com/articles/heroku-redis#redis-credentials](https://devcenter.heroku.com/articles/heroku-redis#redis-credentials) 30 | 31 | ### Development Server 32 | 33 | ```shell 34 | node index.js 35 | ``` 36 | 37 | ## How to start/stop/reset order creation process 38 | 39 | By default, it's stopped. 40 | The worker is subscribing to Redis and you can send a command to channel `generate_orders`. 41 | 42 | `command` takes `start`, `stop`, or `reset`. 43 | 44 | The format is: 45 | 46 | ``` 47 | { 48 | type: 'command', 49 | value: '{start|stop|reset}'' 50 | } 51 | ``` 52 | 53 | - **start** starts making new orders 54 | - **stop** stops the process 55 | - **reset** also stops the process then starts deleting all rows. 56 | 57 | ### User interface 58 | 59 | You can run `viz` and go to `/ordercontrol` and you will see three buttons with start, stop, and delete all(reset). 60 | By pressing these buttons, you can send commands to this woker. 61 | There is a status text you can see to know what it's doing. 62 | 63 | ## Configuration 64 | You can find the config files under generate_orders/config. 65 | 66 | - CATEGORY_LIST 67 | List of categories for orders 68 | 69 | - ORDER_INTERVAL 70 | This value defines how often orders are created 71 | 72 | - ORDER_QUANTITY 73 | This value defines the base amount of orders to create for each categories 74 | 75 | - ORDER_QUANTITY_RANDOMNESS 76 | A random number is generated from 0 to this value and it's either added or subtracted from the base value when order is made. 77 | 78 | - FULFILLMENT_ORDER_MULTIPLY 79 | When a fulfillment order is created, its amount will be multiplied by this value 80 | 81 | - FULFILLMENT_ORDER_RATIO 82 | This value defines how often a fulfillment order is created. 83 | For example, if this value is 5, then it's created about 1 in 5 times. 84 | 85 | - FULFILLMENT_ORDER_TYPE 86 | The name of the fulfilment order type 87 | 88 | - PURCHASE_ORDER_TYPE 89 | The name of the purchase order type 90 | 91 | - REDIS_CHANNEL 92 | The name of the redis channel 93 | 94 | #### Logs shows the current state of the worker 95 | 96 | The logs section of the Heroku dashboard should show what this worker is doing in detail. 97 | The worker's name is `order_maker` in the log. 98 | If you want to test this worker, open this log and the UI. 99 | -------------------------------------------------------------------------------- /generate_orders/config/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "CATEGORY_LIST": [ 3 | "Lawn & Garden", 4 | "Electronics", 5 | "Apparel", 6 | "Home Furnishing", 7 | "Housewares", 8 | "Toys", 9 | "Books" 10 | ], 11 | "ORDER_INTERVAL": 15000, 12 | "ORDER_QUANTITY": 40, 13 | "ORDER_QUANTITY_RANDOMNESS": 30, 14 | "FULFILLMENT_ORDER_MULTIPLY": 3, 15 | "FULFILLMENT_ORDER_RATIO": 5, 16 | "FULFILLMENT_ORDER_TYPE": "Fulfillment Order", 17 | "PURCHASE_ORDER_TYPE": "Purchase Order", 18 | "REDIS_CHANNEL": "generate_orders2" 19 | } 20 | -------------------------------------------------------------------------------- /generate_orders/index.js: -------------------------------------------------------------------------------- 1 | const Redis = require('ioredis') 2 | const runner = require('./runner') 3 | const logger = require('../logger')('generate_orders') 4 | const knex = require('knex')({ 5 | client: 'pg', 6 | connection: `${process.env.DATABASE_URL}?ssl=true` 7 | }) 8 | const config = require('config') 9 | const redisSub = new Redis(process.env.REDIS_URL, { connectTimeout: 10000 }) 10 | const redisPub = new Redis(process.env.REDIS_URL, { connectTimeout: 10000 }) 11 | 12 | let deletePromise = Promise.resolve() 13 | 14 | const initEvents = () => { 15 | redisSub.subscribe(config.REDIS_CHANNEL, () => { 16 | console.log('Subscribing to redis') 17 | }) 18 | 19 | redisSub.on('message', function(channel, _message) { 20 | const message = JSON.parse(_message) 21 | if (message.type !== 'command') { 22 | return 23 | } 24 | 25 | console.log(`Receive command ${message.value}`) 26 | 27 | const command = message.value 28 | switch (command) { 29 | case 'start': 30 | deletePromise.then(runner.startOrderInterval) 31 | break 32 | case 'stop': 33 | runner.stopOrderInterval() 34 | break 35 | case 'reset': 36 | deletePromise = deletePromise.then(runner.deleteAll) 37 | break 38 | default: 39 | logger.info(`Invalid command: ${command}`) 40 | } 41 | }) 42 | } 43 | 44 | const stratStatusPubInterval = () => { 45 | setInterval(() => { 46 | redisPub.publish( 47 | config.REDIS_CHANNEL, 48 | JSON.stringify({ type: 'status', value: runner.getStatus() }) 49 | ) 50 | }, 3000) 51 | } 52 | 53 | const init = () => { 54 | initEvents() 55 | runner.init({ 56 | _knex: knex 57 | }) 58 | logger.info('all ready') 59 | stratStatusPubInterval() 60 | } 61 | 62 | init() 63 | -------------------------------------------------------------------------------- /generate_orders/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "generate_orders", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": {}, 7 | "keywords": [], 8 | "author": "", 9 | "license": "ISC", 10 | "dependencies": { 11 | "config": "^3.2.5", 12 | "express": "^4.17.1", 13 | "ioredis": "^4.14.1", 14 | "knex": "^0.20.1", 15 | "lodash": "^4.17.15", 16 | "moment": "^2.24.0", 17 | "pg": "^7.12.1" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /generate_orders/runner.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash') 2 | const moment = require('moment') 3 | const config = require('config') 4 | const logger = require('../logger')('generate_orders') 5 | 6 | /* 7 | ORDER_QUANTITY_RANDOMNESS - adding randomness to the amount +- this value 8 | FULFILLMENT_ORDER_MULTIPLY - When it's a fulfillment order, this amount is multipled with this number 9 | FULFILLMENT_ORDER_RATIO - 1 in this amount will be the fulfilment order 10 | */ 11 | 12 | let deleting = false 13 | let totallOrdersCreated = 0 14 | let knex = null 15 | let orderInterval = null 16 | 17 | const getOrderCount = (isFulfillmentOrder) => { 18 | let count = 19 | config.ORDER_QUANTITY + 20 | _.sample([1, -1]) * 21 | Math.floor(Math.random() * config.ORDER_QUANTITY_RANDOMNESS) 22 | return parseInt( 23 | isFulfillmentOrder ? count * config.FULFILLMENT_ORDER_MULTIPLY : count 24 | ) 25 | } 26 | 27 | const makeOrder = (category) => { 28 | const isFulfillmentOrder = 29 | Math.floor(Math.random() * 100) % config.FULFILLMENT_ORDER_RATIO === 0 30 | const amount = getOrderCount(isFulfillmentOrder) 31 | 32 | let order = { 33 | category, 34 | amount, 35 | approved: true, 36 | type: isFulfillmentOrder 37 | ? config.FULFILLMENT_ORDER_TYPE 38 | : config.PURCHASE_ORDER_TYPE, 39 | createdat: moment().toISOString() 40 | } 41 | totallOrdersCreated++ 42 | 43 | return knex('orders') 44 | .insert(order) 45 | .returning('*') 46 | } 47 | 48 | const makeOrders = () => { 49 | let promise = Promise.resolve() 50 | let categoryClone = _.shuffle(config.CATEGORY_LIST).splice( 51 | Math.floor(Math.random() * config.CATEGORY_LIST.length) 52 | ) 53 | 54 | _.forEach(categoryClone, (category) => { 55 | promise = promise.then(() => { 56 | return makeOrder(category) 57 | }) 58 | }) 59 | return promise 60 | } 61 | 62 | const init = ({ _knex }) => { 63 | knex = _knex 64 | } 65 | 66 | const deleteAll = () => { 67 | stopOrderInterval() 68 | deleting = true 69 | return knex('orders') 70 | .del() 71 | .then(() => (deleting = false)) 72 | } 73 | 74 | const startOrderInterval = () => { 75 | stopOrderInterval() 76 | orderInterval = setInterval(makeOrders, config.ORDER_INTERVAL) 77 | logger.info('Order creation interval started') 78 | makeOrders() 79 | } 80 | 81 | const stopOrderInterval = () => { 82 | if (orderInterval) { 83 | clearInterval(orderInterval) 84 | orderInterval = null 85 | } 86 | logger.info('Order creation interval stopped') 87 | } 88 | 89 | const getStatus = () => { 90 | let state 91 | if (deleting) { 92 | state = 'Deleting' 93 | } else if (orderInterval) { 94 | state = 'Running' 95 | } else { 96 | state = 'Stopped' 97 | } 98 | 99 | return { 100 | state, 101 | totallOrdersCreated 102 | } 103 | } 104 | 105 | module.exports = { 106 | deleteAll, 107 | startOrderInterval, 108 | stopOrderInterval, 109 | getStatus, 110 | init 111 | } 112 | -------------------------------------------------------------------------------- /logger.js: -------------------------------------------------------------------------------- 1 | const winston = require('winston') 2 | 3 | module.exports = (service, options = {}) => 4 | winston.createLogger( 5 | Object.assign( 6 | { 7 | level: 'debug', 8 | format: winston.format.json(), 9 | defaultMeta: { service }, 10 | transports: [new winston.transports.Console()] 11 | }, 12 | options 13 | ) 14 | ) 15 | -------------------------------------------------------------------------------- /mq_broker/constants.js: -------------------------------------------------------------------------------- 1 | const logger = require('../logger')('mq_broker') 2 | 3 | const prefix = process.env.KAFKA_PREFIX || '' 4 | 5 | module.exports.KAFKA_TOPIC = prefix + process.env.KAFKA_TOPIC 6 | module.exports.KAFKA_QUEUE_TOPIC = prefix + process.env.KAFKA_QUEUE_TOPIC 7 | module.exports.KAFKA_QUEUE_WORKER = prefix + process.env.KAFKA_QUEUE_WORKER 8 | 9 | logger.info(`Kafka topic: ${module.exports.KAFKA_TOPIC}`) 10 | logger.info(`Kafka queue length topic: ${module.exports.KAFKA_QUEUE_TOPIC}`) 11 | logger.info(`Kafka queue worker topic: ${module.exports.KAFKA_QUEUE_WORKER}`) 12 | -------------------------------------------------------------------------------- /mq_broker/index.js: -------------------------------------------------------------------------------- 1 | const Kafka = require('no-kafka') 2 | const constants = require('./constants') 3 | const logger = require('../logger')('mq_broker') 4 | 5 | const kafkaConfig = { 6 | idleTimeout: 1000, 7 | connectionTimeout: 10 * 1000, 8 | clientId: 'mq_broker', 9 | consumer: { 10 | connectionString: process.env.KAFKA_URL.replace(/\+ssl/g, ''), 11 | ssl: { 12 | cert: process.env.KAFKA_CLIENT_CERT, 13 | key: process.env.KAFKA_CLIENT_CERT_KEY 14 | } 15 | } 16 | } 17 | 18 | const consumer = new Kafka.SimpleConsumer(kafkaConfig) 19 | const producer = new Kafka.Producer(kafkaConfig) 20 | 21 | ;(async () => { 22 | await producer.init() 23 | await consumer.init() 24 | 25 | await consumer.subscribe(constants.KAFKA_TOPIC, (messageSet) => { 26 | logger.info(`Message set length: ${messageSet.length}`) 27 | 28 | messageSet.forEach((m) => { 29 | const value = m.message.value.toString('utf8') 30 | 31 | producer.send({ 32 | topic: constants.KAFKA_QUEUE_WORKER, 33 | message: { value }, 34 | partition: 0 35 | }) 36 | 37 | logger.info(`Sent message to ${constants.KAFKA_QUEUE_WORKER}: ${value}`) 38 | }) 39 | }) 40 | })() 41 | -------------------------------------------------------------------------------- /mq_broker/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mq_broker", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@types/bluebird": { 8 | "version": "3.5.0", 9 | "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.0.tgz", 10 | "integrity": "sha1-JjNHCk6r6aR82aRf2yDtX5NAe8o=" 11 | }, 12 | "@types/lodash": { 13 | "version": "4.14.137", 14 | "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.137.tgz", 15 | "integrity": "sha512-g4rNK5SRKloO+sUGbuO7aPtwbwzMgjK+bm9BBhLD7jGUiGR7zhwYEhSln/ihgYQBeIJ5j7xjyaYzrWTcu3UotQ==" 16 | }, 17 | "bin-protocol": { 18 | "version": "3.1.1", 19 | "resolved": "https://registry.npmjs.org/bin-protocol/-/bin-protocol-3.1.1.tgz", 20 | "integrity": "sha512-9vCGfaHC2GBHZwGQdG+DpyXfmLvx9uKtf570wMLwIc9wmTIDgsdCBXQxTZu5X2GyogkfBks2Ode4N0sUVxJ2qQ==", 21 | "requires": { 22 | "lodash": "^4.17.11", 23 | "long": "^4.0.0", 24 | "protocol-buffers-schema": "^3.0.0" 25 | } 26 | }, 27 | "bluebird": { 28 | "version": "3.5.5", 29 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.5.tgz", 30 | "integrity": "sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w==" 31 | }, 32 | "buffer-crc32": { 33 | "version": "0.2.13", 34 | "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", 35 | "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" 36 | }, 37 | "connection-parse": { 38 | "version": "0.0.7", 39 | "resolved": "https://registry.npmjs.org/connection-parse/-/connection-parse-0.0.7.tgz", 40 | "integrity": "sha1-GOcxiqsGppkmc3KxDFIm0locmmk=" 41 | }, 42 | "hashring": { 43 | "version": "3.2.0", 44 | "resolved": "https://registry.npmjs.org/hashring/-/hashring-3.2.0.tgz", 45 | "integrity": "sha1-/aTv3oqiLNuX+x0qZeiEAeHBRM4=", 46 | "requires": { 47 | "connection-parse": "0.0.x", 48 | "simple-lru-cache": "0.0.x" 49 | } 50 | }, 51 | "lodash": { 52 | "version": "4.17.15", 53 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", 54 | "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" 55 | }, 56 | "long": { 57 | "version": "4.0.0", 58 | "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", 59 | "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" 60 | }, 61 | "murmur-hash-js": { 62 | "version": "1.0.0", 63 | "resolved": "https://registry.npmjs.org/murmur-hash-js/-/murmur-hash-js-1.0.0.tgz", 64 | "integrity": "sha1-UEEEkmnJZjPIZjhpYLL0KJ515bA=" 65 | }, 66 | "nice-simple-logger": { 67 | "version": "1.0.1", 68 | "resolved": "https://registry.npmjs.org/nice-simple-logger/-/nice-simple-logger-1.0.1.tgz", 69 | "integrity": "sha1-D55khSe+e+PkmrdvqMjAmK+VG/Y=", 70 | "requires": { 71 | "lodash": "^4.3.0" 72 | } 73 | }, 74 | "no-kafka": { 75 | "version": "3.4.3", 76 | "resolved": "https://registry.npmjs.org/no-kafka/-/no-kafka-3.4.3.tgz", 77 | "integrity": "sha512-hYnkg1OWVdaxORdzVvdQ4ueWYpf7IICObPzd24BBiDyVG5219VkUnRxSH9wZmisFb6NpgABzlSIL1pIZaCKmXg==", 78 | "requires": { 79 | "@types/bluebird": "3.5.0", 80 | "@types/lodash": "^4.14.55", 81 | "bin-protocol": "^3.1.1", 82 | "bluebird": "^3.3.3", 83 | "buffer-crc32": "^0.2.5", 84 | "hashring": "^3.2.0", 85 | "lodash": "=4.17.11", 86 | "murmur-hash-js": "^1.0.0", 87 | "nice-simple-logger": "^1.0.1", 88 | "wrr-pool": "^1.0.3" 89 | }, 90 | "dependencies": { 91 | "lodash": { 92 | "version": "4.17.11", 93 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", 94 | "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" 95 | } 96 | } 97 | }, 98 | "protocol-buffers-schema": { 99 | "version": "3.3.2", 100 | "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.3.2.tgz", 101 | "integrity": "sha512-Xdayp8sB/mU+sUV4G7ws8xtYMGdQnxbeIfLjyO9TZZRJdztBGhlmbI5x1qcY4TG5hBkIKGnc28i7nXxaugu88w==" 102 | }, 103 | "simple-lru-cache": { 104 | "version": "0.0.2", 105 | "resolved": "https://registry.npmjs.org/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz", 106 | "integrity": "sha1-1ZzDoZPBpdAyD4Tucy9uRxPlEd0=" 107 | }, 108 | "wrr-pool": { 109 | "version": "1.1.4", 110 | "resolved": "https://registry.npmjs.org/wrr-pool/-/wrr-pool-1.1.4.tgz", 111 | "integrity": "sha512-+lEdj42HlYqmzhvkZrx6xEymj0wzPBxqr7U1Xh9IWikMzOge03JSQT9YzTGq54SkOh/noViq32UejADZVzrgAg==", 112 | "requires": { 113 | "lodash": "^4.17.11" 114 | } 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /mq_broker/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mq_broker", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "no-kafka": "^3.4.3" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /mq_worker/api.js: -------------------------------------------------------------------------------- 1 | function wait(ms) { 2 | return new Promise((resolve) => setTimeout(resolve, ms)) 3 | } 4 | 5 | function random(min, max) { 6 | return Math.floor(Math.random() * (max - min + 1)) + min 7 | } 8 | 9 | async function getUserInfo(message) { 10 | await wait(random(400, 500)) 11 | return message.session 12 | } 13 | 14 | async function getProductInfo(message) { 15 | await wait(random(400, 500)) 16 | return message.product 17 | } 18 | 19 | async function getCategoryInfo(message) { 20 | await wait(random(400, 500)) 21 | return message.category 22 | } 23 | 24 | async function getCampaignInfo(message) { 25 | await wait(random(400, 500)) 26 | return message.campaign 27 | } 28 | 29 | async function sendEmail(options) { 30 | await wait(random(400, 500)) 31 | return options 32 | } 33 | 34 | module.exports = { 35 | getUserInfo, 36 | getProductInfo, 37 | getCategoryInfo, 38 | getCampaignInfo, 39 | sendEmail 40 | } 41 | -------------------------------------------------------------------------------- /mq_worker/constants.js: -------------------------------------------------------------------------------- 1 | const logger = require('../logger')('mq_broker') 2 | 3 | const prefix = process.env.KAFKA_PREFIX || '' 4 | 5 | module.exports.KAFKA_TOPIC = prefix + process.env.KAFKA_TOPIC 6 | module.exports.KAFKA_QUEUE_TOPIC = prefix + process.env.KAFKA_QUEUE_TOPIC 7 | module.exports.KAFKA_QUEUE_WORKER = prefix + process.env.KAFKA_QUEUE_WORKER 8 | 9 | logger.info(`Kafka topic: ${module.exports.KAFKA_TOPIC}`) 10 | logger.info(`Kafka queue length topic: ${module.exports.KAFKA_QUEUE_TOPIC}`) 11 | logger.info(`Kafka queue worker topic: ${module.exports.KAFKA_QUEUE_WORKER}`) 12 | -------------------------------------------------------------------------------- /mq_worker/index.js: -------------------------------------------------------------------------------- 1 | const Kafka = require('no-kafka') 2 | const logger = require('../logger')('mq_worker') 3 | const processMessage = require('./processMessage') 4 | const constants = require('./constants') 5 | 6 | const convertTime = (hrtime) => { 7 | const nanoseconds = hrtime[0] * 1e9 + hrtime[1] 8 | const seconds = nanoseconds / 1e9 9 | return seconds 10 | } 11 | 12 | const kafkaConfig = { 13 | idleTimeout: 1000, 14 | connectionTimeout: 10 * 1000, 15 | clientId: 'mq_worker', 16 | consumer: { 17 | connectionString: process.env.KAFKA_URL.replace(/\+ssl/g, ''), 18 | ssl: { 19 | cert: process.env.KAFKA_CLIENT_CERT, 20 | key: process.env.KAFKA_CLIENT_CERT_KEY 21 | } 22 | } 23 | } 24 | 25 | const consumer = new Kafka.SimpleConsumer(kafkaConfig) 26 | const producer = new Kafka.Producer(kafkaConfig) 27 | 28 | ;(async () => { 29 | await producer.init() 30 | await consumer.init() 31 | 32 | await consumer.subscribe(constants.KAFKA_QUEUE_WORKER, (messageSet) => { 33 | logger.info(`Message set length: ${messageSet.length}`) 34 | 35 | messageSet.forEach(async (m) => { 36 | const value = m.message.value.toString('utf8') 37 | 38 | logger.info(`Starting processing: ${value}`) 39 | 40 | const start = process.hrtime() 41 | const data = JSON.parse(value) 42 | const processed = await processMessage(data) 43 | const processingTime = convertTime(process.hrtime(start)) 44 | 45 | logger.info(`Processed ${processingTime} - ${JSON.stringify(processed)}`) 46 | 47 | producer.send({ 48 | topic: constants.KAFKA_QUEUE_TOPIC, 49 | partition: 0, 50 | message: { 51 | value: JSON.stringify({ 52 | type: 'queue', 53 | data: { 54 | processingTime, 55 | time: new Date() 56 | } 57 | }) 58 | } 59 | }) 60 | }) 61 | }) 62 | })() 63 | -------------------------------------------------------------------------------- /mq_worker/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mq_worker", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@types/bluebird": { 8 | "version": "3.5.0", 9 | "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.0.tgz", 10 | "integrity": "sha1-JjNHCk6r6aR82aRf2yDtX5NAe8o=" 11 | }, 12 | "@types/lodash": { 13 | "version": "4.14.141", 14 | "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.141.tgz", 15 | "integrity": "sha512-v5NYIi9qEbFEUpCyikmnOYe4YlP8BMUdTcNCAquAKzu+FA7rZ1onj9x80mbnDdOW/K5bFf3Tv5kJplP33+gAbQ==" 16 | }, 17 | "bin-protocol": { 18 | "version": "3.1.1", 19 | "resolved": "https://registry.npmjs.org/bin-protocol/-/bin-protocol-3.1.1.tgz", 20 | "integrity": "sha512-9vCGfaHC2GBHZwGQdG+DpyXfmLvx9uKtf570wMLwIc9wmTIDgsdCBXQxTZu5X2GyogkfBks2Ode4N0sUVxJ2qQ==", 21 | "requires": { 22 | "lodash": "^4.17.11", 23 | "long": "^4.0.0", 24 | "protocol-buffers-schema": "^3.0.0" 25 | } 26 | }, 27 | "bluebird": { 28 | "version": "3.7.0", 29 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.0.tgz", 30 | "integrity": "sha512-aBQ1FxIa7kSWCcmKHlcHFlT2jt6J/l4FzC7KcPELkOJOsPOb/bccdhmIrKDfXhwFrmc7vDoDrrepFvGqjyXGJg==" 31 | }, 32 | "buffer-crc32": { 33 | "version": "0.2.13", 34 | "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", 35 | "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" 36 | }, 37 | "connection-parse": { 38 | "version": "0.0.7", 39 | "resolved": "https://registry.npmjs.org/connection-parse/-/connection-parse-0.0.7.tgz", 40 | "integrity": "sha1-GOcxiqsGppkmc3KxDFIm0locmmk=" 41 | }, 42 | "hashring": { 43 | "version": "3.2.0", 44 | "resolved": "https://registry.npmjs.org/hashring/-/hashring-3.2.0.tgz", 45 | "integrity": "sha1-/aTv3oqiLNuX+x0qZeiEAeHBRM4=", 46 | "requires": { 47 | "connection-parse": "0.0.x", 48 | "simple-lru-cache": "0.0.x" 49 | } 50 | }, 51 | "lodash": { 52 | "version": "4.17.11", 53 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", 54 | "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" 55 | }, 56 | "long": { 57 | "version": "4.0.0", 58 | "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", 59 | "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" 60 | }, 61 | "murmur-hash-js": { 62 | "version": "1.0.0", 63 | "resolved": "https://registry.npmjs.org/murmur-hash-js/-/murmur-hash-js-1.0.0.tgz", 64 | "integrity": "sha1-UEEEkmnJZjPIZjhpYLL0KJ515bA=" 65 | }, 66 | "nice-simple-logger": { 67 | "version": "1.0.1", 68 | "resolved": "https://registry.npmjs.org/nice-simple-logger/-/nice-simple-logger-1.0.1.tgz", 69 | "integrity": "sha1-D55khSe+e+PkmrdvqMjAmK+VG/Y=", 70 | "requires": { 71 | "lodash": "^4.3.0" 72 | } 73 | }, 74 | "no-kafka": { 75 | "version": "3.4.3", 76 | "resolved": "https://registry.npmjs.org/no-kafka/-/no-kafka-3.4.3.tgz", 77 | "integrity": "sha512-hYnkg1OWVdaxORdzVvdQ4ueWYpf7IICObPzd24BBiDyVG5219VkUnRxSH9wZmisFb6NpgABzlSIL1pIZaCKmXg==", 78 | "requires": { 79 | "@types/bluebird": "3.5.0", 80 | "@types/lodash": "^4.14.55", 81 | "bin-protocol": "^3.1.1", 82 | "bluebird": "^3.3.3", 83 | "buffer-crc32": "^0.2.5", 84 | "hashring": "^3.2.0", 85 | "lodash": "=4.17.11", 86 | "murmur-hash-js": "^1.0.0", 87 | "nice-simple-logger": "^1.0.1", 88 | "wrr-pool": "^1.0.3" 89 | } 90 | }, 91 | "protocol-buffers-schema": { 92 | "version": "3.3.2", 93 | "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.3.2.tgz", 94 | "integrity": "sha512-Xdayp8sB/mU+sUV4G7ws8xtYMGdQnxbeIfLjyO9TZZRJdztBGhlmbI5x1qcY4TG5hBkIKGnc28i7nXxaugu88w==" 95 | }, 96 | "simple-lru-cache": { 97 | "version": "0.0.2", 98 | "resolved": "https://registry.npmjs.org/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz", 99 | "integrity": "sha1-1ZzDoZPBpdAyD4Tucy9uRxPlEd0=" 100 | }, 101 | "wrr-pool": { 102 | "version": "1.1.4", 103 | "resolved": "https://registry.npmjs.org/wrr-pool/-/wrr-pool-1.1.4.tgz", 104 | "integrity": "sha512-+lEdj42HlYqmzhvkZrx6xEymj0wzPBxqr7U1Xh9IWikMzOge03JSQT9YzTGq54SkOh/noViq32UejADZVzrgAg==", 105 | "requires": { 106 | "lodash": "^4.17.11" 107 | } 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /mq_worker/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mq_worker", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "no-kafka": "^3.4.3" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /mq_worker/processMessage.js: -------------------------------------------------------------------------------- 1 | const api = require('./api') 2 | 3 | // eslint-disable-next-line no-unused-vars 4 | const processAsync = async (message) => { 5 | const [user, product, category, campaign] = await Promise.all([ 6 | api.getUserInfo(message), 7 | api.getProductInfo(message), 8 | api.getCategoryInfo(message), 9 | api.getCampaignInfo(message) 10 | ]) 11 | return api.sendEmail({ user, product, category, campaign }) 12 | } 13 | 14 | // eslint-disable-next-line no-unused-vars 15 | const processSync = async (message) => { 16 | const user = await api.getUserInfo(message) 17 | const product = await api.getProductInfo(message) 18 | const category = await api.getCategoryInfo(message) 19 | const campaign = await api.getCampaignInfo(message) 20 | return api.sendEmail({ user, product, category, campaign }) 21 | } 22 | 23 | module.exports = processSync 24 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "analytics-with-kafka-redshift-metabase", 3 | "version": "1.0.1", 4 | "dependencies": { 5 | "winston": "^3.2.1" 6 | }, 7 | "devDependencies": { 8 | "eslint": "^6.8.0", 9 | "eslint-config-prettier": "^6.9.0", 10 | "eslint-plugin-prettier": "^3.1.2", 11 | "git-validate": "^2.2.4", 12 | "prettier": "^1.19.1" 13 | }, 14 | "engines": { 15 | "node": "~10.13.0", 16 | "npm": "~6.4.1" 17 | }, 18 | "eslintConfig": { 19 | "extends": [ 20 | "eslint:recommended", 21 | "plugin:prettier/recommended" 22 | ], 23 | "parserOptions": { 24 | "ecmaVersion": 10, 25 | "sourceType": "module" 26 | }, 27 | "env": { 28 | "browser": true, 29 | "node": true, 30 | "es6": true 31 | } 32 | }, 33 | "eslintIgnore": [ 34 | "dist/*", 35 | "public/*", 36 | "sfdx/*" 37 | ], 38 | "license": "ISC", 39 | "main": "index.js", 40 | "pre-commit": [ 41 | "lint" 42 | ], 43 | "prettier": { 44 | "semi": false, 45 | "singleQuote": true, 46 | "arrowParens": "always" 47 | }, 48 | "scripts": { 49 | "build": "cd viz && npm run build", 50 | "eslint": "eslint -c .eslintrc.json --no-eslintrc --fix .", 51 | "get-fixture": "bin/getfixture", 52 | "install": "npm run install-redshift && npm run install-generate && npm run install-viz && npm run install-mq && npm run install-mq-worker && npm run install-orders && npm run get-fixture", 53 | "install-generate": "cd generate_data && npm install", 54 | "install-mq": "cd mq_broker && npm install", 55 | "install-mq-worker": "cd mq_worker && npm install", 56 | "install-redshift": "cd redshift_batch && npm install", 57 | "install-viz": "cd viz && npm install --dev", 58 | "install-orders": "cd generate_orders && npm install", 59 | "lint": "npm run eslint && npm run prettier:list", 60 | "prettier": "prettier --write '**/*.{js,css,json}'", 61 | "prettier:list": "prettier --list-different '**/*.{js,css,json}'", 62 | "test": "echo \"Error: no test specified\" && exit 1" 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /redshift_batch/README.md: -------------------------------------------------------------------------------- 1 | # redshift_batch 2 | 3 | Batches kafka messages into redshift. 4 | 5 | ## Install 6 | 7 | `npm install` 8 | 9 | Copy config/default.json to config/local.json then edit the production.json 10 | 11 | ```js 12 | { 13 | "queueSize": 50, // number of msgs to queue up before inserting 14 | "timeout": 3000, // max time to queue before inserting 15 | "database": "postgres://fritzy@localhost:5432/fritzy", // pg connection string 16 | "kafka": { 17 | "topic": "ecommerce-logs", // kafka topic 18 | "group": "redshift-batch", // consumer group id 19 | "config": { // no-kafka configuration object 20 | "kafkaHost": "kafka://localhost:9092", 21 | "ssl": { 22 | "key": "", 23 | "cert: "" 24 | } 25 | } 26 | } 27 | } 28 | ``` 29 | 30 | ```sql 31 | CREATE TABLE ecommercelogs(id INT IDENTITY(1, 1), time TIMESTAMP WITH TIME ZONE, session VARCHAR(255), action VARCHAR(255), product VARCHAR(255), category VARCHAR(255), campaign VARCHAR(255)) 32 | ``` 33 | 34 | ## Running 35 | 36 | `node index.js` 37 | -------------------------------------------------------------------------------- /redshift_batch/config/default.js: -------------------------------------------------------------------------------- 1 | const error = (message) => { 2 | throw new Error(message) 3 | } 4 | 5 | const config = { 6 | queueSize: 50, 7 | timeout: 3000, 8 | database: 9 | process.env.AWS_DATABASE_URL || 10 | process.env.DATABASE_URL || 11 | error('AWS_DATABASE_URL or DATABASE_URL env var must be defined.'), 12 | kafka: { 13 | topic: 14 | process.env.KAFKA_TOPIC || 15 | error( 16 | 'KAFKA_TOPIC env var must be defined. See `app.json` for default value.' 17 | ), 18 | group: 19 | process.env.KAFKA_CONSUMER_GROUP || 20 | error( 21 | 'KAFKA_CONSUMER_GROUP env var must be defined. See `app.json` for default value.' 22 | ), 23 | config: { 24 | connectionString: 25 | process.env.KAFKA_URL || error('KAFKA_URL env var must be defined.'), 26 | ssl: { 27 | cert: 28 | process.env.KAFKA_CLIENT_CERT || 29 | error('KAFKA_CLIENT_CERT env var must be defined.'), 30 | key: 31 | process.env.KAFKA_CLIENT_CERT_KEY || 32 | error('KAFKA_CLIENT_CERT_KEY env var must be defined.') 33 | } 34 | } 35 | } 36 | } 37 | 38 | if (process.env.KAFKA_PREFIX) { 39 | config.kafka.topic = process.env.KAFKA_PREFIX + config.kafka.topic 40 | config.kafka.group = process.env.KAFKA_PREFIX + config.kafka.group 41 | } 42 | 43 | module.exports = config 44 | -------------------------------------------------------------------------------- /redshift_batch/index.js: -------------------------------------------------------------------------------- 1 | const Postgres = require('pg-promise')({ 2 | capSQL: true 3 | }) 4 | const Config = require('getconfig') 5 | const Kafka = require('no-kafka') 6 | const { performance } = require('perf_hooks') 7 | const logger = require('../logger')('redshift_batch') 8 | 9 | Config.database = `${Config.database}?ssl=true` 10 | const db = Postgres(Config.database) 11 | db.connect() 12 | const ecommTable = new Postgres.helpers.ColumnSet( 13 | ['time', 'session', 'action', 'product', 'category', 'campaign'], 14 | { table: 'ecommercelogs' } 15 | ) 16 | 17 | const consumer = new Kafka.SimpleConsumer({ 18 | ...Config.kafka.config, 19 | groupId: Config.kafka.group 20 | }) 21 | 22 | let queue = [] 23 | let lastUpdate = performance.now() 24 | let lock = false 25 | 26 | const dataHandler = (messageSet, topic, partition) => { 27 | messageSet.forEach((msg) => { 28 | const now = performance.now() 29 | const sinceLast = now - lastUpdate 30 | const value = JSON.parse(msg.message.value) 31 | const offset = msg.offset 32 | const length = queue.push(value) 33 | 34 | if ( 35 | lock === false && 36 | (length >= Config.queueSize || sinceLast > Config.timeout) 37 | ) { 38 | logger.log(queue.length) 39 | lock = true 40 | lastUpdate = now 41 | const query = Postgres.helpers.insert(queue, ecommTable) 42 | db.query(query, queue) 43 | .then(() => { 44 | return consumer.commitOffset({ topic, partition, offset }) 45 | }) 46 | .then(() => { 47 | lock = false 48 | logger.log('unlock') 49 | }) 50 | .catch((err) => { 51 | lock = false 52 | logger.log(err) 53 | }) 54 | queue = [] 55 | } 56 | }) 57 | } 58 | 59 | consumer.init().then(() => { 60 | logger.log(`Subscribing to topic ${Config.kafka.topic}`) 61 | consumer.subscribe(Config.kafka.topic, dataHandler) 62 | }) 63 | -------------------------------------------------------------------------------- /redshift_batch/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "redshift_batch", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "getconfig": "^4.5.0", 13 | "no-kafka": "^3.4.3", 14 | "pg-promise": "^9.1.4" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/.eslintignore: -------------------------------------------------------------------------------- 1 | **/lwc/**/*.css 2 | **/lwc/**/*.html 3 | **/lwc/**/*.json 4 | **/lwc/**/*.svg 5 | **/lwc/**/*.xml 6 | .sfdx -------------------------------------------------------------------------------- /sfdx/order-fulfillment/.forceignore: -------------------------------------------------------------------------------- 1 | # List files or directories below to ignore them when running force:source:push, force:source:pull, and force:source:status 2 | # More information: https://developer.salesforce.com/docs/atlas.en-us.sfdx_dev.meta/sfdx_dev/sfdx_dev_exclude_source.htm 3 | # 4 | 5 | package.xml 6 | 7 | # LWC configuration files 8 | **/jsconfig.json 9 | **/.eslintrc.json 10 | 11 | # LWC Jest 12 | **/__tests__/** -------------------------------------------------------------------------------- /sfdx/order-fulfillment/.gitignore: -------------------------------------------------------------------------------- 1 | # This file is used for Git repositories to specify intentionally untracked files that Git should ignore. 2 | # If you are not using git, you can delete this file. For more information see: https://git-scm.com/docs/gitignore 3 | # For useful gitignore templates see: https://github.com/github/gitignore 4 | 5 | # Salesforce cache 6 | .sfdx/ 7 | .localdevserver/ 8 | 9 | # LWC VSCode autocomplete 10 | **/lwc/jsconfig.json 11 | 12 | # LWC Jest coverage reports 13 | coverage/ 14 | 15 | # Logs 16 | logs 17 | *.log 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* 21 | 22 | # Dependency directories 23 | node_modules/ 24 | 25 | # Eslint cache 26 | .eslintcache 27 | 28 | # MacOS system files 29 | .DS_Store 30 | 31 | # Windows system files 32 | Thumbs.db 33 | ehthumbs.db 34 | [Dd]esktop.ini 35 | $RECYCLE.BIN/ 36 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/.prettierignore: -------------------------------------------------------------------------------- 1 | # List files or directories below to ignore them when running prettier 2 | # More information: https://prettier.io/docs/en/ignore.html 3 | # 4 | 5 | .localdevserver 6 | .sfdx 7 | 8 | coverage/ -------------------------------------------------------------------------------- /sfdx/order-fulfillment/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "none", 3 | "overrides": [ 4 | { 5 | "files": "**/lwc/**/*.html", 6 | "options": { "parser": "lwc" } 7 | }, 8 | { 9 | "files": "*.{cmp,page,component}", 10 | "options": { "parser": "html" } 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/config/project-scratch-def.json: -------------------------------------------------------------------------------- 1 | { 2 | "orgName": "Order Fulfillment", 3 | "edition": "Developer", 4 | "features": [] 5 | } 6 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/applications/Product_Orders.app-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | #0070D2 5 | Box13205680954488989511 6 | 1 7 | false 8 | 9 | Fulfillment and Purchase Orders 10 | Small 11 | Large 12 | false 13 | false 14 | 15 | Standard 16 | public_orders__x 17 | Product_Demand 18 | Lightning 19 | Supply_Demand_UtilityBar 20 | 21 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/contentassets/Box1320568095448898951.asset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/sfdx/order-fulfillment/force-app/main/default/contentassets/Box1320568095448898951.asset -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/contentassets/Box1320568095448898951.asset-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | false 4 | en_US 5 | Box1320568095448898951 6 | 7 | 8 | VIEWER 9 | 10 | 11 | 12 | 13 | 1 14 | Box-1320568095448898951.png 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/contentassets/Box13205680954488989511.asset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/sfdx/order-fulfillment/force-app/main/default/contentassets/Box13205680954488989511.asset -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/contentassets/Box13205680954488989511.asset-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | false 4 | en_US 5 | Box13205680954488989511 6 | 7 | 8 | VIEWER 9 | 10 | 11 | 12 | 13 | 1 14 | Box-1320568095448898951.png 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/dataSources/Heroku_Connect.dataSource-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | {"inlineCountEnabled":"true","csrfTokenName":"","requestCompression":"false","pagination":"CLIENT","noIdMapping":"false","format":"JSON","compatibility":"DEFAULT","csrfTokenEnabled":"false","timeout":"120","searchEnabled":"true","ChangeTrackingEnabled":"false"} 4 | https://odata-2-virginia.heroku.com/odata/v4/b9081e7b362a4e2f8facc4ec2305f0de/ 5 | true 6 | 7 | NamedUser 8 | NoAuthentication 9 | OData4 10 | 11 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/flexipages/Fulfillment_Orders.flexipage-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | cacheAge 7 | 1440 8 | 9 | 10 | reportName 11 | Product_Demand_40o 12 | 13 | 14 | showRefreshButton 15 | true 16 | 17 | flexipage:reportChart 18 | 19 | main 20 | Region 21 | 22 | Fulfillment Orders 23 | 26 | AppPage 27 | 28 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/flexipages/Supply_Demand_UtilityBar.flexipage-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | utilityItems 5 | Region 6 | 7 | 8 | backgroundComponents 9 | Background 10 | 11 | Supply Demand UtilityBar 12 | 15 | UtilityBar 16 | 17 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/lwc/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["@salesforce/eslint-config-lwc/recommended"] 3 | } 4 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/objects/public_orders__x/fields/amount__c.field-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | amount__c 4 | amount 5 | amount 6 | false 7 | false 8 | false 9 | false 10 | 11 | 18 12 | false 13 | 0 14 | Number 15 | false 16 | 17 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/objects/public_orders__x/fields/approved__c.field-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | approved__c 4 | false 5 | approved 6 | approved 7 | false 8 | false 9 | false 10 | false 11 | 12 | Checkbox 13 | 14 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/objects/public_orders__x/fields/category__c.field-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | category__c 4 | category 5 | category 6 | false 7 | false 8 | false 9 | false 10 | 11 | 200 12 | false 13 | Text 14 | false 15 | 16 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/objects/public_orders__x/fields/createdat__c.field-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | createdat__c 4 | createdat 5 | createdat 6 | false 7 | false 8 | false 9 | false 10 | 11 | false 12 | DateTime 13 | 14 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/objects/public_orders__x/fields/id__c.field-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | id__c 4 | id 5 | id 6 | false 7 | false 8 | false 9 | false 10 | 11 | 18 12 | false 13 | 0 14 | Number 15 | false 16 | 17 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/objects/public_orders__x/fields/updatedat__c.field-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | updatedat__c 4 | updatedat 5 | updatedat 6 | false 7 | false 8 | false 9 | false 10 | 11 | false 12 | DateTime 13 | 14 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/objects/public_orders__x/listViews/All.listView-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | All 4 | ExternalId 5 | category__c 6 | amount__c 7 | approved__c 8 | createdat__c 9 | Everything 10 | 11 | approved__c 12 | equals 13 | 0 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/objects/public_orders__x/listViews/Approved_Orders.listView-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Approved_Orders 4 | ExternalId 5 | category__c 6 | amount__c 7 | approved__c 8 | createdat__c 9 | Everything 10 | 11 | approved__c 12 | equals 13 | 1 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/profilePasswordPolicies/PT1_profilePasswordPolicy1585938656989.profilePasswordPolicy-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | false 4 | 15 5 | 10 6 | 8 7 | false 8 | false 9 | 1 10 | 90 11 | 3 12 | 1 13 | Admin 14 | 15 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/profilePasswordPolicies/PT2_profilePasswordPolicy1585938581099.profilePasswordPolicy-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | false 4 | 15 5 | 10 6 | 8 7 | false 8 | false 9 | 1 10 | 90 11 | 3 12 | 1 13 | standard 14 | 15 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/profileSessionSettings/PT1_profileSessionSetting1585938657131.profileSessionSetting-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | false 4 | false 5 | Admin 6 | false 7 | 0 8 | false 9 | 10 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/profileSessionSettings/PT2_profileSessionSetting1585938581288.profileSessionSetting-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | false 4 | false 5 | standard 6 | false 7 | 0 8 | false 9 | 10 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/quickActions/public_orders__x.Approve.quickAction-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | approved__c 5 | true 6 | 7 | false 8 | 9 | TwoColumnsLeftToRight 10 | 11 | 12 | false 13 | category__c 14 | Required 15 | 16 | 17 | false 18 | amount__c 19 | Required 20 | 21 | 22 | false 23 | approved__c 24 | Edit 25 | 26 | 27 | false 28 | createdat__c 29 | Edit 30 | 31 | 32 | 33 | 34 | Update 35 | Update 36 | 37 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/reports/unfiled$public/Product_Demand_40o.report-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | #FFFFFF 5 | #FFFFFF 6 | Diagonal 7 | 8 | Sum 9 | y 10 | public_orders__x.amount__c 11 | 12 | Donut 13 | false 14 | false 15 | public_orders__x.category__c 16 | Right 17 | CHART_BOTTOM 18 | false 19 | false 20 | true 21 | false 22 | Medium 23 | Auto 24 | #000000 25 | 12 26 | Product Demand 27 | #000000 28 | 18 29 | 30 | 31 | Sum 32 | public_orders__x.amount__c 33 | 34 | Product Demand 35 | Summary 36 | 37 | Day 38 | public_orders__x.category__c 39 | Asc 40 | 41 | Product Demand 42 | 43 | co 44 | 1 45 | 46 | CustomEntity$public_orders__x 47 | organization 48 | false 49 | true 50 | true 51 | 52 | public_orders__x.createdat__c 53 | INTERVAL_CUSTOM 54 | 55 | 56 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/tabs/Product_Demand.tab-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Created by Lightning App Builder 4 | Fulfillment_Orders 5 | 6 | Custom4: Hexagon 7 | 8 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/force-app/main/default/tabs/public_orders__x.tab-meta.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | true 4 | Custom13: Box 5 | 6 | -------------------------------------------------------------------------------- /sfdx/order-fulfillment/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "salesforce-app", 3 | "private": true, 4 | "version": "1.0.0", 5 | "description": "Salesforce App", 6 | "scripts": { 7 | "lint": "npm run lint:lwc", 8 | "lint:lwc": "eslint force-app/main/default/lwc", 9 | "test": "npm run test:unit", 10 | "test:unit": "sfdx-lwc-jest", 11 | "test:unit:watch": "sfdx-lwc-jest --watch", 12 | "test:unit:debug": "sfdx-lwc-jest --debug", 13 | "test:unit:coverage": "sfdx-lwc-jest --coverage", 14 | "prettier": "prettier --write \"**/*.{cls,cmp,component,css,html,js,json,md,page,trigger,xml,yaml,yml}\"", 15 | "prettier:verify": "prettier --list-different \"**/*.{cls,cmp,component,css,html,js,json,md,page,trigger,xml,yaml,yml}\"" 16 | }, 17 | "devDependencies": { 18 | "@prettier/plugin-xml": "^0.7.0", 19 | "@salesforce/eslint-config-lwc": "^0.4.0", 20 | "@salesforce/sfdx-lwc-jest": "^0.7.0", 21 | "eslint": "^5.16.0", 22 | "prettier": "^1.19.1", 23 | "prettier-plugin-apex": "^1.0.0" 24 | } 25 | } -------------------------------------------------------------------------------- /sfdx/order-fulfillment/scripts/apex/hello.apex: -------------------------------------------------------------------------------- 1 | // Use .apex files to store anonymous Apex. 2 | // You can execute anonymous Apex in VS Code by selecting the 3 | // apex text and running the command: 4 | // SFDX: Execute Anonymous Apex with Currently Selected Text 5 | // You can also execute the entire file by running the command: 6 | // SFDX: Execute Anonymous Apex with Editor Contents 7 | 8 | string tempvar = 'Enter_your_name_here'; 9 | System.debug('Hello World!'); 10 | System.debug('My name is ' + tempvar); -------------------------------------------------------------------------------- /sfdx/order-fulfillment/scripts/soql/account.soql: -------------------------------------------------------------------------------- 1 | // Use .soql files to store SOQL queries. 2 | // You can execute queries in VS Code by selecting the 3 | // query text and running the command: 4 | // SFDX: Execute SOQL Query with Currently Selected Text 5 | 6 | SELECT Id, Name FROM Account; -------------------------------------------------------------------------------- /sfdx/order-fulfillment/sfdx-project.json: -------------------------------------------------------------------------------- 1 | { 2 | "packageDirectories": [ 3 | { 4 | "path": "force-app", 5 | "default": true 6 | } 7 | ], 8 | "namespace": "", 9 | "sfdcLoginUrl": "https://login.salesforce.com", 10 | "sourceApiVersion": "48.0" 11 | } -------------------------------------------------------------------------------- /viz/.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = spaces 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | 15 | [*.pug] 16 | trim_trailing_whitespace = false 17 | -------------------------------------------------------------------------------- /viz/.prettierignore: -------------------------------------------------------------------------------- 1 | package.json 2 | package-lock.json 3 | dist/ 4 | public/ 5 | config/ -------------------------------------------------------------------------------- /viz/README.md: -------------------------------------------------------------------------------- 1 | ## kafka-stream-viz 2 | 3 | A simple app that streams tweets containing a specified set of keywords to web browser clients. 4 | 5 | Keywords are specified in the kafka-tweets app. They are read from a Kafka topic named 'test' from the 0th (zeroth) partition in that topic. 6 | 7 | #### Development Setup 8 | 9 | ```shell 10 | npm install 11 | ``` 12 | 13 | Additionally these environment variables need to be defined: 14 | 15 | - `KAFKA_URL`: A comma separated list of SSL URLs to the Kafka brokers making up the cluster. 16 | - `KAFKA_CLIENT_CERT`: The required client certificate (in PEM format) to authenticate clients against the broker. 17 | - `KAFKA_CLIENT_CERT_KEY`: The required client certificate key (in PEM format) to authenticate clients against the broker. 18 | - `KAFKA_TOPIC`: The Kafka topics to subscribe to. 19 | - `KAFKA_PREFIX`: (optional) This is only used by [Heroku's multi-tenant Apache Kafka plans](https://devcenter.heroku.com/articles/multi-tenant-kafka-on-heroku) (i.e. `basic` plans) 20 | 21 | #### Development Server 22 | 23 | ```shell 24 | npm run dev 25 | ``` 26 | 27 | Open http://localhost:3000 in a browser and watch data stream in... 28 | -------------------------------------------------------------------------------- /viz/bin/env: -------------------------------------------------------------------------------- 1 | # You can replace the value of these variables with something else if you do not have a Heroku app from which to populate them. 2 | # See README.md or app.json for a description of each var 3 | export KAFKA_URL="$(heroku config:get KAFKA_URL)" 4 | export KAFKA_CLIENT_CERT="$(heroku config:get KAFKA_CLIENT_CERT)" 5 | export KAFKA_CLIENT_CERT_KEY="$(heroku config:get KAFKA_CLIENT_CERT_KEY)" 6 | export KAFKA_TOPIC="$(heroku config:get KAFKA_TOPIC)" 7 | export KAFKA_CMD_TOPIC="$(heroku config:get KAFKA_CMD_TOPIC)" 8 | export KAFKA_WEIGHT_TOPIC="$(heroku config:get KAFKA_WEIGHT_TOPIC)" 9 | export KAFKA_QUEUE_TOPIC="$(heroku config:get KAFKA_QUEUE_TOPIC)" 10 | export KAFKA_QUEUE_WORKER="$(heroku config:get KAFKA_QUEUE_WORKER)" 11 | export KAFKA_CONSUMER_GROUP="$(heroku config:get KAFKA_CONSUMER_GROUP)" 12 | export FIXTURE_DATA_S3="$(heroku config:get FIXTURE_DATA_S3)" 13 | export AWS_DATABASE_URL="$(heroku config:get AWS_DATABASE_URL)" 14 | export DATABASE_URL="$(heroku config:get DATABASE_URL)" 15 | export KAFKA_PREFIX="$(heroku config:get KAFKA_PREFIX)" 16 | export HEROKU_CONNECT_ACCOUNT_ID="$(heroku config:get HEROKU_CONNECT_ACCOUNT_ID)" 17 | export HEROKU_CONNECT_FULFILLMENT_TYPE_ID="$(heroku config:get HEROKU_CONNECT_FULFILLMENT_TYPE_ID)" 18 | export HEROKU_CONNECT_PRICEBOOK_ID="$(heroku config:get HEROKU_CONNECT_PRICEBOOK_ID)" 19 | export REDIS_URL="$(heroku config:get REDIS_URL)" -------------------------------------------------------------------------------- /viz/bin/start: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eo pipefail 4 | 5 | function cleanup { 6 | rm -f client.crt client.key 7 | echo "" 8 | echo "-----> Exiting." 9 | exit 10 | } 11 | trap cleanup SIGINT SIGTERM 12 | 13 | if [ "${NODE_ENV}" = "development" ]; then 14 | # Redefine the env vars exported in this file if you don't have a Herok app from which to populate them. 15 | # See README.md or app.json for a description of each var 16 | source ./bin/env 17 | fi 18 | 19 | if [[ -z "${KAFKA_CLIENT_CERT}" ]]; then 20 | echo "KAFKA_CLIENT_CERT is not set. Aborting" 21 | exit 1 22 | fi 23 | 24 | if [[ -z "${KAFKA_CLIENT_CERT_KEY}" ]]; then 25 | echo "KAFKA_CLIENT_CERT_KEY is not set. Aborting" 26 | exit 1 27 | fi 28 | 29 | if [[ -z "${KAFKA_URL}" ]]; then 30 | echo "KAFKA_URL is not set. Aborting" 31 | exit 1 32 | fi 33 | 34 | if [[ -z "${KAFKA_TOPIC}" ]]; then 35 | echo "KAFKA_TOPIC is not set. Aborting" 36 | exit 1 37 | fi 38 | 39 | # Setup cert and cert key 40 | rm -f client.crt client.key 41 | echo -n "$KAFKA_CLIENT_CERT" > client.crt 42 | echo -n "$KAFKA_CLIENT_CERT_KEY" > client.key 43 | 44 | # Start it up 45 | node app.js 46 | -------------------------------------------------------------------------------- /viz/config/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "supplyDemand": { 3 | "chart": { 4 | "CHART_VISIBLE_MINS": 2, 5 | "CHART_COLOR_LIST": [ 6 | "rgba(255, 99, 132, 0.9)", 7 | "rgba(255, 159, 64, 0.9)", 8 | "rgba(255, 205, 86, 0.9)", 9 | "rgba(75, 192, 192, 0.9)", 10 | "rgba(54, 162, 235, 0.9)", 11 | "rgb(153, 102, 255)", 12 | "rgb(201, 203, 207)" 13 | ], 14 | "CHART_REFRESH_DURATION": 15000, 15 | "CHART_DELAY": 15000, 16 | "CHART_LINE_THICKNESS": 6 17 | }, 18 | "CATEGORY_LIST": [ 19 | "Lawn & Garden", 20 | "Electronics", 21 | "Apparel", 22 | "Home Furnishing", 23 | "Housewares", 24 | "Toys","Books" 25 | ], 26 | "FULFILLMENT_ORDER_TYPE": "Fulfillment Order", 27 | "PURCHASE_ORDER_TYPE": "Purchase Order", 28 | "MAX_SNAPSHOTS_PAST_MINUTES": 30, 29 | "DEFAULT_DATA_PERIOD": "1 week", 30 | "REDIS_CHANNEL": "generate_orders2", 31 | "UPDATE_INTERVAL": 10000 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /viz/consumer/constants.js: -------------------------------------------------------------------------------- 1 | const logger = require('../../logger')('viz') 2 | 3 | // Millisecond interval to expect new data 4 | const interval = (module.exports.INTERVAL = 1000) 5 | 6 | // Max items to display for rolling data 7 | // Use a 10% buffer to prevent transitions from being too small for the chart bounds 8 | const maxSize = (module.exports.MAX_SIZE = 1 * 60 * (1000 / interval)) 9 | module.exports.MAX_BUFFER_SIZE = Math.floor(maxSize * 1.1) 10 | 11 | const prefix = process.env.KAFKA_PREFIX || '' 12 | 13 | module.exports.KAFKA_TOPIC = prefix + process.env.KAFKA_TOPIC 14 | module.exports.KAFKA_CMD_TOPIC = prefix + process.env.KAFKA_CMD_TOPIC 15 | module.exports.KAFKA_WEIGHT_TOPIC = prefix + process.env.KAFKA_WEIGHT_TOPIC 16 | module.exports.KAFKA_QUEUE_TOPIC = prefix + process.env.KAFKA_QUEUE_TOPIC 17 | module.exports.KAFKA_QUEUE_WORKER = prefix + process.env.KAFKA_QUEUE_WORKER 18 | 19 | logger.info(`Kafka topic: ${module.exports.KAFKA_TOPIC}`) 20 | logger.info(`Kafka cmd topic: ${module.exports.KAFKA_CMD_TOPIC}`) 21 | logger.info(`Kafka weight topic: ${module.exports.KAFKA_WEIGHT_TOPIC}`) 22 | logger.info(`Kafka queue length topic: ${module.exports.KAFKA_QUEUE_TOPIC}`) 23 | logger.info(`Kafka queue worker topic: ${module.exports.KAFKA_QUEUE_WORKER}`) 24 | -------------------------------------------------------------------------------- /viz/consumer/index.js: -------------------------------------------------------------------------------- 1 | const Kafka = require('no-kafka') 2 | const Moment = require('moment') 3 | 4 | module.exports = class Consumer { 5 | constructor({ interval, broadcast, topic, consumer }) { 6 | this._broadcast = broadcast 7 | this._interval = interval 8 | 9 | this.startTime = null 10 | this.latestTime = null 11 | this.categories = {} 12 | 13 | this._consumer = new Kafka.SimpleConsumer({ 14 | idleTimeout: this._interval, 15 | connectionTimeout: 10 * 1000, 16 | clientId: topic, 17 | ...consumer 18 | }) 19 | } 20 | 21 | init() { 22 | const { _consumer: consumer } = this 23 | const { clientId: topic } = consumer.options 24 | 25 | return consumer 26 | .init() 27 | .then(() => consumer.subscribe(topic, this.onMessage.bind(this))) 28 | .then(() => setInterval(this.cullAndBroadcast.bind(this), this._interval)) 29 | } 30 | 31 | onMessage(messageSet) { 32 | const items = messageSet 33 | .map((m) => JSON.parse(m.message.value.toString('utf8'))) 34 | .filter(({ action }) => action === 'BROWSE') 35 | 36 | for (const item of items) { 37 | const time = Moment(item.time) 38 | 39 | if ( 40 | !Object.prototype.hasOwnProperty.call(this.categories, item.category) 41 | ) { 42 | if (this.startTime === null) { 43 | this.startTime = time 44 | } 45 | 46 | this.categories[item.category] = { 47 | id: item.category, 48 | times: [], 49 | first: time, 50 | count: 0 51 | } 52 | } 53 | this.categories[item.category].times.push(time) 54 | this.categories[item.category].count++ 55 | if (this.latestTime === null || time.isAfter(this.latestTime)) { 56 | this.latestTime = time.clone() 57 | } 58 | } 59 | } 60 | 61 | cullAndBroadcast() { 62 | const items = [] 63 | for (const category of Object.keys(this.categories)) { 64 | const collate = this.categories[category] 65 | const cullFrom = this.latestTime.clone().subtract(5, 'seconds') 66 | let idx = 0 67 | let cullCount = 0 68 | do { 69 | if ( 70 | !Moment.isMoment(collate.times[idx]) || 71 | collate.times[idx].isBefore(cullFrom) 72 | ) { 73 | cullCount++ 74 | } else { 75 | break 76 | } 77 | idx++ 78 | } while (idx < collate.times.length) 79 | for (idx = 0; idx < cullCount; idx++) { 80 | collate.times.shift() 81 | } 82 | 83 | let timeRange = 5 84 | let avgPerSecond = 0 85 | if (this.latestTime !== null && collate.times.length > 0) { 86 | if (collate.first.isAfter(cullFrom)) { 87 | timeRange = this.latestTime.diff(collate.times[0], 'seconds') 88 | } 89 | avgPerSecond = collate.times.length / timeRange 90 | } 91 | items.push({ 92 | id: collate.id, 93 | count: collate.count, 94 | avgPerSecond: avgPerSecond 95 | }) 96 | } 97 | const time = new Date().valueOf() 98 | const bcast = { 99 | data: items.reduce((res, cat) => { 100 | res[cat.id] = [{ ...cat, time }] 101 | return res 102 | }, {}) 103 | } 104 | this._broadcast(bcast) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /viz/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kafka-stream-viz", 3 | "description": "A demo showing Heroku+AWS integration using Apache Kafka on Heroku and AWS RedShift.", 4 | "version": "1.0.0", 5 | "bugs": { 6 | "url": "https://github.com/heroku-examples/kafka-stream-viz/issues" 7 | }, 8 | "dependencies": { 9 | "axios": "^0.19.0", 10 | "body-parser": "^1.19.0", 11 | "c3": "^0.7.11", 12 | "chart.js": "^2.9.0", 13 | "chartjs-plugin-datalabels": "^0.7.0", 14 | "chartjs-plugin-streaming": "^1.8.0", 15 | "config": "^3.2.5", 16 | "d3": "^5.12.0", 17 | "dateformat": "^3.0.3", 18 | "dotenv": "^8.2.0", 19 | "express": "^4.17.1", 20 | "express-basic-auth": "^1.2.0", 21 | "ioredis": "^4.14.1", 22 | "jquery": "^3.4.1", 23 | "knex": "^0.20.0", 24 | "lodash": "^4.17.15", 25 | "moment": "^2.24.0", 26 | "moving-average": "^1.0.0", 27 | "no-kafka": "^3.4.3", 28 | "normalize.css": "^8.0.1", 29 | "optimist": "^0.6.1", 30 | "pg-promise": "^9.1.4", 31 | "pug": "^2.0.4", 32 | "reconnecting-websocket": "^4.2.0", 33 | "ws": "^7.1.2" 34 | }, 35 | "devDependencies": { 36 | "@babel/core": "^7.6.0", 37 | "@babel/preset-env": "^7.6.0", 38 | "acorn": "^7.1.1", 39 | "babel-loader": "^8.0.6", 40 | "babel-plugin-lodash": "^3.3.4", 41 | "clean-webpack-plugin": "^3.0.0", 42 | "connect-history-api-fallback": "^1.6.0", 43 | "css-loader": "^3.2.0", 44 | "html-webpack-plugin": "^3.2.0", 45 | "mini-css-extract-plugin": "^0.8.0", 46 | "optimize-css-assets-webpack-plugin": "^5.0.3", 47 | "postcss-import": "^12.0.1", 48 | "postcss-loader": "^3.0.0", 49 | "postcss-preset-env": "^6.7.0", 50 | "precss": "^4.0.0", 51 | "pug-loader": "^2.4.0", 52 | "style-loader": "^1.0.0", 53 | "uglifyjs-webpack-plugin": "^2.2.0", 54 | "webpack": "^4.40.2", 55 | "webpack-cli": "^3.3.9", 56 | "webpack-dev-middleware": "^3.7.1", 57 | "webpack-livereload-plugin": "^2.2.0" 58 | }, 59 | "license": "MIT", 60 | "main": "bin/start", 61 | "private": true, 62 | "repository": { 63 | "type": "git", 64 | "url": "git@github.com:crcastle/kafka-stream-viz.git" 65 | }, 66 | "scripts": { 67 | "build": "NODE_ENV=production webpack", 68 | "dev": "NODE_ENV=development bin/start", 69 | "local-nokafka": "NODE_ENV=development node app.js --nokafka", 70 | "local-nodb": "NODE_ENV=development node app.js --nodb --nokafka", 71 | "start": "NODE_ENV=production bin/start" 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /viz/public/diagram/heroku-connect-diagram.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | heroku-connect-diagram 7 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 |
35 | 36 |
37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /viz/public/heroku-connect-diagram/heroku-connect.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | kafka-salesforce-data-demo-animation 7 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 |
27 | 28 |
29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /viz/public/heroku-connect-diagram/kafka-salesforce-data-demo-animation.hyperesources/1F13B8-restorable.plist: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/viz/public/heroku-connect-diagram/kafka-salesforce-data-demo-animation.hyperesources/1F13B8-restorable.plist -------------------------------------------------------------------------------- /viz/public/heroku-connect-diagram/kafka-salesforce-data-demo-animation.hyperesources/arrow-one.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | arrow-one 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /viz/public/heroku-connect-diagram/kafka-salesforce-data-demo-animation.hyperesources/arrow-two.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /viz/public/heroku-connect-diagram/kafka-salesforce-data-demo-animation.hyperesources/blank.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/viz/public/heroku-connect-diagram/kafka-salesforce-data-demo-animation.hyperesources/blank.gif -------------------------------------------------------------------------------- /viz/public/heroku-connect-diagram/kafka-salesforce-data-demo-animation.hyperesources/heroku-app.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | heroku-app 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /viz/public/heroku-connect-diagram/kafka-salesforce-data-demo-animation.hyperesources/mobile-arrow-1.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | mobile-arrow-1 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /viz/public/heroku-connect-diagram/kafka-salesforce-data-demo-animation.hyperesources/mobile-arrow-2.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | mobile-arrow-2 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /viz/public/heroku-connect-diagram/kafka-salesforce-data-demo-animation.hyperesources/mobile-heroku-app.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | mobile-heroku-app 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /viz/public/heroku-connect-diagram/kafka-salesforce-data-demo-animation.hyperesources/mobile-outline.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | mobile-outline 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /viz/public/heroku-connect-diagram/kafka-salesforce-data-demo-animation.hyperesources/outline.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | outline 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /viz/public/images/add-mark-28.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | product/add-mark-28 5 | Created with Sketch. 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /viz/public/images/layout-list-28.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | product/layout-list-28 5 | Created with Sketch. 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /viz/public/images/loading.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/viz/public/images/loading.gif -------------------------------------------------------------------------------- /viz/public/images/metrics-28.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | product/metrics-28 5 | Created with Sketch. 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /viz/public/images/remove-mark-28.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | product/remove-mark-28 5 | Created with Sketch. 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /viz/public/images/salesforce-heroku-connect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/viz/public/images/salesforce-heroku-connect.png -------------------------------------------------------------------------------- /viz/public/images/usage-28.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | product/usage-28 5 | Created with Sketch. 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | kafka-diagram-v2 7 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 |
27 | 28 |
29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/9EF7D6-restorable.plist: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/9EF7D6-restorable.plist -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/Asset 14_2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/Asset 14_2x.png -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/amazon-arrow-mobile-1.svg: -------------------------------------------------------------------------------- 1 | amazon-arrow-mobile -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/amazon-arrow-mobile.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | amazon-arrow-mobile 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/amazon-arrow-new.svg: -------------------------------------------------------------------------------- 1 | amazon-arrow -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/amazon-arrow.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | amazon-arrow 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/amazon-container-mobile.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 12 | 13 | amazon-container-mobile 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/amazon-container-new.svg: -------------------------------------------------------------------------------- 1 | amazon-container -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/amazon-container.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 12 | 13 | amazon-container 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/blank.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/blank.gif -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/data-arrow-mobile.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | data-arrow-mobile 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/data-prod-arrow.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | data-prod-arrow 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/data-prod-container-mobile.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 11 | 12 | data-prod-container-mobile 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/data-prod-container.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 11 | 12 | data-prod-container 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/data-producers.svg: -------------------------------------------------------------------------------- 1 | data-producers -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/data-viz-arrow.svg: -------------------------------------------------------------------------------- 1 | data-viz-arrow -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/data-writer.svg: -------------------------------------------------------------------------------- 1 | data-writer -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/heroku-postgres.svg: -------------------------------------------------------------------------------- 1 | heroku-postgres -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/kafka-container-mobile.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 15 | 16 | kafka-container-mobile 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/kafka-container.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 15 | 16 | kafka-container 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/kafka-icon.svg: -------------------------------------------------------------------------------- 1 | kafka-icon-NEW.svg -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/redshift-icon.svg: -------------------------------------------------------------------------------- 1 | redshift-iconREDSHIFTBATCHWRITER -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/viz-arrow-mobile.svg: -------------------------------------------------------------------------------- 1 | viz-arrow-mobile -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/viz-container-mobile.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 11 | 12 | viz-container-mobile 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /viz/public/kafka-diagram/kafka-diagram-v2.hyperesources/viz-container.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 11 | 12 | viz-container 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /viz/public/logo-heroku-white.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /viz/sql/create_pg.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS ecommercelogs(id SERIAL PRIMARY KEY, time TIMESTAMP WITH TIME ZONE, session VARCHAR(255), action VARCHAR(255), product VARCHAR(255), category VARCHAR(255), campaign VARCHAR(255)); 2 | -------------------------------------------------------------------------------- /viz/sql/load.sql: -------------------------------------------------------------------------------- 1 | COPY ecommercelogs (time, session, action, product, category, campaign) 2 | FROM $1 3 | ACCESS_KEY_ID $2 4 | SECRET_ACCESS_KEY $3 5 | CSV 6 | IGNOREHEADER 1 7 | TIMEFORMAT 'auto'; 8 | -------------------------------------------------------------------------------- /viz/sql/load_pg.sql: -------------------------------------------------------------------------------- 1 | COPY ecommercelogs (time, session, action, product, category, campaign) FROM 'fixture.sql' DELIMITER ',' CSV HEADER; 2 | -------------------------------------------------------------------------------- /viz/sql/truncate.sql: -------------------------------------------------------------------------------- 1 | TRUNCATE ecommercelogs; 2 | -------------------------------------------------------------------------------- /viz/src/demand/DemandChart.js: -------------------------------------------------------------------------------- 1 | import _ from 'lodash' 2 | import moment from 'moment' 3 | import Chart from 'chart.js' 4 | import ChartDataLabels from 'chartjs-plugin-datalabels' 5 | import 'chartjs-plugin-streaming' 6 | 7 | Chart.plugins.unregister(ChartDataLabels) 8 | 9 | export default class DemandChart { 10 | constructor(options) { 11 | this.config = options.config.chart 12 | this.categories = options.config.CATEGORY_LIST 13 | this.prevValue = {} 14 | this.categories.forEach((cat) => (this.prevValue[cat] = -1)) 15 | this.render(this.generateDatasets(options.originalData || {})) 16 | } 17 | 18 | onRefresh() { 19 | this.newData = this.newData || {} 20 | this.chart.config.data.datasets.forEach((dataset) => { 21 | let clone = _.clone(dataset.data) 22 | clone.push({ 23 | x: Date.now(), 24 | y: _.isUndefined(this.newData[dataset.label]) 25 | ? dataset.data[dataset.data.length - 1].y 26 | : this.newData[dataset.label] 27 | }) 28 | dataset.data = _.reverse(_.sortBy(clone, 'x')) 29 | }) 30 | } 31 | 32 | generateDatasets(originalData) { 33 | console.log('original data', originalData) 34 | let chartColors = this.config.CHART_COLOR_LIST 35 | return this.categories.map((categoryName, index) => { 36 | originalData[categoryName] = 37 | originalData[categoryName] || 38 | _.times(this.config.CHART_VISIBLE_MINS + 1, () => 0) 39 | let currentData = originalData[categoryName].map((value, i) => { 40 | return { 41 | x: moment() 42 | .subtract(originalData[categoryName].length - i, 'minute') 43 | .valueOf(), 44 | y: value 45 | } 46 | }) 47 | //adding one more data value to the end so it looks continuous 48 | currentData.push({ 49 | x: moment().valueOf(), 50 | y: currentData[currentData.length - 1].y 51 | }) 52 | 53 | return { 54 | label: categoryName, 55 | backgroundColor: chartColors[index], 56 | borderColor: chartColors[index], 57 | borderWidth: this.config.CHART_LINE_THICKNESS, 58 | fill: false, 59 | lineTension: 0.1, 60 | data: currentData, 61 | datalabels: { 62 | color: chartColors[index] 63 | } 64 | } 65 | }) 66 | } 67 | 68 | render(datasets) { 69 | let config = { 70 | plugins: [ChartDataLabels], 71 | type: 'line', 72 | data: { 73 | datasets: datasets 74 | }, 75 | options: { 76 | plugins: { 77 | datalabels: { 78 | clamp: true, 79 | align: (context) => { 80 | //controlling the position of the data label 81 | let dataList = config.data.datasets[context.datasetIndex].data 82 | const curData = dataList[context.dataIndex] 83 | const nextData = dataList[context.dataIndex + 1] 84 | const prevData = dataList[context.dataIndex - 1] 85 | let align = 'top' 86 | 87 | if (prevData) { 88 | if (prevData.y > curData.y) { 89 | align = 'bottom' 90 | } 91 | } 92 | 93 | if (nextData) { 94 | if (nextData.y > curData.y) { 95 | align = 'bottom' 96 | } 97 | } 98 | return align 99 | }, 100 | offset: 10, 101 | font: { 102 | size: 20 103 | }, 104 | formatter: (value, data) => { 105 | //removing redundant label - basically if there is no change in the amount, it doens't show any label 106 | let prevData = data.dataset.data[data.dataIndex + 1] 107 | if (prevData && prevData.y === value.y) { 108 | return null 109 | } else { 110 | return value.y 111 | } 112 | } 113 | } 114 | }, 115 | layout: { 116 | padding: { 117 | left: 0, 118 | right: 0, 119 | top: 40, 120 | bottom: 40 121 | } 122 | }, 123 | aspectRatio: 3, 124 | responsive: true, 125 | maintainAspectRatio: true, 126 | elements: { 127 | point: { 128 | radius: 0 129 | } 130 | }, 131 | legend: { 132 | display: false 133 | }, 134 | title: { 135 | display: false 136 | }, 137 | scales: { 138 | xAxes: [ 139 | { 140 | gridLines: { 141 | display: false 142 | }, 143 | type: 'realtime', 144 | display: false, 145 | realtime: { 146 | duration: this.config.CHART_VISIBLE_MINS * 60 * 1000, 147 | refresh: this.config.CHART_REFRESH_DURATION, 148 | delay: this.config.CHART_DELAY, 149 | onRefresh: this.onRefresh.bind(this) 150 | } 151 | } 152 | ], 153 | yAxes: [ 154 | { 155 | gridLines: { 156 | display: false 157 | }, 158 | // display: false, 159 | ticks: { 160 | suggestedMin: -100, 161 | suggestedMax: 100 162 | } 163 | } 164 | ] 165 | }, 166 | tooltips: { 167 | enabled: false 168 | } 169 | } 170 | } 171 | 172 | const ctx = document.getElementById('demand-chart').getContext('2d') 173 | this.chart = new Chart(ctx, config) 174 | } 175 | 176 | updateChart() { 177 | this.chart.load({ 178 | columns: this.chartData.data 179 | }) 180 | } 181 | 182 | /** 183 | * 184 | * new data is coming from the server 185 | */ 186 | update(newData) { 187 | if (!this.chart) { 188 | return 189 | } 190 | console.log(newData) 191 | this.newData = newData 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /viz/src/demand/DemandControls.js: -------------------------------------------------------------------------------- 1 | import _ from 'lodash' 2 | import axios from 'axios' 3 | import DemandChart from './DemandChart' 4 | import DemandFulfillmentForm from './DemandFulfillmentForm' 5 | 6 | export default class DemandControls { 7 | constructor(options) { 8 | if (document.querySelectorAll('.order-control-buttons').length > 0) { 9 | require('./order-control')() 10 | } 11 | 12 | const chartEl = document.querySelectorAll(options.chartSelector)[0] 13 | this.isDisabled = !chartEl 14 | if (this.isDisabled) { 15 | return 16 | } 17 | 18 | this.isAllReady = axios 19 | .get('/demand/chart-config') 20 | .then((chartConfig) => { 21 | this.config = chartConfig.data 22 | this.categories = chartConfig.data.CATEGORY_LIST 23 | return this.getCurrentChartData() 24 | }) 25 | .then((chartData) => { 26 | this.chart = new DemandChart({ 27 | originalData: chartData.data, 28 | config: this.config 29 | }) 30 | this.renderCategories() 31 | this.renderXticks() 32 | this.fulfillmentForm = new DemandFulfillmentForm({ 33 | openButtonSelector: options.formOpenSelector, 34 | categories: this.categories 35 | }) 36 | }) 37 | } 38 | 39 | init(ws) { 40 | if (this.isDisabled) { 41 | return 42 | } 43 | this.isAllReady.then(() => { 44 | this.initUpdateCycle(ws) 45 | }) 46 | } 47 | 48 | initUpdateCycle(ws) { 49 | ws.addEventListener('message', (e) => { 50 | const msg = JSON.parse(e.data) 51 | if (msg.type === 'orders') { 52 | this.chart.update(msg.data) 53 | } 54 | }) 55 | } 56 | 57 | initCategories() { 58 | return axios.get('/demand/categories').then((res) => { 59 | console.log(res) 60 | this.categories = res.data 61 | }) 62 | } 63 | 64 | renderXticks() { 65 | let tickLabels = _.reverse( 66 | _.times(this.config.chart.CHART_VISIBLE_MINS + 1, (x) => { 67 | return x === 0 ? 'Now' : x + 'mins' 68 | }) 69 | ) 70 | 71 | let template = _.template(` 72 | <% tickLabels.forEach( label => { %> 73 | <%- label %> 74 | <% }); %> 75 | `) 76 | 77 | let html = template({ 78 | tickLabels 79 | }) 80 | 81 | document.querySelectorAll('.demand-chart--x-ticks')[0].innerHTML = html 82 | } 83 | 84 | renderCategories() { 85 | let template = _.template(` 86 | 95 | `) 96 | 97 | let html = template({ 98 | categories: this.categories, 99 | colorList: this.config.chart.CHART_COLOR_LIST 100 | }) 101 | document.querySelectorAll('.demand-category-container')[0].innerHTML = html 102 | } 103 | 104 | getCurrentChartData() { 105 | return axios.get('/demand/data', { 106 | params: { 107 | period: this.config.chart.CHART_VISIBLE_MINS 108 | } 109 | }) 110 | } 111 | 112 | initCategorySelectList() { 113 | let categorySelection = document.querySelectorAll( 114 | 'select.co--field-input' 115 | )[0] 116 | categorySelection.innerHTML = '' 117 | this.categories.forEach((categoryName) => { 118 | let option = document.createElement('option') 119 | option.innerHTML = categoryName 120 | categorySelection.appendChild(option) 121 | }) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /viz/src/demand/order-control.js: -------------------------------------------------------------------------------- 1 | module.exports = () => { 2 | const $ = require('jquery') 3 | 4 | let ajaxing = false 5 | 6 | const generateAjaxHandler = (command) => { 7 | return (e) => { 8 | e.preventDefault() 9 | if (ajaxing) { 10 | return 11 | } 12 | ajaxing = true 13 | 14 | $.ajax({ 15 | type: 'POST', 16 | url: '/demand/command', 17 | data: JSON.stringify({ command }), 18 | dataType: 'json', 19 | contentType: 'application/json' 20 | }) 21 | .then((data) => { 22 | console.log(data) 23 | ajaxing = false 24 | alert(`${command} comamnd has been sent. Check the status.`) 25 | }) 26 | .catch((e) => { 27 | console.log(e) 28 | ajaxing = false 29 | alert('Error: ' + e) 30 | }) 31 | } 32 | } 33 | 34 | ;['start', 'stop', 'reset'].forEach((key) => { 35 | $(`[${key}-button]`).click(generateAjaxHandler(key)) 36 | }) 37 | 38 | let prevClass = '' 39 | let prevState = null 40 | const bodyClassList = document.body.classList 41 | 42 | const updateStatus = () => { 43 | $.get('/demand/worker-status').then((res) => { 44 | if (prevState === res.state) { 45 | return 46 | } 47 | 48 | let state = res.state || '' 49 | if (prevClass) { 50 | bodyClassList.remove(prevClass) 51 | } 52 | prevState = res.state 53 | prevClass = `worker-${state.toLowerCase()}` 54 | bodyClassList.add(prevClass) 55 | $('[status]').text(res.state) 56 | }) 57 | } 58 | 59 | setInterval(updateStatus, 3000) 60 | 61 | updateStatus() 62 | } 63 | -------------------------------------------------------------------------------- /viz/src/index.js: -------------------------------------------------------------------------------- 1 | import '../styles/style.css' 2 | 3 | import Stream from './lib/stream' 4 | import Queue from './lib/queue' 5 | import Nav from './lib/nav' 6 | import { MAX_SIZE, MAX_BUFFER_SIZE, INTERVAL } from '../consumer/constants' 7 | import AudienceControl from './lib/audienceControls' 8 | import BoothController from './lib/boothControls' 9 | import DemandControls from './demand/DemandControls' 10 | import ReconnectingWebSocket from 'reconnecting-websocket' 11 | 12 | let navConfig = { 13 | legend: '.footer-legend ul', 14 | architecture: '.architecture-link', 15 | iframeUrl: '/public/kafka-diagram/kafka-diagram-v2.html' 16 | } 17 | 18 | if (document.querySelector('#demand-chart')) { 19 | navConfig.iframeUrl = '/public/heroku-connect-diagram/heroku-connect.html' 20 | navConfig.type = 'herokuConnect' 21 | } 22 | 23 | const aggregate = [ 24 | new Nav(navConfig), 25 | new Stream({ 26 | selector: '.chart-stream .chart', 27 | transition: INTERVAL, 28 | x: 'time', 29 | y: 'avgPerSecond', 30 | maxSize: MAX_BUFFER_SIZE, 31 | maxDisplaySize: MAX_SIZE 32 | }) 33 | ] 34 | 35 | const QueueGraph = new Queue({ 36 | selector: '.chart-line .chart', 37 | countSelector: '.chart-line .queue-count', 38 | transition: INTERVAL, 39 | x: 'time', 40 | y: 'processingTime', 41 | maxSize: MAX_BUFFER_SIZE, 42 | maxDisplaySize: MAX_SIZE 43 | }) 44 | 45 | const AudienceControls = new AudienceControl({}) 46 | const BoothControls = new BoothController({ selector: '.big-button' }) 47 | 48 | const demandControls = new DemandControls({ 49 | chartSelector: '#demand-chart', 50 | formOpenSelector: '.demand--fulfillment-button' 51 | }) 52 | 53 | const url = `ws${window.location.href.match(/^http(s?:\/\/.*)\/.*$/)[1]}` 54 | const ws = new ReconnectingWebSocket(url, null, { 55 | reconnectInterval: 1000, 56 | reconnectDecay: 1 57 | }) 58 | 59 | window.ws = ws 60 | 61 | aggregate.forEach((a) => a.init()) 62 | AudienceControls.init(ws) 63 | BoothControls.init(ws) 64 | QueueGraph.init() 65 | demandControls.init(ws) 66 | 67 | ws.onmessage = (e) => { 68 | const msg = JSON.parse(e.data) 69 | if (msg.type === 'weights') { 70 | AudienceControls.update(msg) 71 | } else if (msg.type === 'ecommerce') { 72 | aggregate.forEach((a) => a.update(msg.data)) 73 | } else if (msg.type === 'queue') { 74 | QueueGraph.update(msg.data) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /viz/src/lib/audienceControls.js: -------------------------------------------------------------------------------- 1 | import kafkaConfig from '../../../generate_data/kafka.js' 2 | // { 3 | // "type": "cmd", 4 | // "cmd": "weight", 5 | // "category": "XFLK", 6 | // "change": 1, // or -1 7 | // } 8 | export default class AudienceControl { 9 | constructor() { 10 | this.ws = null 11 | 12 | // this.init = this.init.bind(this) 13 | } 14 | change({ category, type }) { 15 | this.ws.send( 16 | JSON.stringify({ 17 | type: 'cmd', 18 | cmd: 'weight', 19 | category: category, 20 | change: type === 'increment' ? 1 : -1 21 | }) 22 | ) 23 | } 24 | 25 | makeCategoryControl({ category, weight, totalProducts }) { 26 | const wrapper = document.querySelector('.category-controls-wrapper') 27 | const section = document.createElement('section') 28 | const minusButton = document.createElement('a') 29 | const plusButton = document.createElement('a') 30 | const progressBarWrapper = document.createElement('div') 31 | const progressBar = document.createElement('progress') 32 | const categoryName = document.createElement('h2') 33 | 34 | if (!wrapper) return 35 | 36 | section.classList.add('category-controls', category) 37 | 38 | minusButton.classList.add('minus') 39 | plusButton.classList.add('plus') 40 | minusButton.innerHTML = 41 | '' 42 | plusButton.innerHTML = 43 | '' 44 | 45 | plusButton.onclick = () => this.change({ category, type: 'increment' }) 46 | minusButton.onclick = () => this.change({ category, type: 'decrement' }) 47 | // Prevent double tap zoom on buttons 48 | plusButton.ontouchend = function(e) { 49 | e.preventDefault() 50 | this.click() 51 | } 52 | minusButton.ontouchend = function(e) { 53 | e.preventDefault() 54 | this.click() 55 | } 56 | 57 | categoryName.textContent = kafkaConfig.categories[category].name 58 | progressBarWrapper.classList.add('progress-bar') 59 | progressBarWrapper.appendChild(categoryName) 60 | progressBar.setAttribute('max', '100') 61 | progressBar.setAttribute('value', weight) 62 | progressBarWrapper.appendChild(progressBar) 63 | 64 | section.appendChild(minusButton) 65 | section.appendChild(plusButton) 66 | section.appendChild(progressBarWrapper) 67 | 68 | // If we've already initialized and have all the nodes then this is an update 69 | // Re-render the wrapper 70 | if (wrapper.children.length === totalProducts) { 71 | wrapper.innerHTML = '' 72 | } 73 | wrapper.appendChild(section) 74 | } 75 | 76 | update(msg) { 77 | if (msg.type === 'weights') { 78 | const categoryKeys = Object.keys(msg.weights) 79 | categoryKeys.forEach((cat) => 80 | this.makeCategoryControl({ 81 | category: cat, 82 | weight: msg.weights[cat], 83 | totalProducts: categoryKeys.length 84 | }) 85 | ) 86 | } 87 | } 88 | 89 | init(ws) { 90 | this.ws = ws 91 | this.ws.onopen = () => { 92 | this.ws.send(JSON.stringify({ type: 'cmd', cmd: 'weight' })) 93 | } 94 | } 95 | } 96 | // "{ 'type': 'cmd', cmd: 'weight'}" 97 | -------------------------------------------------------------------------------- /viz/src/lib/boothControls.js: -------------------------------------------------------------------------------- 1 | import kafkaConfig from '../../../generate_data/kafka' 2 | export default class BoothController { 3 | constructor(options) { 4 | this.button = document.querySelector(options.selector) 5 | 6 | this.init = this.init.bind(this) 7 | this.handleButtonClick = this.handleButtonClick.bind(this) 8 | this.currentScenario = 'no_sale' 9 | } 10 | 11 | handleButtonClick() { 12 | this.getRandomScenario() 13 | this.ws.send( 14 | JSON.stringify({ 15 | type: 'cmd', 16 | cmd: 'scenario', 17 | name: this.currentScenario 18 | }) 19 | ) 20 | this.setSaleHeader() 21 | } 22 | 23 | addHandlers() { 24 | if (!this.button) return 25 | this.button.onclick = this.handleButtonClick 26 | } 27 | 28 | getRandomScenario() { 29 | const scenarios = Object.keys(kafkaConfig.scenarios) 30 | const getRandomNotCurrent = () => { 31 | const random = scenarios[Math.floor(Math.random() * scenarios.length)] 32 | if (random !== this.currentScenario) return random 33 | else return getRandomNotCurrent() 34 | } 35 | this.currentScenario = getRandomNotCurrent() 36 | this.currentText = kafkaConfig.scenarios[this.currentScenario].text 37 | } 38 | 39 | setSaleHeader() { 40 | document.querySelector('.current-sale').textContent = this.currentText 41 | } 42 | 43 | init(ws) { 44 | this.ws = ws 45 | this.addHandlers() 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /viz/src/lib/nav.js: -------------------------------------------------------------------------------- 1 | const kafkaConfig = require('../../../generate_data/kafka') 2 | 3 | module.exports = class Nav { 4 | constructor(options) { 5 | this.legend = document.querySelector(options.legend) 6 | this.architectureLink = document.querySelector(options.architecture) 7 | this.main = document.querySelector('main') 8 | this.architectureFrame = document.querySelector('.architecture-iframe') 9 | this.type = options.type 10 | this.iframeUrl = options.iframeUrl 11 | if (this.architectureLink) { 12 | this.architecture() 13 | } 14 | this.toggleView() 15 | } 16 | 17 | formatData(data) { 18 | return Object.keys(data) 19 | } 20 | 21 | architecture() { 22 | this.architectureLink.addEventListener('click', () => { 23 | const toggleables = document.querySelectorAll('.toggleable') 24 | const isOpen = this.main.classList.contains('open') 25 | if (isOpen) { 26 | this.architectureFrame.removeAttribute('src') 27 | this.main.classList.remove('open') 28 | } else { 29 | this.architectureFrame.setAttribute('src', this.iframeUrl) 30 | this.main.classList.add('open') 31 | } 32 | toggleables.forEach((toggleable) => { 33 | if (toggleable.classList.contains('show')) { 34 | toggleable.classList.remove('show') 35 | } 36 | }) 37 | }) 38 | } 39 | 40 | toggleView() { 41 | const toggleLinks = document.querySelectorAll('.toggle') 42 | toggleLinks.forEach((toggleLink) => { 43 | toggleLink.addEventListener('click', (event) => { 44 | const currentLink = event.currentTarget.getAttribute('name') // toggle button 45 | const currentToggleable = document.querySelector( 46 | `.toggleable[name="${currentLink}"]` 47 | ) // element to toggle 48 | const isShown = currentToggleable.classList.contains('show') 49 | const isOpen = this.main.classList.contains('open') // if this is true, then the architecture diagram is open and we need to close it 50 | if (isShown) { 51 | currentToggleable.classList.remove('show') 52 | } else { 53 | currentToggleable.classList.add('show') 54 | if (isOpen) { 55 | this.architectureFrame.removeAttribute('src') 56 | this.main.classList.remove('open') 57 | } 58 | } 59 | }) 60 | }) 61 | } 62 | 63 | init() {} 64 | 65 | update(data) { 66 | if (!this.architectureLink || this.type === 'herokuConnect') return 67 | this.formatData(data).forEach((topic, index) => { 68 | if (!this.legend.querySelector(`#topic-${topic}`)) { 69 | const li = document.createElement('li') 70 | li.textContent = kafkaConfig.categories[topic].name 71 | li.setAttribute('id', `topic-${topic}`) 72 | li.classList.add(`color-${index + 1}`) 73 | this.legend.appendChild(li) 74 | } 75 | }) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /viz/src/lib/sizedArray.js: -------------------------------------------------------------------------------- 1 | module.exports = class SizedArray { 2 | constructor(size) { 3 | this._size = size 4 | this._items = [] 5 | } 6 | 7 | push(items) { 8 | if (!Array.isArray(items)) { 9 | items = [items] 10 | } 11 | const current = this._items.length 12 | const add = items.length 13 | const max = this._size 14 | 15 | if (current + add >= max) { 16 | this._items = [...this._items, ...items].slice(current + add - max) 17 | } else { 18 | this._items.push(...items) 19 | } 20 | } 21 | 22 | items() { 23 | return this._items 24 | } 25 | 26 | empty() { 27 | return this._items.length === 0 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /viz/styles/components/audience.css: -------------------------------------------------------------------------------- 1 | .audience { 2 | main { 3 | align-items: center; 4 | } 5 | 6 | .chart-container { 7 | height: 0; 8 | min-height: 0; 9 | 10 | &.show { 11 | margin-bottom: 30px; 12 | 13 | .chart { 14 | height: auto; 15 | } 16 | } 17 | 18 | .footer-legend { 19 | margin-top: 50px; 20 | } 21 | } 22 | 23 | h1 { 24 | margin-bottom: 10px; 25 | } 26 | 27 | p { 28 | text-align: center; 29 | margin-bottom: 40px; 30 | } 31 | } 32 | 33 | .category-controls-wrapper { 34 | width: 100%; 35 | display: flex; 36 | flex-direction: column; 37 | align-items: center; 38 | flex-shrink: 0; 39 | flex-grow: 1; 40 | } 41 | 42 | .category-controls { 43 | width: 100%; 44 | display: flex; 45 | justify-content: space-between; 46 | align-items: center; 47 | flex-shrink: 0; /* don't let them overlap */ 48 | flex-grow: 1; /* but do let them fill up the remaining height */ 49 | margin-bottom: 20px; 50 | max-width: 500px; 51 | 52 | .plus, 53 | .minus { 54 | display: flex; 55 | justify-content: center; 56 | align-items: center; 57 | } 58 | 59 | .plus { 60 | order: 1; 61 | } 62 | 63 | .minus { 64 | order: -1; 65 | } 66 | 67 | h2 { 68 | margin: 0 0 10px; 69 | font-size: 12px; 70 | text-transform: uppercase; 71 | } 72 | } 73 | 74 | .category-controls a { 75 | position: relative; 76 | display: flex; 77 | justify-content: center; 78 | align-items: center; 79 | user-select: none; /* fixes dragging bug */ 80 | 81 | border-radius: 50px; 82 | width: 50px; 83 | height: 50px; 84 | border: none; 85 | font-size: 30px; 86 | color: white; 87 | background: #ccc; 88 | 89 | &:focus { 90 | border-radius: 50px; 91 | outline: none; 92 | border: 4px solid rgba(255, 255, 255, 0.5); 93 | } 94 | } 95 | 96 | .category-controls a span { 97 | width: 50px; 98 | height: 50px; 99 | display: block; 100 | border-radius: 60px; 101 | background: rgba(0, 0, 0, 0); 102 | top: 0; 103 | left: 0; 104 | position: absolute; 105 | z-index: -1; 106 | transition: all 0.1s; 107 | } 108 | 109 | .category-controls a:active span { 110 | background: rgba(0, 0, 0, 0.2); 111 | width: 60px; 112 | height: 60px; 113 | top: -5px; 114 | left: -5px; 115 | } 116 | 117 | .progress-bar { 118 | flex: 1; 119 | text-align: center; 120 | margin: 0 10px; 121 | 122 | display: flex; 123 | flex-direction: column; 124 | justify-content: center; 125 | align-content: center; 126 | } 127 | 128 | .category-controls { 129 | progress { 130 | border: 1px solid #dedede; 131 | height: 8px; 132 | border-radius: 50px; 133 | width: 100%; 134 | } 135 | progress::-webkit-progress-bar { 136 | border: 0; 137 | height: 8px; 138 | border-radius: 50px; 139 | background: #eeeeee; 140 | } 141 | progress::-moz-progress-bar { 142 | border: 0; 143 | height: 8px; 144 | border-radius: 50px; 145 | background: #eeeeee; 146 | } 147 | progress::-webkit-progress-value { 148 | border: 0; 149 | height: 8px; 150 | border-radius: 50px; 151 | background: $hColor1; 152 | } 153 | } 154 | 155 | /* Colors */ 156 | 157 | .category-controls:nth-child(1) { 158 | a { 159 | background: $hColor1; 160 | } 161 | progress::-webkit-progress-value { 162 | background: $hColor1; 163 | } 164 | h2 { 165 | color: $hColor1; 166 | } 167 | } 168 | 169 | .category-controls:nth-child(2) { 170 | a { 171 | background: $hColor2; 172 | } 173 | progress::-webkit-progress-value { 174 | background: $hColor2; 175 | } 176 | h2 { 177 | color: $hColor2; 178 | } 179 | } 180 | 181 | .category-controls:nth-child(3) { 182 | a { 183 | background: $hColor3; 184 | } 185 | progress::-webkit-progress-value { 186 | background: $hColor3; 187 | } 188 | h2 { 189 | color: #599551; 190 | } /* this one's a little darker because it's not a suitable text color */ 191 | } 192 | 193 | .category-controls:nth-child(4) { 194 | a { 195 | background: $hColor4; 196 | } 197 | progress::-webkit-progress-value { 198 | background: $hColor4; 199 | } 200 | h2 { 201 | color: #599551; 202 | } /* this one's a little darker because it's not a suitable text color */ 203 | } 204 | 205 | .category-controls:nth-child(5) { 206 | a { 207 | background: $hColor5; 208 | } 209 | progress::-webkit-progress-value { 210 | background: $hColor5; 211 | } 212 | h2 { 213 | color: #d08542; 214 | } /* this one's a little darker because it's not a suitable text color */ 215 | } 216 | 217 | .category-controls:nth-child(6) { 218 | a { 219 | background: $hColor6; 220 | } 221 | progress::-webkit-progress-value { 222 | background: $hColor6; 223 | } 224 | h2 { 225 | color: $hColor6; 226 | } 227 | } 228 | 229 | .category-controls:nth-child(7) { 230 | a { 231 | background: $hColor7; 232 | } 233 | progress::-webkit-progress-value { 234 | background: $hColor7; 235 | } 236 | h2 { 237 | color: $hColor7; 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /viz/styles/components/booth.css: -------------------------------------------------------------------------------- 1 | .booth { 2 | text-align: center; 3 | 4 | .chart-container { 5 | min-height: 150px; 6 | margin-bottom: 50px; 7 | } 8 | 9 | .footer-legend { 10 | padding-bottom: 30px; 11 | } 12 | } 13 | 14 | .big-button { 15 | background: $button-color; 16 | border: none; 17 | border-radius: 3px; 18 | color: white; 19 | font-size: 27px; 20 | padding: 20px 30px; 21 | margin: 15px auto; 22 | flex-shrink: 0; 23 | } 24 | -------------------------------------------------------------------------------- /viz/styles/components/chart-colors.css: -------------------------------------------------------------------------------- 1 | $hColor1: #482d8c; /* purple */ 2 | $hColor2: #3ca1da; /* blue */ 3 | $hColor3: #51854a; /* green */ 4 | $hColor4: #79c36d; /* light green */ 5 | $hColor5: #f9a151; /* orange */ 6 | $hColor6: #d64242; /* red */ 7 | $hColor7: #bc5fb2; /* pink */ 8 | $hColor8: #000000; /* black */ 9 | $hChartText: #757a7d; 10 | 11 | $sfColor1: #18afe5; 12 | $sfColor2: #955aa4; 13 | $sfColor3: #f29e1f; 14 | $sfColor4: #1fbcb8; 15 | $sfColor5: #29347c; 16 | $sfColor6: #60879e; 17 | $sfColor7: #bc5fb2; 18 | $sfChartText: #757a7d; 19 | 20 | body.heroku { 21 | .chart-color-1 { 22 | background: $hColor1; 23 | fill: $hColor1; 24 | } 25 | .chart-color-2 { 26 | background: $hColor2; 27 | fill: $hColor2; 28 | } 29 | .chart-color-3 { 30 | background: $hColor3; 31 | fill: $hColor3; 32 | } 33 | .chart-color-4 { 34 | background: $hColor4; 35 | fill: $hColor4; 36 | } 37 | .chart-color-5 { 38 | background: $hColor5; 39 | fill: $hColor5; 40 | } 41 | .chart-color-6 { 42 | background: $hColor6; 43 | fill: $hColor6; 44 | } 45 | .chart-color-7 { 46 | background: $hColor7; 47 | fill: $hColor7; 48 | } 49 | .chart-color-8 { 50 | background: $hColor8; 51 | fill: $hColor8; 52 | } 53 | .chart-line-color-1 { 54 | fill: none; 55 | stroke: $hColor1; 56 | } 57 | .chart-line-color-2 { 58 | fill: none; 59 | stroke: $hColor2; 60 | } 61 | .chart-line-color-3 { 62 | fill: none; 63 | stroke: $hColor3; 64 | } 65 | .chart-line-color-4 { 66 | fill: none; 67 | stroke: $hColor4; 68 | } 69 | .chart-line-color-5 { 70 | fill: none; 71 | stroke: $hColor5; 72 | } 73 | .chart-line-color-6 { 74 | fill: none; 75 | stroke: $hColor6; 76 | } 77 | .chart-line-color-7 { 78 | fill: none; 79 | stroke: $hColor7; 80 | } 81 | .chart-line-color-8 { 82 | fill: none; 83 | stroke: $hColor8; 84 | } 85 | path.domain { 86 | stroke: $hChartText; 87 | } 88 | g.tick text { 89 | fill: $hChartText; 90 | } 91 | g.tick line { 92 | stroke: $hChartText; 93 | } 94 | text.label { 95 | fill: $hChartText; 96 | font-size: 12px; 97 | } 98 | } 99 | 100 | body.salesforce { 101 | .chart-color-1 { 102 | background: $sfColor1; 103 | fill: $sfColor1; 104 | } 105 | .chart-color-2 { 106 | background: $sfColor2; 107 | fill: $sfColor2; 108 | } 109 | .chart-color-3 { 110 | background: $sfColor3; 111 | fill: $sfColor3; 112 | } 113 | .chart-color-4 { 114 | background: $sfColor4; 115 | fill: $sfColor4; 116 | } 117 | .chart-color-5 { 118 | background: $hColor5; 119 | fill: $hColor5; 120 | } 121 | .chart-color-6 { 122 | background: $hColor6; 123 | fill: $hColor6; 124 | } 125 | .chart-line-color-1 { 126 | fill: none; 127 | stroke: $sfColor1; 128 | } 129 | .chart-line-color-2 { 130 | fill: none; 131 | stroke: $sfColor2; 132 | } 133 | .chart-line-color-3 { 134 | fill: none; 135 | stroke: $sfColor3; 136 | } 137 | .chart-line-color-4 { 138 | fill: none; 139 | stroke: $sfColor4; 140 | } 141 | .chart-line-color-5 { 142 | fill: none; 143 | stroke: $sfColor5; 144 | } 145 | .chart-line-color-6 { 146 | fill: none; 147 | stroke: $sfColor6; 148 | } 149 | path.domain { 150 | stroke: $sfChartText; 151 | } 152 | g.tick text { 153 | fill: $sfChartText; 154 | } 155 | g.tick line { 156 | stroke: $sfChartText; 157 | } 158 | text.label { 159 | fill: $sfChartText; 160 | font-size: 12px; 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /viz/styles/components/demand.css: -------------------------------------------------------------------------------- 1 | .demand-chart { 2 | 3 | display: flex; 4 | flex-direction: column; 5 | justify-content: space-between; 6 | 7 | &-container { 8 | width: 100%; 9 | display: flex; 10 | flex-direction: column; 11 | } 12 | 13 | &--x-ticks { 14 | display: flex; 15 | margin: 20px 0px 20px 0; 16 | justify-content: space-between; 17 | position: relative; 18 | } 19 | 20 | &--x-tick { 21 | font-size: 18px; 22 | } 23 | 24 | &-y-ticks { 25 | width: 38px; 26 | position: relative; 27 | font-size: 18px; 28 | } 29 | 30 | } 31 | 32 | #demand-chart { 33 | /* width: 100%; */ 34 | /* max-width: 1440px; */ 35 | /* max-height: 60vh; */ 36 | border: solid 1px #ccc; 37 | background-color: #FFFFFF; 38 | } 39 | 40 | .demand-category { 41 | text-align: center; 42 | 43 | li { 44 | list-style: none; 45 | display: inline-block; 46 | margin-right: 20px; 47 | padding: 9px 0; 48 | padding-left: 20px; 49 | font-size: 12px; 50 | line-height: 14px; 51 | color: #757a7d; 52 | text-transform: capitalize; 53 | margin-left: 8px; 54 | font-weight: bold; 55 | } 56 | 57 | &--color-box { 58 | display: inline-block; 59 | width: 22px; 60 | height: 22px; 61 | margin-right: 3px; 62 | border-radius: 4px; 63 | top: 7px; 64 | position: relative; 65 | } 66 | } 67 | 68 | .demand--fulfillment-button { 69 | background-color: #42a1d7; 70 | border-radius: 3px; 71 | color: #ffffff; 72 | font-weight: bold; 73 | padding: 10px 15px; 74 | margin-right: 21px; 75 | display: inline-block; 76 | position: absolute; 77 | right: 0; 78 | top: 15px; 79 | cursor: pointer; 80 | 81 | @media screen and (min-width: 730px) { 82 | padding: 15px 55px; 83 | top: 35px; 84 | } 85 | } 86 | 87 | .demand-modal { 88 | display: none; 89 | width: 100%; 90 | height: 100%; 91 | background-color: rgba(0,0,0,.5); 92 | position: fixed; 93 | top: 0; 94 | left: 0; 95 | z-index: 1; 96 | 97 | &__active { 98 | display: block; 99 | } 100 | 101 | &__submitting { 102 | .demand-modal--overlay { 103 | display: block; 104 | } 105 | } 106 | 107 | &--title { 108 | text-align: center; 109 | 110 | &__success, 111 | &__error { 112 | display: none 113 | } 114 | } 115 | 116 | &__success, 117 | &__error { 118 | .demand-form, 119 | .demand-modal--title { 120 | display: none 121 | } 122 | } 123 | 124 | &__success { 125 | .demand-modal--title__success { 126 | display: block; 127 | } 128 | } 129 | 130 | &__error { 131 | .demand-modal--title__error { 132 | display: block; 133 | } 134 | } 135 | 136 | &--overlay { 137 | display: none; 138 | width: 100%; 139 | height: 100%; 140 | position: absolute; 141 | background-color: rgba(256,256,256,0.7); 142 | top: 0; 143 | left: 0; 144 | border-radius: 5px; 145 | 146 | img { 147 | position: absolute; 148 | top: 50%; 149 | left: 50%; 150 | width: 50px; 151 | height: 50px; 152 | margin-left: -25px; 153 | margin-top: -25px; 154 | } 155 | } 156 | } 157 | 158 | .demand-form-container { 159 | border-radius: 5px; 160 | background-color: #ffffff; 161 | padding: 20px 45px; 162 | position: fixed; 163 | width: 500px; 164 | height: 400; 165 | top: calc(50% - 200px); 166 | left: calc(50% - 250px); 167 | } 168 | 169 | .demand-form { 170 | &--fields-container { 171 | display: flex; 172 | label { 173 | display: none; 174 | } 175 | &:first-child { 176 | label { 177 | display: block; 178 | } 179 | } 180 | } 181 | 182 | &--field-item { 183 | display: flex; 184 | flex-direction: column; 185 | } 186 | 187 | &--field-item-category { 188 | width: 70%; 189 | margin-right: 10%; 190 | } 191 | 192 | &--field-item-amount { 193 | width: 20%; 194 | } 195 | 196 | &--field-input { 197 | margin-top: 7px; 198 | border-radius: 1px; 199 | text-align: center; 200 | border-style: solid; 201 | border-color: #ccc; 202 | height: 40px; 203 | } 204 | 205 | &--add-button-container { 206 | padding: 10px 0; 207 | } 208 | 209 | &--add-button { 210 | display: inline-block; 211 | color: #42a1d7; 212 | 213 | &:hover { 214 | text-decoration: underline; 215 | cursor: pointer; 216 | } 217 | 218 | &:before { 219 | content: '⊕'; 220 | margin-right: 4px; 221 | display: inline-block; 222 | font-size: 130%; 223 | } 224 | } 225 | 226 | &--submit-button-container { 227 | text-align: center; 228 | margin: 20px; 229 | } 230 | 231 | &--submit-button { 232 | text-align: center; 233 | padding: 10px 79px; 234 | background-color: #42a1d7; 235 | color: #ffffff; 236 | border-radius: 4px; 237 | cursor: pointer; 238 | } 239 | } 240 | 241 | .worker-running [start-button], 242 | .worker-stopped [stop-button], 243 | .worker-deleting button{ 244 | display: none; 245 | } 246 | 247 | .worker-deleting [status] { 248 | color: red; 249 | } 250 | 251 | .worker-running [status] { 252 | color: green; 253 | } 254 | -------------------------------------------------------------------------------- /viz/styles/components/nav.css: -------------------------------------------------------------------------------- 1 | header { 2 | height: $headerHeight; 3 | 4 | body.salesforce & { 5 | height: $sfHeaderHeight; 6 | } 7 | 8 | .logo { 9 | position: relative; 10 | z-index: 10; 11 | } 12 | 13 | object { 14 | pointer-events: none; /* make objects clickable when they're linked */ 15 | } 16 | 17 | nav { 18 | height: 100%; 19 | padding: 0 0 0 15px; 20 | display: flex; 21 | justify-content: space-between; 22 | align-items: center; 23 | 24 | ul { 25 | margin: 0; 26 | padding: 0; 27 | list-style: none; 28 | } 29 | } 30 | 31 | .nav-items { 32 | display: flex; 33 | align-items: center; 34 | } 35 | 36 | .nav-items > li { 37 | margin-left: 15px; 38 | padding: 17px 0 15px; 39 | position: relative; 40 | display: inline-block; 41 | font-size: 0.9em; 42 | 43 | &.nav-link { 44 | border-left: 1px solid $hLinkBorder; 45 | border-right: 1px solid $hLinkBorder; 46 | margin-left: 0; 47 | padding: 17px 15px; 48 | 49 | &.icon { 50 | padding: 0; 51 | 52 | a { 53 | display: block; 54 | padding: 17px 15px; 55 | } 56 | } 57 | 58 | &:first-of-type { 59 | border-left: 2px solid $hLinkBorder !important; 60 | } 61 | 62 | &:hover { 63 | cursor: pointer; 64 | 65 | .dropdown { 66 | display: block; 67 | } 68 | } 69 | } 70 | 71 | &.has-dropdown { 72 | position: relative; 73 | } 74 | } 75 | 76 | .title { 77 | font-weight: 400; 78 | 79 | body.heroku &, 80 | body.salesforce & { 81 | color: white; 82 | } 83 | 84 | @media screen and (max-width: 600px) { 85 | display: none; 86 | } 87 | 88 | @media screen and (min-width: 601px) { 89 | position: absolute; 90 | left: 50%; 91 | width: 200px; 92 | margin-left: -100px; 93 | text-align: center; 94 | } 95 | } 96 | 97 | .dropdown { 98 | width: 200px; 99 | padding: 10px 20px; 100 | display: none; 101 | position: absolute; 102 | z-index: 10; 103 | right: 10px; 104 | top: 45px; 105 | background: #fff; 106 | border-radius: 4px; 107 | border: 1px solid $hBorder; 108 | box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.05); 109 | line-height: 1.3; 110 | color: $hText; 111 | cursor: default; 112 | 113 | li a { 114 | margin: 10px 0; 115 | display: block; 116 | text-decoration: none; 117 | color: $hText; 118 | } 119 | } 120 | 121 | body.heroku & { 122 | background: $hHeader; 123 | color: $hHeaderText; 124 | 125 | .logo { 126 | width: 108px; 127 | height: 30px; 128 | background: url('/public/logo-heroku-white.svg') no-repeat center center; 129 | } 130 | } 131 | 132 | body.salesforce & { 133 | background: $sfHeader; 134 | color: $sfHeaderText; 135 | 136 | .logo { 137 | width: 93px; 138 | height: 65px; 139 | background: url('/public/logo-salesforce.svg') no-repeat center center; 140 | background-size: 93px 65px; 141 | } 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /viz/styles/components/typography.css: -------------------------------------------------------------------------------- 1 | body { 2 | color: #656565; 3 | } 4 | 5 | h1, 6 | h2, 7 | h3, 8 | h4, 9 | h5 { 10 | line-height: 120%; 11 | } 12 | 13 | p { 14 | line-height: 150%; 15 | } 16 | 17 | h1 { 18 | font-size: 27px; 19 | color: #79599f; 20 | text-align: center; 21 | font-weight: normal; 22 | } 23 | 24 | .title { 25 | text-transform: uppercase; 26 | text-align: center; 27 | margin: 0 0 25px 0; 28 | font-size: 0.8em; 29 | line-height: 1.3; 30 | letter-spacing: 1px; 31 | font-weight: bold; 32 | font-size: 16px; 33 | 34 | body.heroku & { 35 | color: $hText; 36 | } 37 | 38 | body.salesforce & { 39 | color: $sfText; 40 | } 41 | 42 | &.has-text { 43 | margin-bottom: 0; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /viz/styles/style.css: -------------------------------------------------------------------------------- 1 | @import '../node_modules/normalize.css/normalize.css'; 2 | @import '../node_modules/c3/c3.css'; 3 | 4 | @import './components/chart-colors.css'; 5 | @import './components/main.css'; 6 | @import './components/typography.css'; 7 | @import './components/nav.css'; 8 | 9 | @import './components/audience.css'; 10 | @import './components/booth.css'; 11 | @import './components/demand.css'; 12 | -------------------------------------------------------------------------------- /viz/views/audience.pug: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block nav 4 | ul.nav-items 5 | li.nav-link.icon.architecture-link 6 | a(href="#") 7 | img(src='/public/images/metrics-28.svg') 8 | li.nav-link.icon 9 | a(href='#' name='chart').toggle 10 | img(src='/public/images/usage-28.svg') 11 | 12 | block architecture 13 | include includes/architecture.pug 14 | 15 | block content 16 | 17 | .chart-container.chart-stream.toggleable(name='chart') 18 | .chart.loading 19 | footer.footer-legend 20 | ul 21 | 22 | h1.title Wishlist additions by product category 23 | p Press (-) or (+) on a product in order to make it less or more popular. 24 | 25 | //- Category Name 26 | div.category-controls-wrapper 27 | -------------------------------------------------------------------------------- /viz/views/booth.pug: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block nav 4 | include nav-home 5 | 6 | block architecture 7 | include includes/architecture.pug 8 | 9 | block content 10 | p.title Wishlist Additions By Product Category 11 | .chart-container.chart-stream 12 | .chart.loading 13 | 14 | footer.footer-legend 15 | ul 16 | 17 | section.content-container 18 | h1.title.has-text Click the button to create an event 19 | p.current-sale 20 | button.big-button Create an event! 21 | 22 | -------------------------------------------------------------------------------- /viz/views/connect.pug: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block nav 4 | ul.nav-items 5 | li.nav-link.architecture-link 6 | a(href="#") 7 | img(src='/public/images/metrics-28.svg') 8 | li.nav-link.has-dropdown 9 | img(src='/public/images/layout-list-28.svg') 10 | ul.dropdown.nav-learn 11 | li: a(href='https://github.com/heroku-examples/analytics-with-kafka-redshift-metabase') Fork on GitHub 12 | 13 | 14 | block architecture 15 | include includes/architecture.pug 16 | 17 | block content 18 | 19 | h1.title Orders by category 20 | a.demand--fulfillment-button New Fulfillment 21 | 22 | div.demand-chart 23 | canvas#demand-chart 24 | div.demand-chart--x-ticks 25 | 26 | div.demand-category-container -------------------------------------------------------------------------------- /viz/views/includes/architecture.pug: -------------------------------------------------------------------------------- 1 | .architecture 2 | iframe.architecture-iframe(width='100%', height='480px', scrolling='no', seamless) 3 | //- p Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Consectetur a erat nam at lectus urna duis convallis convallis. 4 | -------------------------------------------------------------------------------- /viz/views/includes/charts.pug: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heroku-examples/analytics-with-kafka-redshift-metabase/e4faa2faad033ff8dd622d4b3e9c604ff5de6238/viz/views/includes/charts.pug -------------------------------------------------------------------------------- /viz/views/index.pug: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block vars 4 | - var mainClass = 'index'; 5 | 6 | block nav 7 | include nav-home 8 | 9 | block architecture 10 | include includes/architecture.pug 11 | 12 | block content 13 | .chart-container.chart-stream 14 | .text-wrapper 15 | p.title Wishlist Additions By Product Category 16 | .chart.loading 17 | footer.footer-legend 18 | ul 19 | 20 | .chart-container.chart-line 21 | .text-wrapper 22 | p.title Backlog 23 | .queue-count 24 | .queue-color(data-container='queue-color') 25 | span.queue-title Avg Time 26 | span.queue-text(data-container='queue-text') 27 | .chart.loading 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /viz/views/layout.pug: -------------------------------------------------------------------------------- 1 | doctype html 2 | html 3 | head 4 | meta(http-equiv='Content-type' content='text/html; charset=utf-8') 5 | meta(name='viewport' content='width=device-width, initial-scale=1') 6 | link(rel='shortcut icon' href='https://www.herokucdn.com/favicon.ico' type='image/vnd.microsoft.icon') 7 | link(rel='stylesheet' href='https://www.herokucdn.com/shibori3/latest/shibori3.min.css') 8 | if (!htmlWebpackPlugin.options.production) 9 | script(src='http://localhost:35729/livereload.js') 10 | for css in htmlWebpackPlugin.files.css 11 | link(rel='stylesheet' type='text/css' href=css) 12 | title #{htmlWebpackPlugin.options.title} 13 | block vars 14 | body(class=htmlWebpackPlugin.options.bodyClass) 15 | header 16 | nav 17 | a(href='/') 18 | .logo 19 | if(typeof htmlWebpackPlugin.options.header !== 'undefined') 20 | span.title #{htmlWebpackPlugin.options.header} 21 | else 22 | span.title #{htmlWebpackPlugin.options.title} 23 | 24 | block nav 25 | block architecture 26 | 27 | main(class=mainClass) 28 | block content 29 | 30 | for js in htmlWebpackPlugin.files.js 31 | script(src=js) 32 | -------------------------------------------------------------------------------- /viz/views/nav-home.pug: -------------------------------------------------------------------------------- 1 | ul.nav-items 2 | li.nav-link.architecture-link 3 | a(href="#") 4 | object(type='image/svg+xml' data='/public/images/metrics-28.svg') 5 | img(src='/public/images/metrics-28.svg') 6 | li.nav-link.has-dropdown 7 | object(type='image/svg+xml' data='/public/images/layout-list-28.svg') 8 | img(src='/public/images/layout-list-28.svg') 9 | ul.dropdown.nav-learn 10 | li: a(href='https://github.com/heroku-examples/analytics-with-kafka-redshift-metabase') Fork on GitHub 11 | li: a(href='/connect') Salesforce Integration 12 | li: a(href='https://heroku.com/kafka') Apache Kafka on Heroku 13 | hr 14 | li: a(href='/admin/reload', target='_blank') Truncate and Reload Fixture Data 15 | li: a(href='/admin/start', target='_blank') Start Data Generator 16 | li: a(href='/admin/kill', target='_blank') Stop Data Generator 17 | -------------------------------------------------------------------------------- /viz/views/ordercontrol.pug: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block content 4 | h4 5 | | Order Maker Status: 6 | span(status='') 7 | p 8 | | These buttons will send a requset to the woker. Please look at the status above to see if it worked. 9 | div.order-control-buttons 10 | button(start-button='') 11 | | Start 12 | button(stop-button='') 13 | | Stop 14 | button(reset-button='') 15 | | Delete All 16 | 17 | -------------------------------------------------------------------------------- /viz/views/presentation.pug: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block nav 4 | ul.nav-items 5 | li.nav-link.architecture-link 6 | a(href="#") 7 | object(type='image/svg+xml' data='/public/images/metrics-28.svg') 8 | img(src='/public/images/metrics-28.svg') 9 | 10 | block architecture 11 | include includes/architecture.pug 12 | 13 | block content 14 | .chart-container.chart-stream 15 | h1.title Wishlist Additions By Product Category 16 | .chart.loading 17 | 18 | footer.footer-legend 19 | ul 20 | -------------------------------------------------------------------------------- /viz/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const webpack = require('webpack') 3 | const HtmlPlugin = require('html-webpack-plugin') 4 | const UglifyJsPlugin = require('uglifyjs-webpack-plugin') 5 | const MiniCssExtractPlugin = require('mini-css-extract-plugin') 6 | const OptimizeCSSAssetsPlugin = require('optimize-css-assets-webpack-plugin') 7 | const { CleanWebpackPlugin } = require('clean-webpack-plugin') 8 | const LiveReloadPlugin = require('webpack-livereload-plugin') 9 | 10 | const PRODUCTION = process.env.NODE_ENV === 'production' 11 | 12 | const DEFAULT_THEME = 'heroku' 13 | const THEMES = ['salesforce', DEFAULT_THEME] 14 | const THEME = THEMES.includes(process.env.THEME) 15 | ? process.env.THEME 16 | : DEFAULT_THEME 17 | 18 | const htmlPlugin = (options) => 19 | new HtmlPlugin({ 20 | production: PRODUCTION, 21 | minify: PRODUCTION ? { collapseWhitespace: true } : false, 22 | filename: 'index.html', 23 | title: 'Product Analytics', 24 | inject: false, 25 | template: path.join(__dirname, 'views', 'index.pug'), 26 | ...options 27 | }) 28 | 29 | module.exports = { 30 | devtool: PRODUCTION ? 'source-map' : 'cheap-module-source-map', 31 | mode: PRODUCTION ? 'production' : 'development', 32 | entry: path.join(__dirname, 'src', 'index.js'), 33 | output: { 34 | path: path.join(__dirname, 'dist'), 35 | filename: `app${PRODUCTION ? '.[hash]' : ''}.js` 36 | }, 37 | stats: 'minimal', 38 | module: { 39 | rules: [ 40 | { 41 | test: /.js$/, 42 | exclude: /node_modules/, 43 | use: [ 44 | { 45 | loader: 'babel-loader', 46 | options: { 47 | plugins: ['lodash'], 48 | presets: ['@babel/preset-env'] 49 | } 50 | } 51 | ] 52 | }, 53 | { 54 | test: /.pug$/, 55 | use: ['pug-loader'] 56 | }, 57 | { 58 | test: /\.css$/, 59 | use: [ 60 | PRODUCTION 61 | ? { 62 | loader: MiniCssExtractPlugin.loader 63 | } 64 | : 'style-loader', 65 | { loader: 'css-loader', options: { importLoaders: 1 } }, 66 | { 67 | loader: 'postcss-loader', 68 | options: { 69 | ident: 'postcss', 70 | plugins: (loader) => [ 71 | require('postcss-import')({ root: loader.resourcePath }), 72 | require('postcss-preset-env')(), 73 | require('precss')() 74 | ] 75 | } 76 | } 77 | ] 78 | } 79 | ] 80 | }, 81 | optimization: { 82 | minimizer: [ 83 | new UglifyJsPlugin({ 84 | parallel: true, 85 | sourceMap: true, 86 | uglifyOptions: { 87 | output: { 88 | comments: false 89 | } 90 | } 91 | }), 92 | new OptimizeCSSAssetsPlugin({}) 93 | ] 94 | }, 95 | plugins: [ 96 | new webpack.NormalModuleReplacementPlugin( 97 | /\/logger\.js/, 98 | './clientLogger.js' 99 | ), 100 | htmlPlugin({ 101 | bodyClass: THEME 102 | }), 103 | htmlPlugin({ 104 | filename: 'audience.html', 105 | title: 'Audience', 106 | template: path.join(__dirname, 'views', 'audience.pug'), 107 | bodyClass: `${THEME} audience` 108 | }), 109 | htmlPlugin({ 110 | filename: 'connect.html', 111 | title: 'Heroku Connect', 112 | template: path.join(__dirname, 'views', 'connect.pug'), 113 | bodyClass: `${THEME} connect` 114 | }), 115 | htmlPlugin({ 116 | filename: 'ordercontrol.html', 117 | title: 'Heroku Connect Order Control', 118 | template: path.join(__dirname, 'views', 'ordercontrol.pug'), 119 | bodyClass: `ordercontrol` 120 | }), 121 | htmlPlugin({ 122 | filename: 'presentation.html', 123 | title: 'Presentation', 124 | header: 'Kafka AWS', 125 | template: path.join(__dirname, 'views', 'presentation.pug'), 126 | bodyClass: `${THEME} presentation` 127 | }), 128 | htmlPlugin({ 129 | filename: 'booth.html', 130 | title: 'Booth', 131 | template: path.join(__dirname, 'views', 'booth.pug'), 132 | bodyClass: `${THEME} booth` 133 | }), 134 | new CleanWebpackPlugin(), 135 | new webpack.DefinePlugin( 136 | [ 137 | 'KAFKA_PREFIX', 138 | 'KAFKA_TOPIC', 139 | 'KAFKA_CMD_TOPIC', 140 | 'KAFKA_WEIGHT_TOPIC', 141 | 'KAFKA_QUEUE_TOPIC', 142 | 'KAFKA_QUEUE_WORKER' 143 | ].reduce((acc, key) => { 144 | acc[`process.env.${key}`] = JSON.stringify(process.env[key]) 145 | return acc 146 | }, {}) 147 | ), 148 | new MiniCssExtractPlugin({ 149 | filename: '[name].[contenthash].css' 150 | }), 151 | // Add other themes in dev mode for easier viewing 152 | ...(PRODUCTION 153 | ? [] 154 | : THEMES.map((theme) => 155 | htmlPlugin({ 156 | filename: `${theme}.html`, 157 | bodyClass: theme 158 | }) 159 | )), 160 | !PRODUCTION && new LiveReloadPlugin({ quiet: true }) 161 | ].filter(Boolean) 162 | } 163 | --------------------------------------------------------------------------------