├── .gitignore ├── pubnub.js ├── package.json ├── ingest.js └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .DS_Store 3 | .vscode/ 4 | node_modules/ -------------------------------------------------------------------------------- /pubnub.js: -------------------------------------------------------------------------------- 1 | var PubNub = require('pubnub'); 2 | //var jsonfile = require('jsonfile'); 3 | 4 | var pubnub = new PubNub({ 5 | ssl: true, 6 | //subscribe_key: 'sub-c-5f1b7c8e-fbee-11e3-aa40-02ee2ddab7fe' 7 | subscribe_key: 'sub-c-4377ab04-f100-11e3-bffd-02ee2ddab7fe' 8 | }); 9 | 10 | pubnub.addListener({ 11 | message: function(message) { 12 | console.log(message.message); 13 | //jsonfile.writeFileSync(`test.json`, data); 14 | } 15 | }); 16 | 17 | pubnub.subscribe({ 18 | //channels: ['pubnub-sensor-network'] 19 | channels: ['pubnub-market-orders'] 20 | }); -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "stream-sequelize-node", 3 | "version": "1.0.0", 4 | "description": "Ingest sample PubNub stream using Sequelize to Postgres with TimescaleDB extension installed and enabled for time series analysis", 5 | "main": "ingest.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/sfrechette/stream-sequelize-node.git" 12 | }, 13 | "author": "Stéphane Fréchette", 14 | "license": "ISC", 15 | "bugs": { 16 | "url": "https://github.com/sfrechette/stream-sequelize-node/issues" 17 | }, 18 | "homepage": "https://github.com/sfrechette/stream-sequelize-node#readme", 19 | "dependencies": { 20 | "dotenv": "^8.2.0", 21 | "pg": "^8.11.0", 22 | "pg-hstore": "^2.3.3", 23 | "pubnub": "^9.5.2", 24 | "sequelize": "^6.29.0" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /ingest.js: -------------------------------------------------------------------------------- 1 | var pubnub = require('pubnub'); 2 | var sequelize = require('sequelize'); 3 | var dotenv = require('dotenv'); 4 | dotenv.load(); 5 | 6 | // Initialize connection to database (using environment variables) 7 | var connection = new sequelize('market_orders', process.env.DB_SERVER_USER_NAME, process.env.DB_SERVER_USER_PASSWORD, { 8 | host: process.env.DB_SERVER_HOST, 9 | port: process.env.DB_SERVER_PORT, 10 | dialect: 'postgres', 11 | //operatorsAliases: false, 12 | logging:()=>{} 13 | }); 14 | 15 | // Define model 16 | var order = connection.define('orders', { 17 | order_time: { 18 | type: sequelize.BIGINT, 19 | allowNull: false, 20 | primaryKey: true 21 | }, 22 | trade_type: { 23 | type: sequelize.STRING, 24 | allowNull: false 25 | }, 26 | symbol: { 27 | type: sequelize.STRING, 28 | allowNull: false 29 | }, 30 | order_quantity: { 31 | type: sequelize.DOUBLE, 32 | allowNull: false 33 | }, 34 | bid_price: { 35 | type: sequelize.DOUBLE, 36 | allowNull: false 37 | } 38 | }, { 39 | timestamps: false, 40 | freezeTableName: true 41 | }); 42 | 43 | // Initialize PubNub client 44 | var pubnub = new pubnub({ 45 | ssl: true, 46 | subscribe_key: 'sub-c-4377ab04-f100-11e3-bffd-02ee2ddab7fe' 47 | }); 48 | 49 | // Subscribe (listen on) to channel 50 | pubnub.subscribe({ 51 | channels: ['pubnub-market-orders'] 52 | }); 53 | 54 | // Handle message payload 55 | pubnub.addListener({ 56 | message: function (message) { 57 | console.log(message.message); 58 | connection.sync({ 59 | //logging: ()=>{} 60 | }) 61 | .then(function () { 62 | // Build and Save message stream to database 63 | var orderInstance = order.build({ 64 | order_time: message.message.timestamp, 65 | trade_type: message.message.trade_type, 66 | symbol: message.message.symbol, 67 | order_quantity: message.message.order_quantity, 68 | bid_price: message.message.bid_price 69 | }) 70 | //orderInstance.save() 71 | //if (Math.random() > .9) { 72 | //throw new Error('Something unusual'+new Date().toISOString()) 73 | //} 74 | }) 75 | .catch(function (err) { 76 | console.log(err); 77 | }); 78 | } 79 | }); 80 | 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # stream-sequelize-node 2 | Ingest sample Market Orders Data feed from PubNub to Postgres with TimescaleDB extension installed and enabled for time series analysis. 3 | ### Installation 4 | Clone or download repository 5 | ``` 6 | git clone https://github.com/sfrechette/stream-sequelize-node.git 7 | cd stream-sequilize-node 8 | ``` 9 | Install the dependencies with the following command: 10 | ```javascript 11 | npm install 12 | ``` 13 | ### Configuration 14 | Create an .env file and add the following with your configuration values 15 | ``` 16 | DB_SERVER_USER_NAME = "" 17 | DB_SERVER_USER_PASSWORD = "" 18 | DB_SERVER_HOST = "" 19 | DB_SERVER_PORT = "" 20 | ``` 21 | ### Usage 22 | ```javascript 23 | node ingest.js 24 | ``` 25 | ## Prerequesites 26 | Running this application assumes that you already have an instance of Postgres running and have installed the TimescaleDB extension. 27 | If not please refer the following documentation: 28 | 29 | Getting started, installing and setting up 30 | https://docs.timescale.com/v0.9/getting-started 31 | 32 | ### Database schema 33 | Next you need to execute the following SQL code snippets in your Postgres environment in order to create the database, TimescaleDB extension, schema definition, and hypertable. 34 | 35 | ```sql 36 | 37 | create database market_orders; 38 | ``` 39 | 40 | ```sql 41 | \c market_orders 42 | ``` 43 | 44 | ```sql 45 | create extension if not exists timescaledb cascade; 46 | ``` 47 | 48 | ```sql 49 | create table orders ( 50 | order_time bigint not null, 51 | trade_type text not null, 52 | symbol text not null, 53 | order_quantity double precision not null, 54 | bid_price double precision not null 55 | ); 56 | ``` 57 | 58 | ```sql 59 | create index on orders(order_time desc); 60 | create index on orders(trade_type, order_time desc); 61 | create index on orders(symbol, order_time desc); 62 | create index on orders(order_time desc, order_quantity); 63 | create index on orders(order_time desc, bid_price); 64 | -- 86400000000 is in usecs and is equal to 1 day 65 | select create_hypertable('orders', 'order_time', chunk_time_interval => 86400000000); 66 | ``` 67 | 68 | Once completed, you can now run the application to start ingesting data in your database. 69 | ### Sample queries 70 | Assuming that you have been ingesting data for a while (you can modify the time interval), run some queries to test and validate how Timescale performs nicely in fetching results while data is also being ingested in the database. 71 | ```sql 72 | -- Average bid price by 5 minute intervals for Google and day trade type 73 | select time_bucket('5 minutes', to_timestamp(order_time)) as five_min, 74 | avg(bid_price) as avg_bid_price 75 | from orders 76 | where symbol = 'Google' and trade_type = 'day' 77 | group by five_min 78 | order by five_min limit 20; 79 | ``` 80 | 81 | ```sql 82 | -- Min and Max bid price by 2 minute intervals for all symbols 83 | select time_bucket('2 minutes', to_timestamp(order_time)) as two_min, 84 | symbol, 85 | min(bid_price) as min_bid_price, 86 | max(bid_price) as max_bid_price 87 | from orders 88 | group by two_min, symbol 89 | order by two_min desc limit 20; 90 | ``` 91 | 92 | ```sql 93 | -- Orders with order quantity greater than 975... 94 | select order_time, 95 | trade_type, 96 | order_quantity, 97 | bid_price 98 | from orders 99 | where order_quantity > 975 and 100 | symbol = 'Bespin Gas' and 101 | trade_type = 'market' limit 20; 102 | ``` 103 | 104 | ### Links 105 | **TimescaleDB** 106 | https://www.timescale.com/ 107 | 108 | **Data feed** 109 | Market Orders - artificial data stream that provides the latest market orders for a fictitious marketplace 110 | https://www.pubnub.com/developers/realtime-data-streams/financial-securities-market-orders/ --------------------------------------------------------------------------------