├── .babelrc ├── .codeclimate.yml ├── .eslintignore ├── .eslintrc ├── .gitignore ├── .pglistend-connection.yml.dist ├── .pglistend.yml.dist ├── .pylintrc ├── LICENSE ├── README.md ├── bin └── pglisten ├── index.js ├── listener.js.dist ├── misc └── wiki │ ├── logs-following.png │ ├── logs-instantly.png │ ├── logs-start.png │ ├── refresh-materialized-view.png │ └── setup-output.png ├── package.json ├── setup └── setup.py └── src ├── Listener.js ├── index.js ├── logging └── logger.js ├── messages └── common.js ├── program.js ├── query.js ├── resolver.js └── util.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "es2015" 4 | ], 5 | "plugins": [ 6 | [ 7 | "transform-es2015-for-of", 8 | { 9 | "loose": true 10 | } 11 | ] 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | engines: 2 | duplication: 3 | enabled: true 4 | config: 5 | languages: 6 | - javascript 7 | eslint: 8 | enabled: true 9 | fixme: 10 | enabled: true 11 | pep8: 12 | enabled: true 13 | ratings: 14 | paths: 15 | - '**.js' 16 | - '**.py' 17 | exclude_paths: 18 | - dist/* 19 | - lib/**/* 20 | - tests/**/* 21 | - coverage/**/* 22 | - node_modules/**/* 23 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | node_modules/ 3 | coverage 4 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | parser: 'babel-eslint' 2 | 3 | env: 4 | es6: true 5 | node: true 6 | 7 | parserOptions: 8 | ecmaVersion: 6, 9 | sourceType: 'module' 10 | ecmaFeatures: 11 | modules: true 12 | 13 | # http://eslint.org/docs/rules/ 14 | rules: 15 | # Possible Errors 16 | comma-dangle: [2, never] 17 | no-cond-assign: 2 18 | no-console: 0 19 | no-constant-condition: 2 20 | no-control-regex: 2 21 | no-debugger: 2 22 | no-dupe-args: 2 23 | no-dupe-keys: 2 24 | no-duplicate-case: 2 25 | no-empty: 2 26 | no-empty-character-class: 2 27 | no-ex-assign: 2 28 | no-extra-boolean-cast: 2 29 | no-extra-parens: 0 30 | no-extra-semi: 2 31 | no-func-assign: 2 32 | no-inner-declarations: [2, functions] 33 | no-invalid-regexp: 2 34 | no-irregular-whitespace: 2 35 | no-negated-in-lhs: 2 36 | no-obj-calls: 2 37 | no-regex-spaces: 2 38 | no-sparse-arrays: 2 39 | no-unexpected-multiline: 2 40 | no-unreachable: 2 41 | use-isnan: 2 42 | valid-jsdoc: 0 43 | valid-typeof: 2 44 | 45 | # Best Practices 46 | accessor-pairs: 2 47 | block-scoped-var: 0 48 | complexity: [2, 6] 49 | consistent-return: 0 50 | curly: 0 51 | default-case: 0 52 | dot-location: 0 53 | dot-notation: 0 54 | eqeqeq: 2 55 | guard-for-in: 2 56 | no-alert: 2 57 | no-caller: 2 58 | no-case-declarations: 2 59 | no-div-regex: 2 60 | no-else-return: 0 61 | no-empty-pattern: 2 62 | no-eq-null: 2 63 | no-eval: 2 64 | no-extend-native: 2 65 | no-extra-bind: 2 66 | no-fallthrough: 2 67 | no-floating-decimal: 0 68 | no-implicit-coercion: 0 69 | no-implied-eval: 2 70 | no-invalid-this: 0 71 | no-iterator: 2 72 | no-labels: 0 73 | no-lone-blocks: 2 74 | no-loop-func: 2 75 | no-magic-number: 0 76 | no-multi-spaces: 0 77 | no-multi-str: 0 78 | no-native-reassign: 2 79 | no-new-func: 2 80 | no-new-wrappers: 2 81 | no-new: 2 82 | no-octal-escape: 2 83 | no-octal: 2 84 | no-proto: 2 85 | no-redeclare: 2 86 | no-return-assign: 2 87 | no-script-url: 2 88 | no-self-compare: 2 89 | no-sequences: 0 90 | no-throw-literal: 0 91 | no-unused-expressions: 2 92 | no-useless-call: 2 93 | no-useless-concat: 2 94 | no-void: 2 95 | no-warning-comments: 0 96 | no-with: 2 97 | radix: 2 98 | vars-on-top: 0 99 | wrap-iife: 2 100 | yoda: 0 101 | 102 | # Strict 103 | strict: 0 104 | 105 | # Variables 106 | init-declarations: 0 107 | no-catch-shadow: 2 108 | no-delete-var: 2 109 | no-label-var: 2 110 | no-shadow-restricted-names: 2 111 | no-shadow: 0 112 | no-undef-init: 2 113 | no-undef: [2, {'typeof': true }] 114 | no-undefined: 0 115 | no-unused-vars: 0 116 | no-use-before-define: 0 117 | 118 | # Node.js and CommonJS 119 | callback-return: 2 120 | handle-callback-err: 2 121 | no-mixed-requires: 0 122 | no-new-require: 0 123 | no-path-concat: 2 124 | no-restricted-modules: 0 125 | no-sync: 0 126 | 127 | # Stylistic Issues 128 | array-bracket-spacing: 0 129 | block-spacing: 0 130 | brace-style: 0 131 | camelcase: 0 132 | comma-spacing: 0 133 | comma-style: 0 134 | computed-property-spacing: 0 135 | consistent-this: 0 136 | eol-last: 0 137 | func-names: 0 138 | func-style: 0 139 | id-length: 0 140 | id-match: 0 141 | indent: 0 142 | jsx-quotes: 0 143 | key-spacing: 0 144 | linebreak-style: 0 145 | lines-around-comment: 0 146 | max-depth: 0 147 | max-len: 0 148 | max-nested-callbacks: 0 149 | max-params: 0 150 | max-statements: [2, 30] 151 | new-cap: 0 152 | new-parens: 0 153 | newline-after-var: 0 154 | no-array-constructor: 0 155 | no-bitwise: 0 156 | no-continue: 0 157 | no-inline-comments: 0 158 | no-lonely-if: 0 159 | no-mixed-spaces-and-tabs: 0 160 | no-multiple-empty-lines: 0 161 | no-negated-condition: 0 162 | no-nested-ternary: 0 163 | no-new-object: 0 164 | no-plusplus: 0 165 | no-restricted-syntax: 0 166 | no-spaced-func: 0 167 | no-ternary: 0 168 | no-trailing-spaces: 0 169 | no-underscore-dangle: 0 170 | no-unneeded-ternary: 0 171 | object-curly-spacing: 0 172 | one-var: 0 173 | operator-assignment: 0 174 | operator-linebreak: 0 175 | padded-blocks: 0 176 | quote-props: 0 177 | quotes: [2, 'single'] 178 | require-jsdoc: 0 179 | semi-spacing: 0 180 | semi: [2, 'always'] 181 | sort-vars: 0 182 | space-after-keywords: 0 183 | space-before-blocks: 0 184 | space-before-function-paren: 0 185 | space-before-keywords: 0 186 | space-in-parens: 0 187 | space-infix-ops: 0 188 | space-return-throw-case: 0 189 | space-unary-ops: 0 190 | spaced-comment: 0 191 | wrap-regex: 0 192 | 193 | # ECMAScript 6 194 | arrow-body-style: 0 195 | arrow-parens: 0 196 | arrow-spacing: 0 197 | constructor-super: 0 198 | generator-star-spacing: 0 199 | no-arrow-condition: 0 200 | no-class-assign: 0 201 | no-const-assign: 0 202 | no-dupe-class-members: 0 203 | no-this-before-super: 0 204 | no-var: 0 205 | object-shorthand: 0 206 | prefer-arrow-callback: 0 207 | prefer-const: 0 208 | prefer-reflect: 0 209 | prefer-spread: 0 210 | prefer-template: 0 211 | require-yield: 0 212 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | node_modules 3 | .pglistend*.yml 4 | *.log 5 | -------------------------------------------------------------------------------- /.pglistend-connection.yml.dist: -------------------------------------------------------------------------------- 1 | # database connection 2 | connection: 3 | database: DATABASE_NAME 4 | user: YOUR_USERNAME 5 | password: PASSWORD 6 | port: 5432 7 | max: 2 8 | 9 | # channels to LISTEN to 10 | channels: [CHANNEL_1, CHANNEL_2] 11 | 12 | # list of listener scripts to be included 13 | scripts: 14 | - /path/to/listener/script.js 15 | # - path/to/another/listener-script.js 16 | -------------------------------------------------------------------------------- /.pglistend.yml.dist: -------------------------------------------------------------------------------- 1 | default: 2 | connection: 3 | host: localhost 4 | port: 5432 5 | max: 2 6 | idleTimeoutMillis: 10000 7 | 8 | # Include configuration files database connections you want to use 9 | connections: 10 | - /path/to/.pglistend-connection.yml 11 | - /path/to/another/.pglistend-connection.yml 12 | 13 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [REPORTS] 2 | reports=no 3 | 4 | [MESSAGES CONTROL] 5 | disable=missing-docstring,invalid-name 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Kabir Baidhya 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pglistend 2 | [![npm version](https://img.shields.io/npm/v/pglistend.svg?style=flat-square)](https://www.npmjs.com/package/pglistend) [![npm downloads](https://img.shields.io/npm/dt/pglistend.svg?style=flat-square)](https://www.npmjs.com/package/pglistend) [![Code Climate](https://img.shields.io/codeclimate/github/kabirbaidhya/pglistend.svg?style=flat-square)](https://codeclimate.com/github/kabirbaidhya/pglistend) 3 | 4 | A lightweight Postgres LISTEN Daemon built on top of [node](https://nodejs.org/en/), [node-postgres](https://github.com/brianc/node-postgres) and [systemd](https://wiki.debian.org/systemd). 5 | 6 | It's a very simple yet generic daemon application that could be used in any project that makes use of Postgres' `LISTEN`/`NOTIFY` feature. 7 | 8 | It runs as a background process that does `LISTEN` on the configured channels on a database and allows to perform custom actions on receiving [`NOTIFY`](https://www.postgresql.org/docs/9.1/static/sql-notify.html) signals on those channels. 9 | 10 | Check this [simple tutorial](https://github.com/kabirbaidhya/pglistend/wiki/Tutorial:-Basics) to get started with it. 11 | 12 | ## Installation 13 | 14 | Firstly, install the npm package globally. This will make `pglisten` CLI tool available on your system. 15 | ```bash 16 | $ npm install -g pglistend 17 | ``` 18 | 19 | Now setup the daemon using this command. 20 | 21 | ```bash 22 | $ sudo pglisten setup-daemon 23 | ``` 24 | Or, alternatively you can `curl` the script and run it on the fly. 25 | ```bash 26 | $ curl https://raw.githubusercontent.com/kabirbaidhya/pglistend/master/setup/setup.py | sudo python 27 | ``` 28 | 29 | When it's done, edit your [configuration](https://github.com/kabirbaidhya/pglistend/wiki/Configuration). And finally start the service using 30 | ```bash 31 | $ sudo systemctl start pglistend 32 | ``` 33 | 34 | ## Usage 35 | ### Managing the daemon 36 | You can use `systemd` commands to manage `pglistend`. 37 | ```bash 38 | # Start the service 39 | $ systemctl start pglistend 40 | 41 | # Stop the service 42 | $ systemctl stop pglistend 43 | 44 | # Check service status 45 | $ systemctl status pglistend 46 | 47 | # Enable the service (This will start the service on bootup) 48 | $ systemctl enable pglistend 49 | 50 | # Disable the service (Disable the service to not start on bootup) 51 | $ systemctl disable pglistend 52 | ``` 53 | 54 | For more information about `systemd` check [this](https://wiki.debian.org/systemd#Managing_services_with_systemd) 55 | 56 | ### Logs 57 | All logs are written to `syslog`. 58 | So, you can make use of `journalctl` here 59 | ```bash 60 | $ journalctl -u pglistend 61 | $ journalctl -f -u pglistend 62 | ``` 63 | 64 | Or, you can simply `tail` the logs like this: 65 | ```bash 66 | $ tail /var/log/syslog | grep pglistend 67 | $ tail -f /var/log/syslog | grep pglistend 68 | ``` 69 | Check [this](https://www.digitalocean.com/community/tutorials/how-to-use-journalctl-to-view-and-manipulate-systemd-logs) to read more about journalctl. 70 | 71 | 72 | ## Tutorials 73 | 1. [Getting Started](https://github.com/kabirbaidhya/pglistend/wiki/Tutorial:-Basics) 74 | 2. [Performing custom actions](https://github.com/kabirbaidhya/pglistend/wiki/Tutorial:-Custom-actions) 75 | 76 | ## Testing 77 | 1. Clone repository: `git clone git@github.com:kabirbaidhya/pglistend.git` 78 | 2. Install dependencies: `npm install` 79 | 3. Install other required packages: 80 | - `pycodestyle`: `pip install pycodestyle` or `pip install --upgrade pycodestyle` [[Reference](https://github.com/PyCQA/pycodestyle)] 81 | - `pylint`: `sudo apt-get install pylint` [[Reference](https://www.pylint.org/#install)] 82 | 4. Copy configuration file `config.yml.sample` and rename to `.pglistend.yml` in root directory. Update database credentials, channels and location of scripts. 83 | 5. To prepare a script, copy `listener.js.sample` and save it as `listener.js`, or anything you wish, to any location(recommended to save outside project directory). Update the preferred channels and instructions in the script. Also, update the location of script in `.pglistend.yml`. 84 | 6. From terminal in root directory, run: `npm start`. You can see the logs in terminal as the channels hit the queries when the `notify` operation is called on. 85 | 86 | ## TODOs 87 | * Delegate CPU-intensive tasks (mostly queries) to separate thread or message queue most likely. [Here's why](http://stackoverflow.com/questions/3491811/node-js-and-cpu-intensive-requests/3536183#answer-3491931) 88 | -------------------------------------------------------------------------------- /bin/pglisten: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | require('../dist/index'); 4 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // Just in case the user tries to require() the package. 2 | throw new Error('This package is a CLI tool and was not supposed to be used with require() programatically'); 3 | -------------------------------------------------------------------------------- /listener.js.dist: -------------------------------------------------------------------------------- 1 | module.exports = function(h) { 2 | return { 3 | 'channel_1': function(payload) { 4 | // Do something 5 | }, 6 | 7 | 'channel_2': function(payload) { 8 | // Do something 9 | } 10 | }; 11 | }; 12 | -------------------------------------------------------------------------------- /misc/wiki/logs-following.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabirbaidhya/pglistend/586b565ae2332c8fd6cf25daf5ab47926c63ceeb/misc/wiki/logs-following.png -------------------------------------------------------------------------------- /misc/wiki/logs-instantly.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabirbaidhya/pglistend/586b565ae2332c8fd6cf25daf5ab47926c63ceeb/misc/wiki/logs-instantly.png -------------------------------------------------------------------------------- /misc/wiki/logs-start.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabirbaidhya/pglistend/586b565ae2332c8fd6cf25daf5ab47926c63ceeb/misc/wiki/logs-start.png -------------------------------------------------------------------------------- /misc/wiki/refresh-materialized-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabirbaidhya/pglistend/586b565ae2332c8fd6cf25daf5ab47926c63ceeb/misc/wiki/refresh-materialized-view.png -------------------------------------------------------------------------------- /misc/wiki/setup-output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kabirbaidhya/pglistend/586b565ae2332c8fd6cf25daf5ab47926c63ceeb/misc/wiki/setup-output.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pglistend", 3 | "version": "0.2.1-0", 4 | "description": "pglistend - Postgres LISTEN Daemon using Node.js/Systemd", 5 | "scripts": { 6 | "lint:js": "eslint .", 7 | "py:pep8": "pycodestyle setup/setup.py && pylint setup/setup.py", 8 | "lint": "npm run lint:js && npm run py:pep8", 9 | "dev": "babel-node src/index.js --config=.pglistend.yml", 10 | "start": "nodemon --exec 'npm run lint && npm run dev'", 11 | "clean": "rm -rf dist/", 12 | "build": "npm run clean && babel src/ -d dist/", 13 | "prepublish": "npm run build" 14 | }, 15 | "keywords": [ 16 | "postgresql", 17 | "listen", 18 | "notify", 19 | "daemon", 20 | "pglistend" 21 | ], 22 | "author": "Kabir Baidhya ", 23 | "license": "MIT", 24 | "repository": { 25 | "type": "git", 26 | "url": "git+https://github.com/kabirbaidhya/pglistend.git" 27 | }, 28 | "dependencies": { 29 | "chalk": "^1.1.3", 30 | "commander": "^2.9.0", 31 | "deep-assign": "^2.0.0", 32 | "es6-shim": "^0.35.1", 33 | "lodash.debounce": "^4.0.6", 34 | "lodash.throttle": "^4.0.1", 35 | "pg": "^6.0.1", 36 | "winston": "^2.2.0", 37 | "yamljs": "^0.2.8" 38 | }, 39 | "devDependencies": { 40 | "babel-cli": "^6.10.1", 41 | "babel-eslint": "^5.0.0", 42 | "babel-preset-es2015": "^6.9.0", 43 | "babel-plugin-transform-es2015-for-of": "^6.8.0", 44 | "eslint": "2.2.0", 45 | "nodemon": "^1.9.2" 46 | }, 47 | "preferGlobal": true, 48 | "bin": { 49 | "pglisten": "bin/pglisten" 50 | }, 51 | "files": [ 52 | "bin/", 53 | "dist/", 54 | "setup", 55 | ".pglistend.yml.dist", 56 | ".pglistend-connection.yml.dist", 57 | "listener.js.dist" 58 | ] 59 | } 60 | -------------------------------------------------------------------------------- /setup/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Setup script for setting up the daemon 3 | # Need to run this with sudo access 4 | 5 | import os 6 | import subprocess 7 | from subprocess import CalledProcessError, Popen 8 | 9 | PACKAGE = 'pglistend' 10 | EXEC_NAME = 'pglisten' 11 | BASE_DIRECTORY = '/etc/pglistend' 12 | CONFIG_FILE = BASE_DIRECTORY + '/config.yml' 13 | DEFAULT_LISTENER_FILE = BASE_DIRECTORY + '/listener.js' 14 | 15 | # Systemd unit file template 16 | SYSTEMD_TEMPLATE = ''' 17 | [Unit] 18 | Description={package} - Postgres LISTEN Daemon 19 | After=postgresql.service 20 | 21 | [Service] 22 | WorkingDirectory={directory} 23 | ExecStart={exec_path} --config={config_path} 24 | Restart=always 25 | StandardOutput=syslog 26 | StandardError=syslog 27 | SyslogIdentifier={package} 28 | Environment=NODE_ENV=production 29 | 30 | [Install] 31 | WantedBy=multi-user.target 32 | ''' 33 | 34 | # Configuration file template 35 | CONFIG_TEMPLATE = ''' 36 | default: 37 | connection: 38 | host: localhost 39 | port: 5432 40 | max: 2 41 | idleTimeoutMillis: 10000 42 | 43 | # Include configuration files database connections you want to use 44 | connections: 45 | # - path/to/connection-config.yml 46 | ''' 47 | 48 | # Connection config file template 49 | CONNECTION_CONFIG_TEMPLATE = ''' 50 | # Change the following configuration parameters according to your need. 51 | 52 | # postgresql connection 53 | connection: 54 | user: 'YOUR_USERNAME' 55 | database: 'DATABASE' 56 | password: 'PASSWORD' 57 | port: 5432 58 | max: 10 59 | 60 | # channels to LISTEN to 61 | channels: [CHANNEL_1, CHANNEL_2] 62 | 63 | # list of listener scripts 64 | scripts: 65 | - {default_listener} 66 | # You can add another scripts here 67 | # - path/to/another/listener.js 68 | ''' 69 | 70 | # Listener script template 71 | DEFAULT_LISTENER_TEMPLATE = ''' 72 | // Here you can define handlers for each of the channels 73 | // that are being LISTENed. 74 | module.exports = function(h) { 75 | return { 76 | 'channel_1': function(payload) { 77 | // Do something 78 | }, 79 | 80 | 'channel_2': function(payload) { 81 | // Do something 82 | } 83 | }; 84 | }; 85 | 86 | ''' 87 | 88 | # Constants 89 | DEVNULL = open(os.devnull, 'w') 90 | COLOR_RED = '\033[0;31m' 91 | COLOR_END = '\033[0m' 92 | COLOR_GREY = '\033[0;37m' 93 | COLOR_GREEN = '\033[0;32m' 94 | COLOR_YELLOW = '\033[1;33m' 95 | COLOR_DARKGREY = '\033[1;30m' 96 | COLOR_LIGHTGREEN = '\033[1;32m' 97 | 98 | 99 | def info_t(s): 100 | return '- ' + COLOR_GREEN + s + COLOR_END 101 | 102 | 103 | def ok_t(s): 104 | return COLOR_GREEN + s + COLOR_END 105 | 106 | 107 | def cmd_t(s): 108 | return COLOR_GREY + s + COLOR_END 109 | 110 | 111 | def out_t(s): 112 | return COLOR_DARKGREY + s + COLOR_END 113 | 114 | 115 | def err_t(s): 116 | return COLOR_RED + s + COLOR_END 117 | 118 | 119 | # Just a simple command to execute shell commands and display the output 120 | def exec_cmd(cmd): 121 | print(cmd_t('> {}'.format(cmd))) 122 | 123 | p = Popen(cmd.split(' '), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 124 | 125 | output = '' 126 | 127 | for raw_line in iter(p.stdout.readline, b''): 128 | line = raw_line.decode('utf-8') 129 | print(out_t('| ' + line.rstrip())) 130 | output += line 131 | 132 | returncode = p.wait() 133 | 134 | if returncode != 0: 135 | raise CalledProcessError(returncode, cmd) 136 | 137 | print('') 138 | 139 | return output.rstrip() 140 | 141 | 142 | def ensure_package_installed(package_name): 143 | print(info_t('Ensure {0} is installed'.format(package_name))) 144 | 145 | try: 146 | # Ensure pglistend package is installed on the system globally 147 | exec_cmd('npm list -g {}'.format(package_name)) 148 | except CalledProcessError: 149 | print(err_t('Warning: npm list returned non-zero exit code.')) 150 | 151 | 152 | def get_exec_path(exec_name): 153 | # Get full path of pglisten, to know where it is 154 | try: 155 | return exec_cmd('which {0}'.format(exec_name)) 156 | except CalledProcessError: 157 | raise SystemExit(err_t( 158 | 'Could not find pglisten. ' 159 | 'Make sure you have installed the package correctly using: ' 160 | 'npm install --global {}'.format(PACKAGE) 161 | )) 162 | 163 | 164 | def mkdir(directory): 165 | print(info_t('Create directory {0}'.format(directory))) 166 | 167 | try: 168 | exec_cmd('mkdir -p {0}'.format(directory)) 169 | except CalledProcessError: 170 | raise SystemExit( 171 | err_t('Error creating directory {0}'.format(directory)) 172 | ) 173 | 174 | 175 | def create_file(filename, contents): 176 | print(info_t('Write to file {0}'.format(filename))) 177 | 178 | try: 179 | with open(filename, 'w') as f: 180 | f.write(contents) 181 | except Exception as e: 182 | raise SystemExit(err_t('Error: ' + str(e))) 183 | 184 | 185 | def create_systemd_unit_file(filename, exec_path): 186 | contents = SYSTEMD_TEMPLATE.format( 187 | package=PACKAGE, 188 | directory=BASE_DIRECTORY, 189 | exec_path=exec_path, 190 | config_path=CONFIG_FILE 191 | ) 192 | 193 | create_file(filename, contents) 194 | 195 | 196 | def create_config_file(filename): 197 | config = CONFIG_TEMPLATE.format( 198 | default_listener=DEFAULT_LISTENER_FILE 199 | ) 200 | 201 | create_file(filename, config) 202 | 203 | 204 | def enable_daemon(): 205 | print(info_t('Enable the service')) 206 | 207 | try: 208 | # Ensure pglistend package is installed on the system globally 209 | exec_cmd('systemctl enable {0}'.format(PACKAGE)) 210 | except CalledProcessError: 211 | raise SystemExit(err_t('Error enabling the service')) 212 | 213 | 214 | def check_status(): 215 | print(info_t('Check status')) 216 | 217 | try: 218 | exec_cmd('systemctl is-enabled {0}'.format(PACKAGE)) 219 | exec_cmd('systemctl status {0}'.format(PACKAGE)) 220 | except CalledProcessError: 221 | pass 222 | 223 | 224 | def setup(): 225 | ''' 226 | Setup Process 227 | ''' 228 | print() 229 | print('Setup - pglistend - Postgres LISTEN Daemon') 230 | print('------------------------------------------') 231 | 232 | # Ensure the package is installed globally on the system 233 | ensure_package_installed(PACKAGE) 234 | 235 | # Ensure the pglisten cli command exists and get its path 236 | exec_path = get_exec_path(EXEC_NAME) 237 | 238 | # Create the application directory 239 | mkdir(BASE_DIRECTORY) 240 | 241 | # Create a default listener file 242 | create_file(DEFAULT_LISTENER_FILE, DEFAULT_LISTENER_TEMPLATE) 243 | 244 | # Create application config file 245 | create_config_file(CONFIG_FILE) 246 | 247 | # Create the systemd daemon unit file 248 | create_systemd_unit_file( 249 | '/etc/systemd/system/pglistend.service', exec_path) 250 | 251 | # Finally enable the daemon and check status 252 | enable_daemon() 253 | check_status() 254 | 255 | print(ok_t('\nAll done!!')) 256 | print( 257 | 'Please manually edit the configuration file {0}. \n' 258 | 'And finally start the service using {1}.' 259 | .format(out_t(CONFIG_FILE), out_t('systemctl start ' + PACKAGE)) 260 | ) 261 | 262 | setup() 263 | -------------------------------------------------------------------------------- /src/Listener.js: -------------------------------------------------------------------------------- 1 | import {Client, Pool} from 'pg'; 2 | import {format} from 'util'; 3 | import {yellow, green, red, dim} from 'chalk'; 4 | 5 | import {halt} from './program'; 6 | import {log, error} from './util'; 7 | import logger from './logging/logger'; 8 | import * as msg from './messages/common'; 9 | 10 | class Listener { 11 | 12 | constructor(config, handlers) { 13 | this.config = config; 14 | this.handlers = handlers; 15 | this.client = new Client(config.connection); 16 | } 17 | 18 | listen() { 19 | let {client, config: {connection, channels}} = this; 20 | 21 | channels = Array.isArray(channels) ? channels : []; 22 | 23 | if (channels.length === 0) { 24 | throw new Error(msg.NO_CHANNELS_TO_LISTEN); 25 | } 26 | 27 | client.connect(err => { 28 | if (err instanceof Error) { 29 | halt(err); 30 | } 31 | 32 | logger.info(msg.DATABASE_CONNECTED, yellow(connection.database)); 33 | }); 34 | 35 | client.on('notice', msg => logger.info(msg)); 36 | client.on('notification', notification => this.handle(notification)); 37 | 38 | channels.forEach(channel => this.listenTo(channel)); 39 | } 40 | 41 | /** 42 | * Parses the payload and returns it. 43 | * If it's a valid json, parse it and return the decoded object. 44 | * Otherwise, just return the payload string as it is. 45 | * 46 | * @returns {object|string} 47 | */ 48 | parsePayload(str) { 49 | try { 50 | return JSON.parse(str); 51 | } catch (e) { 52 | return str; 53 | } 54 | } 55 | 56 | invokeHandlers(channel, payload) { 57 | const handlers = this.handlers[channel]; 58 | 59 | if (!Array.isArray(handlers)) { 60 | throw new Error(format(msg.WARN_NO_HANDLERS_FOUND, channel)); 61 | } 62 | 63 | // TODO: Delegate CPU-intensive jobs to a task queue or a separate process. 64 | 65 | handlers.forEach(callback => callback(payload)); 66 | } 67 | 68 | handle(notification) { 69 | const database = this.client.database; 70 | const {channel, payload: str} = notification; 71 | 72 | logger.info(msg.RECEIVED_NOTIFICATION, green(database + ':' + channel), dim(str || '(empty)')); 73 | 74 | try { 75 | const payload = this.parsePayload(str); 76 | 77 | // Invoke all the handlers registered on the channel 78 | this.invokeHandlers(channel, payload); 79 | } catch (e) { 80 | error(e.message); 81 | } 82 | } 83 | 84 | listenTo(channel) { 85 | const database = this.client.database; 86 | 87 | this.client.query(`LISTEN ${channel}`).then(() => { 88 | logger.info(msg.STARTED_LISTENING, green(database + ':' + channel)); 89 | 90 | // Warn if handlers are not registered for the channels being listened to 91 | if (!Array.isArray(this.handlers[channel])) { 92 | logger.warn(msg.WARN_NO_HANDLERS_FOUND, channel); 93 | } 94 | }); 95 | } 96 | } 97 | 98 | export default Listener; 99 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | 2 | import {run, halt} from './program'; 3 | 4 | try { 5 | run(); 6 | } catch (err) { 7 | halt(err); 8 | } 9 | -------------------------------------------------------------------------------- /src/logging/logger.js: -------------------------------------------------------------------------------- 1 | import winston from 'winston'; 2 | 3 | const formatter = (opts) => { 4 | return `[${opts.level}] ${(opts.message || '')}` + 5 | (opts.meta && Object.keys(opts.meta).length ? '\n\t' + JSON.stringify(opts.meta) : '' ); 6 | }; 7 | 8 | const logger = new (winston.Logger)({ 9 | transports: [ 10 | new winston.transports.Console({formatter}) 11 | ] 12 | }); 13 | 14 | export default logger; -------------------------------------------------------------------------------- /src/messages/common.js: -------------------------------------------------------------------------------- 1 | export const WARN_NO_HANDLERS_FOUND = 'Warning: No handlers are registered for channel "%s" yet.'; 2 | export const NO_CHANNELS_TO_LISTEN = 'No channels to LISTEN to'; 3 | export const RECEIVED_NOTIFICATION = 'Received notification on channel %s: %s'; 4 | export const DATABASE_CONNECTED = 'Connected to database %s'; 5 | export const STARTED_LISTENING = 'Started listening to channel %s'; 6 | export const SETUP_ERROR = 'Setup could not be completed.'; 7 | export const GENERIC_ERROR_MESSAGE = 'An error occurred.'; 8 | export const LOADED_CONFIG_FILE = 'Loaded configuration file %s'; 9 | export const ERROR_LOADING_CONFIG_FILE = 'Error loading configuration file %s'; 10 | export const NO_CONNECTIONS_CONFIGURED = 'No database connections are configured. Please do configure properly and start pglistend again.'; 11 | -------------------------------------------------------------------------------- /src/program.js: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import {Client} from 'pg'; 3 | import prog from 'commander'; 4 | import {spawnSync} from 'child_process'; 5 | 6 | import Listener from './Listener'; 7 | import {error, isString} from './util'; 8 | import logger from './logging/logger'; 9 | import * as msg from './messages/common'; 10 | import {resolveConfig, resolveHandlers} from './resolver'; 11 | 12 | /** 13 | * Run the program 14 | */ 15 | export function run() { 16 | prog.version(require('../package').version) 17 | .description('pglisten - Postgres LISTEN CLI tool') 18 | .usage('--config=') 19 | .option('-c, --config ', 'Configuration file to use'); 20 | 21 | prog.command('setup-daemon') 22 | .description('Setup pglistend service on this system') 23 | .option('-C, --configure', 'Configure the daemon during setup') 24 | .action(({configure}) => setupDaemon({configure})); 25 | 26 | prog.parse(process.argv); 27 | 28 | if (prog.config) { 29 | listen({config: prog.config}); 30 | } else { 31 | prog.help(); 32 | } 33 | } 34 | 35 | /** 36 | * Halt the program. 37 | * Note: Should be called only in case of fatal error. 38 | */ 39 | export function halt(err) { 40 | logger.error(isString(err) ? err : (err.message || msg.GENERIC_ERROR_MESSAGE)); 41 | process.exit(1); 42 | } 43 | 44 | function listen(args) { 45 | let config = resolveConfig(args.config); 46 | 47 | if (config.connections.length === 0) { 48 | throw new Error(msg.NO_CONNECTIONS_CONFIGURED); 49 | } 50 | 51 | for (let connection of config.connections) { 52 | let listener = new Listener(connection, resolveHandlers(connection)); 53 | 54 | listener.listen(); 55 | } 56 | } 57 | 58 | function setupDaemon(args) { 59 | let setupPath = path.join(__dirname, '/../setup/setup.py'); 60 | args = args.configure ? ['--configure'] : []; 61 | 62 | // Run the setup script and display the output without buffering 63 | let {status} = spawnSync('python', [setupPath, ...args], {stdio: 'inherit'}); 64 | 65 | if (status !== 0) { 66 | error(msg.SETUP_ERROR); 67 | } 68 | 69 | process.exit(status); 70 | } 71 | -------------------------------------------------------------------------------- /src/query.js: -------------------------------------------------------------------------------- 1 | import {log, error} from './util'; 2 | import logger from './logging/logger'; 3 | 4 | /** 5 | * @return {Promise} 6 | */ 7 | export default function query(sql, values) { 8 | return new Promise((resolve, reject) => { 9 | this.connect((err, client, done) => { 10 | if(err) { 11 | let message = `Connection Error: ${err}`; 12 | 13 | reject(message); 14 | logger.error(message); 15 | 16 | return; 17 | } 18 | 19 | client.query(sql, values, (err, result) => { 20 | done(); 21 | 22 | if (err) { 23 | let message = `Query error: ${err}`; 24 | 25 | reject(message); 26 | logger.error(message); 27 | 28 | return; 29 | } 30 | 31 | logger.info('Executed query: ', sql); 32 | resolve(result); 33 | }); 34 | }); 35 | }); 36 | } 37 | -------------------------------------------------------------------------------- /src/resolver.js: -------------------------------------------------------------------------------- 1 | import {Pool} from 'pg'; 2 | import Yaml from 'yamljs'; 3 | import {yellow} from 'chalk'; 4 | import deepAssign from 'deep-assign'; 5 | 6 | import query from './query'; 7 | import {throttle, debounce} from './util'; 8 | import {isObject, isFunction} from './util'; 9 | import * as msg from './messages/common'; 10 | import logger from './logging/logger'; 11 | 12 | export function resolveConfig(file) { 13 | let config = loadConfig(file); 14 | 15 | config.connections = resolveConnections(config.connections, config.default); 16 | 17 | return config; 18 | } 19 | 20 | function resolveConnections(files, defaults = {}) { 21 | if (!Array.isArray(files)) return []; 22 | 23 | return files.map(path => deepAssign({}, loadConfig(path), defaults)); 24 | } 25 | 26 | function loadConfig(file) { 27 | try { 28 | let config = Yaml.load(file); 29 | 30 | logger.info(msg.LOADED_CONFIG_FILE, yellow(file)); 31 | return config; 32 | } catch (e) { 33 | logger.error(msg.ERROR_LOADING_CONFIG_FILE, file); 34 | throw e; 35 | } 36 | } 37 | 38 | export function resolveHandlers(config) { 39 | let scripts = Array.isArray(config.scripts) ? config.scripts : []; 40 | 41 | if (scripts.length === 0) { 42 | logger.warn('No listener scripts are configured.'); 43 | 44 | return {}; 45 | } 46 | 47 | const helpers = getCallbackHelpers(config); 48 | 49 | return resolveForScripts(scripts, helpers); 50 | } 51 | 52 | /** 53 | * Some helper functions available in the listener scripts 54 | */ 55 | function getCallbackHelpers(config) { 56 | let pool = new Pool(config.connection); 57 | 58 | return { 59 | log: logger.info, 60 | error: logger.error, 61 | throttle, debounce, 62 | query: query.bind(pool) 63 | }; 64 | } 65 | 66 | function resolveHandlersFromFile(file, helpers) { 67 | let func = require(file); 68 | 69 | if (!isFunction(func)) { 70 | throw new Error(`Invalid listener script provided. The script file "${file}" should export a function.`); 71 | } 72 | 73 | let handlers = func(helpers); 74 | 75 | if (!isObject(handlers)) { 76 | throw new Error(`Invalid listener script provided. The exported function in "${file}" should return an object with handlers`); 77 | } 78 | 79 | return handlers; 80 | } 81 | 82 | function resolveForScripts(scripts, helpers) { 83 | let resolved = {}; 84 | 85 | scripts.forEach(file => { 86 | try { 87 | let handlers = resolveHandlersFromFile(file, helpers); 88 | 89 | for (let key of Object.keys(handlers)) { 90 | let callback = handlers[key]; 91 | 92 | if (!isFunction(callback)) { 93 | logger.error(`Invalid callback function specified for key "${key}" on file "${file}"`); 94 | 95 | continue; 96 | } 97 | 98 | if (!Array.isArray(resolved[key])) { 99 | resolved[key] = []; 100 | } 101 | 102 | resolved[key].push(callback); 103 | } 104 | } catch (e) { 105 | logger.error(e.message); 106 | } 107 | }); 108 | 109 | return resolved; 110 | } 111 | -------------------------------------------------------------------------------- /src/util.js: -------------------------------------------------------------------------------- 1 | 2 | import {red} from 'chalk'; 3 | import lodashThrottle from 'lodash.throttle'; 4 | import lodashDebounce from 'lodash.debounce'; 5 | 6 | // Console helpers 7 | export const log = (...params) => console.log(...params); 8 | export const error = (...params) => console.error(...params.map(param => red(param))); 9 | 10 | // Misc helpers 11 | export const isObject = (v) => (typeof v === 'object'); 12 | export const isFunction = (v) => (typeof v === 'function'); 13 | export const isString = (v) => (typeof v === 'string'); 14 | 15 | export function throttle(func, wait = 0, opts = {}) { 16 | return lodashThrottle(func, wait, Object.assign({}, { 17 | leading: true, 18 | trailing: false 19 | }, opts)); 20 | } 21 | 22 | export function debounce(func, wait = 0, opts = {}) { 23 | return lodashDebounce(func, wait, opts); 24 | } 25 | --------------------------------------------------------------------------------