├── .gitignore ├── Dockerfile ├── circle.yml ├── src ├── app_config.rs └── main.rs ├── Cargo.toml └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | target 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM scorpil/rust:1.18-onbuild 2 | MAINTAINER Francois-Guillaume Ribreau 3 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | machine: 2 | services: 3 | - docker 4 | 5 | test: 6 | override: 7 | - docker run -it --rm -v $(pwd):/source -w /source scorpil/rust:1.18 cargo test --release 8 | -------------------------------------------------------------------------------- /src/app_config.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | #[derive(Debug, Clone)] 4 | pub struct AppConfig { 5 | pub postgresql_uri: String, 6 | pub postgresql_channel: String, 7 | pub amqp_uri: String, 8 | pub amqp_queue_name: String 9 | } 10 | 11 | impl AppConfig { 12 | pub fn new() -> AppConfig { 13 | AppConfig { 14 | postgresql_uri: env::var("POSTGRESQL_URI").expect("POSTGRESQL_URI environment variable must be defined"), 15 | postgresql_channel: env::var("POSTGRESQL_CHANNEL").expect("POSTGRESQL_CHANNEL environment variable must be defined"), 16 | amqp_uri: env::var("AMQP_URI").expect("AMQP_URI environment variable must be defined"), 17 | amqp_queue_name: env::var("AMQP_QUEUE_NAME").expect("AMQP_QUEUE_NAME environment variable must be defined"), 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "postgresql-to-amqp" 3 | version = "0.1.4-pre" 4 | authors = ["FG Ribreau "] 5 | 6 | # A short blurb about the package. This is not rendered in any format when 7 | # uploaded to crates.io (aka this is not markdown). 8 | description = "PostgreSQL to AMQP, forward PostgreSQL notifications to an AMQP queue." 9 | 10 | # These URLs point to more information about the repository. These are 11 | # intended to be webviews of the relevant data, not necessarily compatible 12 | # with VCS tools and the like. 13 | documentation = "https://docs.rs/postgresql-to-amqp" 14 | homepage = "https://github.com/FGRibreau/postgresql-to-amqp" 15 | repository = "https://github.com/FGRibreau/postgresql-to-amqp" 16 | 17 | # This points to a file in the repository (relative to this `Cargo.toml`). The 18 | # contents of this file are stored and indexed in the registry. 19 | readme = "README.md" 20 | 21 | # This is a list of up to five keywords that describe this crate. Keywords 22 | # are searchable on crates.io, and you may choose any words that would 23 | # help someone find this crate. 24 | keywords = ["postgresql", "amqp", "rabbitmq", "notify", "postgres"] 25 | 26 | # This is a list of up to five categories where this crate would fit. 27 | # Categories are a fixed list available at crates.io/category_slugs, and 28 | # they must match exactly. 29 | categories = ["database"] 30 | 31 | # This is a string description of the license for this package. Currently 32 | # crates.io will validate the license provided against a whitelist of known 33 | # license identifiers from http://spdx.org/licenses/. Multiple licenses can be 34 | # separated with a `/`. 35 | license = "MIT" 36 | 37 | include = ["src/main.rs","src/app_config.rs", "Cargo.toml"] 38 | 39 | [dependencies] 40 | # https://github.com/sozu-proxy/lapin-futures-tls 41 | lapin-futures-rustls = "^0.7" 42 | 43 | # https://docs.rs/tokio-core/0.1.6/tokio_core/ 44 | tokio-core = "0.1.6" 45 | 46 | # An implementation of futures and streams featuring zero allocations, composability, and iterator-like interfaces. 47 | # https://docs.rs/futures/0.1.12/futures/ 48 | futures = "0.1.12" 49 | 50 | # A lightweight logging facade for Rust 51 | # https://doc.rust-lang.org/log/log/index.html 52 | log = "0.3.7" 53 | 54 | # http://rust-lang-nursery.github.io/log/env_logger/ 55 | # A logger configured via an environment variable which writes to standard error. 56 | env_logger = "0.4.2" 57 | 58 | # https://github.com/sfackler/rust-postgres 59 | # A native PostgreSQL driver for Rust. 60 | postgres = "0.14" 61 | 62 | # https://crates.io/crates/fallible-iterator 63 | # Fallible iterator traits (used for iterate (blocking) through notifications) 64 | fallible-iterator = "0.1.3" 65 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # :warning: Discontinued 2 | 3 | ### @subzerocloud took inspiration from this project and built [pg-amqp-bridge](https://github.com/subzerocloud/pg-amqp-bridge) which now has more options, use it instead :+1: 4 | 5 | --------------------------------------------------------- 6 | 7 | 8 | # 🚇 PostgreSQL to AMQP gateway 9 | #### Forward PostgreSQL `pg_notify` notifications to an AMQP queue. 10 | 11 | [![Cargo version](https://img.shields.io/crates/v/postgresql-to-amqp.svg)](https://crates.io/crates/postgresql-to-amqp) [![Crates.io](https://img.shields.io/crates/l/postgresql-to-amqp.svg)](https://crates.io/crates/postgresql-to-amqp) [![Crates.io](https://img.shields.io/crates/d/postgresql-to-amqp.svg)](https://crates.io/crates/postgresql-to-amqp) [![Docker Automated build](https://img.shields.io/docker/automated/fgribreau/postgresql-to-amqp.svg)](https://hub.docker.com/r/fgribreau/postgresql-to-amqp) [![Docker Pulls](https://img.shields.io/docker/pulls/fgribreau/postgresql-to-amqp.svg)](https://hub.docker.com/r/fgribreau/postgresql-to-amqp) [![Docker Stars](https://img.shields.io/docker/stars/fgribreau/postgresql-to-amqp.svg)](https://hub.docker.com/r/fgribreau/postgresql-to-amqp) [![Slack](https://img.shields.io/badge/Slack-Join%20our%20tech%20community-17202A?logo=slack)](https://join.slack.com/t/fgribreau/shared_invite/zt-edpjwt2t-Zh39mDUMNQ0QOr9qOj~jrg) 12 | ================== 13 | 14 |

15 | 16 | 17 | 18 | ## ⛴ Cargo 19 | 20 | ```shell 21 | cargo install postgresql-to-amqp 22 | ``` 23 | 24 | ## 🐳 Docker 25 | ```shell 26 | docker run --rm -it \ 27 | -e POSTGRESQL_URI=postgresql://username:password@domain.tld:port/database \ 28 | -e POSTGRESQL_CHANNEL=foo \ 29 | -e AMQP_URI=amqp://127.0.0.1:5672/ \ 30 | -e AMQP_QUEUE_NAME=queueName fgribreau/postgresql-to-amqp 31 | ``` 32 | 33 | ## ⚙ Configuration 34 | 35 | Configuration is done through environment variables: 36 | 37 | - **POSTGRESQL_URI**: e.g. `postgresql://username:password@domain.tld:port/database` 38 | - **POSTGRESQL_CHANNEL**: e.g. `foo` 39 | - **AMQP_URI**: e.g. `amqp://127.0.0.1:5672/` 40 | - **AMQP_QUEUE_NAME**: e.g. `queueName` 41 | 42 | ## 🎩 Usage 43 | 44 | Start the forwarder: 45 | 46 | ```bash 47 | POSTGRESQL_URI="postgresql://username:password@domain.tld:port/database" POSTGRESQL_CHANNEL="foo" AMQP_URI="amqp://127.0.0.1:5672/" AMQP_QUEUE_NAME="queueName" postgresql-to-amqp 48 | ``` 49 | 50 | 51 | Execute in psql: 52 | 53 | ```sql 54 | SELECT pg_notify('foo', 'payload'); 55 | ``` 56 | 57 | The forwarder will log and forward the notification to the amqp queue: 58 | 59 | ``` 60 | Forwarding Notification { process_id: 31694, channel: "foo", payload: "payload" } to queue "queueName" 61 | ``` 62 | 63 | ## 👁 Philosophy 64 | 65 | - Low memory consumption (1,9Mo) 66 | - Single binary 67 | - No dependency 68 | - Predictable performance 69 | 70 | 71 | ## 🔫 Todo 72 | 73 | I will happily accept PRs for this: 74 | 75 | - [ ] AMQP connection string (AMQP authentication support) 👻 76 | - [ ] Support JSON message 77 | - [ ] Publish to exchange 78 | - [ ] Add original channel as message property 79 | - [ ] Add postgresql-to-amqp `version` as message property 80 | - [ ] Let environment variables specify additional message properties 81 | - [ ] Handle AMQP disconnection/reconnection 82 | - [ ] Handle PostgreSQL disconnection/reconnection 83 | - [ ] Health check route 84 | - [ ] Metric route 85 | - [x] Docker support 86 | - [ ] Kubernetes support 😍 87 | - [ ] Make a first major release with tests ☝️ 88 | 89 | ## Related work 90 | 91 | - [pgsql-listen-exchange](https://github.com/gmr/pgsql-listen-exchange) - RabbitMQ Exchange that publishes messages received from PostgreSQL Notifications 92 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate log; 3 | extern crate env_logger; 4 | 5 | extern crate futures; 6 | extern crate tokio_core; 7 | extern crate lapin_futures_rustls; 8 | extern crate postgres; 9 | extern crate fallible_iterator; 10 | 11 | mod app_config; 12 | 13 | use lapin_futures_rustls::lapin; 14 | use lapin_futures_rustls::AMQPConnectionRustlsExt; 15 | 16 | use futures::future::Future; 17 | use tokio_core::reactor::Core; 18 | use lapin::channel::{BasicPublishOptions, BasicProperties, QueueDeclareOptions}; 19 | use lapin::types::FieldTable; 20 | 21 | use postgres::{Connection, TlsMode}; 22 | use fallible_iterator::FallibleIterator; 23 | 24 | fn main() { 25 | env_logger::init().unwrap(); 26 | 27 | // load configuration 28 | let config = app_config::AppConfig::new(); 29 | 30 | // create the reactor 31 | let mut core = Core::new().unwrap(); 32 | let handle = core.handle(); 33 | 34 | let pg_conn = Connection::connect(config.postgresql_uri.clone(), TlsMode::None).expect("Could not connect to PostgreSQL"); 35 | 36 | core.run( 37 | config.amqp_uri.connect(&handle) 38 | .and_then(|client| client.create_channel()) 39 | .and_then(|channel| { 40 | let id = channel.id; 41 | info!("created channel with id: {}", id); 42 | 43 | channel.queue_declare(config.clone().amqp_queue_name.as_str(), &QueueDeclareOptions::default(), &FieldTable::new()).and_then(move |_| { 44 | info!("channel {} declared queue {}", id, config.amqp_queue_name.as_str()); 45 | 46 | // https://www.postgresql.org/docs/7.4/static/sql-listen.html 47 | let listen_command = format!("LISTEN {}", config.postgresql_channel.as_str()); 48 | pg_conn.execute(listen_command.as_str(), &[]).expect("Could not send LISTEN"); 49 | 50 | let notifications = pg_conn.notifications(); 51 | 52 | // https://sfackler.github.io/rust-postgres/doc/v0.11.11/postgres/notification/struct.BlockingIter.html 53 | let mut it = notifications.blocking_iter(); 54 | 55 | println!("Waiting for notifications..."); 56 | 57 | // could not use 'loop' here because it does not compile in --release mode 58 | // since Ok() is unreachable. 59 | #[allow(while_true)] 60 | while true { 61 | 62 | // it.next() -> Result> 63 | match it.next() { 64 | Ok(Some(notification)) => { 65 | // https://github.com/sfackler/rust-postgres/blob/master/postgres-shared/src/lib.rs 66 | println!("Forwarding {:?} to queue {:?}", notification, config.amqp_queue_name.as_str()); 67 | channel.basic_publish( 68 | "", 69 | config.amqp_queue_name.as_str(), 70 | // @todo we might want to send it as JSON (configurable) 71 | // https://doc.rust-lang.org/1.12.0/std/fmt/ 72 | format!("{}!", notification.payload).as_bytes(), 73 | &BasicPublishOptions::default(), 74 | // @todo make this configurable through environment variables 75 | BasicProperties::default().with_user_id("guest".to_string()).with_reply_to("foobar".to_string()) 76 | ); 77 | }, 78 | Err(err) => println!("Got err {:?}", err), 79 | _ => panic!("Unexpected state.") 80 | } 81 | } 82 | 83 | 84 | Ok(channel) 85 | }) 86 | }).map_err(|err| { 87 | println!("Could not connect to AMQP: {}", err); 88 | err 89 | }) 90 | ).expect("Could not run reactor"); 91 | } 92 | --------------------------------------------------------------------------------