├── .gitignore ├── History.md ├── Makefile ├── Readme.md ├── client ├── client.go └── config.go ├── config.yml ├── docker-compose.yml ├── handler └── handler.go └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | log.json 2 | -------------------------------------------------------------------------------- /History.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tj/nsq_to_postgres/75785f7d11371d388578a217c19605ed80eb506e/History.md -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | compose: 3 | @docker-compose up 4 | 5 | client: 6 | @PGPASSWORD=sloth psql -h 192.168.59.103 -U tj -p 5432 7 | 8 | .PHONY: client compose -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | 2 | ## nsq_to_postgres 3 | 4 | Pull messages from NSQ and write to a Postgres JSONB column. 5 | 6 | ## Example 7 | 8 | Run: 9 | 10 | ``` 11 | $ nsq_to_postgres --config config.yml 12 | ``` 13 | 14 | Query: 15 | 16 | ```sql 17 | select 18 | log->'program' as program, 19 | log->'level' as level, 20 | count(*) 21 | from logs 22 | group by program, level 23 | order by count desc; 24 | ``` 25 | 26 | Result: 27 | 28 | ``` 29 | program | level | count 30 | ---------+---------+-------- 31 | "site" | "info" | 142276 32 | "api" | "info" | 8176 33 | "api" | "error" | 1638 34 | ``` 35 | 36 | ## Configuration 37 | 38 | A configuration file _must_ be specified via `--config`, I prefer making the path explicit so no one is left guessing of its whereabouts (unlike most programs, grr!). 39 | 40 | ### Postgres 41 | 42 | Two sections are available for tweaking, first the `postgres` section which defines the connection information, the target table name, target column name, and verbosity. For example: 43 | 44 | ```yml 45 | postgres: 46 | connection: user=tj password=sloth host=localhost port=5432 sslmode=disable 47 | table: logs 48 | column: log 49 | max_open_connections: 10 50 | verbose: no 51 | ``` 52 | 53 | When nsq_to_postgres first establishes a connect the table will be automatically created for you, if you have not already done so. 54 | 55 | ### NSQ 56 | 57 | The next section available is `nsq` which defines the topic to consume from, the nsqd or nsqlookupd addresses, max number of retry attempts and so on. 58 | 59 | For more nsq configuration options visit [segmentio/go-queue](https://github.com/segmentio/go-queue). 60 | 61 | ```yml 62 | nsq: 63 | topic: logs 64 | nsqd: localhost:4150 65 | max_attempts: 5 66 | msg_timeout: 15s 67 | max_in_flight: 300 68 | concurrency: 50 69 | ``` 70 | 71 | ## Development 72 | 73 | Boot postgres, nsqd, and nsqlookupd (requires docker and docker-compose): 74 | 75 | ``` 76 | $ make 77 | ``` 78 | 79 | Connect via `psql`: 80 | 81 | ``` 82 | $ make client 83 | ``` 84 | 85 | # License 86 | 87 | MIT -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import "github.com/jmoiron/sqlx" 4 | import "github.com/lib/pq" 5 | import "fmt" 6 | import "log" 7 | 8 | // Client connection. 9 | type Client struct { 10 | *Config 11 | *sqlx.DB 12 | stmt *sqlx.Stmt 13 | } 14 | 15 | // New client with the given `config`. 16 | func New(config *Config) (*Client, error) { 17 | c := &Client{ 18 | Config: config, 19 | } 20 | 21 | if c.Verbose { 22 | log.Printf("connecting to %s (max connections: %d)", c.Connection, c.MaxOpenConns) 23 | } 24 | 25 | err := c.connect() 26 | return c, err 27 | } 28 | 29 | // Establish connection. 30 | func (c *Client) connect() error { 31 | db, err := sqlx.Connect("postgres", c.Connection) 32 | if err != nil { 33 | return err 34 | } 35 | c.DB = db 36 | db.SetMaxOpenConns(c.MaxOpenConns) 37 | 38 | stmt, err := c.Preparex(fmt.Sprintf(`insert into %s values ($1)`, c.Table)) 39 | if err != nil { 40 | return err 41 | } 42 | c.stmt = stmt 43 | 44 | return nil 45 | } 46 | 47 | // Bootstrap: 48 | // 49 | // - create "events" table 50 | // 51 | func (c *Client) Bootstrap() error { 52 | return c.CreateEventsTable() 53 | } 54 | 55 | // CreateEventsTable creates the "events" table and noop when it exists. 56 | func (c *Client) CreateEventsTable() error { 57 | _, err := c.Exec(fmt.Sprintf(`create table %s (%s jsonb)`, c.Table, c.Column)) 58 | 59 | if err, ok := err.(*pq.Error); ok { 60 | if err.Code == "42P07" { 61 | return nil 62 | } 63 | } 64 | 65 | return err 66 | } 67 | 68 | // Insert event json blob. 69 | func (c *Client) Insert(e []byte) error { 70 | if c.Verbose { 71 | log.Printf("inserting %s", e) 72 | } 73 | 74 | _, err := c.stmt.Exec(e) 75 | return err 76 | } 77 | -------------------------------------------------------------------------------- /client/config.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import "fmt" 4 | 5 | // Config. 6 | type Config struct { 7 | Connection string `yaml:"connection"` 8 | Table string `yaml:"table"` 9 | Column string `yaml:"column"` 10 | MaxOpenConns int `yaml:"max_open_connections"` 11 | Verbose bool `yaml:"verbose"` 12 | } 13 | 14 | // Validate the configuration and apply defaults. 15 | func (c *Config) Validate() error { 16 | if c.Connection == "" { 17 | return fmt.Errorf(`"connection" required`) 18 | } 19 | 20 | if c.Table == "" { 21 | return fmt.Errorf(`"table" required`) 22 | } 23 | 24 | if c.Column == "" { 25 | return fmt.Errorf(`"column" required`) 26 | } 27 | 28 | if c.MaxOpenConns == 0 { 29 | c.MaxOpenConns = 10 30 | } 31 | 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /config.yml: -------------------------------------------------------------------------------- 1 | 2 | postgres: 3 | connection: user=tj password=sloth host=192.168.59.103 port=5432 sslmode=disable 4 | table: logs 5 | column: log 6 | max_open_connections: 10 7 | verbose: no 8 | 9 | nsq: 10 | topic: logs 11 | channel: postgres_logger 12 | nsqd: 192.168.59.103:4150 13 | max_attempts: 5 14 | msg_timeout: 15s 15 | max_in_flight: 300 16 | concurrency: 50 17 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | 2 | nsqd: 3 | image: segment/nsqd 4 | ports: 5 | - "4150:4150" 6 | - "4151:4151" 7 | links: 8 | - nsqlookupd 9 | 10 | nsqadmin: 11 | image: segment/nsqadmin 12 | ports: 13 | - "4171:4171" 14 | links: 15 | - nsqlookupd 16 | - nsqd 17 | 18 | nsqlookupd: 19 | image: segment/nsqlookupd 20 | ports: 21 | - "4160:4160" 22 | - "4161:4161" 23 | 24 | postgres: 25 | image: postgres:9.4 26 | ports: 27 | - "5432:5432" 28 | environment: 29 | POSTGRES_PASSWORD: sloth 30 | POSTGRES_USER: tj -------------------------------------------------------------------------------- /handler/handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import "github.com/tj/nsq_to_postgres/client" 4 | import "github.com/segmentio/go-stats" 5 | import "github.com/bitly/go-nsq" 6 | import "time" 7 | 8 | // Handler. 9 | type Handler struct { 10 | stats *stats.Stats 11 | db *client.Client 12 | } 13 | 14 | // New Handler with the given db client. 15 | func New(db *client.Client) *Handler { 16 | stats := stats.New() 17 | go stats.TickEvery(10 * time.Second) 18 | return &Handler{ 19 | stats: stats, 20 | db: db, 21 | } 22 | } 23 | 24 | // HandleMessage inserts the message body into postgres. 25 | func (h *Handler) HandleMessage(msg *nsq.Message) error { 26 | h.stats.Incr("inserts") 27 | return h.db.Insert(msg.Body) 28 | } 29 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/tj/nsq_to_postgres/handler" 4 | import "github.com/tj/nsq_to_postgres/client" 5 | import "github.com/segmentio/go-queue" 6 | import "github.com/tj/go-gracefully" 7 | import "github.com/tj/docopt" 8 | import "gopkg.in/yaml.v2" 9 | import "io/ioutil" 10 | import "log" 11 | 12 | var Version = "0.0.1" 13 | 14 | const Usage = ` 15 | Usage: 16 | nsq_to_postgres --config file 17 | nsq_to_postgres -h | --help 18 | nsq_to_postgres --version 19 | 20 | Options: 21 | -c, --config file configuration file path 22 | -h, --help output help information 23 | -v, --version output version 24 | 25 | ` 26 | 27 | type Config struct { 28 | Postgres *client.Config 29 | Nsq map[string]interface{} 30 | } 31 | 32 | func main() { 33 | args, err := docopt.Parse(Usage, nil, true, Version, false) 34 | if err != nil { 35 | log.Fatalf("error: %s", err) 36 | } 37 | 38 | log.Printf("starting nsq_to_postgres version %s", Version) 39 | 40 | // Read config 41 | file := args["--config"].(string) 42 | b, err := ioutil.ReadFile(file) 43 | if err != nil { 44 | log.Fatalf("error reading config: %s", err) 45 | } 46 | 47 | // Unmarshal config 48 | config := new(Config) 49 | err = yaml.Unmarshal(b, config) 50 | if err != nil { 51 | log.Fatalf("error unmarshalling config: %s", err) 52 | } 53 | 54 | // Validate config 55 | err = config.Postgres.Validate() 56 | if err != nil { 57 | log.Fatalf("configuration error: %s", err) 58 | } 59 | 60 | // Apply nsq config 61 | c := queue.NewConsumer("", "nsq_to_postgres") 62 | 63 | for k, v := range config.Nsq { 64 | c.Set(k, v) 65 | } 66 | 67 | // Connect 68 | log.Printf("connecting to postgres") 69 | db, err := client.New(config.Postgres) 70 | if err != nil { 71 | log.Fatalf("error connecting: %s", err) 72 | } 73 | 74 | // Bootstrap with table 75 | err = db.Bootstrap() 76 | if err != nil { 77 | log.Printf("error bootstrapping: %s", err) 78 | } 79 | 80 | // Start consumer 81 | log.Printf("starting consumer") 82 | c.Start(handler.New(db)) 83 | gracefully.Shutdown() 84 | log.Printf("stopping consumer") 85 | c.Stop() 86 | 87 | log.Printf("bye :)") 88 | } 89 | --------------------------------------------------------------------------------