├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── formats ├── csv.go ├── export.go ├── inserts.go ├── json.go ├── jsonlines.go ├── template.go ├── xlsx.go └── xml.go ├── logo.png ├── pg └── postgres.go ├── pgclimb.go └── test.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | *.swp 6 | 7 | # Folders 8 | _obj 9 | _test 10 | samples 11 | 12 | # Architecture specific extensions/prefixes 13 | *.[568vq] 14 | [568vq].out 15 | 16 | *.cgo1.go 17 | *.cgo2.c 18 | _cgo_defun.c 19 | _cgo_gotypes.go 20 | _cgo_export.* 21 | 22 | _testmain.go 23 | 24 | *.exe 25 | *.test 26 | *.prof 27 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: false 3 | addons: 4 | postgresql: 9.3 5 | go: 6 | - 1.8 7 | - 1.9 8 | - tip 9 | install: 10 | - go get github.com/urfave/cli 11 | - go get github.com/lib/pq 12 | - go get github.com/jmoiron/sqlx 13 | - go get github.com/tealeg/xlsx 14 | - go get github.com/lukasmartinelli/pgfutter 15 | - go get github.com/andrew-d/go-termutil 16 | script: 17 | - go install 18 | - ./test.sh 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Lukas Martinelli 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pgclimb [![Build Status](https://travis-ci.org/lukasmartinelli/pgclimb.svg?branch=master)](https://travis-ci.org/lukasmartinelli/pgclimb) [![Go Report Card](https://goreportcard.com/badge/github.com/lukasmartinelli/pgclimb)](https://goreportcard.com/report/github.com/lukasmartinelli/pgclimb) ![License](https://img.shields.io/badge/license-MIT-blue.svg) 2 | 3 | Climbing elephant 4 | 5 | A PostgreSQL utility to export data into different data formats with 6 | support for templates. 7 | 8 | Features: 9 | - Export data to [JSON](#json-document), [JSON Lines](#json-lines), [CSV](#csv-and-tsv), [XLSX](#xlsx), [XML](#xml) 10 | - Use [Templates](#templates) to support custom formats (HTML, Markdown, Text) 11 | 12 | Use Cases: 13 | - `psql` alternative for getting data out of PostgreSQL 14 | - Publish data sets 15 | - Create Excel reports from the database 16 | - Generate HTML reports 17 | - Export XML data for further processing with XSLT 18 | - Transform data to JSON for graphing it with JavaScript libraries 19 | - Generate readonly JSON APIs 20 | 21 | ## Install 22 | 23 | You can download a single binary for Linux, OSX or Windows. 24 | 25 | **OSX** 26 | 27 | ```bash 28 | wget -O pgclimb https://github.com/lukasmartinelli/pgclimb/releases/download/v0.3/pgclimb_darwin_amd64 29 | chmod +x pgclimb 30 | 31 | ./pgclimb --help 32 | ``` 33 | 34 | **Linux** 35 | 36 | ```bash 37 | wget -O pgclimb https://github.com/lukasmartinelli/pgclimb/releases/download/v0.3/pgclimb_linux_amd64 38 | chmod +x pgclimb 39 | 40 | ./pgclimb --help 41 | ``` 42 | 43 | **Install from source** 44 | 45 | ```bash 46 | go get github.com/lukasmartinelli/pgclimb 47 | ``` 48 | 49 | If you are using Windows or 32-bit architectures you need to [download the appropriate binary 50 | yourself](https://github.com/lukasmartinelli/pgclimb/releases/latest). 51 | 52 | ## Supported Formats 53 | 54 | The example queries operate on the open data [employee salaries of Montgomery County Maryland](https://data.montgomerycountymd.gov/api/views/54rh-89p8/rows.csv). You can import CSV files into your database using [my PostgreSQL import tool pgfutter](http://github.com/lukasmartinelli/pgfutter). 55 | To connect to your beloved PostgreSQL database set the [appropriate connection options](#database-connection). 56 | 57 | ### CSV and TSV 58 | 59 | Exporting CSV and TSV files is very similar to using `psql` and the `COPY TO` statement. 60 | 61 | ```bash 62 | # Write CSV file to stdout with comma as default delimiter 63 | pgclimb -c "SELECT * FROM employee_salaries" csv 64 | 65 | # Save CSV file with custom delimiter and header row to file 66 | pgclimb -o salaries.csv \ 67 | -c "SELECT full_name, position_title FROM employee_salaries" \ 68 | csv --delimiter ";" --header 69 | 70 | # Create TSV file with SQL query from stdin 71 | pgclimb -o positions.tsv tsv < employees_by_position.sql 91 | SELECT s.position_title, json_agg(s) AS employees 92 | FROM employee_salaries s 93 | GROUP BY s.position_title 94 | ORDER BY 1 95 | EOF 96 | 97 | # Load query from file and store it as JSON array in file 98 | pgclimb -f employees_by_position.sql \ 99 | -o employees_by_position.json \ 100 | json 101 | ``` 102 | 103 | ### JSON Lines 104 | 105 | [Newline delimited JSON](http://jsonlines.org/) is a good format to exchange 106 | structured data in large quantities which does not fit well into the CSV format. 107 | Instead of storing the entire JSON array each line is a valid JSON object. 108 | 109 | ```bash 110 | # Query all salaries as separate JSON objects 111 | pgclimb -c "SELECT * FROM employee_salaries" jsonlines 112 | 113 | # In this example we interface with jq to pluck the first employee of each position 114 | pgclimb -f employees_by_position.sql jsonlines | jq '.employees[0].full_name' 115 | ``` 116 | 117 | ### XLSX 118 | 119 | Excel files are really useful to exchange data with non programmers 120 | and create graphs and filters. You can fill different datasets into different spreedsheets and distribute one single Excel file. 121 | 122 | ```bash 123 | # Store all salaries in XLSX file 124 | pgclimb -o salaries.xlsx -c "SELECT * FROM employee_salaries" xlsx 125 | 126 | # Create XLSX file with multiple sheets 127 | pgclimb -o salary_report.xlsx \ 128 | -c "SELECT DISTINCT position_title FROM employee_salaries" \ 129 | xlsx --sheet "positions" 130 | pgclimb -o salary_report.xlsx \ 131 | -c "SELECT full_name FROM employee_salaries" \ 132 | xlsx --sheet "employees" 133 | ``` 134 | 135 | ### XML 136 | 137 | You can output XML to process it with other programs like [XSLT](http://www.w3schools.com/xml/xsl_intro.asp). 138 | To have more control over the XML output you should use the `pgclimb` template functionality directly to generate XML or build your own XML document with [XML functions in PostgreSQL](https://wiki.postgresql.org/wiki/XML_Support). 139 | 140 | ```bash 141 | # Output XML for each row 142 | pgclimb -o salaries.xml -c "SELECT * FROM employee_salaries" xml 143 | ``` 144 | 145 | A good default XML export is currently lacking because the XML format 146 | can be controlled using templates. 147 | If there is enough demand I will implement a solid 148 | default XML support without relying on templates. 149 | 150 | ## Templates 151 | 152 | Templates are the most powerful feature of `pgclimb` and allow you to implement 153 | other formats that are not built in. In this example we will create a 154 | HTML report of the salaries. 155 | 156 | Create a template `salaries.tpl`. 157 | 158 | ```html 159 | 160 | 161 | Montgomery County MD Employees 162 | 163 |

Employees

164 | 169 | 170 | 171 | ``` 172 | 173 | And now run the template. 174 | 175 | ``` 176 | pgclimb -o salaries.html \ 177 | -c "SELECT * FROM employee_salaries" \ 178 | template salaries.tpl 179 | ``` 180 | 181 | ## Database Connection 182 | 183 | Database connection details can be provided via environment variables 184 | or as separate flags (same flags as `psql`). 185 | 186 | name | default | flags | description 187 | ------------|-------------|---------------------|----------------- 188 | `DB_NAME` | `postgres` | `-d`, `--dbname` | database name 189 | `DB_HOST` | `localhost` | `--host` | host name 190 | `DB_PORT` | `5432` | `-p`, `--port` | port 191 | `DB_USER` | `postgres` | `-U`, `--username` | database user 192 | `DB_PASS` | | `--pass` | password (or empty if none) 193 | 194 | ## Advanced Use Cases 195 | 196 | ### Different ways of Querying 197 | 198 | Like `psql` you can specify a query at different places. 199 | 200 | ```bash 201 | # Read query from stdin 202 | echo "SELECT * FROM employee_salaries" | pgclimb 203 | # Specify simple queries directly as arguments 204 | pgclimb -c "SELECT * FROM employee_salaries" 205 | # Load query from file 206 | pgclimb -f query.sql 207 | ``` 208 | 209 | ### Control Output 210 | 211 | `pgclimb` will write the result to `stdout` by default. 212 | By specifying the `-o` option you can write the output to a file. 213 | 214 | ```bash 215 | pgclimb -o salaries.tsv -c "SELECT * FROM employee_salaries" tsv 216 | ``` 217 | 218 | ### Using JSON aggregation 219 | 220 | This is not a `pgclimb` feature but shows you how to create more complex 221 | JSON objects by using the [PostgreSQL JSON functions](http://www.postgresql.org/docs/9.5/static/functions-json.html). 222 | 223 | Let's query communities and join an additional birth rate table. 224 | 225 | ```bash 226 | pgclimb -c "SELECT id, name, \\ 227 | (SELECT array_to_json(array_agg(t)) FROM ( \\ 228 | SELECT year, births FROM public.births \\ 229 | WHERE community_id = c.id \\ 230 | ORDER BY year ASC \\ 231 | ) AS t \\ 232 | ) AS births, \\ 233 | FROM communities) AS c" jsonlines 234 | ``` 235 | 236 | # Contribute 237 | 238 | ## Dependencies 239 | 240 | Go get the required dependencies for building `pgclimb`. 241 | 242 | ```bash 243 | go get github.com/codegangsta/cli 244 | go get github.com/lib/pq 245 | go get github.com/jmoiron/sqlx 246 | go get github.com/tealeg/xlsx 247 | go get github.com/andrew-d/go-termutil 248 | ``` 249 | 250 | ## Cross-compiling 251 | 252 | We use [gox](https://github.com/mitchellh/gox) to create distributable 253 | binaries for Windows, OSX and Linux. 254 | 255 | ```bash 256 | docker run --rm -v "$(pwd)":/usr/src/pgclimb -w /usr/src/pgclimb tcnksm/gox:1.9 257 | ``` 258 | 259 | ## Integration Tests 260 | 261 | Run `test.sh` to run integration tests of the program with a PostgreSQL server. Take a look at the `.travis.yml`. 262 | -------------------------------------------------------------------------------- /formats/csv.go: -------------------------------------------------------------------------------- 1 | package formats 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "io" 7 | "strconv" 8 | "time" 9 | ) 10 | 11 | type CsvFormat struct { 12 | writer *csv.Writer 13 | columns []string 14 | headerRow bool 15 | } 16 | 17 | func NewCsvFormat(w io.Writer, delimiter rune, headerRow bool) *CsvFormat { 18 | writer := csv.NewWriter(w) 19 | writer.Comma = delimiter 20 | return &CsvFormat{ 21 | writer: writer, 22 | columns: make([]string, 0), 23 | headerRow: headerRow, 24 | } 25 | } 26 | 27 | func (f *CsvFormat) WriteHeader(columns []string) error { 28 | f.columns = columns 29 | if f.headerRow { 30 | return f.writer.Write(columns) 31 | } else { 32 | return nil 33 | } 34 | } 35 | 36 | func (f *CsvFormat) Flush() error { return nil } 37 | 38 | func (f *CsvFormat) WriteRow(values map[string]interface{}) error { 39 | record := []string{} 40 | for _, col := range f.columns { 41 | switch value := (values[col]).(type) { 42 | case string: 43 | record = append(record, value) 44 | case []byte: 45 | record = append(record, string(value)) 46 | case int64: 47 | record = append(record, fmt.Sprintf("%d", value)) 48 | case float64: 49 | record = append(record, strconv.FormatFloat(value, 'f', -1, 64)) 50 | case time.Time: 51 | record = append(record, value.Format(time.RFC3339)) 52 | case bool: 53 | if value == true { 54 | record = append(record, "true") 55 | } else { 56 | record = append(record, "false") 57 | } 58 | } 59 | } 60 | err := f.writer.Write(record) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | f.writer.Flush() 66 | err = f.writer.Error() 67 | return err 68 | } 69 | -------------------------------------------------------------------------------- /formats/export.go: -------------------------------------------------------------------------------- 1 | package formats 2 | 3 | import "github.com/lukasmartinelli/pgclimb/pg" 4 | 5 | // Supports storing data in different formats 6 | type DataFormat interface { 7 | WriteHeader(columns []string) error 8 | WriteRow(map[string]interface{}) error 9 | Flush() error 10 | } 11 | 12 | func Export(query string, connStr string, format DataFormat) error { 13 | db, err := pg.Connect(connStr) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | defer db.Close() 19 | 20 | rows, err := db.Queryx(query) 21 | if err != nil { 22 | return err 23 | } 24 | defer rows.Close() 25 | 26 | columnNames, err := rows.Columns() 27 | if err != nil { 28 | return err 29 | } 30 | 31 | if err = format.WriteHeader(columnNames); err != nil { 32 | return err 33 | } 34 | 35 | for rows.Next() { 36 | values := make(map[string]interface{}) 37 | if err = rows.MapScan(values); err != nil { 38 | return err 39 | } 40 | 41 | if err = format.WriteRow(values); err != nil { 42 | return err 43 | } 44 | } 45 | 46 | if err = format.Flush(); err != nil { 47 | return err 48 | } 49 | 50 | return rows.Err() 51 | } 52 | -------------------------------------------------------------------------------- /formats/inserts.go: -------------------------------------------------------------------------------- 1 | // Implements INSERTS output for exported rows 2 | // e.g. 3 | // INSERT INTO (name, last_name, something); 4 | package formats 5 | 6 | import ( 7 | "bufio" 8 | "bytes" 9 | "fmt" 10 | "io" 11 | "strconv" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | const ( 17 | insertQuery = "INSERT INTO %s (%s) " 18 | ) 19 | 20 | // very simple quoting for sql values 21 | func quote(val string) string { 22 | buf := bytes.NewBufferString("'") 23 | buf.WriteString(strings.Replace(val, "'", "''", -1)) 24 | buf.WriteString("'") 25 | return buf.String() 26 | } 27 | 28 | type InsertsFormat struct { 29 | DataFormat 30 | tableName string 31 | columns []string 32 | multiInsert bool 33 | writer *bufio.Writer 34 | } 35 | 36 | func NewInsertsFormat(w io.Writer, fileName string, tableName string) (*InsertsFormat, error) { 37 | return &InsertsFormat{ 38 | writer: bufio.NewWriter(w), 39 | tableName: tableName, 40 | columns: make([]string, 0), 41 | multiInsert: false, 42 | }, nil 43 | } 44 | 45 | func (f *InsertsFormat) WriteHeader(columns []string) error { 46 | f.columns = columns 47 | return nil 48 | } 49 | 50 | func (f *InsertsFormat) Flush() error { return nil } 51 | 52 | func (f *InsertsFormat) WriteRow(values map[string]interface{}) error { 53 | columnsVal := strings.Join(f.columns, ",") 54 | queryStr := fmt.Sprintf(insertQuery, f.tableName, columnsVal) 55 | buf := bytes.NewBufferString(queryStr) 56 | record := []string{} 57 | for _, col := range f.columns { 58 | switch value := (values[col]).(type) { 59 | case string: 60 | record = append(record, quote(value)) 61 | case []byte: 62 | record = append(record, quote(string(value))) 63 | case int64: 64 | record = append(record, fmt.Sprintf("%d", value)) 65 | case float64: 66 | record = append(record, strconv.FormatFloat(value, 'f', -1, 64)) 67 | case time.Time: 68 | record = append(record, quote(value.Format(time.RFC3339))) 69 | case bool: 70 | if value == true { 71 | record = append(record, "true") 72 | } else { 73 | record = append(record, "false") 74 | } 75 | case nil: 76 | record = append(record, "null") 77 | } 78 | } 79 | buf.WriteString("VALUES (") 80 | buf.WriteString(strings.Join(record, ",")) 81 | buf.WriteString(");\n") 82 | 83 | _, err := f.writer.Write(buf.Bytes()) 84 | if err != nil { 85 | return err 86 | } 87 | err = f.writer.Flush() 88 | // defer f.writer.Close() 89 | return err 90 | } 91 | -------------------------------------------------------------------------------- /formats/json.go: -------------------------------------------------------------------------------- 1 | package formats 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | ) 7 | 8 | type JSONArrayFormat struct { 9 | //Rows are stored in Memory until they are serialized into one Document 10 | //this only works because JSON documents are not supposed to be big 11 | //which would make them complicated to parse as well 12 | rows []map[string]interface{} 13 | encoder *json.Encoder 14 | } 15 | 16 | func NewJSONArrayFormat(w io.Writer) *JSONArrayFormat { 17 | return &JSONArrayFormat{make([]map[string]interface{}, 0), json.NewEncoder(w)} 18 | } 19 | 20 | // Writing header for JSON is a NOP 21 | func (e *JSONArrayFormat) WriteHeader(columns []string) error { 22 | return nil 23 | } 24 | 25 | func (e *JSONArrayFormat) Flush() error { 26 | err := e.encoder.Encode(e.rows) 27 | return err 28 | } 29 | 30 | func (e *JSONArrayFormat) WriteRow(rows map[string]interface{}) error { 31 | e.rows = append(e.rows, convertToJSON(rows)) 32 | return nil 33 | } 34 | 35 | func convertToJSON(rows map[string]interface{}) map[string]interface{} { 36 | for k, v := range rows { 37 | switch v := (v).(type) { 38 | case []byte: 39 | var jsonVal interface{} 40 | err := json.Unmarshal(v, &jsonVal) 41 | if err == nil { 42 | rows[k] = jsonVal 43 | } else { 44 | rows[k] = string(v) 45 | } 46 | default: 47 | rows[k] = v 48 | } 49 | } 50 | return rows 51 | } 52 | 53 | // Try to JSON decode the bytes 54 | func tryUnmarshal(b []byte) error { 55 | var v interface{} 56 | err := json.Unmarshal(b, &v) 57 | return err 58 | } 59 | 60 | func convertBytesToString(rows map[string]interface{}) map[string]interface{} { 61 | for k, v := range rows { 62 | switch v := (v).(type) { 63 | case []byte: 64 | rows[k] = string(v) 65 | default: 66 | rows[k] = v 67 | } 68 | } 69 | return rows 70 | } 71 | -------------------------------------------------------------------------------- /formats/jsonlines.go: -------------------------------------------------------------------------------- 1 | package formats 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | ) 7 | 8 | type JSONLinesFormat struct { 9 | encoder *json.Encoder 10 | } 11 | 12 | func NewJSONLinesFormat(w io.Writer) *JSONLinesFormat { 13 | return &JSONLinesFormat{json.NewEncoder(w)} 14 | } 15 | 16 | // Writing header for JSON is a NOP 17 | func (e *JSONLinesFormat) WriteHeader(columns []string) error { 18 | return nil 19 | } 20 | 21 | func (e *JSONLinesFormat) Flush() error { return nil } 22 | 23 | func (e *JSONLinesFormat) WriteRow(rows map[string]interface{}) error { 24 | rows = convertToJSON(rows) 25 | err := e.encoder.Encode(rows) 26 | return err 27 | } 28 | -------------------------------------------------------------------------------- /formats/template.go: -------------------------------------------------------------------------------- 1 | package formats 2 | 3 | import ( 4 | "io" 5 | "text/template" 6 | ) 7 | 8 | type TemplateFormat struct { 9 | rows []map[string]interface{} 10 | template *template.Template 11 | writer io.Writer 12 | } 13 | 14 | func NewTemplateFormat(w io.Writer, rawTemplate string) *TemplateFormat { 15 | t := template.Must(template.New("climbtemplate").Parse(rawTemplate)) 16 | return &TemplateFormat{ 17 | rows: make([]map[string]interface{}, 0), 18 | template: t, 19 | writer: w, 20 | } 21 | } 22 | 23 | func (e *TemplateFormat) Flush() error { 24 | err := e.template.Execute(e.writer, e.rows) 25 | return err 26 | } 27 | 28 | func (e *TemplateFormat) WriteHeader(columns []string) error { 29 | return nil 30 | } 31 | 32 | func (e *TemplateFormat) WriteRow(values map[string]interface{}) error { 33 | e.rows = append(e.rows, convertToJSON(values)) 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /formats/xlsx.go: -------------------------------------------------------------------------------- 1 | package formats 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/tealeg/xlsx" 9 | ) 10 | 11 | type XlsxFormat struct { 12 | file *xlsx.File 13 | sheet *xlsx.Sheet 14 | fileName string 15 | columns []string 16 | } 17 | 18 | func NewXlsxFormat(fileName string, sheetName string) (*XlsxFormat, error) { 19 | file := xlsx.NewFile() 20 | if _, err := os.Stat(fileName); fileName != "" && err == nil { 21 | file, err = xlsx.OpenFile(fileName) 22 | if err != nil { 23 | fmt.Println("Errord file") 24 | return nil, err 25 | } 26 | } 27 | 28 | sheet, err := file.AddSheet(sheetName) 29 | if err != nil { 30 | // Sheet already exists - empty it first 31 | sheet = file.Sheet[sheetName] 32 | sheet.Rows = make([]*xlsx.Row, 0) 33 | } 34 | 35 | return &XlsxFormat{ 36 | file: file, 37 | fileName: fileName, 38 | sheet: sheet, 39 | columns: make([]string, 0), 40 | }, nil 41 | } 42 | 43 | func (f *XlsxFormat) Flush() error { 44 | if f.fileName == "" { 45 | return f.file.Write(os.Stdout) 46 | } else { 47 | return f.file.Save(f.fileName) 48 | } 49 | } 50 | 51 | func (f *XlsxFormat) WriteHeader(columns []string) error { 52 | f.columns = columns 53 | row := f.sheet.AddRow() 54 | for _, col := range columns { 55 | cell := row.AddCell() 56 | cell.SetString(col) 57 | } 58 | return nil 59 | } 60 | 61 | func (f *XlsxFormat) WriteRow(values map[string]interface{}) error { 62 | row := f.sheet.AddRow() 63 | 64 | for _, col := range f.columns { 65 | cell := row.AddCell() 66 | switch value := (values[col]).(type) { 67 | case string: 68 | cell.SetString(value) 69 | case []byte: 70 | cell.SetString(string(value)) 71 | case int64: 72 | cell.SetInt64(value) 73 | case float64: 74 | cell.SetFloat(value) 75 | case time.Time: 76 | cell.SetDate(value) 77 | case bool: 78 | cell.SetBool(value) 79 | } 80 | } 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /formats/xml.go: -------------------------------------------------------------------------------- 1 | package formats 2 | 3 | import ( 4 | "encoding/xml" 5 | "fmt" 6 | "io" 7 | "strconv" 8 | "time" 9 | ) 10 | 11 | type XMLFormat struct { 12 | encoder *xml.Encoder 13 | } 14 | 15 | func NewXMLFormat(w io.Writer) *XMLFormat { 16 | e := xml.NewEncoder(w) 17 | e.Indent(" ", " ") 18 | return &XMLFormat{e} 19 | } 20 | 21 | // Writing header for XML is a NOP 22 | func (e *XMLFormat) WriteHeader(columns []string) error { 23 | return nil 24 | } 25 | 26 | func (e *XMLFormat) Flush() error { return nil } 27 | 28 | func (e *XMLFormat) WriteRow(values map[string]interface{}) error { 29 | row := xml.StartElement{Name: xml.Name{"", "row"}} 30 | tokens := []xml.Token{row} 31 | for key, value := range values { 32 | var charData xml.CharData 33 | 34 | t := xml.StartElement{Name: xml.Name{"", key}} 35 | 36 | switch value := (value).(type) { 37 | case string: 38 | charData = xml.CharData(value) 39 | case []byte: 40 | charData = xml.CharData(string(value)) 41 | case int64: 42 | charData = xml.CharData(fmt.Sprintf("%d", value)) 43 | case float64: 44 | charData = xml.CharData(strconv.FormatFloat(value, 'f', -1, 64)) 45 | case time.Time: 46 | charData = xml.CharData(value.Format(time.RFC3339)) 47 | case bool: 48 | if value == true { 49 | charData = xml.CharData("true") 50 | } else { 51 | charData = xml.CharData("false") 52 | } 53 | } 54 | tokens = append(tokens, t, charData, t.End()) 55 | } 56 | tokens = append(tokens, row.End()) 57 | 58 | for _, t := range tokens { 59 | err := e.encoder.EncodeToken(t) 60 | if err != nil { 61 | return err 62 | } 63 | } 64 | 65 | err := e.encoder.Flush() 66 | if err != nil { 67 | return err 68 | } 69 | 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lukasmartinelli/pgclimb/78d4bc6ecfc2e6eccff7fa70f8437284bede9432/logo.png -------------------------------------------------------------------------------- /pg/postgres.go: -------------------------------------------------------------------------------- 1 | package pg 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/jmoiron/sqlx" 7 | 8 | _ "github.com/lib/pq" 9 | "github.com/urfave/cli" 10 | ) 11 | 12 | //setup a database connection and create the import schema 13 | func Connect(connStr string) (*sqlx.DB, error) { 14 | db, err := sqlx.Open("postgres", connStr) 15 | if err != nil { 16 | return db, err 17 | } 18 | 19 | err = db.Ping() 20 | if err != nil { 21 | return db, err 22 | } 23 | 24 | return db, nil 25 | } 26 | 27 | //parse sql connection string from cli flags 28 | func ParseConnStr(c *cli.Context) string { 29 | otherParams := "sslmode=disable connect_timeout=5" 30 | if c.GlobalBool("ssl") { 31 | otherParams = "sslmode=require connect_timeout=5" 32 | } 33 | return fmt.Sprintf("user=%s dbname=%s password='%s' host=%s port=%s %s", 34 | c.GlobalString("username"), 35 | c.GlobalString("dbname"), 36 | c.GlobalString("pass"), 37 | c.GlobalString("host"), 38 | c.GlobalString("port"), 39 | otherParams, 40 | ) 41 | } 42 | -------------------------------------------------------------------------------- /pgclimb.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "strings" 10 | "unicode/utf8" 11 | 12 | "github.com/andrew-d/go-termutil" 13 | "github.com/lukasmartinelli/pgclimb/formats" 14 | "github.com/lukasmartinelli/pgclimb/pg" 15 | "github.com/urfave/cli" 16 | ) 17 | 18 | func changeHelpTemplateArgs(args string) { 19 | cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", args, -1) 20 | } 21 | 22 | func parseTemplate(filename string) string { 23 | rawTemplate, err := ioutil.ReadFile(filename) 24 | if err != nil { 25 | log.Fatalln(err) 26 | } 27 | return string(rawTemplate) 28 | } 29 | 30 | func parseWriter(c *cli.Context) io.Writer { 31 | outputFilename := c.GlobalString("output") 32 | 33 | if outputFilename != "" { 34 | f, err := os.Create(outputFilename) 35 | exitOnError(err) 36 | return f 37 | } 38 | return os.Stdout 39 | } 40 | 41 | func exportFormat(c *cli.Context, format formats.DataFormat) { 42 | connStr := pg.ParseConnStr(c) 43 | query, err := parseQuery(c) 44 | exitOnError(err) 45 | err = formats.Export(query, connStr, format) 46 | exitOnError(err) 47 | } 48 | 49 | func parseQuery(c *cli.Context) (string, error) { 50 | filename := c.GlobalString("file") 51 | if filename != "" { 52 | query, err := ioutil.ReadFile(filename) 53 | return string(query), err 54 | } 55 | 56 | command := c.GlobalString("command") 57 | if command != "" { 58 | return command, nil 59 | } 60 | 61 | if !termutil.Isatty(os.Stdin.Fd()) { 62 | query, err := ioutil.ReadAll(os.Stdin) 63 | return string(query), err 64 | } 65 | 66 | return "", errors.New("You need to specify a SQL query.") 67 | } 68 | 69 | func exitOnError(err error) { 70 | log.SetFlags(0) 71 | if err != nil { 72 | log.Fatalln(err) 73 | } 74 | } 75 | 76 | func main() { 77 | app := cli.NewApp() 78 | app.Name = "pgclimb" 79 | app.Version = "0.2" 80 | app.Usage = "Export data from PostgreSQL into different data formats" 81 | 82 | app.Flags = []cli.Flag{ 83 | cli.StringFlag{ 84 | Name: "dbname, d", 85 | Value: "postgres", 86 | Usage: "database", 87 | EnvVar: "DB_NAME", 88 | }, 89 | cli.StringFlag{ 90 | Name: "host", 91 | Value: "localhost", 92 | Usage: "host name", 93 | EnvVar: "DB_HOST", 94 | }, 95 | cli.StringFlag{ 96 | Name: "port, p", 97 | Value: "5432", 98 | Usage: "port", 99 | EnvVar: "DB_PORT", 100 | }, 101 | cli.StringFlag{ 102 | Name: "username, U", 103 | Value: "postgres", 104 | Usage: "username", 105 | EnvVar: "DB_USER", 106 | }, 107 | cli.BoolFlag{ 108 | Name: "ssl", 109 | Usage: "require ssl mode", 110 | }, 111 | cli.StringFlag{ 112 | Name: "password, pass", 113 | Value: "", 114 | Usage: "password", 115 | EnvVar: "DB_PASS", 116 | }, 117 | cli.StringFlag{ 118 | Name: "query, command, c", 119 | Value: "", 120 | Usage: "SQL query to execute", 121 | EnvVar: "DB_QUERY", 122 | }, 123 | cli.StringFlag{ 124 | Name: "file, f", 125 | Value: "", 126 | Usage: "SQL query filename", 127 | }, 128 | cli.StringFlag{ 129 | Name: "output, o", 130 | Value: "", 131 | Usage: "Output filename", 132 | }, 133 | } 134 | 135 | app.Commands = []cli.Command{ 136 | { 137 | Name: "template", 138 | Usage: "Export data with custom template", 139 | Action: func(c *cli.Context) error { 140 | changeHelpTemplateArgs("