├── .gitignore
├── .travis.yml
├── LICENSE
├── README.md
├── formats
├── csv.go
├── export.go
├── inserts.go
├── json.go
├── jsonlines.go
├── template.go
├── xlsx.go
└── xml.go
├── logo.png
├── pg
└── postgres.go
├── pgclimb.go
└── test.sh
/.gitignore:
--------------------------------------------------------------------------------
1 | # Compiled Object files, Static and Dynamic libs (Shared Objects)
2 | *.o
3 | *.a
4 | *.so
5 | *.swp
6 |
7 | # Folders
8 | _obj
9 | _test
10 | samples
11 |
12 | # Architecture specific extensions/prefixes
13 | *.[568vq]
14 | [568vq].out
15 |
16 | *.cgo1.go
17 | *.cgo2.c
18 | _cgo_defun.c
19 | _cgo_gotypes.go
20 | _cgo_export.*
21 |
22 | _testmain.go
23 |
24 | *.exe
25 | *.test
26 | *.prof
27 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: go
2 | sudo: false
3 | addons:
4 | postgresql: 9.3
5 | go:
6 | - 1.8
7 | - 1.9
8 | - tip
9 | install:
10 | - go get github.com/urfave/cli
11 | - go get github.com/lib/pq
12 | - go get github.com/jmoiron/sqlx
13 | - go get github.com/tealeg/xlsx
14 | - go get github.com/lukasmartinelli/pgfutter
15 | - go get github.com/andrew-d/go-termutil
16 | script:
17 | - go install
18 | - ./test.sh
19 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2015 Lukas Martinelli
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
23 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # pgclimb [](https://travis-ci.org/lukasmartinelli/pgclimb) [](https://goreportcard.com/report/github.com/lukasmartinelli/pgclimb) 
2 |
3 |
4 |
5 | A PostgreSQL utility to export data into different data formats with
6 | support for templates.
7 |
8 | Features:
9 | - Export data to [JSON](#json-document), [JSON Lines](#json-lines), [CSV](#csv-and-tsv), [XLSX](#xlsx), [XML](#xml)
10 | - Use [Templates](#templates) to support custom formats (HTML, Markdown, Text)
11 |
12 | Use Cases:
13 | - `psql` alternative for getting data out of PostgreSQL
14 | - Publish data sets
15 | - Create Excel reports from the database
16 | - Generate HTML reports
17 | - Export XML data for further processing with XSLT
18 | - Transform data to JSON for graphing it with JavaScript libraries
19 | - Generate readonly JSON APIs
20 |
21 | ## Install
22 |
23 | You can download a single binary for Linux, OSX or Windows.
24 |
25 | **OSX**
26 |
27 | ```bash
28 | wget -O pgclimb https://github.com/lukasmartinelli/pgclimb/releases/download/v0.3/pgclimb_darwin_amd64
29 | chmod +x pgclimb
30 |
31 | ./pgclimb --help
32 | ```
33 |
34 | **Linux**
35 |
36 | ```bash
37 | wget -O pgclimb https://github.com/lukasmartinelli/pgclimb/releases/download/v0.3/pgclimb_linux_amd64
38 | chmod +x pgclimb
39 |
40 | ./pgclimb --help
41 | ```
42 |
43 | **Install from source**
44 |
45 | ```bash
46 | go get github.com/lukasmartinelli/pgclimb
47 | ```
48 |
49 | If you are using Windows or 32-bit architectures you need to [download the appropriate binary
50 | yourself](https://github.com/lukasmartinelli/pgclimb/releases/latest).
51 |
52 | ## Supported Formats
53 |
54 | The example queries operate on the open data [employee salaries of Montgomery County Maryland](https://data.montgomerycountymd.gov/api/views/54rh-89p8/rows.csv). You can import CSV files into your database using [my PostgreSQL import tool pgfutter](http://github.com/lukasmartinelli/pgfutter).
55 | To connect to your beloved PostgreSQL database set the [appropriate connection options](#database-connection).
56 |
57 | ### CSV and TSV
58 |
59 | Exporting CSV and TSV files is very similar to using `psql` and the `COPY TO` statement.
60 |
61 | ```bash
62 | # Write CSV file to stdout with comma as default delimiter
63 | pgclimb -c "SELECT * FROM employee_salaries" csv
64 |
65 | # Save CSV file with custom delimiter and header row to file
66 | pgclimb -o salaries.csv \
67 | -c "SELECT full_name, position_title FROM employee_salaries" \
68 | csv --delimiter ";" --header
69 |
70 | # Create TSV file with SQL query from stdin
71 | pgclimb -o positions.tsv tsv < employees_by_position.sql
91 | SELECT s.position_title, json_agg(s) AS employees
92 | FROM employee_salaries s
93 | GROUP BY s.position_title
94 | ORDER BY 1
95 | EOF
96 |
97 | # Load query from file and store it as JSON array in file
98 | pgclimb -f employees_by_position.sql \
99 | -o employees_by_position.json \
100 | json
101 | ```
102 |
103 | ### JSON Lines
104 |
105 | [Newline delimited JSON](http://jsonlines.org/) is a good format to exchange
106 | structured data in large quantities which does not fit well into the CSV format.
107 | Instead of storing the entire JSON array each line is a valid JSON object.
108 |
109 | ```bash
110 | # Query all salaries as separate JSON objects
111 | pgclimb -c "SELECT * FROM employee_salaries" jsonlines
112 |
113 | # In this example we interface with jq to pluck the first employee of each position
114 | pgclimb -f employees_by_position.sql jsonlines | jq '.employees[0].full_name'
115 | ```
116 |
117 | ### XLSX
118 |
119 | Excel files are really useful to exchange data with non programmers
120 | and create graphs and filters. You can fill different datasets into different spreedsheets and distribute one single Excel file.
121 |
122 | ```bash
123 | # Store all salaries in XLSX file
124 | pgclimb -o salaries.xlsx -c "SELECT * FROM employee_salaries" xlsx
125 |
126 | # Create XLSX file with multiple sheets
127 | pgclimb -o salary_report.xlsx \
128 | -c "SELECT DISTINCT position_title FROM employee_salaries" \
129 | xlsx --sheet "positions"
130 | pgclimb -o salary_report.xlsx \
131 | -c "SELECT full_name FROM employee_salaries" \
132 | xlsx --sheet "employees"
133 | ```
134 |
135 | ### XML
136 |
137 | You can output XML to process it with other programs like [XSLT](http://www.w3schools.com/xml/xsl_intro.asp).
138 | To have more control over the XML output you should use the `pgclimb` template functionality directly to generate XML or build your own XML document with [XML functions in PostgreSQL](https://wiki.postgresql.org/wiki/XML_Support).
139 |
140 | ```bash
141 | # Output XML for each row
142 | pgclimb -o salaries.xml -c "SELECT * FROM employee_salaries" xml
143 | ```
144 |
145 | A good default XML export is currently lacking because the XML format
146 | can be controlled using templates.
147 | If there is enough demand I will implement a solid
148 | default XML support without relying on templates.
149 |
150 | ## Templates
151 |
152 | Templates are the most powerful feature of `pgclimb` and allow you to implement
153 | other formats that are not built in. In this example we will create a
154 | HTML report of the salaries.
155 |
156 | Create a template `salaries.tpl`.
157 |
158 | ```html
159 |
160 |
161 | Montgomery County MD Employees
162 |
163 | Employees
164 |
165 | {{range .}}
166 | - {{.full_name}}
167 | {{end}}
168 |
169 |
170 |
171 | ```
172 |
173 | And now run the template.
174 |
175 | ```
176 | pgclimb -o salaries.html \
177 | -c "SELECT * FROM employee_salaries" \
178 | template salaries.tpl
179 | ```
180 |
181 | ## Database Connection
182 |
183 | Database connection details can be provided via environment variables
184 | or as separate flags (same flags as `psql`).
185 |
186 | name | default | flags | description
187 | ------------|-------------|---------------------|-----------------
188 | `DB_NAME` | `postgres` | `-d`, `--dbname` | database name
189 | `DB_HOST` | `localhost` | `--host` | host name
190 | `DB_PORT` | `5432` | `-p`, `--port` | port
191 | `DB_USER` | `postgres` | `-U`, `--username` | database user
192 | `DB_PASS` | | `--pass` | password (or empty if none)
193 |
194 | ## Advanced Use Cases
195 |
196 | ### Different ways of Querying
197 |
198 | Like `psql` you can specify a query at different places.
199 |
200 | ```bash
201 | # Read query from stdin
202 | echo "SELECT * FROM employee_salaries" | pgclimb
203 | # Specify simple queries directly as arguments
204 | pgclimb -c "SELECT * FROM employee_salaries"
205 | # Load query from file
206 | pgclimb -f query.sql
207 | ```
208 |
209 | ### Control Output
210 |
211 | `pgclimb` will write the result to `stdout` by default.
212 | By specifying the `-o` option you can write the output to a file.
213 |
214 | ```bash
215 | pgclimb -o salaries.tsv -c "SELECT * FROM employee_salaries" tsv
216 | ```
217 |
218 | ### Using JSON aggregation
219 |
220 | This is not a `pgclimb` feature but shows you how to create more complex
221 | JSON objects by using the [PostgreSQL JSON functions](http://www.postgresql.org/docs/9.5/static/functions-json.html).
222 |
223 | Let's query communities and join an additional birth rate table.
224 |
225 | ```bash
226 | pgclimb -c "SELECT id, name, \\
227 | (SELECT array_to_json(array_agg(t)) FROM ( \\
228 | SELECT year, births FROM public.births \\
229 | WHERE community_id = c.id \\
230 | ORDER BY year ASC \\
231 | ) AS t \\
232 | ) AS births, \\
233 | FROM communities) AS c" jsonlines
234 | ```
235 |
236 | # Contribute
237 |
238 | ## Dependencies
239 |
240 | Go get the required dependencies for building `pgclimb`.
241 |
242 | ```bash
243 | go get github.com/codegangsta/cli
244 | go get github.com/lib/pq
245 | go get github.com/jmoiron/sqlx
246 | go get github.com/tealeg/xlsx
247 | go get github.com/andrew-d/go-termutil
248 | ```
249 |
250 | ## Cross-compiling
251 |
252 | We use [gox](https://github.com/mitchellh/gox) to create distributable
253 | binaries for Windows, OSX and Linux.
254 |
255 | ```bash
256 | docker run --rm -v "$(pwd)":/usr/src/pgclimb -w /usr/src/pgclimb tcnksm/gox:1.9
257 | ```
258 |
259 | ## Integration Tests
260 |
261 | Run `test.sh` to run integration tests of the program with a PostgreSQL server. Take a look at the `.travis.yml`.
262 |
--------------------------------------------------------------------------------
/formats/csv.go:
--------------------------------------------------------------------------------
1 | package formats
2 |
3 | import (
4 | "encoding/csv"
5 | "fmt"
6 | "io"
7 | "strconv"
8 | "time"
9 | )
10 |
11 | type CsvFormat struct {
12 | writer *csv.Writer
13 | columns []string
14 | headerRow bool
15 | }
16 |
17 | func NewCsvFormat(w io.Writer, delimiter rune, headerRow bool) *CsvFormat {
18 | writer := csv.NewWriter(w)
19 | writer.Comma = delimiter
20 | return &CsvFormat{
21 | writer: writer,
22 | columns: make([]string, 0),
23 | headerRow: headerRow,
24 | }
25 | }
26 |
27 | func (f *CsvFormat) WriteHeader(columns []string) error {
28 | f.columns = columns
29 | if f.headerRow {
30 | return f.writer.Write(columns)
31 | } else {
32 | return nil
33 | }
34 | }
35 |
36 | func (f *CsvFormat) Flush() error { return nil }
37 |
38 | func (f *CsvFormat) WriteRow(values map[string]interface{}) error {
39 | record := []string{}
40 | for _, col := range f.columns {
41 | switch value := (values[col]).(type) {
42 | case string:
43 | record = append(record, value)
44 | case []byte:
45 | record = append(record, string(value))
46 | case int64:
47 | record = append(record, fmt.Sprintf("%d", value))
48 | case float64:
49 | record = append(record, strconv.FormatFloat(value, 'f', -1, 64))
50 | case time.Time:
51 | record = append(record, value.Format(time.RFC3339))
52 | case bool:
53 | if value == true {
54 | record = append(record, "true")
55 | } else {
56 | record = append(record, "false")
57 | }
58 | }
59 | }
60 | err := f.writer.Write(record)
61 | if err != nil {
62 | return err
63 | }
64 |
65 | f.writer.Flush()
66 | err = f.writer.Error()
67 | return err
68 | }
69 |
--------------------------------------------------------------------------------
/formats/export.go:
--------------------------------------------------------------------------------
1 | package formats
2 |
3 | import "github.com/lukasmartinelli/pgclimb/pg"
4 |
5 | // Supports storing data in different formats
6 | type DataFormat interface {
7 | WriteHeader(columns []string) error
8 | WriteRow(map[string]interface{}) error
9 | Flush() error
10 | }
11 |
12 | func Export(query string, connStr string, format DataFormat) error {
13 | db, err := pg.Connect(connStr)
14 | if err != nil {
15 | return err
16 | }
17 |
18 | defer db.Close()
19 |
20 | rows, err := db.Queryx(query)
21 | if err != nil {
22 | return err
23 | }
24 | defer rows.Close()
25 |
26 | columnNames, err := rows.Columns()
27 | if err != nil {
28 | return err
29 | }
30 |
31 | if err = format.WriteHeader(columnNames); err != nil {
32 | return err
33 | }
34 |
35 | for rows.Next() {
36 | values := make(map[string]interface{})
37 | if err = rows.MapScan(values); err != nil {
38 | return err
39 | }
40 |
41 | if err = format.WriteRow(values); err != nil {
42 | return err
43 | }
44 | }
45 |
46 | if err = format.Flush(); err != nil {
47 | return err
48 | }
49 |
50 | return rows.Err()
51 | }
52 |
--------------------------------------------------------------------------------
/formats/inserts.go:
--------------------------------------------------------------------------------
1 | // Implements INSERTS output for exported rows
2 | // e.g.
3 | // INSERT INTO (name, last_name, something);
4 | package formats
5 |
6 | import (
7 | "bufio"
8 | "bytes"
9 | "fmt"
10 | "io"
11 | "strconv"
12 | "strings"
13 | "time"
14 | )
15 |
16 | const (
17 | insertQuery = "INSERT INTO %s (%s) "
18 | )
19 |
20 | // very simple quoting for sql values
21 | func quote(val string) string {
22 | buf := bytes.NewBufferString("'")
23 | buf.WriteString(strings.Replace(val, "'", "''", -1))
24 | buf.WriteString("'")
25 | return buf.String()
26 | }
27 |
28 | type InsertsFormat struct {
29 | DataFormat
30 | tableName string
31 | columns []string
32 | multiInsert bool
33 | writer *bufio.Writer
34 | }
35 |
36 | func NewInsertsFormat(w io.Writer, fileName string, tableName string) (*InsertsFormat, error) {
37 | return &InsertsFormat{
38 | writer: bufio.NewWriter(w),
39 | tableName: tableName,
40 | columns: make([]string, 0),
41 | multiInsert: false,
42 | }, nil
43 | }
44 |
45 | func (f *InsertsFormat) WriteHeader(columns []string) error {
46 | f.columns = columns
47 | return nil
48 | }
49 |
50 | func (f *InsertsFormat) Flush() error { return nil }
51 |
52 | func (f *InsertsFormat) WriteRow(values map[string]interface{}) error {
53 | columnsVal := strings.Join(f.columns, ",")
54 | queryStr := fmt.Sprintf(insertQuery, f.tableName, columnsVal)
55 | buf := bytes.NewBufferString(queryStr)
56 | record := []string{}
57 | for _, col := range f.columns {
58 | switch value := (values[col]).(type) {
59 | case string:
60 | record = append(record, quote(value))
61 | case []byte:
62 | record = append(record, quote(string(value)))
63 | case int64:
64 | record = append(record, fmt.Sprintf("%d", value))
65 | case float64:
66 | record = append(record, strconv.FormatFloat(value, 'f', -1, 64))
67 | case time.Time:
68 | record = append(record, quote(value.Format(time.RFC3339)))
69 | case bool:
70 | if value == true {
71 | record = append(record, "true")
72 | } else {
73 | record = append(record, "false")
74 | }
75 | case nil:
76 | record = append(record, "null")
77 | }
78 | }
79 | buf.WriteString("VALUES (")
80 | buf.WriteString(strings.Join(record, ","))
81 | buf.WriteString(");\n")
82 |
83 | _, err := f.writer.Write(buf.Bytes())
84 | if err != nil {
85 | return err
86 | }
87 | err = f.writer.Flush()
88 | // defer f.writer.Close()
89 | return err
90 | }
91 |
--------------------------------------------------------------------------------
/formats/json.go:
--------------------------------------------------------------------------------
1 | package formats
2 |
3 | import (
4 | "encoding/json"
5 | "io"
6 | )
7 |
8 | type JSONArrayFormat struct {
9 | //Rows are stored in Memory until they are serialized into one Document
10 | //this only works because JSON documents are not supposed to be big
11 | //which would make them complicated to parse as well
12 | rows []map[string]interface{}
13 | encoder *json.Encoder
14 | }
15 |
16 | func NewJSONArrayFormat(w io.Writer) *JSONArrayFormat {
17 | return &JSONArrayFormat{make([]map[string]interface{}, 0), json.NewEncoder(w)}
18 | }
19 |
20 | // Writing header for JSON is a NOP
21 | func (e *JSONArrayFormat) WriteHeader(columns []string) error {
22 | return nil
23 | }
24 |
25 | func (e *JSONArrayFormat) Flush() error {
26 | err := e.encoder.Encode(e.rows)
27 | return err
28 | }
29 |
30 | func (e *JSONArrayFormat) WriteRow(rows map[string]interface{}) error {
31 | e.rows = append(e.rows, convertToJSON(rows))
32 | return nil
33 | }
34 |
35 | func convertToJSON(rows map[string]interface{}) map[string]interface{} {
36 | for k, v := range rows {
37 | switch v := (v).(type) {
38 | case []byte:
39 | var jsonVal interface{}
40 | err := json.Unmarshal(v, &jsonVal)
41 | if err == nil {
42 | rows[k] = jsonVal
43 | } else {
44 | rows[k] = string(v)
45 | }
46 | default:
47 | rows[k] = v
48 | }
49 | }
50 | return rows
51 | }
52 |
53 | // Try to JSON decode the bytes
54 | func tryUnmarshal(b []byte) error {
55 | var v interface{}
56 | err := json.Unmarshal(b, &v)
57 | return err
58 | }
59 |
60 | func convertBytesToString(rows map[string]interface{}) map[string]interface{} {
61 | for k, v := range rows {
62 | switch v := (v).(type) {
63 | case []byte:
64 | rows[k] = string(v)
65 | default:
66 | rows[k] = v
67 | }
68 | }
69 | return rows
70 | }
71 |
--------------------------------------------------------------------------------
/formats/jsonlines.go:
--------------------------------------------------------------------------------
1 | package formats
2 |
3 | import (
4 | "encoding/json"
5 | "io"
6 | )
7 |
8 | type JSONLinesFormat struct {
9 | encoder *json.Encoder
10 | }
11 |
12 | func NewJSONLinesFormat(w io.Writer) *JSONLinesFormat {
13 | return &JSONLinesFormat{json.NewEncoder(w)}
14 | }
15 |
16 | // Writing header for JSON is a NOP
17 | func (e *JSONLinesFormat) WriteHeader(columns []string) error {
18 | return nil
19 | }
20 |
21 | func (e *JSONLinesFormat) Flush() error { return nil }
22 |
23 | func (e *JSONLinesFormat) WriteRow(rows map[string]interface{}) error {
24 | rows = convertToJSON(rows)
25 | err := e.encoder.Encode(rows)
26 | return err
27 | }
28 |
--------------------------------------------------------------------------------
/formats/template.go:
--------------------------------------------------------------------------------
1 | package formats
2 |
3 | import (
4 | "io"
5 | "text/template"
6 | )
7 |
8 | type TemplateFormat struct {
9 | rows []map[string]interface{}
10 | template *template.Template
11 | writer io.Writer
12 | }
13 |
14 | func NewTemplateFormat(w io.Writer, rawTemplate string) *TemplateFormat {
15 | t := template.Must(template.New("climbtemplate").Parse(rawTemplate))
16 | return &TemplateFormat{
17 | rows: make([]map[string]interface{}, 0),
18 | template: t,
19 | writer: w,
20 | }
21 | }
22 |
23 | func (e *TemplateFormat) Flush() error {
24 | err := e.template.Execute(e.writer, e.rows)
25 | return err
26 | }
27 |
28 | func (e *TemplateFormat) WriteHeader(columns []string) error {
29 | return nil
30 | }
31 |
32 | func (e *TemplateFormat) WriteRow(values map[string]interface{}) error {
33 | e.rows = append(e.rows, convertToJSON(values))
34 | return nil
35 | }
36 |
--------------------------------------------------------------------------------
/formats/xlsx.go:
--------------------------------------------------------------------------------
1 | package formats
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "time"
7 |
8 | "github.com/tealeg/xlsx"
9 | )
10 |
11 | type XlsxFormat struct {
12 | file *xlsx.File
13 | sheet *xlsx.Sheet
14 | fileName string
15 | columns []string
16 | }
17 |
18 | func NewXlsxFormat(fileName string, sheetName string) (*XlsxFormat, error) {
19 | file := xlsx.NewFile()
20 | if _, err := os.Stat(fileName); fileName != "" && err == nil {
21 | file, err = xlsx.OpenFile(fileName)
22 | if err != nil {
23 | fmt.Println("Errord file")
24 | return nil, err
25 | }
26 | }
27 |
28 | sheet, err := file.AddSheet(sheetName)
29 | if err != nil {
30 | // Sheet already exists - empty it first
31 | sheet = file.Sheet[sheetName]
32 | sheet.Rows = make([]*xlsx.Row, 0)
33 | }
34 |
35 | return &XlsxFormat{
36 | file: file,
37 | fileName: fileName,
38 | sheet: sheet,
39 | columns: make([]string, 0),
40 | }, nil
41 | }
42 |
43 | func (f *XlsxFormat) Flush() error {
44 | if f.fileName == "" {
45 | return f.file.Write(os.Stdout)
46 | } else {
47 | return f.file.Save(f.fileName)
48 | }
49 | }
50 |
51 | func (f *XlsxFormat) WriteHeader(columns []string) error {
52 | f.columns = columns
53 | row := f.sheet.AddRow()
54 | for _, col := range columns {
55 | cell := row.AddCell()
56 | cell.SetString(col)
57 | }
58 | return nil
59 | }
60 |
61 | func (f *XlsxFormat) WriteRow(values map[string]interface{}) error {
62 | row := f.sheet.AddRow()
63 |
64 | for _, col := range f.columns {
65 | cell := row.AddCell()
66 | switch value := (values[col]).(type) {
67 | case string:
68 | cell.SetString(value)
69 | case []byte:
70 | cell.SetString(string(value))
71 | case int64:
72 | cell.SetInt64(value)
73 | case float64:
74 | cell.SetFloat(value)
75 | case time.Time:
76 | cell.SetDate(value)
77 | case bool:
78 | cell.SetBool(value)
79 | }
80 | }
81 | return nil
82 | }
83 |
--------------------------------------------------------------------------------
/formats/xml.go:
--------------------------------------------------------------------------------
1 | package formats
2 |
3 | import (
4 | "encoding/xml"
5 | "fmt"
6 | "io"
7 | "strconv"
8 | "time"
9 | )
10 |
11 | type XMLFormat struct {
12 | encoder *xml.Encoder
13 | }
14 |
15 | func NewXMLFormat(w io.Writer) *XMLFormat {
16 | e := xml.NewEncoder(w)
17 | e.Indent(" ", " ")
18 | return &XMLFormat{e}
19 | }
20 |
21 | // Writing header for XML is a NOP
22 | func (e *XMLFormat) WriteHeader(columns []string) error {
23 | return nil
24 | }
25 |
26 | func (e *XMLFormat) Flush() error { return nil }
27 |
28 | func (e *XMLFormat) WriteRow(values map[string]interface{}) error {
29 | row := xml.StartElement{Name: xml.Name{"", "row"}}
30 | tokens := []xml.Token{row}
31 | for key, value := range values {
32 | var charData xml.CharData
33 |
34 | t := xml.StartElement{Name: xml.Name{"", key}}
35 |
36 | switch value := (value).(type) {
37 | case string:
38 | charData = xml.CharData(value)
39 | case []byte:
40 | charData = xml.CharData(string(value))
41 | case int64:
42 | charData = xml.CharData(fmt.Sprintf("%d", value))
43 | case float64:
44 | charData = xml.CharData(strconv.FormatFloat(value, 'f', -1, 64))
45 | case time.Time:
46 | charData = xml.CharData(value.Format(time.RFC3339))
47 | case bool:
48 | if value == true {
49 | charData = xml.CharData("true")
50 | } else {
51 | charData = xml.CharData("false")
52 | }
53 | }
54 | tokens = append(tokens, t, charData, t.End())
55 | }
56 | tokens = append(tokens, row.End())
57 |
58 | for _, t := range tokens {
59 | err := e.encoder.EncodeToken(t)
60 | if err != nil {
61 | return err
62 | }
63 | }
64 |
65 | err := e.encoder.Flush()
66 | if err != nil {
67 | return err
68 | }
69 |
70 | return nil
71 | }
72 |
--------------------------------------------------------------------------------
/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lukasmartinelli/pgclimb/78d4bc6ecfc2e6eccff7fa70f8437284bede9432/logo.png
--------------------------------------------------------------------------------
/pg/postgres.go:
--------------------------------------------------------------------------------
1 | package pg
2 |
3 | import (
4 | "fmt"
5 |
6 | "github.com/jmoiron/sqlx"
7 |
8 | _ "github.com/lib/pq"
9 | "github.com/urfave/cli"
10 | )
11 |
12 | //setup a database connection and create the import schema
13 | func Connect(connStr string) (*sqlx.DB, error) {
14 | db, err := sqlx.Open("postgres", connStr)
15 | if err != nil {
16 | return db, err
17 | }
18 |
19 | err = db.Ping()
20 | if err != nil {
21 | return db, err
22 | }
23 |
24 | return db, nil
25 | }
26 |
27 | //parse sql connection string from cli flags
28 | func ParseConnStr(c *cli.Context) string {
29 | otherParams := "sslmode=disable connect_timeout=5"
30 | if c.GlobalBool("ssl") {
31 | otherParams = "sslmode=require connect_timeout=5"
32 | }
33 | return fmt.Sprintf("user=%s dbname=%s password='%s' host=%s port=%s %s",
34 | c.GlobalString("username"),
35 | c.GlobalString("dbname"),
36 | c.GlobalString("pass"),
37 | c.GlobalString("host"),
38 | c.GlobalString("port"),
39 | otherParams,
40 | )
41 | }
42 |
--------------------------------------------------------------------------------
/pgclimb.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "errors"
5 | "io"
6 | "io/ioutil"
7 | "log"
8 | "os"
9 | "strings"
10 | "unicode/utf8"
11 |
12 | "github.com/andrew-d/go-termutil"
13 | "github.com/lukasmartinelli/pgclimb/formats"
14 | "github.com/lukasmartinelli/pgclimb/pg"
15 | "github.com/urfave/cli"
16 | )
17 |
18 | func changeHelpTemplateArgs(args string) {
19 | cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", args, -1)
20 | }
21 |
22 | func parseTemplate(filename string) string {
23 | rawTemplate, err := ioutil.ReadFile(filename)
24 | if err != nil {
25 | log.Fatalln(err)
26 | }
27 | return string(rawTemplate)
28 | }
29 |
30 | func parseWriter(c *cli.Context) io.Writer {
31 | outputFilename := c.GlobalString("output")
32 |
33 | if outputFilename != "" {
34 | f, err := os.Create(outputFilename)
35 | exitOnError(err)
36 | return f
37 | }
38 | return os.Stdout
39 | }
40 |
41 | func exportFormat(c *cli.Context, format formats.DataFormat) {
42 | connStr := pg.ParseConnStr(c)
43 | query, err := parseQuery(c)
44 | exitOnError(err)
45 | err = formats.Export(query, connStr, format)
46 | exitOnError(err)
47 | }
48 |
49 | func parseQuery(c *cli.Context) (string, error) {
50 | filename := c.GlobalString("file")
51 | if filename != "" {
52 | query, err := ioutil.ReadFile(filename)
53 | return string(query), err
54 | }
55 |
56 | command := c.GlobalString("command")
57 | if command != "" {
58 | return command, nil
59 | }
60 |
61 | if !termutil.Isatty(os.Stdin.Fd()) {
62 | query, err := ioutil.ReadAll(os.Stdin)
63 | return string(query), err
64 | }
65 |
66 | return "", errors.New("You need to specify a SQL query.")
67 | }
68 |
69 | func exitOnError(err error) {
70 | log.SetFlags(0)
71 | if err != nil {
72 | log.Fatalln(err)
73 | }
74 | }
75 |
76 | func main() {
77 | app := cli.NewApp()
78 | app.Name = "pgclimb"
79 | app.Version = "0.2"
80 | app.Usage = "Export data from PostgreSQL into different data formats"
81 |
82 | app.Flags = []cli.Flag{
83 | cli.StringFlag{
84 | Name: "dbname, d",
85 | Value: "postgres",
86 | Usage: "database",
87 | EnvVar: "DB_NAME",
88 | },
89 | cli.StringFlag{
90 | Name: "host",
91 | Value: "localhost",
92 | Usage: "host name",
93 | EnvVar: "DB_HOST",
94 | },
95 | cli.StringFlag{
96 | Name: "port, p",
97 | Value: "5432",
98 | Usage: "port",
99 | EnvVar: "DB_PORT",
100 | },
101 | cli.StringFlag{
102 | Name: "username, U",
103 | Value: "postgres",
104 | Usage: "username",
105 | EnvVar: "DB_USER",
106 | },
107 | cli.BoolFlag{
108 | Name: "ssl",
109 | Usage: "require ssl mode",
110 | },
111 | cli.StringFlag{
112 | Name: "password, pass",
113 | Value: "",
114 | Usage: "password",
115 | EnvVar: "DB_PASS",
116 | },
117 | cli.StringFlag{
118 | Name: "query, command, c",
119 | Value: "",
120 | Usage: "SQL query to execute",
121 | EnvVar: "DB_QUERY",
122 | },
123 | cli.StringFlag{
124 | Name: "file, f",
125 | Value: "",
126 | Usage: "SQL query filename",
127 | },
128 | cli.StringFlag{
129 | Name: "output, o",
130 | Value: "",
131 | Usage: "Output filename",
132 | },
133 | }
134 |
135 | app.Commands = []cli.Command{
136 | {
137 | Name: "template",
138 | Usage: "Export data with custom template",
139 | Action: func(c *cli.Context) error {
140 | changeHelpTemplateArgs("")
141 |
142 | templateArg := c.Args().First()
143 | if templateArg == "" {
144 | cli.ShowCommandHelp(c, "template")
145 | os.Exit(1)
146 | }
147 |
148 | rawTemplate := parseTemplate(templateArg)
149 | writer := parseWriter(c)
150 | exportFormat(c, formats.NewTemplateFormat(writer, rawTemplate))
151 | return nil
152 | },
153 | },
154 | {
155 | Name: "jsonlines",
156 | Usage: "Export newline-delimited JSON objects",
157 | Action: func(c *cli.Context) error {
158 | format := formats.NewJSONLinesFormat(parseWriter(c))
159 | exportFormat(c, format)
160 | return nil
161 | },
162 | },
163 | {
164 | Name: "json",
165 | Usage: "Export JSON document",
166 | Action: func(c *cli.Context) error {
167 | format := formats.NewJSONArrayFormat(parseWriter(c))
168 | exportFormat(c, format)
169 | return nil
170 | },
171 | },
172 | {
173 | Name: "csv",
174 | Usage: "Export CSV",
175 | Flags: []cli.Flag{
176 | cli.StringFlag{
177 | Name: "delimiter",
178 | Value: ",",
179 | Usage: "column delimiter",
180 | },
181 | cli.BoolFlag{
182 | Name: "header",
183 | Usage: "output header row",
184 | },
185 | },
186 | Action: func(c *cli.Context) error {
187 | delimiter, _ := utf8.DecodeRuneInString(c.String("delimiter"))
188 | format := formats.NewCsvFormat(
189 | parseWriter(c),
190 | delimiter,
191 | c.Bool("header"),
192 | )
193 | exportFormat(c, format)
194 | return nil
195 | },
196 | },
197 | {
198 | Name: "tsv",
199 | Usage: "Export TSV",
200 | Flags: []cli.Flag{
201 | cli.BoolFlag{
202 | Name: "header",
203 | Usage: "output header row",
204 | },
205 | },
206 | Action: func(c *cli.Context) error {
207 | format := formats.NewCsvFormat(
208 | parseWriter(c),
209 | '\t',
210 | c.Bool("header"),
211 | )
212 | exportFormat(c, format)
213 | return nil
214 | },
215 | },
216 | {
217 | Name: "xml",
218 | Usage: "Export XML",
219 | Action: func(c *cli.Context) error {
220 | format := formats.NewXMLFormat(parseWriter(c))
221 | exportFormat(c, format)
222 | return nil
223 | },
224 | },
225 | {
226 | Name: "xlsx",
227 | Usage: "Export XLSX spreadsheets",
228 | Flags: []cli.Flag{
229 | cli.StringFlag{
230 | Name: "sheet",
231 | Value: "data",
232 | Usage: "spreadsheet name",
233 | },
234 | },
235 | Action: func(c *cli.Context) error {
236 | format, err := formats.NewXlsxFormat(
237 | c.GlobalString("output"),
238 | c.String("sheet"),
239 | )
240 | exitOnError(err)
241 | exportFormat(c, format)
242 | return nil
243 | },
244 | },
245 | {
246 | Name: "inserts",
247 | Usage: "Export INSERT statements",
248 | Flags: []cli.Flag{
249 | cli.StringFlag{
250 | Name: "table",
251 | Value: "data",
252 | Usage: "table name",
253 | },
254 | },
255 | Action: func(c *cli.Context) error {
256 | format, err := formats.NewInsertsFormat(
257 | parseWriter(c),
258 | c.GlobalString("output"),
259 | c.String("table"),
260 | )
261 | exitOnError(err)
262 | exportFormat(c, format)
263 | return nil
264 | },
265 | },
266 | }
267 |
268 | app.Run(os.Args)
269 | }
270 |
--------------------------------------------------------------------------------
/test.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -o errexit
3 | set -o pipefail
4 | set -o nounset
5 |
6 | readonly CWD=$(pwd)
7 | readonly SAMPLES_DIR="$CWD/samples"
8 | readonly DB_USER=${DB_USER:-postgres}
9 | readonly DB_NAME="integration_test"
10 | readonly DB_SCHEMA="public"
11 | readonly GITHUB_SAMPLE="$SAMPLES_DIR/2015-01-01-15.json"
12 | readonly MONTGOMERY_SALARIES_SAMPLE="$SAMPLES_DIR/employee_salaries.csv"
13 |
14 | function download_montgomery_county_samples() {
15 | if [ ! -f "$MONTGOMERY_SALARIES_SAMPLE" ]; then
16 | wget -O "$MONTGOMERY_SALARIES_SAMPLE" https://data.montgomerycountymd.gov/api/views/54rh-89p8/rows.csv
17 | fi
18 | }
19 |
20 | function download_github_samples() {
21 | if [ ! -f "$GITHUB_SAMPLE" ]; then
22 | mkdir -p $SAMPLES_DIR
23 | cd $SAMPLES_DIR
24 | wget http://data.githubarchive.org/2015-01-01-15.json.gz && gunzip -f 2015-01-01-15.json.gz
25 | cd $CWD
26 | fi
27 | }
28 |
29 | function recreate_db() {
30 | psql -U ${DB_USER} -c "drop database if exists ${DB_NAME};"
31 | psql -U ${DB_USER} -c "create database ${DB_NAME};"
32 | }
33 |
34 | function import_csv() {
35 | local table=$1
36 | local filename=$2
37 | pgfutter --table "$table" --schema $DB_SCHEMA --db $DB_NAME --user $DB_USER csv "$filename"
38 | if [ $? -ne 0 ]; then
39 | echo "pgfutter could not import $filename"
40 | exit 300
41 | else
42 | echo "Imported $filename into $table"
43 | fi
44 | }
45 |
46 | function import_json() {
47 | local table=$1
48 | local filename=$2
49 | pgfutter --table "$table" --schema $DB_SCHEMA --db $DB_NAME --user $DB_USER json "$filename"
50 | if [ $? -ne 0 ]; then
51 | echo "pgfutter could not import $filename"
52 | exit 300
53 | else
54 | echo "Imported $filename into $table"
55 | fi
56 | }
57 |
58 | function import_montgomery_county_samples() {
59 | download_montgomery_county_samples
60 | import_csv "employee_salaries" "$MONTGOMERY_SALARIES_SAMPLE"
61 | }
62 |
63 | function import_github_samples() {
64 | download_github_samples
65 | import_json "github_events" "$GITHUB_SAMPLE"
66 | }
67 |
68 | function test_json_lines_export() {
69 | local query="SELECT e.data->'repo'->>'name' as name, json_agg(c->>'sha') as commmits FROM github_events AS e, json_array_elements(e.data->'payload'->'commits') AS c WHERE e.data->>'type' = 'PushEvent' GROUP BY e.data->'repo'->>'name'"
70 | local filename="push_events.json"
71 | pgclimb -d $DB_NAME -U $DB_USER -c "$query" -o "$filename" jsonlines
72 | echo "Exported JSON lines to $filename"
73 | }
74 |
75 | function test_xml() {
76 | local query="SELECT * FROM employee_salaries"
77 | local filename="salaries.xml"
78 | pgclimb -d $DB_NAME -U $DB_USER -c "$query" -o "$filename" xml
79 | echo "Exported XML to $filename"
80 | }
81 |
82 | function test_excel_export() {
83 | local query1="SELECT * FROM employee_salaries"
84 | local query2="SELECT full_name FROM employee_salaries"
85 | local filename="montgomery_positions.xlsx"
86 | pgclimb -d $DB_NAME -U $DB_USER -c "$query1" -o "$filename" xlsx --sheet salaries
87 | echo "Exported salaries sheet to $filename"
88 | pgclimb -d $DB_NAME -U $DB_USER -c "$query2" -o "$filename" xlsx --sheet employees
89 | echo "Exported Excel employees sheet to $filename"
90 | }
91 |
92 | function test_templates() {
93 | local query="SELECT * FROM employee_salaries"
94 | local template="salaries_report.tpl"
95 | local filename="salaries_report.html"
96 |
97 | echo -e '' > $template
98 | echo -e 'Montgomery County MD Employees' >> $template
99 | echo -e '' >> $template
100 | echo -e 'Employees
' >> $template
101 | echo -e '' >> $template
102 | echo -e '{{range .}}' >> $template
103 | echo -e '- {{.full_name}}
' >> $template
104 | echo -e '{{end}}' >> $template
105 | echo -e '
' >> $template
106 | echo -e '' >> $template
107 | echo -e '' >> $template
108 |
109 | pgclimb -d $DB_NAME -U $DB_USER -c "$query" -o "$filename" template "$template"
110 | echo "Exported template $template to $filename"
111 | }
112 |
113 |
114 | function test_json_doc_export {
115 | local query="SELECT e.data FROM github_events e WHERE e.data->>'type' = 'PushEvent'"
116 | local filename="push_event_docs.json"
117 | pgclimb --dbname $DB_NAME --username $DB_USER --command "$query" -o "$filename" json
118 | echo "Exported JSON to $filename"
119 |
120 | }
121 |
122 | function test_csv_export() {
123 | local query="SELECT position_title, COUNT(*) AS employees, round(AVG(replace(current_annual_salary, '$', '')::numeric)) AS avg_salary FROM employee_salaries GROUP BY position_title ORDER BY 3 DESC"
124 | local filename="montgomery_average_salaries.csv"
125 | echo "$query" | pgclimb -d $DB_NAME -U $DB_USER -o "$filename" csv --delimiter ";" --header
126 | echo "Exported CSV to $filename"
127 | }
128 |
129 | function main() {
130 | recreate_db
131 | import_github_samples
132 | import_montgomery_county_samples
133 | test_csv_export
134 | test_json_lines_export
135 | test_json_doc_export
136 | test_templates
137 | test_excel_export
138 | test_xml
139 | }
140 |
141 | main
142 |
--------------------------------------------------------------------------------