├── .circleci └── config.yml ├── .gitignore ├── Godeps ├── Godeps.json └── Readme ├── LICENSE ├── Makefile ├── README.md ├── client.go ├── client_test.go ├── drivers ├── conn_string.go ├── conn_string_test.go ├── mssql.go ├── mysql.go ├── postgres.go └── sqlite.go ├── example.yml ├── go.mod ├── go.sum ├── integration_test.go ├── main.go ├── models ├── config.go ├── config_test.go ├── dataset.go ├── dataset_test.go ├── errors.go ├── fixtures │ ├── ca.cert.pem │ ├── ca.key.pem │ ├── db.sqlite │ ├── invalid_config.yml │ ├── mssql.sql │ ├── mysql.sql │ ├── postgres.sql │ ├── sqlite.sql │ ├── test.crt │ ├── test.key │ ├── valid_config.yml │ ├── valid_config2.yml │ ├── valid_config_all_envs.yml │ └── valid_config_with_missing_envs.yml ├── number.go ├── sql.go └── sql_test.go └── scripts ├── wait_for_mysql └── wait_for_postgres /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | codecov: codecov/codecov@1.2.5 5 | 6 | executors: 7 | ubuntu: 8 | machine: 9 | image: "ubuntu-2004:202010-01" 10 | working_directory: /home/circleci/go/src/github.com/geckoboard/sql-dataset 11 | environment: 12 | GOPATH: "/home/circleci/go" 13 | 14 | commands: 15 | install_go: 16 | description: "Exports go bin path and installs specific go version" 17 | steps: 18 | - run: echo 'export PATH=$GOPATH/bin:$PATH' >> $BASH_ENV 19 | - run: 20 | name: "Remove old go directory" 21 | command: sudo rm -rf /usr/local/go 22 | - run: 23 | name: "Upgrade go to 1.16" 24 | command: | 25 | cd $HOME 26 | curl https://dl.google.com/go/go1.16.7.linux-amd64.tar.gz -o golang.tar.gz 27 | sudo tar -C /usr/local -xzf golang.tar.gz 28 | 29 | jobs: 30 | test: 31 | executor: ubuntu 32 | steps: 33 | - install_go 34 | - checkout 35 | - run: make pull-docker-images run-containers 36 | - run: make setup-db 37 | - run: make test 38 | - codecov/upload 39 | workflows: 40 | version: 2 41 | test_build: 42 | jobs: 43 | - test 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | builds 10 | 11 | # Architecture specific extensions/prefixes 12 | *.[568vq] 13 | [568vq].out 14 | 15 | *.cgo1.go 16 | *.cgo2.c 17 | _cgo_defun.c 18 | _cgo_gotypes.go 19 | _cgo_export.* 20 | 21 | _testmain.go 22 | 23 | *.exe 24 | *.test 25 | *.prof 26 | coverage.txt 27 | -------------------------------------------------------------------------------- /Godeps/Godeps.json: -------------------------------------------------------------------------------- 1 | { 2 | "ImportPath": "github.com/geckoboard/sql-dataset", 3 | "GoVersion": "go1.8", 4 | "GodepVersion": "v80", 5 | "Deps": [ 6 | { 7 | "ImportPath": "github.com/denisenkom/go-mssqldb", 8 | "Rev": "f77039e4e9bc788cb0406a666f410abee0ca1580" 9 | }, 10 | { 11 | "ImportPath": "github.com/go-sql-driver/mysql", 12 | "Comment": "v1.4.0-6-g361f66e", 13 | "Rev": "361f66ef3b53de1f16b7f2af9ef38a6c159ceb3e" 14 | }, 15 | { 16 | "ImportPath": "github.com/lib/pq", 17 | "Comment": "go1.0-cutoff-166-g2704adc", 18 | "Rev": "2704adc878c21e1329f46f6e56a1c387d788ff94" 19 | }, 20 | { 21 | "ImportPath": "github.com/lib/pq/oid", 22 | "Comment": "go1.0-cutoff-166-g2704adc", 23 | "Rev": "2704adc878c21e1329f46f6e56a1c387d788ff94" 24 | }, 25 | { 26 | "ImportPath": "github.com/mattn/go-sqlite3", 27 | "Comment": "v1.2.0-80-gcf7286f", 28 | "Rev": "cf7286f069c3ef596efcc87781a4653a2e7607bd" 29 | }, 30 | { 31 | "ImportPath": "golang.org/x/crypto/md4", 32 | "Rev": "e1a4589e7d3ea14a3352255d04b6f1a418845e5e" 33 | }, 34 | { 35 | "ImportPath": "golang.org/x/net/context", 36 | "Rev": "da118f7b8e5954f39d0d2130ab35d4bf0e3cb344" 37 | }, 38 | { 39 | "ImportPath": "gopkg.in/guregu/null.v3", 40 | "Comment": "v3.1", 41 | "Rev": "41961cea0328defc5f95c1c473f89ebf0d1813f6" 42 | }, 43 | { 44 | "ImportPath": "gopkg.in/yaml.v2", 45 | "Rev": "cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b" 46 | } 47 | ] 48 | } 49 | -------------------------------------------------------------------------------- /Godeps/Readme: -------------------------------------------------------------------------------- 1 | This directory tree is generated automatically by godep. 2 | 3 | Please do not edit. 4 | 5 | See https://github.com/tools/godep for more information. 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Jon Normington 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BUILD_DIR=builds 2 | VERSION=0.2.5 3 | GIT_SHA=$(shell git rev-parse --short HEAD) 4 | PASSWORD=root 5 | MSPASS=zebra-IT-32 6 | 7 | DOCKER_MYSQL=mysql/mysql-server:5.7 8 | DOCKER_POSTGRES=postgres:9.6 9 | DOCKER_MSSQL=mcr.microsoft.com/mssql/server:2017-latest 10 | DB_NAME=testdb 11 | 12 | BUILD_PREFIX=builds/sql-dataset 13 | LDFLAGS="-X main.version=$(VERSION) -X main.gitSHA=$(GIT_SHA)" 14 | 15 | build-darwin: 16 | rm builds/* || true 17 | CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -o ${BUILD_PREFIX}-darwin-amd64 -ldflags=${LDFLAGS} 18 | CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -o ${BUILD_PREFIX}-darwin-arm64 -ldflags=${LDFLAGS} 19 | 20 | build-unix: 21 | CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o ${BUILD_PREFIX}-linux-amd64 -ldflags=${LDFLAGS} 22 | CGO_ENABLED=1 GOOS=linux GOARCH=386 go build -o ${BUILD_PREFIX}-linux-386 -ldflags=${LDFLAGS} 23 | 24 | build-win: 25 | set GOARCH=amd64; go build -o ${BUILD_PREFIX}-windows-10.0-amd64.exe -ldflags=${LDFLAGS} 26 | set GOARCH=386; go build -o ${BUILD_PREFIX}-windows-10.0-386.exe -ldflags=${LDFLAGS} 27 | 28 | pull-docker-images: 29 | docker pull ${DOCKER_MYSQL} 30 | docker pull ${DOCKER_POSTGRES} 31 | docker pull ${DOCKER_MSSQL} 32 | 33 | run-containers: 34 | docker rm -f sd-mysql sd-postgres sd-mssql || true 35 | docker run --name sd-mssql -e ACCEPT_EULA=Y -e SA_PASSWORD=${MSPASS} -p 1433:1433 -d ${DOCKER_MSSQL} || true 36 | # MySQL 37 | docker run --name sd-mysql -e MYSQL_ROOT_PASSWORD=${PASSWORD} -p 3307:3306 -d ${DOCKER_MYSQL} || true 38 | scripts/wait_for_mysql sd-mysql 39 | # Postgres 40 | docker run --name sd-postgres -e POSTGRES_PASSWORD=${PASSWORD} -p 5433:5432 -d ${DOCKER_POSTGRES} || true 41 | scripts/wait_for_postgres sd-postgres 42 | 43 | setup-db: 44 | # Mysql ensure root can access from anywhere 45 | docker exec -it sd-mysql mysql -uroot -proot -e "CREATE USER 'root'@'%' IDENTIFIED BY '${PASSWORD}'" || true 46 | docker exec -it sd-mysql mysql -uroot -proot -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%'" || true 47 | docker exec -it sd-mysql mysql -uroot -proot -e "CREATE DATABASE ${DB_NAME};" || true 48 | # Postgres db creation 49 | docker exec --user postgres -it sd-postgres psql -c "CREATE DATABASE ${DB_NAME}" || true 50 | # MSSQL db creation 51 | docker exec -it sd-mssql /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P ${MSPASS} -Q "CREATE DATABASE testdb" || true 52 | 53 | 54 | # 55 | # Running the more full integration tests requires docker containers to be 56 | # running database servers. The setup target helps with call the other targets 57 | # which download docker images, run-containers and setup-db 58 | # 59 | setup: pull-docker-images run-containers setup-db 60 | 61 | # 62 | # Once you run setup target the docker containers will continously 63 | # run the tests should handle different states already with SQL fixtures 64 | # using REPLACE for example instead of INSERT, but eventually you will want 65 | # to stop the containers running this target supports that 66 | # 67 | teardown: 68 | @docker stop sd-mysql > /dev/null 2>&1 || true 69 | @docker rm -v sd-mysql > /dev/null 2>&1 || true 70 | @docker stop sd-mssql > /dev/null 2>&1 || true 71 | @docker rm -v sd-mssql > /dev/null 2>&1 || true 72 | @docker stop sd-postgres > /dev/null 2>&1 || true 73 | @docker rm -v sd-postgres > /dev/null 2>&1 || true 74 | 75 | test: 76 | MYSQL_URL="root:${PASSWORD}@tcp(localhost:3307)/testdb?parseTime=true" \ 77 | POSTGRES_URL=postgres://postgres:${PASSWORD}@localhost:5433/testdb?sslmode=disable \ 78 | MSSQL_URL="odbc:server=localhost;port=1433;user id=sa;password=${MSPASS};database=${DB_NAME}" \ 79 | go test ./... -race -covermode=atomic -coverprofile=coverage.txt 80 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SQL-Dataset, by Geckoboard 2 | 3 | [![CircleCI](https://circleci.com/gh/geckoboard/sql-dataset.svg?style=svg)](https://circleci.com/gh/geckoboard/sql-dataset) [![codecov](https://codecov.io/gh/geckoboard/sql-dataset/branch/master/graph/badge.svg)](https://codecov.io/gh/geckoboard/sql-dataset) 4 | 5 | Quickly and easily send data from Microsoft SQL Server, MySQL, Postgres and SQLite databases to Geckoboard Datasets. 6 | 7 | SQL-Dataset is a command line app that takes the hassle out of integrating your database with Geckoboard. Rather than having to work with client libraries and write a bunch of code to connect to and query your database, with SQL-Dataset all you need to do is fill out a simple config file. 8 | 9 | SQL-Dataset is available for macOS, Linux, and Windows. 10 | 11 | ## Quickstart 12 | 13 | ### 1. Download the app 14 | 15 | * macOS [x64](https://github.com/geckoboard/sql-dataset/releases/download/v0.2.5/sql-dataset-darwin-amd64) / [arm64](https://github.com/geckoboard/sql-dataset/releases/download/v0.2.5/sql-dataset-darwin-arm64) 16 | * Linux [x86](https://github.com/geckoboard/sql-dataset/releases/download/v0.2.5/sql-dataset-linux-386) / [x64](https://github.com/geckoboard/sql-dataset/releases/download/v0.2.5/sql-dataset-linux-amd64) 17 | * Windows [x86](https://github.com/geckoboard/sql-dataset/releases/download/v0.2.5/sql-dataset-windows-386.exe) / [x64](https://github.com/geckoboard/sql-dataset/releases/download/v0.2.5/sql-dataset-windows-amd64.exe) 18 | 19 | #### Make it executable (macOS / Linux) 20 | 21 | On macOS and Linux you'll need to open a terminal and run `chmod u+x path/to/file` (replacing `path/to/file` with the actual path to your downloaded app) in order to make the app executable. 22 | 23 | ### 2. Create a config file 24 | 25 | SQL-Datasets works by reading all of the information it needs from a YAML file. We've prepared an [example one](example.yml) for you so you can get started quickly. The fields are fairly self-explanatory, but you can learn more about them [below](README.md#building-your-config-file). 26 | 27 | ### 3. Run the script 28 | 29 | Make sure that the SQL-Dataset app and your config file are in the same folder, then from the command line navigate to that folder and run 30 | 31 | ``` 32 | ./sql-dataset -config config.yml 33 | ``` 34 | 35 | Where `config.yml` is the name of your config file. Once you see confirmation that everything ran successfully, head over to Geckoboard and [start using your new Dataset to build widgets](https://support.geckoboard.com/hc/en-us/articles/223190488-Guide-to-using-datasets)! 36 | 37 | ## Building your config file 38 | 39 | Here's what an example config file looks like: 40 | 41 | ```yaml 42 | geckoboard_api_key: your_api_key 43 | database: 44 | driver: mysql 45 | host: xxxx 46 | port: xxxx 47 | username: xxxx 48 | password: xxxx 49 | name: xxxx 50 | tls_config: 51 | ca_file: xxxx 52 | key_file: xxxx 53 | cert_file: xxxx 54 | ssl_mode: xxxx 55 | refresh_time_sec: 60 56 | datasets: 57 | - name: dataset.name 58 | update_type: replace 59 | sql: > 60 | SELECT 1, 0.34, source 61 | FROM table 62 | fields: 63 | - type: number 64 | name: Signups 65 | - type: percentage 66 | name: Conversion rate 67 | - type: string 68 | name: Source 69 | ``` 70 | 71 | #### Environment variables 72 | 73 | If you wish, you can provide any of `geckoboard_api_key`, `host`, `port`, `username`, `password` and (database) `name` as environment variables with the syntax `"{{ YOUR_CUSTOM_ENV }}"`. Make sure to keep the quotes in there! For example: 74 | 75 | ```yaml 76 | geckoboard_api_key: "{{ GB_API_KEY }}" 77 | ``` 78 | 79 | ### geckoboard_api_key 80 | 81 | Hopefully this is obvious, but this is where your Geckoboard API key goes. You can find yours [here](https://app.geckoboard.com/account/details). 82 | 83 | ### database 84 | 85 | Enter the type of database you're connecting to in the `driver` field. SQL-Dataset supports: 86 | 87 | - `mssql` 88 | - `mysql` 89 | - `postgres` 90 | - `sqlite` 91 | 92 | If you'd like to see support for another type of database, please raise a [support ticket](https://www.geckoboard.com/about/contact/) or, if you're technically inclined, make the change and submit a pull request! 93 | 94 | Only three parameters are required: 95 | 96 | - `driver` 97 | - `username` 98 | - `name` 99 | 100 | The other attributes, such as `host` and `port`, will default to their driver-specific values unless overridden. 101 | 102 | #### SSL 103 | 104 | If your database requires a CA cert or a x509 key/cert pair, you can supply this in `tls_config` under the database key. 105 | 106 | ```yaml 107 | tls_config: 108 | ca_file: /path/to/file.pem 109 | key_file: /path/to/file.key 110 | cert_file: /path/to/cert.crt 111 | ssl_mode: (optional) 112 | ``` 113 | 114 | The possible values for `ssl_mode` depend on the database you're using: 115 | 116 | - MSSQL: `disable`, `false`, `true` - try disable option if you experience connection issues 117 | - MySQL: `true`, `skip-verify` 118 | - Postgres: `disable`, `require`, `verify-ca`, `verify-full` 119 | - SQLite: N/A 120 | 121 | 122 | #### A note on user permissions 123 | 124 | We _strongly_ recommend that the user account you use with SQL-Dataset has the lowest level of permission necessary. For example, one which is only permitted to perform `SELECT` statements on the tables you're going to be using. Like any SQL program, SQL-Dataset will run any query you give it, which includes destructive operations such as overwriting existing data, removing records, and dropping tables. We accept no responsibility for any adverse changes to your database due to accidentally running such a query. 125 | 126 | ### refresh_time_sec 127 | 128 | Once started, SQL-Dataset can run your queries periodically and push the results to Geckoboard. Use this field to specify the time, in seconds, between refreshes. 129 | 130 | If you do not wish for SQL-Dataset to run on a schedule, omit this option from your config. 131 | 132 | ### datasets 133 | 134 | Here's where the magic happens - specify the SQL queries you want to run, and the Datasets you want to push their results into. 135 | 136 | - `name`: The name of your Dataset 137 | - `sql`: Your SQL query 138 | - `fields`: The schema of the Dataset into which the results of your SQL query will be parsed 139 | - `update_type`: Either `replace`, which overwrites the contents of the Dataset with new data on each update, or `append`, which merges the latest update with your existing data. 140 | - `unique_by`: An optional array of one or more field names whose values will be unique across all your records. When using the `append` update method, the fields in `unique_by` will be used to determine whether new data should update any existing records. 141 | 142 | #### fields 143 | 144 | A Dataset can hold up to 10 fields. The fields you declare should map directly to the columns that result from your `SELECT` query, in the **same order**. 145 | 146 | For example: 147 | 148 | ```yaml 149 | sql: SELECT date, orders, refunds FROM sales 150 | fields: 151 | - name: Date 152 | type: date 153 | - name: Orders 154 | type: number 155 | - name: Refunds 156 | type: number 157 | ``` 158 | 159 | SQL-Dataset supports all of the field types supported by the [Datasets API](https://developer.geckoboard.com): 160 | 161 | - date 162 | - datetime 163 | - duration 164 | - number 165 | - percentage 166 | - string 167 | - money 168 | 169 | The `money` field type requires a `currency_code` to be provided: 170 | 171 | ```yaml 172 | fields 173 | - name: MRR 174 | type: money 175 | currency_code: USD 176 | ``` 177 | 178 | The `duration` field type requires a `time_unit` to be provided: 179 | With a value one of: milliseconds, seconds, minutes, hours 180 | 181 | ```yaml 182 | fields 183 | - name: Time until support ticket resolved 184 | type: duration 185 | time_unit: minutes 186 | ``` 187 | 188 | Numeric field types can support null values. For a field to support this, pass the `optional` key: 189 | 190 | ```yaml 191 | fields: 192 | - name: A field which might be NULL 193 | type: number 194 | optional: true 195 | ``` 196 | 197 | The Datasets API requires both a `name` and a `key` for each field, but SQL-Dataset will infer a `key` for you. Sometimes, however, the inferred `key` might not be permitted by the API. If you encounter such a case, you can supply a specific `key` value for that field. 198 | 199 | ```yaml 200 | fields: 201 | - name: Your awesome field 202 | key: some_unique_key 203 | type: number 204 | ``` 205 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/geckoboard/sql-dataset/models" 12 | ) 13 | 14 | type Client struct { 15 | apiKey string 16 | client *http.Client 17 | } 18 | 19 | type Error struct { 20 | Detail `json:"error"` 21 | } 22 | 23 | type Detail struct { 24 | Message string `json:"message"` 25 | } 26 | 27 | type DataPayload struct { 28 | Data models.DatasetRows `json:"data"` 29 | } 30 | 31 | var ( 32 | gbHost = "https://api.geckoboard.com" 33 | userAgent = fmt.Sprintf("SQL-Dataset/%s-%s", version, gitSHA) 34 | maxRows = 500 35 | 36 | errInvalidPayload = "There was an error sending the data to Geckoboard's API: %s" 37 | 38 | errUnexpectedResponse = errors.New("Sorry, there seems to be a problem with " + 39 | "Geckoboard's servers. Please try again, or check" + 40 | "https://geckoboard.statuspage.io") 41 | 42 | errMoreRowsToSend = "You're trying to send %d records, but 'replace' " + 43 | "mode only supports sending %d. To send more, please change " + 44 | "your dataset's update_type to 'append'\n" 45 | ) 46 | 47 | func NewClient(apiKey string) *Client { 48 | return &Client{ 49 | apiKey: apiKey, 50 | client: &http.Client{Timeout: time.Second * 10}, 51 | } 52 | } 53 | 54 | func (c *Client) FindOrCreateDataset(ds *models.Dataset) error { 55 | if err := ds.BuildSchemaFields(); err != nil { 56 | return err 57 | } 58 | 59 | resp, err := c.makeRequest(http.MethodPut, fmt.Sprintf("/datasets/%s", ds.Name), ds) 60 | 61 | if err != nil { 62 | return err 63 | } 64 | 65 | defer resp.Body.Close() 66 | return handleResponse(resp) 67 | } 68 | 69 | func (c *Client) DeleteDataset(name string) (err error) { 70 | resp, err := c.makeRequest(http.MethodDelete, fmt.Sprintf("/datasets/%s", name), nil) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | defer resp.Body.Close() 76 | return handleResponse(resp) 77 | } 78 | 79 | func (c *Client) sendData(ds *models.Dataset, data models.DatasetRows) (err error) { 80 | method := http.MethodPost 81 | 82 | if ds.UpdateType == models.Replace { 83 | method = http.MethodPut 84 | } 85 | 86 | resp, err := c.makeRequest(method, fmt.Sprintf("/datasets/%s/data", ds.Name), DataPayload{data}) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | defer resp.Body.Close() 92 | return handleResponse(resp) 93 | } 94 | 95 | // SendAllData determines how to send the data to Geckoboard and returns an error 96 | // if there is too much data for replace dataset and batches requests for append 97 | func (c *Client) SendAllData(ds *models.Dataset, data models.DatasetRows) (err error) { 98 | switch ds.UpdateType { 99 | case models.Replace: 100 | if len(data) > maxRows { 101 | err = c.sendData(ds, data[0:maxRows]) 102 | if err == nil { 103 | err = fmt.Errorf(errMoreRowsToSend, len(data), maxRows) 104 | } 105 | } else { 106 | err = c.sendData(ds, data) 107 | } 108 | case models.Append: 109 | grps := len(data) / maxRows 110 | 111 | for i := 0; i <= grps; i++ { 112 | batch := maxRows * i 113 | 114 | if i == grps { 115 | if batch+1 <= len(data) { 116 | err = c.sendData(ds, data[batch:]) 117 | } 118 | } else { 119 | err = c.sendData(ds, data[batch:maxRows*(i+1)]) 120 | } 121 | 122 | if err != nil { 123 | return err 124 | } 125 | } 126 | } 127 | 128 | return err 129 | } 130 | 131 | func (c *Client) makeRequest(method, path string, body interface{}) (resp *http.Response, err error) { 132 | var buf bytes.Buffer 133 | 134 | if body != nil { 135 | if err := json.NewEncoder(&buf).Encode(body); err != nil { 136 | return nil, err 137 | } 138 | } 139 | 140 | url := gbHost + path 141 | req, err := http.NewRequest(method, url, &buf) 142 | if err != nil { 143 | return nil, err 144 | } 145 | 146 | req.Header.Set("User-Agent", userAgent) 147 | 148 | req.SetBasicAuth(c.apiKey, "") 149 | return c.client.Do(req) 150 | } 151 | 152 | func handleResponse(resp *http.Response) error { 153 | res := resp.StatusCode 154 | 155 | switch { 156 | case res >= 200 && res < 300: 157 | return nil 158 | case res >= 400 && res < 500: 159 | var err Error 160 | json.NewDecoder(resp.Body).Decode(&err) 161 | return fmt.Errorf(errInvalidPayload, err.Detail.Message) 162 | default: 163 | return errUnexpectedResponse 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /client_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "net/http/httptest" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/geckoboard/sql-dataset/models" 12 | ) 13 | 14 | type request struct { 15 | Method string 16 | Headers map[string]string 17 | Path string 18 | Body string 19 | } 20 | 21 | type response struct { 22 | code int 23 | body string 24 | } 25 | 26 | const ( 27 | apiKey = "ap1K3Y" 28 | expAuth = "Basic YXAxSzNZOg==" 29 | ) 30 | 31 | func TestFindOrCreateDataset(t *testing.T) { 32 | userAgent = "SQL-Dataset/test-fake" 33 | 34 | testCases := []struct { 35 | dataset models.Dataset 36 | request request 37 | response *response 38 | err string 39 | }{ 40 | { 41 | dataset: models.Dataset{ 42 | Name: "active.users.count", 43 | Fields: []models.Field{ 44 | { 45 | Name: "mrr", 46 | Type: models.MoneyType, 47 | CurrencyCode: "USD", 48 | }, 49 | { 50 | Name: "signups", 51 | Type: models.NumberType, 52 | }, 53 | }, 54 | }, 55 | request: request{ 56 | Method: http.MethodPut, 57 | Path: "/datasets/active.users.count", 58 | Body: `{"id":"active.users.count","fields":{"mrr":{"type":"money","name":"mrr","currency_code":"USD"},"signups":{"type":"number","name":"signups"}}}`, 59 | }, 60 | }, 61 | { 62 | dataset: models.Dataset{ 63 | Name: "active.users.count", 64 | Fields: []models.Field{ 65 | { 66 | Name: "day", 67 | Type: models.DatetimeType, 68 | }, 69 | { 70 | Name: "mrr Percent", 71 | Type: models.PercentageType, 72 | Optional: true, 73 | }, 74 | }, 75 | }, 76 | request: request{ 77 | Method: http.MethodPut, 78 | Path: "/datasets/active.users.count", 79 | Body: `{"id":"active.users.count","fields":{"day":{"type":"datetime","name":"day"},"mrr_percent":{"type":"percentage","name":"mrr Percent","optional":true}}}`, 80 | }, 81 | }, 82 | { 83 | dataset: models.Dataset{ 84 | Name: "builds.count.by.day", 85 | UniqueBy: []string{"day"}, 86 | Fields: []models.Field{ 87 | { 88 | Name: "Builds", 89 | Type: models.NumberType, 90 | Optional: true, 91 | }, 92 | { 93 | Name: "day", 94 | Type: models.DateType, 95 | }, 96 | }, 97 | }, 98 | request: request{ 99 | Method: http.MethodPut, 100 | Path: "/datasets/builds.count.by.day", 101 | Body: `{"id":"builds.count.by.day","unique_by":["day"],"fields":{"builds":{"type":"number","name":"Builds","optional":true},"day":{"type":"date","name":"day"}}}`, 102 | }, 103 | }, 104 | { 105 | // Verify 50x just returns generic error 106 | dataset: models.Dataset{ 107 | Name: "active.users.count", 108 | Fields: []models.Field{ 109 | { 110 | Name: "mrr", 111 | Type: models.MoneyType, 112 | CurrencyCode: "USD", 113 | }, 114 | { 115 | Name: "signups", 116 | Type: models.NumberType, 117 | }, 118 | }, 119 | }, 120 | response: &response{ 121 | code: 500, 122 | body: "Internal error", 123 | }, 124 | err: errUnexpectedResponse.Error(), 125 | }, 126 | { 127 | // Verify 40x response unmarshalled and return message 128 | dataset: models.Dataset{ 129 | Name: "active.users.count", 130 | Fields: []models.Field{ 131 | { 132 | Name: "mrr", 133 | Type: models.MoneyType, 134 | CurrencyCode: "USD", 135 | }, 136 | { 137 | Name: "signups", 138 | Type: models.NumberType, 139 | }, 140 | }, 141 | }, 142 | response: &response{ 143 | code: 400, 144 | body: `{"error":{"type":"ErrResourceInvalid","message":"Field name too short"}}`, 145 | }, 146 | err: fmt.Sprintf(errInvalidPayload, "Field name too short"), 147 | }, 148 | { 149 | // Verify 201 response correctly handled 150 | dataset: models.Dataset{ 151 | Name: "active.users.count", 152 | Fields: []models.Field{ 153 | { 154 | Name: "mrr", 155 | Type: models.MoneyType, 156 | CurrencyCode: "USD", 157 | }, 158 | { 159 | Name: "signups", 160 | Type: models.NumberType, 161 | }, 162 | }, 163 | }, 164 | response: &response{ 165 | code: 201, 166 | body: `{}`, 167 | }, 168 | }, 169 | } 170 | 171 | for _, tc := range testCases { 172 | reqCount := 0 173 | 174 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 175 | reqCount++ 176 | 177 | if tc.response != nil { 178 | w.WriteHeader(tc.response.code) 179 | fmt.Fprintf(w, tc.response.body) 180 | return 181 | } 182 | 183 | auth := r.Header.Get("Authorization") 184 | if auth != expAuth { 185 | t.Errorf("Expected authorization header '%s' but got '%s'", expAuth, auth) 186 | } 187 | 188 | ua := r.Header.Get("User-Agent") 189 | if ua != userAgent { 190 | t.Errorf("Expected user header '%s' but got '%s'", userAgent, ua) 191 | } 192 | 193 | if r.URL.Path != tc.request.Path { 194 | t.Errorf("Expected path '%s' but got '%s'", tc.request.Path, r.URL.Path) 195 | } 196 | 197 | if r.Method != tc.request.Method { 198 | t.Errorf("Expected method '%s' but got '%s'", tc.request.Method, r.Method) 199 | } 200 | 201 | b, err := ioutil.ReadAll(r.Body) 202 | if err != nil { 203 | t.Errorf("Errored while consuming body %s", err) 204 | } 205 | 206 | body := strings.Trim(string(b), "\n") 207 | 208 | if body != tc.request.Body { 209 | t.Errorf("Expected body '%s' but got '%s'", tc.request.Body, body) 210 | } 211 | })) 212 | 213 | gbHost = server.URL 214 | 215 | c := NewClient(apiKey) 216 | err := c.FindOrCreateDataset(&tc.dataset) 217 | 218 | if err != nil && tc.err == "" { 219 | t.Errorf("Expected no error but got '%s'", err) 220 | } 221 | 222 | if err == nil && tc.err != "" { 223 | t.Errorf("Expected error '%s' but got none", tc.err) 224 | } 225 | 226 | if err != nil && err.Error() != tc.err { 227 | t.Errorf("Expected error '%s' but got '%s'", tc.err, err) 228 | } 229 | 230 | if reqCount != 1 { 231 | t.Errorf("Expected one request but got %d", reqCount) 232 | } 233 | 234 | server.Close() 235 | } 236 | } 237 | 238 | func TestDeleteDataset(t *testing.T) { 239 | userAgent = "SQL-Dataset/test-fake" 240 | 241 | testCases := []struct { 242 | name string 243 | request request 244 | response *response 245 | err string 246 | }{ 247 | { 248 | // Dataset not existing response 249 | name: "active.users.count", 250 | request: request{ 251 | Path: "/datasets/active.users.count", 252 | }, 253 | response: &response{ 254 | code: 404, 255 | body: `{"error":{"message":"Dataset not found"}}`, 256 | }, 257 | err: fmt.Sprintf(errInvalidPayload, "Dataset not found"), 258 | }, 259 | { 260 | // Verify bad response 261 | name: "active.users.count", 262 | request: request{ 263 | Path: "/datasets/active.users.count", 264 | }, 265 | response: &response{ 266 | code: 500, 267 | body: "Internal error", 268 | }, 269 | err: errUnexpectedResponse.Error(), 270 | }, 271 | { 272 | // Verify 200 response correctly handled 273 | name: "active.users.count", 274 | request: request{ 275 | Path: "/datasets/active.users.count", 276 | }, 277 | response: &response{ 278 | code: 200, 279 | body: `{}`, 280 | }, 281 | }, 282 | } 283 | 284 | for _, tc := range testCases { 285 | reqCount := 0 286 | 287 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 288 | reqCount++ 289 | 290 | auth := r.Header.Get("Authorization") 291 | if auth != expAuth { 292 | t.Errorf("Expected authorization header '%s' but got '%s'", expAuth, auth) 293 | } 294 | 295 | ua := r.Header.Get("User-Agent") 296 | if ua != userAgent { 297 | t.Errorf("Expected user header '%s' but got '%s'", userAgent, ua) 298 | } 299 | 300 | if r.URL.Path != tc.request.Path { 301 | t.Errorf("Expected path '%s' but got '%s'", tc.request.Path, r.URL.Path) 302 | } 303 | 304 | if r.Method != http.MethodDelete { 305 | t.Errorf("Expected method '%s' but got '%s'", http.MethodDelete, r.Method) 306 | } 307 | 308 | if tc.response != nil { 309 | w.WriteHeader(tc.response.code) 310 | fmt.Fprintf(w, tc.response.body) 311 | return 312 | } 313 | })) 314 | 315 | gbHost = server.URL 316 | 317 | c := NewClient(apiKey) 318 | err := c.DeleteDataset(tc.name) 319 | 320 | switch { 321 | case err == nil && tc.err != "": 322 | t.Errorf("Expected error %q but got none", tc.err) 323 | case err != nil && (tc.err == "" || tc.err != err.Error()): 324 | t.Errorf("Expected error '%s' but got '%s'", tc.err, err) 325 | } 326 | 327 | if reqCount != 1 { 328 | t.Errorf("Expected one request but got %d", reqCount) 329 | } 330 | 331 | server.Close() 332 | } 333 | } 334 | 335 | // Preset the dataset rows and test we batch correctly return errors on status codes 336 | func TestSendAllData(t *testing.T) { 337 | testCases := []struct { 338 | dataset models.Dataset 339 | data models.DatasetRows 340 | requests []request 341 | maxRows int 342 | response *response 343 | err string 344 | }{ 345 | { 346 | // Error with 40x 347 | dataset: models.Dataset{ 348 | Name: "app.build.costs", 349 | UpdateType: models.Replace, 350 | Fields: []models.Field{ 351 | {Name: "App", Type: models.StringType}, 352 | {Name: "Run time", Type: models.NumberType}, 353 | }, 354 | }, 355 | data: models.DatasetRows{ 356 | { 357 | "app": "acceptance", 358 | "run_time": 4421, 359 | }, 360 | }, 361 | requests: []request{{}}, 362 | response: &response{ 363 | code: 400, 364 | body: `{"error": {"type":"ErrMissingData", "message": "Missing data for 'app'"}}`, 365 | }, 366 | err: fmt.Sprintf(errInvalidPayload, "Missing data for 'app'"), 367 | }, 368 | { 369 | // Error with 50x 370 | dataset: models.Dataset{ 371 | Name: "app.build.costs", 372 | UpdateType: models.Replace, 373 | Fields: []models.Field{ 374 | {Name: "App", Type: models.StringType}, 375 | {Name: "Run time", Type: models.NumberType}, 376 | }, 377 | }, 378 | data: models.DatasetRows{ 379 | { 380 | "app": "acceptance", 381 | "run_time": 4421, 382 | }, 383 | }, 384 | requests: []request{{}}, 385 | response: &response{ 386 | code: 500, 387 | body: "Internal Server error", 388 | }, 389 | err: errUnexpectedResponse.Error(), 390 | }, 391 | { 392 | //Replace dataset with no data 393 | dataset: models.Dataset{ 394 | Name: "app.no.data", 395 | UpdateType: models.Replace, 396 | Fields: []models.Field{ 397 | {Name: "App", Type: models.StringType}, 398 | {Name: "Percent", Type: models.PercentageType}, 399 | }, 400 | }, 401 | data: models.DatasetRows{}, 402 | requests: []request{ 403 | { 404 | Method: http.MethodPut, 405 | Path: "/datasets/app.no.data/data", 406 | Body: `{"data":[]}`, 407 | }, 408 | }, 409 | }, 410 | { 411 | //Replace dataset under the batch rows limit doesn't error 412 | dataset: models.Dataset{ 413 | Name: "app.reliable.percent", 414 | UpdateType: models.Replace, 415 | Fields: []models.Field{ 416 | {Name: "App", Type: models.StringType}, 417 | {Name: "Percent", Type: models.PercentageType}, 418 | }, 419 | }, 420 | data: models.DatasetRows{ 421 | { 422 | "app": "acceptance", 423 | "percent": 0.43, 424 | }, 425 | { 426 | "app": "redis", 427 | "percent": 0.22, 428 | }, 429 | { 430 | "app": "api", 431 | "percent": 0.66, 432 | }, 433 | }, 434 | requests: []request{ 435 | { 436 | Method: http.MethodPut, 437 | Path: "/datasets/app.reliable.percent/data", 438 | Body: `{"data":[{"app":"acceptance","percent":0.43},{"app":"redis","percent":0.22},{"app":"api","percent":0.66}]}`, 439 | }, 440 | }, 441 | }, 442 | { 443 | //Append with no data makes no requests 444 | dataset: models.Dataset{ 445 | Name: "append.no.data", 446 | UpdateType: models.Append, 447 | Fields: []models.Field{ 448 | {Name: "App", Type: models.StringType}, 449 | {Name: "Count", Type: models.NumberType}, 450 | }, 451 | }, 452 | data: models.DatasetRows{}, 453 | requests: []request{}, 454 | }, 455 | { 456 | //Append dataset under the batch rows limit 457 | dataset: models.Dataset{ 458 | Name: "app.builds.count", 459 | UpdateType: models.Append, 460 | Fields: []models.Field{ 461 | {Name: "App", Type: models.StringType}, 462 | {Name: "Count", Type: models.NumberType}, 463 | }, 464 | }, 465 | data: models.DatasetRows{ 466 | { 467 | "app": "acceptance", 468 | "count": 88, 469 | }, 470 | { 471 | "app": "redis", 472 | "count": 55, 473 | }, 474 | { 475 | "app": "api", 476 | "count": 214, 477 | }, 478 | }, 479 | requests: []request{ 480 | { 481 | Method: http.MethodPost, 482 | Path: "/datasets/app.builds.count/data", 483 | Body: `{"data":[{"app":"acceptance","count":88},{"app":"redis","count":55},{"app":"api","count":214}]}`, 484 | }, 485 | }, 486 | }, 487 | { 488 | //Replace dataset over the batch rows limit sends first 3 and errors 489 | dataset: models.Dataset{ 490 | Name: "app.build.costs", 491 | UpdateType: models.Replace, 492 | Fields: []models.Field{ 493 | {Name: "App", Type: models.StringType}, 494 | {Name: "Cost", Type: models.MoneyType}, 495 | }, 496 | }, 497 | data: models.DatasetRows{ 498 | { 499 | "app": "acceptance", 500 | "cost": 4421, 501 | }, 502 | { 503 | "app": "redis", 504 | "cost": 221, 505 | }, 506 | { 507 | "app": "api", 508 | "cost": 212, 509 | }, 510 | { 511 | "app": "integration", 512 | "cost": 121, 513 | }, 514 | }, 515 | requests: []request{ 516 | { 517 | Method: http.MethodPut, 518 | Path: "/datasets/app.build.costs/data", 519 | Body: `{"data":[{"app":"acceptance","cost":4421},{"app":"redis","cost":221},{"app":"api","cost":212}]}`, 520 | }, 521 | }, 522 | maxRows: 3, 523 | err: fmt.Sprintf(errMoreRowsToSend, 4, 3), 524 | }, 525 | { 526 | //Append dataset sends all data in batches 527 | dataset: models.Dataset{ 528 | Name: "animal.run.time", 529 | UpdateType: models.Append, 530 | Fields: []models.Field{ 531 | {Name: "animal", Type: models.StringType}, 532 | {Name: "Run time", Type: models.NumberType}, 533 | }, 534 | }, 535 | data: models.DatasetRows{ 536 | { 537 | "animal": "worm", 538 | "run_time": 621, 539 | }, 540 | { 541 | "animal": "snail", 542 | "run_time": 521, 543 | }, 544 | { 545 | "animal": "duck", 546 | "run_time": 41, 547 | }, 548 | { 549 | "animal": "geese", 550 | "run_time": 44, 551 | }, 552 | { 553 | "animal": "terrapin", 554 | "run_time": 444, 555 | }, 556 | { 557 | "animal": "bird", 558 | "run_time": 22, 559 | }, 560 | }, 561 | requests: []request{ 562 | { 563 | Method: http.MethodPost, 564 | Path: "/datasets/animal.run.time/data", 565 | Body: `{"data":[{"animal":"worm","run_time":621},{"animal":"snail","run_time":521},{"animal":"duck","run_time":41}]}`, 566 | }, 567 | { 568 | Method: http.MethodPost, 569 | Path: "/datasets/animal.run.time/data", 570 | Body: `{"data":[{"animal":"geese","run_time":44},{"animal":"terrapin","run_time":444},{"animal":"bird","run_time":22}]}`, 571 | }, 572 | }, 573 | maxRows: 3, 574 | }, 575 | { 576 | //Append dataset sends all data in batches remaining one 577 | dataset: models.Dataset{ 578 | Name: "animal.run.time", 579 | UpdateType: models.Append, 580 | Fields: []models.Field{ 581 | {Name: "Animal", Type: models.StringType}, 582 | {Name: "Run time", Type: models.NumberType}, 583 | }, 584 | }, 585 | data: models.DatasetRows{ 586 | { 587 | "animal": "worm", 588 | "run_time": 621, 589 | }, 590 | { 591 | "animal": "snail", 592 | "run_time": 521, 593 | }, 594 | { 595 | "animal": "duck", 596 | "run_time": 41, 597 | }, 598 | { 599 | "animal": "geese", 600 | "run_time": 44, 601 | }, 602 | { 603 | "animal": "terrapin", 604 | "run_time": 444, 605 | }, 606 | { 607 | "animal": "bird", 608 | "run_time": 22, 609 | }, 610 | { 611 | "animal": "squirrel", 612 | "run_time": 88, 613 | }, 614 | }, 615 | requests: []request{ 616 | { 617 | Method: http.MethodPost, 618 | Path: "/datasets/animal.run.time/data", 619 | Body: `{"data":[{"animal":"worm","run_time":621},{"animal":"snail","run_time":521},{"animal":"duck","run_time":41}]}`, 620 | }, 621 | { 622 | Method: http.MethodPost, 623 | Path: "/datasets/animal.run.time/data", 624 | Body: `{"data":[{"animal":"geese","run_time":44},{"animal":"terrapin","run_time":444},{"animal":"bird","run_time":22}]}`, 625 | }, 626 | { 627 | Method: http.MethodPost, 628 | Path: "/datasets/animal.run.time/data", 629 | Body: `{"data":[{"animal":"squirrel","run_time":88}]}`, 630 | }, 631 | }, 632 | maxRows: 3, 633 | }, 634 | } 635 | 636 | for _, tc := range testCases { 637 | reqCount := 0 638 | 639 | if tc.maxRows != 0 { 640 | maxRows = tc.maxRows 641 | } else { 642 | maxRows = 500 643 | } 644 | 645 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 646 | reqCount++ 647 | 648 | if tc.response != nil { 649 | w.WriteHeader(tc.response.code) 650 | fmt.Fprintf(w, tc.response.body) 651 | return 652 | } 653 | 654 | if reqCount > len(tc.requests) { 655 | t.Errorf("Got unexpected extra requests") 656 | return 657 | } 658 | 659 | tcReq := tc.requests[reqCount-1] 660 | 661 | auth := r.Header.Get("Authorization") 662 | if auth != expAuth { 663 | t.Errorf("Expected authorization header '%s' but got '%s'", expAuth, auth) 664 | } 665 | 666 | if r.URL.Path != tcReq.Path { 667 | t.Errorf("Expected path '%s' but got '%s'", tcReq.Path, r.URL.Path) 668 | } 669 | 670 | if r.Method != tcReq.Method { 671 | t.Errorf("Expected method '%s' but got '%s'", tcReq.Method, r.Method) 672 | } 673 | 674 | b, err := ioutil.ReadAll(r.Body) 675 | if err != nil { 676 | t.Errorf("Errored while consuming body %s", err) 677 | } 678 | 679 | body := strings.Trim(string(b), "\n") 680 | 681 | if body != tcReq.Body { 682 | t.Errorf("Expected body '%s' but got '%s'", tcReq.Body, body) 683 | } 684 | })) 685 | 686 | gbHost = server.URL 687 | 688 | c := NewClient(apiKey) 689 | err := c.SendAllData(&tc.dataset, tc.data) 690 | 691 | if err != nil && tc.err == "" { 692 | t.Errorf("Expected no error but got '%s'", err) 693 | } 694 | 695 | if err == nil && tc.err != "" { 696 | t.Errorf("Expected error '%s' but got none", tc.err) 697 | } 698 | 699 | if err != nil && err.Error() != tc.err { 700 | t.Errorf("Expected error '%s' but got '%s'", tc.err, err) 701 | } 702 | 703 | if reqCount != len(tc.requests) { 704 | t.Errorf("Expected %d requests but got %d", len(tc.requests), reqCount) 705 | } 706 | 707 | server.Close() 708 | } 709 | } 710 | -------------------------------------------------------------------------------- /drivers/conn_string.go: -------------------------------------------------------------------------------- 1 | package drivers 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "sort" 8 | 9 | "github.com/geckoboard/sql-dataset/models" 10 | ) 11 | 12 | const ( 13 | defaultHost = "localhost" 14 | tcpConn = "tcp" 15 | ) 16 | 17 | var ( 18 | errDatabaseRequired = errors.New("No database provided.") 19 | errUsernameRequired = errors.New("No username provided.") 20 | ) 21 | 22 | type ConnStringBuilder interface { 23 | Build(*models.DatabaseConfig) (string, error) 24 | } 25 | 26 | func NewConnStringBuilder(driver string) (ConnStringBuilder, error) { 27 | switch driver { 28 | case models.PostgresDriver: 29 | return postgres{}, nil 30 | case models.MySQLDriver: 31 | return mysql{}, nil 32 | case models.SQLiteDriver: 33 | return sqlite{}, nil 34 | case models.MSSQLDriver: 35 | return mssql{}, nil 36 | default: 37 | return nil, fmt.Errorf("%s is not supported driver. SQL-Dataset supports %s", driver, models.SupportedDrivers) 38 | } 39 | } 40 | 41 | func buildParams(buf *bytes.Buffer, str string) { 42 | if str == "" || str == "=" { 43 | return 44 | } 45 | 46 | if buf.Len() > 0 { 47 | buf.WriteString("&") 48 | } 49 | 50 | buf.WriteString(str) 51 | } 52 | 53 | func orderKeys(kv map[string]string) []string { 54 | var keys []string 55 | 56 | for k := range kv { 57 | keys = append(keys, k) 58 | } 59 | 60 | sort.Strings(keys) 61 | return keys 62 | } 63 | -------------------------------------------------------------------------------- /drivers/conn_string_test.go: -------------------------------------------------------------------------------- 1 | package drivers 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/geckoboard/sql-dataset/models" 9 | ) 10 | 11 | func TestNewConnStringBuilder(t *testing.T) { 12 | testCases := []struct { 13 | in models.DatabaseConfig 14 | out string 15 | err string 16 | isDriverErr bool 17 | }{ 18 | //SQLite Driver 19 | { 20 | in: models.DatabaseConfig{ 21 | Driver: models.SQLiteDriver, 22 | }, 23 | err: errDatabaseRequired.Error(), 24 | }, 25 | { 26 | in: models.DatabaseConfig{ 27 | Driver: models.SQLiteDriver, 28 | Database: "models/fixtures/db.sqlite", 29 | }, 30 | out: "models/fixtures/db.sqlite", 31 | }, 32 | { 33 | in: models.DatabaseConfig{ 34 | Driver: models.SQLiteDriver, 35 | Database: "dir/db.sqlite", 36 | Password: "blah123", 37 | Params: map[string]string{ 38 | "cache": "shared", 39 | "mode": "rwc", 40 | }, 41 | }, 42 | out: "file:dir/db.sqlite?password=blah123&cache=shared&mode=rwc", 43 | }, 44 | //Mysql Driver 45 | { 46 | in: models.DatabaseConfig{ 47 | Driver: models.MySQLDriver, 48 | }, 49 | err: errDatabaseRequired.Error(), 50 | }, 51 | { 52 | in: models.DatabaseConfig{ 53 | Driver: models.MySQLDriver, 54 | Database: "some_name", 55 | }, 56 | err: errUsernameRequired.Error(), 57 | }, 58 | { 59 | in: models.DatabaseConfig{ 60 | Driver: models.MySQLDriver, 61 | Username: "root", 62 | Database: "someDB", 63 | }, 64 | out: "root@tcp(localhost:3306)/someDB?parseTime=true", 65 | }, 66 | { 67 | in: models.DatabaseConfig{ 68 | Driver: models.MySQLDriver, 69 | Username: "root", 70 | Password: "fp123", 71 | Database: "someDB", 72 | }, 73 | out: "root:fp123@tcp(localhost:3306)/someDB?parseTime=true", 74 | }, 75 | { 76 | in: models.DatabaseConfig{ 77 | Driver: models.MySQLDriver, 78 | Username: "root", 79 | Password: "fp123", 80 | Database: "someDB", 81 | Host: "fake-host", 82 | }, 83 | out: "root:fp123@tcp(fake-host:3306)/someDB?parseTime=true", 84 | }, 85 | { 86 | in: models.DatabaseConfig{ 87 | Driver: models.MySQLDriver, 88 | Username: "root", 89 | Password: "fp123", 90 | Database: "someDB", 91 | Host: "fake-host", 92 | Port: "3366", 93 | }, 94 | out: "root:fp123@tcp(fake-host:3366)/someDB?parseTime=true", 95 | }, 96 | { 97 | //Unix socket connection 98 | in: models.DatabaseConfig{ 99 | Driver: models.MySQLDriver, 100 | Username: "root", 101 | Password: "fp123", 102 | Database: "someDB", 103 | Host: "/tmp/mysql", 104 | Protocol: "unix", 105 | }, 106 | out: "root:fp123@unix(/tmp/mysql)/someDB?parseTime=true", 107 | }, 108 | { 109 | //IPv6 needs to be in square brackets 110 | in: models.DatabaseConfig{ 111 | Driver: models.MySQLDriver, 112 | Username: "root", 113 | Password: "fp123", 114 | Database: "someDB", 115 | Host: "de:ad:be:ef::ca:fe", 116 | }, 117 | out: "root:fp123@tcp([de:ad:be:ef::ca:fe]:3306)/someDB?parseTime=true", 118 | }, 119 | { 120 | in: models.DatabaseConfig{ 121 | Driver: models.MySQLDriver, 122 | Username: "root", 123 | Password: "fp123", 124 | Database: "someDB", 125 | Host: "project-id:region:instance", 126 | Protocol: "cloudsql", 127 | }, 128 | out: "root:fp123@cloudsql(project-id:region:instance)/someDB?parseTime=true", 129 | }, 130 | { 131 | in: models.DatabaseConfig{ 132 | Driver: models.MySQLDriver, 133 | Username: "root", 134 | Password: "fp123", 135 | Database: "someDB", 136 | Params: map[string]string{ 137 | "charset": "utf8mb4,utf8", 138 | "loc": "US/Pacific", 139 | }, 140 | }, 141 | out: "root:fp123@tcp(localhost:3306)/someDB?charset=utf8mb4,utf8&loc=US/Pacific&parseTime=true", 142 | }, 143 | { 144 | // ca cert file path 145 | in: models.DatabaseConfig{ 146 | Driver: models.MySQLDriver, 147 | Username: "root", 148 | Password: "fp123", 149 | Database: "someDB", 150 | TLSConfig: &models.TLSConfig{ 151 | CAFile: filepath.Join("..", "models", "fixtures", "ca.cert.pem"), 152 | }, 153 | }, 154 | out: "root:fp123@tcp(localhost:3306)/someDB?parseTime=true&tls=customCert", 155 | }, 156 | { 157 | // invalid ca cert file 158 | in: models.DatabaseConfig{ 159 | Driver: models.MySQLDriver, 160 | Username: "root", 161 | Password: "fp123", 162 | Database: "someDB", 163 | TLSConfig: &models.TLSConfig{ 164 | CAFile: filepath.Join("..", "models", "fixtures", "ca.key.pem"), 165 | }, 166 | }, 167 | err: "SSL error: Failed to append PEM. Please check that it's a valid CA certificate.", 168 | }, 169 | { 170 | // ssl only 171 | in: models.DatabaseConfig{ 172 | Driver: models.MySQLDriver, 173 | Username: "root", 174 | Password: "fp123", 175 | Database: "someDB", 176 | TLSConfig: &models.TLSConfig{ 177 | SSLMode: "true", 178 | }, 179 | }, 180 | out: "root:fp123@tcp(localhost:3306)/someDB?parseTime=true&tls=true", 181 | }, 182 | { 183 | // key and cert file path 184 | in: models.DatabaseConfig{ 185 | Driver: models.MySQLDriver, 186 | Username: "root", 187 | Password: "fp123", 188 | Database: "someDB", 189 | TLSConfig: &models.TLSConfig{ 190 | KeyFile: filepath.Join("..", "models", "fixtures", "test.key"), 191 | CertFile: filepath.Join("..", "models", "fixtures", "test.crt"), 192 | }, 193 | }, 194 | out: "root:fp123@tcp(localhost:3306)/someDB?parseTime=true&tls=customCert", 195 | }, 196 | //Postgres Driver 197 | { 198 | in: models.DatabaseConfig{ 199 | Driver: models.PostgresDriver, 200 | }, 201 | err: errDatabaseRequired.Error(), 202 | }, 203 | { 204 | in: models.DatabaseConfig{ 205 | Driver: models.PostgresDriver, 206 | Database: "some_name", 207 | }, 208 | err: errUsernameRequired.Error(), 209 | }, 210 | { 211 | in: models.DatabaseConfig{ 212 | Driver: models.PostgresDriver, 213 | Username: "root", 214 | Database: "someDB", 215 | }, 216 | out: "postgres://root@localhost:5432/someDB", 217 | }, 218 | { 219 | in: models.DatabaseConfig{ 220 | Driver: models.PostgresDriver, 221 | Username: "root", 222 | Password: "fp123", 223 | Database: "someDB", 224 | }, 225 | out: "postgres://root:fp123@localhost:5432/someDB", 226 | }, 227 | { 228 | in: models.DatabaseConfig{ 229 | Driver: models.PostgresDriver, 230 | Username: "root", 231 | Password: "fp123", 232 | Database: "someDB", 233 | Host: "fake-host", 234 | }, 235 | out: "postgres://root:fp123@fake-host:5432/someDB", 236 | }, 237 | { 238 | in: models.DatabaseConfig{ 239 | Driver: models.PostgresDriver, 240 | Username: "root", 241 | Password: "fp123", 242 | Database: "someDB", 243 | Host: "fake-host", 244 | Port: "5433", 245 | }, 246 | out: "postgres://root:fp123@fake-host:5433/someDB", 247 | }, 248 | { 249 | //Unix socket connection 250 | in: models.DatabaseConfig{ 251 | Driver: models.PostgresDriver, 252 | Username: "root", 253 | Password: "fp123", 254 | Database: "someDB", 255 | Host: "/var/run/postgresql/.s.PGSQL.5432", 256 | Protocol: "unix", 257 | }, 258 | out: "postgres://root:fp123@/var/run/postgresql/.s.PGSQL.5432/someDB", 259 | }, 260 | { 261 | //IPv6 needs to be in square brackets 262 | in: models.DatabaseConfig{ 263 | Driver: models.PostgresDriver, 264 | Username: "root", 265 | Password: "fp123", 266 | Database: "someDB", 267 | Host: "de:ad:be:ef::ca:fe", 268 | }, 269 | out: "postgres://root:fp123@[de:ad:be:ef::ca:fe]:5432/someDB", 270 | }, 271 | { 272 | in: models.DatabaseConfig{ 273 | Driver: models.PostgresDriver, 274 | Username: "root", 275 | Password: "fp123", 276 | Database: "someDB", 277 | Params: map[string]string{ 278 | "client_encoding": "utf8mb4", 279 | "datestyle": "ISO, MDY", 280 | }, 281 | }, 282 | out: "postgres://root:fp123@localhost:5432/someDB?client_encoding=utf8mb4&datestyle=ISO, MDY", 283 | }, 284 | { 285 | // ca cert file path 286 | in: models.DatabaseConfig{ 287 | Driver: models.PostgresDriver, 288 | Username: "root", 289 | Password: "fp123", 290 | Database: "someDB", 291 | Host: "fake-host", 292 | TLSConfig: &models.TLSConfig{ 293 | CAFile: filepath.Join("models", "fixtures", "ca.cert.pem"), 294 | }, 295 | }, 296 | out: fmt.Sprintf("postgres://root:fp123@fake-host:5432/someDB?sslrootcert=%s", 297 | filepath.Join("models", "fixtures", "ca.cert.pem"), 298 | ), 299 | }, 300 | { 301 | // key and cert file path 302 | in: models.DatabaseConfig{ 303 | Driver: models.PostgresDriver, 304 | Username: "root", 305 | Password: "fp123", 306 | Database: "someDB", 307 | TLSConfig: &models.TLSConfig{ 308 | KeyFile: filepath.Join("models", "fixtures", "test.key"), 309 | CertFile: filepath.Join("models", "fixtures", "test.crt"), 310 | SSLMode: "verify-full", 311 | }, 312 | }, 313 | out: fmt.Sprintf("postgres://root:fp123@localhost:5432/someDB?sslcert=%s&sslkey=%s&sslmode=%s", 314 | filepath.Join("models", "fixtures", "test.crt"), 315 | filepath.Join("models", "fixtures", "test.key"), 316 | "verify-full", 317 | ), 318 | }, 319 | // MSSQL Driver 320 | { 321 | in: models.DatabaseConfig{ 322 | Driver: models.MSSQLDriver, 323 | }, 324 | err: errDatabaseRequired.Error(), 325 | }, 326 | { 327 | in: models.DatabaseConfig{ 328 | Driver: models.MSSQLDriver, 329 | Database: "some_name", 330 | }, 331 | err: errUsernameRequired.Error(), 332 | }, 333 | { 334 | in: models.DatabaseConfig{ 335 | Driver: models.MSSQLDriver, 336 | Username: "root", 337 | Database: "someDB", 338 | }, 339 | out: "odbc:server={localhost};port=1433;user id={root};;database=someDB;ApplicationIntent=ReadOnly", 340 | }, 341 | { 342 | in: models.DatabaseConfig{ 343 | Driver: models.MSSQLDriver, 344 | Username: "root", 345 | Password: "fp123", 346 | Database: "someDB", 347 | }, 348 | out: "odbc:server={localhost};port=1433;user id={root};password={fp123};database=someDB;ApplicationIntent=ReadOnly", 349 | }, 350 | { 351 | in: models.DatabaseConfig{ 352 | Driver: models.MSSQLDriver, 353 | Username: "root", 354 | Password: "fp123", 355 | Database: "someDB", 356 | Host: "fake-host", 357 | }, 358 | out: "odbc:server={fake-host};port=1433;user id={root};password={fp123};database=someDB;ApplicationIntent=ReadOnly", 359 | }, 360 | { 361 | in: models.DatabaseConfig{ 362 | Driver: models.MSSQLDriver, 363 | Username: "root", 364 | Password: "fp123", 365 | Database: "someDB", 366 | Host: "fake-host", 367 | Port: "5433", 368 | }, 369 | out: "odbc:server={fake-host};port=5433;user id={root};password={fp123};database=someDB;ApplicationIntent=ReadOnly", 370 | }, 371 | { 372 | in: models.DatabaseConfig{ 373 | Driver: models.MSSQLDriver, 374 | Username: "root", 375 | Password: "fp123", 376 | Database: "someDB", 377 | Params: map[string]string{ 378 | "": "", 379 | }, 380 | }, 381 | out: "odbc:server={localhost};port=1433;user id={root};password={fp123};database=someDB;ApplicationIntent=ReadOnly", 382 | }, 383 | { 384 | in: models.DatabaseConfig{ 385 | Driver: models.MSSQLDriver, 386 | Username: "root", 387 | Password: "fp123", 388 | Database: "someDB", 389 | Params: map[string]string{ 390 | "connection timeout": "10", 391 | "dial timeout": "2", 392 | }, 393 | }, 394 | out: "odbc:server={localhost};port=1433;user id={root};password={fp123};database=someDB;ApplicationIntent=ReadOnly;connection timeout=10;dial timeout=2", 395 | }, 396 | { 397 | // ca cert file path 398 | in: models.DatabaseConfig{ 399 | Driver: models.MSSQLDriver, 400 | Username: "root", 401 | Password: "fp123", 402 | Database: "someDB", 403 | Host: "fake-host", 404 | TLSConfig: &models.TLSConfig{ 405 | CAFile: filepath.Join("models", "fixtures", "ca.cert.pem"), 406 | }, 407 | }, 408 | out: fmt.Sprintf("odbc:server={fake-host};port=1433;user id={root};password={fp123};database=someDB;ApplicationIntent=ReadOnly;certificate=%s", filepath.Join("models", "fixtures", "ca.cert.pem")), 409 | }, 410 | { 411 | // ca cert file path 412 | in: models.DatabaseConfig{ 413 | Driver: models.MSSQLDriver, 414 | Username: "root", 415 | Password: "fp123", 416 | Database: "someDB", 417 | Host: "fake-host", 418 | TLSConfig: &models.TLSConfig{ 419 | CAFile: "fakeCAFile", 420 | SSLMode: "true", 421 | }, 422 | Params: map[string]string{ 423 | "hostNameInCertificate": "overriddenHost", 424 | }, 425 | }, 426 | out: "odbc:server={fake-host};port=1433;user id={root};password={fp123};database=someDB;ApplicationIntent=ReadOnly;certificate=fakeCAFile;encrypt=true;hostNameInCertificate=overriddenHost", 427 | }, 428 | { 429 | // key file path supplied not permitted 430 | in: models.DatabaseConfig{ 431 | Driver: models.MSSQLDriver, 432 | Username: "root", 433 | Password: "fp123", 434 | Database: "someDB", 435 | TLSConfig: &models.TLSConfig{ 436 | KeyFile: filepath.Join("models", "fixtures", "test.key"), 437 | }, 438 | }, 439 | err: "Key file not supported, only ca_file is for MSSQL Driver", 440 | }, 441 | { 442 | // key file path supplied not permitted 443 | in: models.DatabaseConfig{ 444 | Driver: models.MSSQLDriver, 445 | Username: "root", 446 | Password: "fp123", 447 | Database: "someDB", 448 | TLSConfig: &models.TLSConfig{ 449 | CertFile: filepath.Join("models", "fixtures", "test.crt"), 450 | }, 451 | }, 452 | err: "Cert file not supported, only ca_file is for MSSQL Driver", 453 | }, 454 | // None existing driver 455 | // This really should never happen because of config validation 456 | { 457 | in: models.DatabaseConfig{ 458 | Driver: "PearDB", 459 | }, 460 | err: "PearDB is not supported driver. SQL-Dataset supports [mssql mysql postgres sqlite3]", 461 | isDriverErr: true, 462 | }, 463 | } 464 | 465 | for i, tc := range testCases { 466 | n, err := NewConnStringBuilder(tc.in.Driver) 467 | if err != nil { 468 | if tc.isDriverErr { 469 | if tc.err != err.Error() { 470 | t.Errorf("Expected driver error %s but got %s", tc.err, err) 471 | } 472 | } else { 473 | t.Error(err) 474 | } 475 | 476 | continue 477 | } 478 | 479 | if tc.isDriverErr && err == nil { 480 | t.Errorf("Expected driver error %s but got none", tc.err) 481 | } 482 | 483 | conn, err := n.Build(&tc.in) 484 | 485 | if tc.err == "" && err != nil { 486 | t.Errorf("[%d] Expected no error but got %s", i, err) 487 | } 488 | 489 | if tc.err != "" && err == nil { 490 | t.Errorf("[%d] Expected error %s but got nothing", i, tc.err) 491 | } 492 | 493 | if err != nil && tc.err != err.Error() { 494 | t.Errorf("[%d] Expected error %s but got %s", i, tc.err, err) 495 | } 496 | 497 | if conn != tc.out { 498 | t.Errorf("[%d] Expected dsn connection string '%s' but got '%s'", i, tc.out, conn) 499 | } 500 | } 501 | } 502 | -------------------------------------------------------------------------------- /drivers/mssql.go: -------------------------------------------------------------------------------- 1 | package drivers 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/geckoboard/sql-dataset/models" 9 | ) 10 | 11 | type mssql struct{} 12 | 13 | const ( 14 | mssqlPort = "1433" 15 | ) 16 | 17 | /* 18 | SSL Supported Modes 19 | https://github.com/denisenkom/go-mssqldb/blob/master/README.md 20 | 21 | disable - Data send between client and server is not encrypted. 22 | false - Data sent between client and server is not encrypted beyond the login packet. (Default) 23 | true - Data sent between client and server is encrypted. 24 | */ 25 | 26 | func (ms mssql) Build(dc *models.DatabaseConfig) (string, error) { 27 | ms.setDefaults(dc) 28 | 29 | if dc.Database == "" { 30 | return "", errDatabaseRequired 31 | } 32 | 33 | // It might be possible to support Windows single sign on 34 | // however this means username can be empty. For now lets not support 35 | // not sure what is involved - I think it needs SPN (Kerberos) :( 36 | if dc.Username == "" { 37 | return "", errUsernameRequired 38 | } 39 | 40 | if err := ms.buildTLSParams(dc); err != nil { 41 | return "", err 42 | } 43 | 44 | return ms.buildConnString(dc), nil 45 | } 46 | 47 | func (ms mssql) buildConnString(dc *models.DatabaseConfig) string { 48 | var buf bytes.Buffer 49 | var password string 50 | 51 | // Shouldn't be the case with password policies 52 | if dc.Password != "" { 53 | password = fmt.Sprintf("password={%s}", dc.Password) 54 | } 55 | 56 | keys := orderKeys(dc.Params) 57 | for _, k := range keys { 58 | ms.buildParams(&buf, fmt.Sprintf("%s=%s", k, dc.Params[k])) 59 | } 60 | 61 | conn := fmt.Sprintf("odbc:server={%s};port=%s;user id={%s};%s;database=%s", 62 | dc.Host, dc.Port, dc.Username, password, dc.Database) 63 | 64 | if buf.Len() > 0 { 65 | conn = fmt.Sprintf(conn+";%s", buf.String()) 66 | } 67 | 68 | return conn 69 | } 70 | 71 | func (ms mssql) buildTLSParams(dc *models.DatabaseConfig) error { 72 | tc := dc.TLSConfig 73 | 74 | if tc == nil { 75 | return nil 76 | } 77 | 78 | if tc.SSLMode != "" { 79 | dc.Params["encrypt"] = tc.SSLMode 80 | } 81 | 82 | if tc.CAFile != "" { 83 | dc.Params["certificate"] = tc.CAFile 84 | } 85 | 86 | if tc.KeyFile != "" { 87 | return errors.New("Key file not supported, only ca_file is for MSSQL Driver") 88 | } 89 | 90 | if tc.CertFile != "" { 91 | return errors.New("Cert file not supported, only ca_file is for MSSQL Driver") 92 | } 93 | 94 | return nil 95 | } 96 | 97 | func (ms mssql) setDefaults(dc *models.DatabaseConfig) { 98 | if dc.Params == nil { 99 | dc.Params = make(map[string]string) 100 | } 101 | 102 | if dc.Host == "" { 103 | dc.Host = defaultHost 104 | } 105 | 106 | if dc.Port == "" { 107 | dc.Port = mssqlPort 108 | } 109 | 110 | // Set the application intent to readonly connection 111 | dc.Params["ApplicationIntent"] = "ReadOnly" 112 | } 113 | 114 | /* buildParams builds a buffer of query parameters, specified by the 115 | user or system defaults, seperated by semi-colon in the following format. 116 | 117 | ApplicationIntent=ReadOnly;connection timeout=10;dial timeout=2 118 | */ 119 | func (ms mssql) buildParams(buf *bytes.Buffer, str string) { 120 | if str == "=" { 121 | return 122 | } 123 | 124 | if buf.Len() > 0 { 125 | buf.WriteString(";") 126 | } 127 | 128 | buf.WriteString(str) 129 | } 130 | -------------------------------------------------------------------------------- /drivers/mysql.go: -------------------------------------------------------------------------------- 1 | package drivers 2 | 3 | import ( 4 | "bytes" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "fmt" 8 | "io/ioutil" 9 | "net" 10 | 11 | "github.com/geckoboard/sql-dataset/models" 12 | msql "github.com/go-sql-driver/mysql" 13 | ) 14 | 15 | /* 16 | SSL Supported Modes 17 | https://github.com/go-sql-driver/mysql#tls 18 | 19 | false - no SSL (default) 20 | true - use ssl connection 21 | skip-verify - use self-signed or invalid server cert 22 | customCert - name of the registered tls config (automatic when supplying ssl (ca,cert,key) 23 | */ 24 | 25 | type mysql struct{} 26 | 27 | const ( 28 | mysqlTLSKey = "customCert" 29 | mysqlPort = "3306" 30 | ) 31 | 32 | func (m mysql) Build(dc *models.DatabaseConfig) (string, error) { 33 | var buf bytes.Buffer 34 | 35 | m.setDefaults(dc) 36 | 37 | if dc.Database == "" { 38 | return "", errDatabaseRequired 39 | } 40 | 41 | if dc.Username == "" { 42 | return "", errUsernameRequired 43 | } 44 | 45 | if dc.TLSConfig != nil { 46 | str, err := m.registerTLS(dc.TLSConfig) 47 | 48 | if err != nil { 49 | return "", err 50 | } 51 | 52 | dc.Params["tls"] = str 53 | } 54 | 55 | keys := orderKeys(dc.Params) 56 | for _, k := range keys { 57 | buildParams(&buf, fmt.Sprintf("%s=%s", k, dc.Params[k])) 58 | } 59 | 60 | return fmt.Sprintf("%s?%s", m.buildConnString(dc), buf.String()), nil 61 | } 62 | 63 | func (m mysql) loadCerts(keyFile, certFile, caFile string) (*x509.CertPool, []tls.Certificate, error) { 64 | var rootCertPool *x509.CertPool 65 | 66 | if caFile != "" { 67 | rootCertPool = x509.NewCertPool() 68 | 69 | pem, err := ioutil.ReadFile(caFile) 70 | if err != nil { 71 | return nil, nil, err 72 | } 73 | 74 | if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { 75 | return nil, nil, fmt.Errorf("SSL error: Failed to append PEM. " + 76 | "Please check that it's a valid CA certificate.") 77 | } 78 | } 79 | 80 | var clientCert []tls.Certificate 81 | 82 | if certFile != "" && keyFile != "" { 83 | certs, err := tls.LoadX509KeyPair(certFile, keyFile) 84 | if err != nil { 85 | return nil, nil, fmt.Errorf("There was an error while "+ 86 | "loading your x509 key pair: %s", err) 87 | } 88 | 89 | clientCert = append(clientCert, certs) 90 | } 91 | 92 | return rootCertPool, clientCert, nil 93 | } 94 | 95 | func (m mysql) registerTLS(tlsConfig *models.TLSConfig) (string, error) { 96 | if tlsConfig.KeyFile == "" && tlsConfig.CertFile == "" && 97 | tlsConfig.CAFile == "" && tlsConfig.SSLMode != "" { 98 | return tlsConfig.SSLMode, nil 99 | } 100 | 101 | rootCertPool, clientCert, err := m.loadCerts( 102 | tlsConfig.KeyFile, 103 | tlsConfig.CertFile, 104 | tlsConfig.CAFile, 105 | ) 106 | 107 | if err != nil { 108 | return "", err 109 | } 110 | 111 | return mysqlTLSKey, msql.RegisterTLSConfig(mysqlTLSKey, &tls.Config{ 112 | RootCAs: rootCertPool, 113 | Certificates: clientCert, 114 | }) 115 | } 116 | 117 | func (m mysql) buildConnString(dc *models.DatabaseConfig) string { 118 | var auth, netHost string 119 | 120 | if dc.Password == "" { 121 | auth = dc.Username 122 | } else { 123 | auth = fmt.Sprintf("%s:%s", dc.Username, dc.Password) 124 | } 125 | 126 | if dc.Protocol == tcpConn { 127 | netHost = net.JoinHostPort(dc.Host, dc.Port) 128 | } else { 129 | netHost = dc.Host 130 | } 131 | 132 | return fmt.Sprintf("%s@%s(%s)/%s", auth, dc.Protocol, netHost, dc.Database) 133 | } 134 | 135 | func (m mysql) setDefaults(dc *models.DatabaseConfig) { 136 | if dc.Params == nil { 137 | dc.Params = make(map[string]string) 138 | } 139 | 140 | if dc.Host == "" { 141 | dc.Host = defaultHost 142 | } 143 | 144 | if dc.Protocol == "" { 145 | dc.Protocol = tcpConn 146 | } 147 | 148 | if dc.Port == "" && dc.Protocol == tcpConn { 149 | dc.Port = mysqlPort 150 | } 151 | 152 | dc.Params["parseTime"] = "true" 153 | } 154 | -------------------------------------------------------------------------------- /drivers/postgres.go: -------------------------------------------------------------------------------- 1 | package drivers 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net" 7 | 8 | "github.com/geckoboard/sql-dataset/models" 9 | ) 10 | 11 | type postgres struct{} 12 | 13 | const ( 14 | postgresPort = "5432" 15 | connPrefix = "postgres://" 16 | ) 17 | 18 | /* 19 | SSL Supported Modes 20 | https://github.com/lib/pq/blob/068cb1c8e4be77b9bdef4d0d91f162160537779e/doc.go 21 | 22 | disable - No SSL 23 | require - Always SSL (skip verification) 24 | verify-ca - Always SSL (verify server cert trusted CA) 25 | verify-full - Same as verify-ca and server host on cert matches 26 | */ 27 | 28 | func (p postgres) Build(dc *models.DatabaseConfig) (string, error) { 29 | var buf bytes.Buffer 30 | 31 | if dc.Database == "" { 32 | return "", errDatabaseRequired 33 | } 34 | 35 | if dc.Username == "" { 36 | return "", errUsernameRequired 37 | } 38 | 39 | p.setDefaults(dc) 40 | p.buildTLSParams(dc) 41 | 42 | keys := orderKeys(dc.Params) 43 | for _, k := range keys { 44 | buildParams(&buf, fmt.Sprintf("%s=%s", k, dc.Params[k])) 45 | } 46 | 47 | var paramSplit string 48 | if buf.Len() > 0 { 49 | paramSplit = "?" 50 | } 51 | 52 | return fmt.Sprintf("%s%s%s%s", 53 | connPrefix, 54 | p.buildConnString(dc), 55 | paramSplit, 56 | buf.String(), 57 | ), nil 58 | } 59 | 60 | func (p postgres) buildTLSParams(dc *models.DatabaseConfig) { 61 | tc := dc.TLSConfig 62 | 63 | if tc == nil { 64 | return 65 | } 66 | 67 | if tc.SSLMode != "" { 68 | dc.Params["sslmode"] = tc.SSLMode 69 | } 70 | 71 | if tc.CAFile != "" { 72 | dc.Params["sslrootcert"] = tc.CAFile 73 | } 74 | 75 | if tc.KeyFile != "" { 76 | dc.Params["sslkey"] = tc.KeyFile 77 | } 78 | 79 | if tc.CertFile != "" { 80 | dc.Params["sslcert"] = tc.CertFile 81 | } 82 | } 83 | 84 | func (p postgres) buildConnString(dc *models.DatabaseConfig) string { 85 | var auth, netHost string 86 | 87 | if dc.Password == "" { 88 | auth = dc.Username 89 | } else { 90 | auth = fmt.Sprintf("%s:%s", dc.Username, dc.Password) 91 | } 92 | 93 | if dc.Protocol == tcpConn { 94 | netHost = net.JoinHostPort(dc.Host, dc.Port) 95 | } else { 96 | netHost = dc.Host 97 | } 98 | 99 | return fmt.Sprintf("%s@%s/%s", auth, netHost, dc.Database) 100 | } 101 | 102 | func (p postgres) setDefaults(dc *models.DatabaseConfig) { 103 | if dc.Params == nil { 104 | dc.Params = make(map[string]string) 105 | } 106 | 107 | if dc.Host == "" { 108 | dc.Host = defaultHost 109 | } 110 | 111 | if dc.Protocol == "" { 112 | dc.Protocol = tcpConn 113 | } 114 | 115 | if dc.Port == "" && dc.Protocol == tcpConn { 116 | dc.Port = postgresPort 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /drivers/sqlite.go: -------------------------------------------------------------------------------- 1 | package drivers 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "github.com/geckoboard/sql-dataset/models" 8 | ) 9 | 10 | type sqlite struct{} 11 | 12 | func (s sqlite) Build(dc *models.DatabaseConfig) (conn string, err error) { 13 | var buf bytes.Buffer 14 | 15 | if dc.Database == "" { 16 | return "", errDatabaseRequired 17 | } 18 | 19 | if dc.Password != "" { 20 | buildParams(&buf, fmt.Sprintf("password=%s", dc.Password)) 21 | } 22 | 23 | keys := orderKeys(dc.Params) 24 | for _, k := range keys { 25 | buildParams(&buf, fmt.Sprintf("%s=%s", k, dc.Params[k])) 26 | } 27 | 28 | if buf.Len() > 0 { 29 | conn = fmt.Sprintf("file:%s?%s", dc.Database, buf.String()) 30 | } else { 31 | conn = dc.Database 32 | } 33 | 34 | return conn, err 35 | } 36 | -------------------------------------------------------------------------------- /example.yml: -------------------------------------------------------------------------------- 1 | geckoboard_api_key: your_api_key 2 | database: 3 | driver: mysql 4 | host: xxxx 5 | port: xxxx 6 | username: xxxx 7 | password: xxxx 8 | name: xxxx 9 | tls_config: 10 | ca_file: xxxx 11 | key_file: xxxx 12 | cert_file: xxxx 13 | ssl_mode: xxxx 14 | refresh_time_sec: 60 15 | datasets: 16 | - name: dataset.name 17 | update_type: replace 18 | sql: > 19 | SELECT 1, 0.34, string 20 | FROM table 21 | fields: 22 | - type: number 23 | name: Count 24 | - type: percentage 25 | name: Some Percent 26 | - type: string 27 | name: Some Label 28 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/geckoboard/sql-dataset 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/denisenkom/go-mssqldb v0.10.0 7 | github.com/go-sql-driver/mysql v1.6.0 8 | github.com/lib/pq v1.10.9 9 | github.com/mattn/go-sqlite3 v1.14.8 10 | golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e // indirect 11 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 12 | gopkg.in/guregu/null.v3 v3.5.0 13 | gopkg.in/yaml.v2 v2.4.0 14 | ) 15 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/denisenkom/go-mssqldb v0.10.0 h1:QykgLZBorFE95+gO3u9esLd0BmbvpWp0/waNNZfHBM8= 2 | github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= 3 | github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= 4 | github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 5 | github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= 6 | github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= 7 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= 8 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 9 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 10 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 11 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 12 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 13 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 14 | github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU= 15 | github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 16 | golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 17 | golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e h1:VvfwVmMH40bpMeizC9/K7ipM5Qjucuu16RWfneFPyhQ= 18 | golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 19 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 20 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 21 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 22 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 23 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 24 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 25 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 26 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 27 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 28 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 29 | gopkg.in/guregu/null.v3 v3.5.0 h1:xTcasT8ETfMcUHn0zTvIYtQud/9Mx5dJqD554SZct0o= 30 | gopkg.in/guregu/null.v3 v3.5.0/go.mod h1:E4tX2Qe3h7QdL+uZ3a0vqvYwKQsRSQKM5V4YltdgH9Y= 31 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 32 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 33 | -------------------------------------------------------------------------------- /integration_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "net/http/httptest" 8 | "path/filepath" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/geckoboard/sql-dataset/models" 13 | ) 14 | 15 | var originalBatchRows = 500 16 | 17 | type GBRequest struct { 18 | Path string 19 | Body string 20 | } 21 | 22 | func TestEndToEndFlow(t *testing.T) { 23 | testCases := []struct { 24 | config models.Config 25 | maxRows int 26 | expectError bool 27 | gbHits int 28 | gbReqs []GBRequest 29 | }{ 30 | { 31 | config: models.Config{ 32 | DatabaseConfig: &models.DatabaseConfig{ 33 | Driver: models.SQLiteDriver, 34 | URL: filepath.Join("models", "fixtures", "db.sqlite"), 35 | }, 36 | Datasets: []models.Dataset{ 37 | { 38 | Name: "app.counts", 39 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 40 | UpdateType: models.Append, 41 | Fields: []models.Field{ 42 | {Name: "App", Type: models.StringType}, 43 | {Name: "Build Count", Type: models.NumberType}, 44 | }, 45 | }, 46 | { 47 | Name: "app.build.costs", 48 | UpdateType: models.Append, 49 | SQL: "SELECT app_name, SUM(CAST(build_cost*100 AS INTEGER)) FROM builds GROUP BY app_name order by app_name", 50 | Fields: []models.Field{ 51 | {Name: "App", Type: models.StringType}, 52 | {Name: "Build Cost", Type: models.MoneyType, CurrencyCode: "USD"}, 53 | }, 54 | }, 55 | }, 56 | }, 57 | gbReqs: []GBRequest{ 58 | { 59 | Path: "/datasets/app.counts", 60 | Body: `{"id":"app.counts","fields":{"app":{"type":"string","name":"App"},"build_count":{"type":"number","name":"Build Count"}}}`, 61 | }, 62 | { 63 | Path: "/datasets/app.counts/data", 64 | Body: `{"data":[{"app":"","build_count":2},{"app":"everdeen","build_count":2},{"app":"geckoboard-ruby","build_count":3},{"app":"react","build_count":1},{"app":"westworld","build_count":1}]}`, 65 | }, 66 | { 67 | Path: "/datasets/app.build.costs", 68 | Body: `{"id":"app.build.costs","fields":{"app":{"type":"string","name":"App"},"build_cost":{"type":"money","name":"Build Cost","currency_code":"USD"}}}`, 69 | }, 70 | { 71 | Path: "/datasets/app.build.costs/data", 72 | Body: `{"data":[{"app":"","build_cost":1132},{"app":"everdeen","build_cost":198},{"app":"geckoboard-ruby","build_cost":116},{"app":"react","build_cost":111},{"app":"westworld","build_cost":264}]}`, 73 | }, 74 | }, 75 | }, 76 | { 77 | config: models.Config{ 78 | DatabaseConfig: &models.DatabaseConfig{ 79 | Driver: models.SQLiteDriver, 80 | URL: filepath.Join("models", "fixtures", "db.sqlite"), 81 | }, 82 | Datasets: []models.Dataset{ 83 | { 84 | Name: "app.durations", 85 | SQL: "SELECT app_name, run_time FROM builds ORDER BY app_name", 86 | UpdateType: models.Append, 87 | Fields: []models.Field{ 88 | {Name: "App", Type: models.StringType}, 89 | {Name: "Build runtime", Type: models.DurationType, TimeUnit: "minutes"}, 90 | }, 91 | }, 92 | }, 93 | }, 94 | gbReqs: []GBRequest{ 95 | { 96 | Path: "/datasets/app.durations", 97 | Body: `{"id":"app.durations","fields":{"app":{"type":"string","name":"App"},"build_runtime":{"type":"duration","name":"Build runtime","time_unit":"minutes"}}}`, 98 | }, 99 | { 100 | Path: "/datasets/app.durations/data", 101 | Body: `{"data":[{"app":"","build_runtime":0.12349876543},{"app":"","build_runtime":46.432763287},{"app":"everdeen","build_runtime":0.31882276212},{"app":"everdeen","build_runtime":144.31838122382},{"app":"geckoboard-ruby","build_runtime":0.21882232124},{"app":"geckoboard-ruby","build_runtime":77.21381276421},{"app":"geckoboard-ruby","build_runtime":0},{"app":"react","build_runtime":118.18382961212},{"app":"westworld","build_runtime":321.93774373}]}`, 102 | }, 103 | }, 104 | }, 105 | { 106 | // Replace update type sends the first batch only 107 | config: models.Config{ 108 | DatabaseConfig: &models.DatabaseConfig{ 109 | Driver: models.SQLiteDriver, 110 | URL: filepath.Join("models", "fixtures", "db.sqlite"), 111 | }, 112 | Datasets: []models.Dataset{ 113 | { 114 | Name: "apps.run.time", 115 | SQL: "SELECT app_name, run_time FROM builds ORDER BY app_name", 116 | UpdateType: models.Replace, 117 | Fields: []models.Field{ 118 | {Name: "App", Type: models.StringType}, 119 | {Name: "Run time", Type: models.NumberType}, 120 | }, 121 | }, 122 | }, 123 | }, 124 | maxRows: 4, 125 | expectError: true, 126 | gbReqs: []GBRequest{ 127 | { 128 | Path: "/datasets/apps.run.time", 129 | Body: `{"id":"apps.run.time","fields":{"app":{"type":"string","name":"App"},"run_time":{"type":"number","name":"Run time"}}}`, 130 | }, 131 | { 132 | Path: "/datasets/apps.run.time/data", 133 | Body: `{"data":[{"app":"","run_time":0.12349876543},{"app":"","run_time":46.432763287},{"app":"everdeen","run_time":0.31882276212},{"app":"everdeen","run_time":144.31838122382}]}`, 134 | }, 135 | }, 136 | }, 137 | { 138 | // Append update type sends multiple requests in batches of batchRow limit when more rows exist 139 | config: models.Config{ 140 | DatabaseConfig: &models.DatabaseConfig{ 141 | Driver: models.SQLiteDriver, 142 | URL: filepath.Join("models", "fixtures", "db.sqlite"), 143 | }, 144 | Datasets: []models.Dataset{ 145 | { 146 | Name: "apps.run.time", 147 | SQL: "SELECT app_name, run_time FROM builds ORDER BY app_name", 148 | UpdateType: models.Append, 149 | Fields: []models.Field{ 150 | {Name: "App", Type: models.StringType}, 151 | {Name: "Run time", Type: models.NumberType}, 152 | }, 153 | }, 154 | }, 155 | }, 156 | maxRows: 4, 157 | gbReqs: []GBRequest{ 158 | { 159 | Path: "/datasets/apps.run.time", 160 | Body: `{"id":"apps.run.time","fields":{"app":{"type":"string","name":"App"},"run_time":{"type":"number","name":"Run time"}}}`, 161 | }, 162 | { 163 | Path: "/datasets/apps.run.time/data", 164 | Body: `{"data":[{"app":"","run_time":0.12349876543},{"app":"","run_time":46.432763287},{"app":"everdeen","run_time":0.31882276212},{"app":"everdeen","run_time":144.31838122382}]}`, 165 | }, 166 | { 167 | Path: "/datasets/apps.run.time/data", 168 | Body: `{"data":[{"app":"geckoboard-ruby","run_time":0.21882232124},{"app":"geckoboard-ruby","run_time":77.21381276421},{"app":"geckoboard-ruby","run_time":0},{"app":"react","run_time":118.18382961212}]}`, 169 | }, 170 | { 171 | Path: "/datasets/apps.run.time/data", 172 | Body: `{"data":[{"app":"westworld","run_time":321.93774373}]}`, 173 | }, 174 | }, 175 | }, 176 | { 177 | // Unique by correctly used and sent - doesn't do validation used with the correct update type 178 | config: models.Config{ 179 | DatabaseConfig: &models.DatabaseConfig{ 180 | Driver: models.SQLiteDriver, 181 | URL: filepath.Join("models", "fixtures", "db.sqlite"), 182 | }, 183 | Datasets: []models.Dataset{ 184 | { 185 | Name: "app.counts", 186 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 187 | UpdateType: models.Append, 188 | UniqueBy: []string{"app"}, 189 | Fields: []models.Field{ 190 | {Name: "App", Type: models.StringType}, 191 | {Name: "Build Count", Type: models.NumberType}, 192 | }, 193 | }, 194 | }, 195 | }, 196 | gbReqs: []GBRequest{ 197 | { 198 | Path: "/datasets/app.counts", 199 | Body: `{"id":"app.counts","unique_by":["app"],"fields":{"app":{"type":"string","name":"App"},"build_count":{"type":"number","name":"Build Count"}}}`, 200 | }, 201 | { 202 | Path: "/datasets/app.counts/data", 203 | Body: `{"data":[{"app":"","build_count":2},{"app":"everdeen","build_count":2},{"app":"geckoboard-ruby","build_count":3},{"app":"react","build_count":1},{"app":"westworld","build_count":1}]}`, 204 | }, 205 | }, 206 | }, 207 | { 208 | // Unique by without a matching field errors makes no requests 209 | config: models.Config{ 210 | DatabaseConfig: &models.DatabaseConfig{ 211 | Driver: models.SQLiteDriver, 212 | URL: filepath.Join("models", "fixtures", "db.sqlite"), 213 | }, 214 | Datasets: []models.Dataset{ 215 | { 216 | Name: "app.counts", 217 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 218 | UpdateType: models.Append, 219 | UniqueBy: []string{"app_name"}, 220 | Fields: []models.Field{ 221 | {Name: "App", Type: models.StringType}, 222 | {Name: "Build Count", Type: models.NumberType}, 223 | }, 224 | }, 225 | }, 226 | }, 227 | gbReqs: []GBRequest{}, 228 | expectError: true, 229 | }, 230 | { 231 | // Optional field correctly sent as null 232 | config: models.Config{ 233 | DatabaseConfig: &models.DatabaseConfig{ 234 | Driver: models.SQLiteDriver, 235 | URL: filepath.Join("models", "fixtures", "db.sqlite"), 236 | }, 237 | Datasets: []models.Dataset{ 238 | { 239 | Name: "app.counts", 240 | SQL: `SELECT "test", null FROM builds limit 1`, 241 | UpdateType: models.Append, 242 | Fields: []models.Field{ 243 | {Name: "App", Type: models.StringType}, 244 | {Name: "Build Count", Type: models.NumberType, Optional: true}, 245 | }, 246 | }, 247 | }, 248 | }, 249 | gbReqs: []GBRequest{ 250 | { 251 | Path: "/datasets/app.counts", 252 | Body: `{"id":"app.counts","fields":{"app":{"type":"string","name":"App"},"build_count":{"type":"number","name":"Build Count","optional":true}}}`, 253 | }, 254 | { 255 | Path: "/datasets/app.counts/data", 256 | Body: `{"data":[{"app":"test","build_count":null}]}`, 257 | }, 258 | }, 259 | }, 260 | { 261 | // No data rows retrieved - so should send {'data': []} when type replace 262 | config: models.Config{ 263 | DatabaseConfig: &models.DatabaseConfig{ 264 | Driver: models.SQLiteDriver, 265 | URL: filepath.Join("models", "fixtures", "db.sqlite"), 266 | }, 267 | Datasets: []models.Dataset{ 268 | { 269 | Name: "empty.sql.rows", 270 | SQL: `SELECT "test", null FROM builds WHERE id < 0`, 271 | UpdateType: models.Replace, 272 | Fields: []models.Field{ 273 | {Name: "App", Type: models.StringType}, 274 | {Name: "Build Count", Type: models.NumberType, Optional: true}, 275 | }, 276 | }, 277 | }, 278 | }, 279 | gbReqs: []GBRequest{ 280 | { 281 | Path: "/datasets/empty.sql.rows", 282 | Body: `{"id":"empty.sql.rows","fields":{"app":{"type":"string","name":"App"},"build_count":{"type":"number","name":"Build Count","optional":true}}}`, 283 | }, 284 | { 285 | Path: "/datasets/empty.sql.rows/data", 286 | Body: `{"data":[]}`, 287 | }, 288 | }, 289 | }, 290 | } 291 | 292 | for i, tc := range testCases { 293 | maxRows = originalBatchRows 294 | 295 | if tc.maxRows != 0 { 296 | maxRows = tc.maxRows 297 | } 298 | 299 | gbWS := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 300 | tc.gbHits++ 301 | 302 | if tc.gbHits-1 >= len(tc.gbReqs) { 303 | t.Errorf("[%d] Got unexpected extra request for geckoboard unable to process", i) 304 | return 305 | } 306 | 307 | tcReq := tc.gbReqs[tc.gbHits-1] 308 | 309 | if tcReq.Path != r.URL.Path { 310 | t.Errorf("[%d] Expected geckoboard request path %s but got %s", i, tcReq.Path, r.URL.Path) 311 | } 312 | 313 | b, err := ioutil.ReadAll(r.Body) 314 | if err != nil { 315 | t.Fatal("Failed to consume body", err) 316 | } 317 | 318 | if strings.TrimSpace(string(b)) != tcReq.Body { 319 | t.Errorf("[%d] Expected geckoboard request body %s but got %s", i, tcReq.Body, string(b)) 320 | } 321 | 322 | fmt.Fprintf(w, `{}`) 323 | })) 324 | 325 | client := NewClient("fakeKey") 326 | dc := tc.config.DatabaseConfig 327 | db, err := newDBConnection(dc.Driver, dc.URL) 328 | if err != nil { 329 | t.Fatal(err) 330 | } 331 | 332 | gbHost = gbWS.URL 333 | 334 | bol := processAllDatasets(&tc.config, client, db) 335 | 336 | if tc.expectError != bol { 337 | t.Errorf("[%d] Expected hasErrors to be %t but got %t", i, tc.expectError, bol) 338 | } 339 | 340 | if tc.gbHits != len(tc.gbReqs) { 341 | t.Errorf("Expected %d requests but got %d", len(tc.gbReqs), tc.gbHits) 342 | } 343 | gbWS.Close() 344 | } 345 | } 346 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "database/sql" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "os" 10 | "strings" 11 | "time" 12 | 13 | "github.com/geckoboard/sql-dataset/drivers" 14 | "github.com/geckoboard/sql-dataset/models" 15 | ) 16 | 17 | var ( 18 | configFile = flag.String("config", "sql-dataset.yml", "Config file to load") 19 | deleteDataset = flag.String("delete-dataset", "", "Pass a dataset name you want to delete") 20 | displayVersion = flag.Bool("version", false, "Displays version info") 21 | version = "" 22 | gitSHA = "" 23 | ) 24 | 25 | func main() { 26 | flag.Parse() 27 | 28 | if *displayVersion { 29 | fmt.Printf("Version: %s\nGitSHA: %s\n", version, gitSHA) 30 | os.Exit(0) 31 | } 32 | 33 | config, err := models.LoadConfig(*configFile) 34 | if err != nil { 35 | log.Fatal(err) 36 | } 37 | 38 | if errs := config.Validate(); errs != nil { 39 | fmt.Println("\nThere are errors in your config:") 40 | 41 | for i, err := range errs { 42 | fmt.Println(" -", err) 43 | 44 | if i == len(errs)-1 { 45 | fmt.Println("") 46 | } 47 | } 48 | 49 | os.Exit(1) 50 | } 51 | 52 | if *deleteDataset != "" { 53 | if err := deleteDatasetSwitch(*deleteDataset, config); err != nil { 54 | fmt.Println(err) 55 | os.Exit(1) 56 | } 57 | 58 | os.Exit(0) 59 | } 60 | 61 | // Build the connection string 62 | dc := config.DatabaseConfig 63 | b, err := drivers.NewConnStringBuilder(dc.Driver) 64 | if err != nil { 65 | fmt.Println(err) 66 | os.Exit(1) 67 | } 68 | 69 | dsn, err := b.Build(dc) 70 | 71 | if err != nil { 72 | fmt.Println("There was an error while trying to build "+ 73 | "your database connection string:", err) 74 | os.Exit(1) 75 | } 76 | 77 | client := NewClient(config.GeckoboardAPIKey) 78 | db, err := newDBConnection(dc.Driver, dsn) 79 | if err != nil { 80 | fmt.Println(err) 81 | os.Exit(1) 82 | } 83 | 84 | if config.RefreshTimeSec == 0 { 85 | processAllDatasets(config, client, db) 86 | } else { 87 | fmt.Printf("Running every %d seconds, until interrupted.\n\n", config.RefreshTimeSec) 88 | for { 89 | processAllDatasets(config, client, db) 90 | time.Sleep(time.Duration(config.RefreshTimeSec) * time.Second) 91 | } 92 | } 93 | } 94 | 95 | func processAllDatasets(config *models.Config, client *Client, db *sql.DB) (hasErrored bool) { 96 | for _, ds := range config.Datasets { 97 | datasetRecs, err := ds.BuildDataset(config.DatabaseConfig, db) 98 | if err != nil { 99 | printErrorMsg(ds.Name, err) 100 | hasErrored = true 101 | continue 102 | } 103 | 104 | err = client.FindOrCreateDataset(&ds) 105 | if err != nil { 106 | printErrorMsg(ds.Name, err) 107 | hasErrored = true 108 | continue 109 | } 110 | 111 | err = client.SendAllData(&ds, datasetRecs) 112 | if err != nil { 113 | printErrorMsg(ds.Name, err) 114 | hasErrored = true 115 | continue 116 | } 117 | 118 | fmt.Printf("Successfully updated \"%s\"\n", ds.Name) 119 | } 120 | 121 | return hasErrored 122 | } 123 | 124 | func printErrorMsg(name string, err error) { 125 | fmt.Printf("There was an error while trying to update %s: %s\n", name, err) 126 | } 127 | 128 | func newDBConnection(driver, url string) (*sql.DB, error) { 129 | // Ignore this error which just checks we have the driver loaded 130 | pool, _ := sql.Open(driver, url) 131 | err := pool.Ping() 132 | 133 | if err != nil { 134 | return nil, fmt.Errorf("Failed to open database connection. "+ 135 | "This is the error received: %s", err) 136 | } 137 | 138 | pool.SetMaxOpenConns(5) 139 | 140 | return pool, err 141 | } 142 | 143 | func deleteDatasetSwitch(name string, config *models.Config) error { 144 | fmt.Printf("Delete dataset %q (y/N): ", name) 145 | 146 | v, err := bufio.NewReader(os.Stdin).ReadString('\n') 147 | if err != nil { 148 | return err 149 | } 150 | 151 | // Remove newline and carriage return for windows 152 | v = strings.TrimRight(v, "\n") 153 | v = strings.TrimRight(v, "\r") 154 | 155 | switch strings.ToLower(v) { 156 | case "y": 157 | client := NewClient(config.GeckoboardAPIKey) 158 | if err := client.DeleteDataset(*deleteDataset); err != nil { 159 | return err 160 | } 161 | 162 | fmt.Println("Dataset deleted successfully") 163 | default: 164 | fmt.Println("Cancelled action") 165 | } 166 | 167 | return nil 168 | } 169 | -------------------------------------------------------------------------------- /models/config.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "regexp" 9 | 10 | "gopkg.in/yaml.v2" 11 | ) 12 | 13 | const ( 14 | MySQLDriver = "mysql" 15 | PostgresDriver = "postgres" 16 | SQLiteDriver = "sqlite3" 17 | MSSQLDriver = "mssql" 18 | ) 19 | 20 | var ( 21 | SupportedDrivers = []string{MSSQLDriver, MySQLDriver, PostgresDriver, SQLiteDriver} 22 | interpolateRegex = regexp.MustCompile(`{{\s*([a-zA-Z0-9_]+)\s*}}`) 23 | ) 24 | 25 | type Config struct { 26 | GeckoboardAPIKey string `yaml:"geckoboard_api_key"` 27 | DatabaseConfig *DatabaseConfig `yaml:"database"` 28 | RefreshTimeSec uint16 `yaml:"refresh_time_sec"` 29 | Datasets []Dataset `yaml:"datasets"` 30 | } 31 | 32 | // DatabaseConfig holds the db type, url 33 | // and other custom options such as tls config 34 | type DatabaseConfig struct { 35 | Driver string `yaml:"driver"` 36 | URL string `yaml:"-"` 37 | Host string `yaml:"host"` 38 | Port string `yaml:"port"` 39 | Protocol string `yaml:"protocol"` 40 | Database string `yaml:"name"` 41 | Username string `yaml:"username"` 42 | Password string `yaml:"password"` 43 | TLSConfig *TLSConfig `yaml:"tls_config"` 44 | Params map[string]string `yaml:"params"` 45 | } 46 | 47 | type TLSConfig struct { 48 | KeyFile string `yaml:"key_file"` 49 | CertFile string `yaml:"cert_file"` 50 | CAFile string `yaml:"ca_file"` 51 | SSLMode string `yaml:"ssl_mode"` 52 | } 53 | 54 | func LoadConfig(filepath string) (config *Config, err error) { 55 | var b []byte 56 | 57 | if filepath == "" { 58 | return nil, errors.New(errNoConfigFound) 59 | } 60 | 61 | if b, err = ioutil.ReadFile(filepath); err != nil { 62 | return nil, err 63 | } 64 | 65 | if err = yaml.Unmarshal(b, &config); err != nil { 66 | return nil, fmt.Errorf(errParseConfigFile, err) 67 | } 68 | 69 | config.replaceSupportedInterpolatedValues() 70 | 71 | return config, nil 72 | } 73 | 74 | func (c Config) Validate() (errors []string) { 75 | if c.GeckoboardAPIKey == "" { 76 | errors = append(errors, errMissingAPIKey) 77 | } 78 | 79 | if c.DatabaseConfig == nil { 80 | errors = append(errors, errMissingDBConfig) 81 | } else { 82 | errors = append(errors, c.DatabaseConfig.Validate()...) 83 | } 84 | 85 | if len(c.Datasets) == 0 { 86 | errors = append(errors, errNoDatasets) 87 | } 88 | 89 | for _, ds := range c.Datasets { 90 | errors = append(errors, ds.Validate()...) 91 | } 92 | 93 | return errors 94 | } 95 | 96 | func (dc DatabaseConfig) Validate() (errors []string) { 97 | if dc.Driver == "" { 98 | errors = append(errors, errMissingDBDriver) 99 | } else { 100 | var matched bool 101 | 102 | for _, d := range SupportedDrivers { 103 | if d == dc.Driver { 104 | matched = true 105 | break 106 | } 107 | } 108 | 109 | if !matched { 110 | errors = append(errors, fmt.Sprintf(errDriverNotSupported, dc.Driver, SupportedDrivers)) 111 | } 112 | } 113 | 114 | return errors 115 | } 116 | 117 | func (c *Config) replaceSupportedInterpolatedValues() { 118 | c.GeckoboardAPIKey = convertEnvToValue(c.GeckoboardAPIKey) 119 | 120 | if c.DatabaseConfig != nil { 121 | dc := c.DatabaseConfig 122 | 123 | dc.Username = convertEnvToValue(dc.Username) 124 | dc.Password = convertEnvToValue(dc.Password) 125 | dc.Host = convertEnvToValue(dc.Host) 126 | dc.Database = convertEnvToValue(dc.Database) 127 | dc.Port = convertEnvToValue(dc.Port) 128 | } 129 | } 130 | 131 | func convertEnvToValue(value string) string { 132 | if value == "" { 133 | return "" 134 | } 135 | 136 | keys := interpolateRegex.FindStringSubmatch(value) 137 | 138 | if len(keys) != 2 { 139 | return value 140 | } 141 | 142 | return os.Getenv(keys[1]) 143 | } 144 | -------------------------------------------------------------------------------- /models/config_test.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "reflect" 8 | "testing" 9 | ) 10 | 11 | func TestValidate(t *testing.T) { 12 | testCases := []struct { 13 | config Config 14 | err []string 15 | }{ 16 | { 17 | Config{}, 18 | []string{ 19 | errMissingAPIKey, 20 | errMissingDBConfig, 21 | errNoDatasets, 22 | }, 23 | }, 24 | { 25 | Config{ 26 | GeckoboardAPIKey: "1234", 27 | DatabaseConfig: &DatabaseConfig{Driver: "mysql"}, 28 | }, 29 | []string{ 30 | errNoDatasets, 31 | }, 32 | }, 33 | { 34 | Config{ 35 | DatabaseConfig: &DatabaseConfig{}, 36 | Datasets: []Dataset{ 37 | { 38 | Name: "dataset.x", 39 | UpdateType: Replace, 40 | SQL: "SELECT count(*) FROM table", 41 | Fields: []Field{ 42 | {Name: "count", Type: NumberType}, 43 | }, 44 | }, 45 | }, 46 | }, 47 | []string{ 48 | errMissingAPIKey, 49 | errMissingDBDriver, 50 | }, 51 | }, 52 | { 53 | Config{ 54 | GeckoboardAPIKey: "123", 55 | DatabaseConfig: &DatabaseConfig{ 56 | Driver: "pear", 57 | URL: "pear://localhost/test", 58 | }, 59 | Datasets: []Dataset{ 60 | { 61 | Name: "dataset.x", 62 | UpdateType: Replace, 63 | SQL: "SELECT count(*) FROM table", 64 | Fields: []Field{ 65 | {Name: "count", Type: NumberType}, 66 | }, 67 | }, 68 | }, 69 | }, 70 | []string{ 71 | fmt.Sprintf(errDriverNotSupported, "pear", SupportedDrivers), 72 | }, 73 | }, 74 | { 75 | Config{ 76 | GeckoboardAPIKey: "1234-12345", 77 | DatabaseConfig: &DatabaseConfig{ 78 | Driver: PostgresDriver, 79 | URL: "mysql://localhost/testdb", 80 | }, 81 | Datasets: []Dataset{ 82 | { 83 | Name: "dataset.x", 84 | UpdateType: Replace, 85 | SQL: "SELECT count(*) FROM table", 86 | Fields: []Field{ 87 | {Name: "count", Type: NumberType}, 88 | }, 89 | }, 90 | }, 91 | }, 92 | nil, 93 | }, 94 | { 95 | Config{ 96 | GeckoboardAPIKey: "1234-12345", 97 | RefreshTimeSec: 120, 98 | DatabaseConfig: &DatabaseConfig{ 99 | Driver: MySQLDriver, 100 | URL: "mysql://localhost/testdb", 101 | }, 102 | Datasets: []Dataset{ 103 | { 104 | Name: "dataset.x", 105 | UpdateType: Replace, 106 | SQL: "SELECT count(*) FROM table", 107 | Fields: []Field{ 108 | {Name: "count", Type: NumberType}, 109 | }, 110 | }, 111 | }, 112 | }, 113 | nil, 114 | }, 115 | { 116 | Config{ 117 | GeckoboardAPIKey: "1234-12345", 118 | RefreshTimeSec: 120, 119 | DatabaseConfig: &DatabaseConfig{ 120 | Driver: MSSQLDriver, 121 | URL: "odbc:server={name};user id={userb};password=test;database=dbname", 122 | }, 123 | Datasets: []Dataset{ 124 | { 125 | Name: "dataset.x", 126 | UpdateType: Replace, 127 | SQL: "SELECT count(*) FROM table", 128 | Fields: []Field{ 129 | {Name: "count", Type: NumberType}, 130 | }, 131 | }, 132 | }, 133 | }, 134 | nil, 135 | }, 136 | { 137 | Config{ 138 | GeckoboardAPIKey: "1234-12345", 139 | RefreshTimeSec: 120, 140 | DatabaseConfig: &DatabaseConfig{ 141 | Driver: MySQLDriver, 142 | URL: "mysql://localhost/testdb", 143 | }, 144 | Datasets: []Dataset{ 145 | { 146 | Name: "users.count", 147 | UpdateType: "wrong", 148 | SQL: "fake sql", 149 | Fields: []Field{{Name: "count", Type: "number"}}, 150 | }, 151 | }, 152 | }, 153 | []string{ 154 | fmt.Sprintf(errInvalidDatasetUpdateType, "wrong"), 155 | }, 156 | }, 157 | } 158 | 159 | for i, tc := range testCases { 160 | err := tc.config.Validate() 161 | 162 | if tc.err == nil && err != nil { 163 | t.Errorf("[%d] Expected no error but got %s", i, err) 164 | } 165 | 166 | if tc.err != nil && err == nil { 167 | t.Errorf("[%d] Expected error %s but got none", i, tc.err) 168 | } 169 | 170 | if len(err) != len(tc.err) { 171 | t.Errorf("[%d] Expected error count %d but got %d", i, len(tc.err), len(err)) 172 | } 173 | 174 | if !reflect.DeepEqual(err, tc.err) { 175 | t.Errorf("[%d] Expected errors %s but got %s", i, tc.err, err) 176 | } 177 | } 178 | } 179 | 180 | func TestLoadConfig(t *testing.T) { 181 | testCases := []struct { 182 | in string 183 | envs map[string]string 184 | config *Config 185 | err string 186 | }{ 187 | { 188 | "", 189 | nil, 190 | nil, 191 | errNoConfigFound, 192 | }, 193 | { 194 | filepath.Join("fixtures", "invalid_config.yml"), 195 | nil, 196 | nil, 197 | fmt.Sprintf(errParseConfigFile, "yaml: did not find expected key"), 198 | }, 199 | { 200 | filepath.Join("fixtures", "valid_config.yml"), 201 | nil, 202 | &Config{ 203 | GeckoboardAPIKey: "1234dsfd21322", 204 | DatabaseConfig: &DatabaseConfig{ 205 | Driver: PostgresDriver, 206 | Username: "root", 207 | Password: "pass234", 208 | Host: "/var/postgres/POSTGRES.5543", 209 | Protocol: "unix", 210 | Database: "someDB", 211 | TLSConfig: &TLSConfig{ 212 | KeyFile: "path/test.key", 213 | CertFile: "path/test.crt", 214 | }, 215 | Params: map[string]string{ 216 | "charset": "utf-8", 217 | }, 218 | }, 219 | RefreshTimeSec: 60, 220 | Datasets: []Dataset{ 221 | { 222 | Name: "active.users.by.org.plan", 223 | UpdateType: Replace, 224 | SQL: "SELECT o.plan_type, count(*) user_count FROM users u, organisation o where o.user_id = u.id AND o.plan_type <> 'trial' order by user_count DESC limit 10", 225 | Fields: []Field{ 226 | {Name: "count", Type: NumberType}, 227 | {Name: "org", Type: StringType, Key: "custom_org"}, 228 | }, 229 | }, 230 | }, 231 | }, 232 | "", 233 | }, 234 | { 235 | filepath.Join("fixtures", "valid_config2.yml"), 236 | nil, 237 | &Config{ 238 | GeckoboardAPIKey: "1234dsfd21322", 239 | DatabaseConfig: &DatabaseConfig{ 240 | Driver: PostgresDriver, 241 | Host: "fake-host", 242 | Port: "5433", 243 | Database: "someDB", 244 | TLSConfig: &TLSConfig{ 245 | CAFile: "path/cert.pem", 246 | SSLMode: "verify-full", 247 | }, 248 | }, 249 | RefreshTimeSec: 60, 250 | Datasets: []Dataset{ 251 | { 252 | Name: "active.users.by.org.plan", 253 | UpdateType: Replace, 254 | SQL: "SELECT o.plan_type, count(*) user_count FROM users u, organisation o where o.user_id = u.id AND o.plan_type <> 'trial' order by user_count DESC limit 10", 255 | Fields: []Field{ 256 | {Name: "count", Type: NumberType, Optional: true}, 257 | {Name: "org", Type: StringType}, 258 | {Name: "Total Earnings", Type: MoneyType, CurrencyCode: "USD"}, 259 | }, 260 | }, 261 | }, 262 | }, 263 | "", 264 | }, 265 | { 266 | filepath.Join("fixtures", "valid_config_all_envs.yml"), 267 | map[string]string{ 268 | "TEST_API_KEY": "1234abc", 269 | "TEST_DB_HOST": "some-host", 270 | "TEST_DB_USER": "joe_bloggs", 271 | "TEST_DB_PASS": "pa331$", 272 | "TEST_DB_PORT": "4403", 273 | "TEST_DB_NAME": "myDBName", 274 | }, 275 | &Config{ 276 | GeckoboardAPIKey: "1234abc", 277 | DatabaseConfig: &DatabaseConfig{ 278 | Driver: PostgresDriver, 279 | Host: "some-host", 280 | Port: "4403", 281 | Database: "myDBName", 282 | Username: "joe_bloggs", 283 | Password: "pa331$", 284 | TLSConfig: &TLSConfig{ 285 | SSLMode: "verify-full", 286 | }, 287 | }, 288 | RefreshTimeSec: 60, 289 | Datasets: []Dataset{ 290 | { 291 | Name: "some.number", 292 | UpdateType: Replace, 293 | SQL: "SELECT 124", 294 | Fields: []Field{ 295 | {Name: "count", Type: NumberType}, 296 | }, 297 | }, 298 | }, 299 | }, 300 | "", 301 | }, 302 | { 303 | filepath.Join("fixtures", "valid_config_with_missing_envs.yml"), 304 | nil, 305 | &Config{ 306 | DatabaseConfig: &DatabaseConfig{ 307 | Driver: PostgresDriver, 308 | Protocol: "unix", 309 | Host: "{{ }}", 310 | Database: "{{ INVAL^&ID }}", 311 | Username: "{{ AGAIN INVALID }}", 312 | Password: "{{ NOT-VALID }}", 313 | }, 314 | RefreshTimeSec: 60, 315 | Datasets: []Dataset{ 316 | { 317 | Name: "some.number", 318 | UpdateType: Replace, 319 | SQL: "SELECT 124", 320 | Fields: []Field{ 321 | {Name: "count", Type: NumberType}, 322 | }, 323 | }, 324 | }, 325 | }, 326 | "", 327 | }, 328 | } 329 | 330 | for i, tc := range testCases { 331 | if len(tc.envs) > 0 { 332 | for k, v := range tc.envs { 333 | os.Setenv(k, v) 334 | } 335 | } 336 | 337 | c, err := LoadConfig(tc.in) 338 | 339 | if tc.err == "" && err != nil { 340 | t.Errorf("[%d] Expected no error but got %s", i, err) 341 | continue 342 | } 343 | 344 | if !reflect.DeepEqual(tc.config, c) { 345 | t.Errorf("[%d] Expected config %#v but got %#v", i, tc.config, c) 346 | } 347 | 348 | if err != nil && tc.err != err.Error() { 349 | t.Errorf("[%d] Expected error %s but got %s", i, tc.err, err.Error()) 350 | } 351 | } 352 | } 353 | -------------------------------------------------------------------------------- /models/dataset.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | ) 8 | 9 | type DatasetType string 10 | type FieldType string 11 | 12 | const ( 13 | Append DatasetType = "append" 14 | Replace DatasetType = "replace" 15 | 16 | NumberType FieldType = "number" 17 | DateType FieldType = "date" 18 | DatetimeType FieldType = "datetime" 19 | MoneyType FieldType = "money" 20 | PercentageType FieldType = "percentage" 21 | StringType FieldType = "string" 22 | DurationType FieldType = "duration" 23 | ) 24 | 25 | var ( 26 | datasetNameRegexp = regexp.MustCompile(`(?)^[0-9a-z][0-9a-z._\-]{1,}[0-9a-z]$`) 27 | fieldIdRegexp = regexp.MustCompile(`[^a-z0-9 ]+|[\W]+$|^[\W]+`) 28 | ) 29 | 30 | var fieldTypes = []FieldType{ 31 | NumberType, 32 | DateType, 33 | DatetimeType, 34 | MoneyType, 35 | PercentageType, 36 | StringType, 37 | DurationType, 38 | } 39 | 40 | type Dataset struct { 41 | Name string `json:"id" yaml:"name"` 42 | UpdateType DatasetType `json:"-" yaml:"update_type"` 43 | UniqueBy []string `json:"unique_by,omitempty" yaml:"unique_by,omitempty"` 44 | SQL string `json:"-" yaml:"sql"` 45 | Fields []Field `json:"-" yaml:"fields"` 46 | SchemaFields map[string]Field `json:"fields" yaml:"-"` 47 | } 48 | 49 | type Field struct { 50 | Type FieldType `json:"type" yaml:"type"` 51 | Key string `json:"-" yaml:"key"` 52 | Name string `json:"name" yaml:"name"` 53 | CurrencyCode string `json:"currency_code,omitempty" yaml:"currency_code"` 54 | TimeUnit string `json:"time_unit,omitempty" yaml:"time_unit"` 55 | Optional bool `json:"optional,omitempty" yaml:"optional,omitempty"` 56 | } 57 | 58 | // KeyValue returns the field key if present 59 | // otherwise by default returns the field name underscored 60 | func (f Field) KeyValue() string { 61 | if f.Key != "" { 62 | return f.Key 63 | } 64 | 65 | // Remove any characters not alphanumeric or space and replace with nothing 66 | key := fieldIdRegexp.ReplaceAllString(strings.ToLower(f.Name), "") 67 | 68 | return strings.Replace(key, " ", "_", -1) 69 | } 70 | 71 | // BuildSchemaFields creates a map[string]Field of 72 | // the dataset fields for sending over to Geckoboard 73 | func (ds *Dataset) BuildSchemaFields() error { 74 | if ds.SchemaFields != nil { 75 | return nil 76 | } 77 | 78 | fields := make(map[string]Field) 79 | 80 | for _, f := range ds.Fields { 81 | fields[f.KeyValue()] = f 82 | } 83 | 84 | ds.SchemaFields = fields 85 | uniqueByKeys, err := ds.updateUniqueByKeys(fields) 86 | if err != nil { 87 | return err 88 | } 89 | 90 | ds.UniqueBy = uniqueByKeys 91 | return nil 92 | } 93 | 94 | func (ds Dataset) updateUniqueByKeys(newFields map[string]Field) ([]string, error) { 95 | // We don't want to modify the unique by inline just yet 96 | // so that if we need to error to the user we can 97 | // This should maintain the same order as the original 98 | var newUniqueByKeys []string 99 | 100 | for i, ub := range ds.UniqueBy { 101 | matched := false 102 | for k := range newFields { 103 | if ub == k { 104 | matched = true 105 | newUniqueByKeys = append(newUniqueByKeys, ub) 106 | break 107 | } 108 | } 109 | 110 | // Do the same the under the hood key update to match 111 | // the new field key that may have been generated 112 | if !matched { 113 | key := fieldIdRegexp.ReplaceAllString(strings.ToLower(ds.UniqueBy[i]), "") 114 | newUniqueByKeys = append(newUniqueByKeys, strings.Replace(key, " ", "_", -1)) 115 | } 116 | } 117 | 118 | // Now double check all unique by have matching field 119 | for i, ub := range newUniqueByKeys { 120 | matched := false 121 | for k := range newFields { 122 | if ub == k { 123 | matched = true 124 | break 125 | } 126 | } 127 | 128 | if !matched { 129 | return newUniqueByKeys, fmt.Errorf( 130 | "Following unique by '%s' for dataset '%s' has no matching field", 131 | ds.UniqueBy[i], 132 | ds.Name, 133 | ) 134 | } 135 | } 136 | 137 | return newUniqueByKeys, nil 138 | } 139 | 140 | func (ds Dataset) Validate() (errors []string) { 141 | if ds.Name == "" { 142 | errors = append(errors, errMissingDatasetName) 143 | } 144 | 145 | if ds.Name != "" && !datasetNameRegexp.MatchString(ds.Name) { 146 | errors = append(errors, errInvalidDatasetName) 147 | } 148 | 149 | if ds.UpdateType != Append && ds.UpdateType != Replace { 150 | errors = append(errors, 151 | fmt.Sprintf(errInvalidDatasetUpdateType, ds.UpdateType)) 152 | } 153 | 154 | if ds.SQL == "" { 155 | errors = append(errors, errMissingDatasetSQL) 156 | } 157 | 158 | if len(ds.Fields) == 0 { 159 | errors = append(errors, errMissingDatasetFields) 160 | } 161 | 162 | for _, f := range ds.Fields { 163 | errors = append(errors, f.Validate()...) 164 | } 165 | 166 | if err := ds.validateGeneratedFieldKeysUnique(); err != "" { 167 | errors = append(errors, err) 168 | } 169 | 170 | return errors 171 | } 172 | 173 | func (f Field) Validate() (errors []string) { 174 | validType := false 175 | 176 | for _, t := range fieldTypes { 177 | if t == f.Type { 178 | validType = true 179 | break 180 | } 181 | } 182 | 183 | if !validType { 184 | errors = append(errors, fmt.Sprintf(errInvalidFieldType, f.Type, fieldTypes)) 185 | } 186 | 187 | if f.Name == "" { 188 | errors = append(errors, errMissingFieldName) 189 | } 190 | 191 | if f.Type == MoneyType && f.CurrencyCode == "" { 192 | errors = append(errors, errMissingCurrency) 193 | } 194 | 195 | if f.Type == DurationType && f.TimeUnit == "" { 196 | errors = append(errors, errMissingTimeUnit) 197 | } 198 | 199 | return errors 200 | } 201 | 202 | func (ds Dataset) validateGeneratedFieldKeysUnique() string { 203 | uniqueNameMap := make(map[string]interface{}) 204 | var names []string 205 | 206 | for _, f := range ds.Fields { 207 | k := f.KeyValue() 208 | if uniqueNameMap[k] == nil { 209 | uniqueNameMap[k] = struct{}{} 210 | continue 211 | } 212 | 213 | names = append(names, f.Name) 214 | } 215 | 216 | if len(names) > 0 { 217 | return fmt.Sprintf(errDuplicateFieldNames, strings.Join(names, `", "`)) 218 | } 219 | 220 | return "" 221 | } 222 | -------------------------------------------------------------------------------- /models/dataset_test.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func TestBuildSchemaFields(t *testing.T) { 10 | testCases := []struct { 11 | in *Dataset 12 | out *Dataset 13 | err string 14 | }{ 15 | { 16 | // Field name unaltered 17 | in: &Dataset{ 18 | Name: "users.count", 19 | Fields: []Field{ 20 | { 21 | Name: "count", 22 | Type: "datetime", 23 | }, 24 | }, 25 | }, 26 | out: &Dataset{ 27 | Name: "users.count", 28 | Fields: []Field{ 29 | { 30 | Name: "count", 31 | Type: "datetime", 32 | }, 33 | }, 34 | SchemaFields: map[string]Field{ 35 | "count": Field{ 36 | Name: "count", 37 | Type: "datetime", 38 | }, 39 | }, 40 | }, 41 | }, 42 | { 43 | // Field has key provided 44 | in: &Dataset{ 45 | Name: "users.count", 46 | Fields: []Field{ 47 | { 48 | Name: "Count All", 49 | Key: "not_matching", 50 | Type: "datetime", 51 | }, 52 | }, 53 | }, 54 | out: &Dataset{ 55 | Name: "users.count", 56 | Fields: []Field{ 57 | { 58 | Name: "Count All", 59 | Key: "not_matching", 60 | Type: "datetime", 61 | }, 62 | }, 63 | SchemaFields: map[string]Field{ 64 | "not_matching": Field{ 65 | Name: "Count All", 66 | Key: "not_matching", 67 | Type: "datetime", 68 | }, 69 | }, 70 | }, 71 | }, 72 | { 73 | // Multiple fields 74 | in: &Dataset{ 75 | Name: "users.count", 76 | Fields: []Field{ 77 | { 78 | Name: "Count All", 79 | Type: "datetime", 80 | }, 81 | { 82 | Name: "Service", 83 | Key: "newkey", 84 | Type: "string", 85 | }, 86 | }, 87 | }, 88 | out: &Dataset{ 89 | Name: "users.count", 90 | Fields: []Field{ 91 | { 92 | Name: "Count All", 93 | Type: "datetime", 94 | }, 95 | { 96 | Name: "Service", 97 | Key: "newkey", 98 | Type: "string", 99 | }, 100 | }, 101 | SchemaFields: map[string]Field{ 102 | "count_all": Field{ 103 | Name: "Count All", 104 | Type: "datetime", 105 | }, 106 | "newkey": { 107 | Name: "Service", 108 | Key: "newkey", 109 | Type: "string", 110 | }, 111 | }, 112 | }, 113 | }, 114 | { 115 | // Unique not matching any fields 116 | in: &Dataset{ 117 | Name: "users.count", 118 | Fields: []Field{ 119 | { 120 | Name: "count", 121 | Type: "datetime", 122 | }, 123 | }, 124 | UniqueBy: []string{"count", "blah"}, 125 | }, 126 | out: &Dataset{}, 127 | err: "Following unique by 'blah' for dataset 'users.count' has no matching field", 128 | }, 129 | { 130 | // Unique by errors when the user doesn't use custom key supplied 131 | in: &Dataset{ 132 | Name: "users.count", 133 | Fields: []Field{ 134 | { 135 | Name: "App name", 136 | Key: "name_of_appy", 137 | Type: "string", 138 | }, 139 | { 140 | Name: "Count All", 141 | Type: "number", 142 | }, 143 | }, 144 | UniqueBy: []string{"APP name", "Count All"}, 145 | }, 146 | out: &Dataset{}, 147 | err: "Following unique by 'APP name' for dataset 'users.count' has no matching field", 148 | }, 149 | { 150 | // Unique by errors with the users original input 151 | in: &Dataset{ 152 | Name: "users.count", 153 | Fields: []Field{ 154 | { 155 | Name: "App name", 156 | Key: "name_of_appy", 157 | Type: "string", 158 | }, 159 | { 160 | Name: "Count All", 161 | Type: "number", 162 | }, 163 | }, 164 | UniqueBy: []string{"App name", "Count All"}, 165 | }, 166 | out: &Dataset{}, 167 | err: "Following unique by 'App name' for dataset 'users.count' has no matching field", 168 | }, 169 | { 170 | // Unique by converted correctly to match generated field keys 171 | in: &Dataset{ 172 | Name: "users.count", 173 | Fields: []Field{ 174 | { 175 | Name: "App name", 176 | Type: "string", 177 | }, 178 | { 179 | Name: "Count All", 180 | Type: "number", 181 | }, 182 | }, 183 | UniqueBy: []string{"App name", "Count All"}, 184 | }, 185 | out: &Dataset{ 186 | Name: "users.count", 187 | Fields: []Field{ 188 | { 189 | Name: "App name", 190 | Type: "string", 191 | }, 192 | { 193 | Name: "Count All", 194 | Type: "number", 195 | }, 196 | }, 197 | UniqueBy: []string{"app_name", "count_all"}, 198 | SchemaFields: map[string]Field{ 199 | "app_name": Field{ 200 | Name: "App name", 201 | Type: "string", 202 | }, 203 | "count_all": Field{ 204 | Name: "Count All", 205 | Type: "number", 206 | }, 207 | }, 208 | }, 209 | }, 210 | { 211 | // Unique by works with users supplied custom key 212 | in: &Dataset{ 213 | Name: "users.count", 214 | Fields: []Field{ 215 | { 216 | Name: "App name", 217 | Key: "name_of_appy", 218 | Type: "string", 219 | }, 220 | { 221 | Name: "Count All", 222 | Type: "number", 223 | }, 224 | }, 225 | UniqueBy: []string{"name_of_appy", "Count All"}, 226 | }, 227 | out: &Dataset{ 228 | Name: "users.count", 229 | Fields: []Field{ 230 | { 231 | Name: "App name", 232 | Key: "name_of_appy", 233 | Type: "string", 234 | }, 235 | { 236 | Name: "Count All", 237 | Type: "number", 238 | }, 239 | }, 240 | UniqueBy: []string{"name_of_appy", "count_all"}, 241 | SchemaFields: map[string]Field{ 242 | "name_of_appy": Field{ 243 | Name: "App name", 244 | Key: "name_of_appy", 245 | Type: "string", 246 | }, 247 | "count_all": Field{ 248 | Name: "Count All", 249 | Type: "number", 250 | }, 251 | }, 252 | }, 253 | }, 254 | } 255 | 256 | for i, tc := range testCases { 257 | err := tc.in.BuildSchemaFields() 258 | 259 | if tc.err == "" && err != nil { 260 | t.Errorf("[%d] Expected no error but got %s", i, err) 261 | } 262 | 263 | if tc.err != "" && err == nil { 264 | t.Errorf("[%d] Expected error %s but got none", i, tc.err) 265 | } 266 | 267 | if err != nil && tc.err != err.Error() { 268 | t.Errorf("[%d] Expected error %s but got %s", i, tc.err, err) 269 | } 270 | 271 | if tc.err == "" && !reflect.DeepEqual(tc.in, tc.out) { 272 | t.Errorf("[%d] Expected dataset %#v but got %#v", i, tc.in, tc.out) 273 | } 274 | } 275 | } 276 | 277 | func TestDatasetValidate(t *testing.T) { 278 | testCases := []struct { 279 | dataset Dataset 280 | err []string 281 | }{ 282 | { 283 | Dataset{}, 284 | []string{ 285 | errMissingDatasetName, 286 | fmt.Sprintf(errInvalidDatasetUpdateType, ""), 287 | errMissingDatasetSQL, 288 | errMissingDatasetFields, 289 | }, 290 | }, 291 | { 292 | Dataset{Fields: []Field{{}}}, 293 | []string{ 294 | errMissingDatasetName, 295 | fmt.Sprintf(errInvalidDatasetUpdateType, ""), 296 | errMissingDatasetSQL, 297 | fmt.Sprintf(errInvalidFieldType, "", fieldTypes), 298 | errMissingFieldName, 299 | }, 300 | }, 301 | { 302 | Dataset{ 303 | Name: "c", 304 | UpdateType: Replace, 305 | SQL: "SELECT 1", 306 | Fields: []Field{{Name: "count", Type: "number"}}, 307 | }, 308 | []string{errInvalidDatasetName}, 309 | }, 310 | { 311 | Dataset{ 312 | Name: "cd", 313 | UpdateType: Replace, 314 | SQL: "SELECT 1", 315 | Fields: []Field{{Name: "count", Type: "number"}}, 316 | }, 317 | []string{errInvalidDatasetName}, 318 | }, 319 | { 320 | Dataset{ 321 | Name: ".bbc", 322 | UpdateType: Replace, 323 | SQL: "SELECT 1", 324 | Fields: []Field{{Name: "count", Type: "number"}}, 325 | }, 326 | []string{errInvalidDatasetName}, 327 | }, 328 | { 329 | Dataset{ 330 | Name: "ABCwat", 331 | UpdateType: Replace, 332 | SQL: "SELECT 1", 333 | Fields: []Field{{Name: "count", Type: "number"}}, 334 | }, 335 | []string{errInvalidDatasetName}, 336 | }, 337 | { 338 | Dataset{Name: "abc wat", 339 | UpdateType: Replace, 340 | SQL: "SELECT 1", 341 | Fields: []Field{{Name: "count", Type: "number"}}, 342 | }, 343 | []string{errInvalidDatasetName}, 344 | }, 345 | { 346 | Dataset{ 347 | Name: "-wat", 348 | UpdateType: Replace, 349 | SQL: "SELECT 1", 350 | Fields: []Field{{Name: "count", Type: "number"}}, 351 | }, 352 | []string{errInvalidDatasetName}, 353 | }, 354 | { 355 | Dataset{ 356 | Name: "users.count", 357 | UpdateType: Replace, 358 | SQL: "SELECT * FROM some_funky_table;", 359 | Fields: []Field{{Name: "count", Type: "numbre"}}, 360 | }, 361 | []string{fmt.Sprintf(errInvalidFieldType, "numbre", fieldTypes)}, 362 | }, 363 | { 364 | Dataset{ 365 | Name: "some.dataset", 366 | UpdateType: Replace, 367 | SQL: "SELECT 1;", 368 | Fields: []Field{ 369 | { 370 | Name: "Unique", 371 | Type: NumberType, 372 | }, 373 | { 374 | Name: "counts", 375 | Type: NumberType, 376 | }, 377 | { 378 | Name: "Count's", 379 | Type: NumberType, 380 | }, 381 | { 382 | Name: "Total Cost", 383 | Type: MoneyType, 384 | CurrencyCode: "USD", 385 | }, 386 | }, 387 | }, 388 | []string{fmt.Sprintf(errDuplicateFieldNames, "Count's")}, 389 | }, 390 | { 391 | Dataset{ 392 | Name: "some.dataset", 393 | UpdateType: Replace, 394 | SQL: "SELECT 1;", 395 | Fields: []Field{ 396 | { 397 | Name: "Unique", 398 | Type: NumberType, 399 | }, 400 | { 401 | Name: "counts", 402 | Type: NumberType, 403 | }, 404 | { 405 | Name: "Count's", 406 | Type: NumberType, 407 | }, 408 | { 409 | Name: "Total Cost.", 410 | Type: MoneyType, 411 | CurrencyCode: "USD", 412 | }, 413 | { 414 | Name: "Total C.o.S.t", 415 | Type: MoneyType, 416 | CurrencyCode: "USD", 417 | }, 418 | }, 419 | }, 420 | []string{fmt.Sprintf(errDuplicateFieldNames, `Count's", "Total C.o.S.t`)}, 421 | }, 422 | { 423 | Dataset{ 424 | Name: "app", 425 | UpdateType: Replace, 426 | SQL: "SELECT * FROM some_funky_table;", 427 | Fields: []Field{{Name: "count", Type: MoneyType, CurrencyCode: "USD"}}, 428 | }, 429 | nil, 430 | }, 431 | { 432 | Dataset{ 433 | Name: "a-abc", 434 | UpdateType: Replace, 435 | SQL: "SELECT * FROM some_funky_table;", 436 | Fields: []Field{{Name: "count", Type: MoneyType, CurrencyCode: "USD"}}, 437 | }, 438 | nil, 439 | }, 440 | { 441 | Dataset{ 442 | Name: "abc_abc", 443 | UpdateType: Replace, 444 | SQL: "SELECT * FROM some_funky_table;", 445 | Fields: []Field{{Name: "count", Type: MoneyType, CurrencyCode: "USD"}}, 446 | }, 447 | nil, 448 | }, 449 | { 450 | Dataset{ 451 | Name: "12abc", 452 | UpdateType: Replace, 453 | SQL: "SELECT * FROM some_funky_table;", 454 | Fields: []Field{{Name: "count", Type: MoneyType, CurrencyCode: "USD"}}, 455 | }, 456 | nil, 457 | }, 458 | { 459 | Dataset{ 460 | Name: "app.build.cost", 461 | UpdateType: Replace, 462 | SQL: "SELECT * FROM some_funky_table;", 463 | Fields: []Field{{Name: "count", Type: MoneyType}}, 464 | }, 465 | []string{errMissingCurrency}, 466 | }, 467 | { 468 | Dataset{ 469 | Name: "app.build.cost", 470 | UpdateType: Replace, 471 | SQL: "SELECT * FROM some_funky_table;", 472 | Fields: []Field{{Name: "duration", Type: DurationType}}, 473 | }, 474 | []string{errMissingTimeUnit}, 475 | }, 476 | { 477 | Dataset{ 478 | Name: "app.build.cost", 479 | UpdateType: Replace, 480 | SQL: "SELECT * FROM some_funky_table;", 481 | Fields: []Field{{Name: "duration", Type: DurationType, TimeUnit: "milliseconds"}}, 482 | }, 483 | nil, 484 | }, 485 | { 486 | Dataset{ 487 | Name: "app.build.cost", 488 | UpdateType: Replace, 489 | SQL: "SELECT * FROM some_funky_table;", 490 | Fields: []Field{{Name: "count", Type: MoneyType, CurrencyCode: "USD"}}, 491 | }, 492 | nil, 493 | }, 494 | { 495 | Dataset{ 496 | Name: "users.count", 497 | UpdateType: Replace, 498 | SQL: "SELECT * FROM some_funky_table;", 499 | Fields: []Field{{Name: "count", Type: "number"}}, 500 | }, 501 | nil, 502 | }, 503 | { 504 | Dataset{ 505 | Name: "users.count", 506 | UpdateType: Replace, 507 | SQL: "SELECT * FROM some_funky_table;", 508 | Fields: []Field{{Name: "count", Type: "datetime"}}, 509 | }, 510 | nil, 511 | }, 512 | } 513 | 514 | for i, tc := range testCases { 515 | err := tc.dataset.Validate() 516 | 517 | if tc.err == nil && err != nil { 518 | t.Errorf("[%d] Expected no error but got %s", i, err) 519 | } 520 | 521 | if tc.err != nil && err == nil { 522 | t.Errorf("[%d] Expected error %s but got none", i, tc.err) 523 | } 524 | 525 | if len(err) != len(tc.err) { 526 | t.Errorf("[%d] Expected error count %d but got %d", i, len(tc.err), len(err)) 527 | } 528 | 529 | if !reflect.DeepEqual(err, tc.err) { 530 | t.Errorf("[%d] Expected errors %#v but got %#v", i, tc.err, err) 531 | } 532 | } 533 | } 534 | 535 | func TestFieldKeyValue(t *testing.T) { 536 | testCases := []struct { 537 | field Field 538 | out string 539 | }{ 540 | { 541 | Field{ 542 | Key: "customKey", 543 | Name: "Percent Complete", 544 | Type: PercentageType, 545 | }, 546 | "customKey", 547 | }, 548 | { 549 | Field{ 550 | Name: "Total Cost", 551 | Type: MoneyType, 552 | }, 553 | "total_cost", 554 | }, 555 | { 556 | Field{ 557 | Name: "Total's", 558 | Type: MoneyType, 559 | }, 560 | "totals", 561 | }, 562 | { 563 | Field{ 564 | Name: "MRR. Tot", 565 | Type: MoneyType, 566 | }, 567 | "mrr_tot", 568 | }, 569 | { 570 | Field{ 571 | Name: "_MRR. T-", 572 | Type: MoneyType, 573 | }, 574 | "mrr_t", 575 | }, 576 | { 577 | Field{ 578 | Name: "_MRR. Tot_", 579 | Type: MoneyType, 580 | }, 581 | "mrr_tot", 582 | }, 583 | { 584 | Field{ 585 | Name: "Random Names'", 586 | Type: MoneyType, 587 | }, 588 | "random_names", 589 | }, 590 | { 591 | Field{ 592 | Name: "2nd stage", 593 | Type: MoneyType, 594 | }, 595 | "2nd_stage", 596 | }, 597 | { 598 | Field{ 599 | Name: " extra whitespace ", 600 | Type: MoneyType, 601 | }, 602 | "extra_whitespace", 603 | }, 604 | { 605 | Field{ 606 | Name: " extra whitespace ", 607 | Type: MoneyType, 608 | }, 609 | "extra__whitespace", 610 | }, 611 | { 612 | // Let the server validate length 613 | Field{ 614 | Name: "mr", 615 | Type: MoneyType, 616 | }, 617 | "mr", 618 | }, 619 | } 620 | 621 | for _, tc := range testCases { 622 | if key := tc.field.KeyValue(); key != tc.out { 623 | t.Errorf("Expected keyvalue '%s' but got '%s'", tc.out, key) 624 | } 625 | } 626 | } 627 | -------------------------------------------------------------------------------- /models/errors.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | const ( 4 | errParseConfigFile = "There are errors in your config: %s" 5 | 6 | errNoConfigFound = "No config file provided. Use -config path/to/file " + 7 | "to specify the location of your config" 8 | 9 | // Config 10 | errMissingDBConfig = "No database config provided." 11 | errDriverNotSupported = `"%s" is not a supported driver. SQL-Dataset supports %s` 12 | errMissingDBDriver = "No dataset driver provided." 13 | errMissingAPIKey = "No Geckoboard API key provided." 14 | 15 | // SQL 16 | errFailedSQLQuery = "Query failed. This is the error received: %s" 17 | errParseSQLResultSet = "Parsing query results failed. " + 18 | "This is the error received: %s" 19 | 20 | // Dataset validations 21 | errNoDatasets = "At least one dataset is required to run" 22 | errMissingDatasetName = "No dataset name provided." 23 | errMissingDatasetSQL = "No SQL query provided." 24 | errMissingDatasetFields = "No dataset fields provided." 25 | 26 | errInvalidDatasetName = "Invalid dataset name. Dataset names must be at " + 27 | "least 3 characters in length, and use only lowercase letters, " + 28 | "numbers, dots, hyphens, and underscores." 29 | 30 | errInvalidDatasetUpdateType = `"%s" is not a valid update type. ` + 31 | `Update type must be either append or replace.` 32 | 33 | // Dataset field validations 34 | errMissingFieldName = "No field name provided." 35 | 36 | errInvalidFieldType = `"%s" is not a valid field type. ` + 37 | `Supported field types are %s.` 38 | 39 | errMissingCurrency = "No currency_code provided for the money field %s. " + 40 | "Please provide an ISO4217 currency code." 41 | 42 | errMissingTimeUnit = "No time_unit provided for the duration field %s. " + 43 | "Please provide one of milliseconds, seconds, minutes or hours" 44 | 45 | errDuplicateFieldNames = `The field names "%s" will create duplicate keys. ` + 46 | `Please revise using a unique combination of letters and numbers.` 47 | ) 48 | -------------------------------------------------------------------------------- /models/fixtures/ca.cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIGMDCCBBigAwIBAgIJALma9zGnbJDsMA0GCSqGSIb3DQEBCwUAMG0xCzAJBgNV 3 | BAYTAmdiMQ0wCwYDVQQIEwR0ZXN0MQ0wCwYDVQQHEwR0ZXN0MQ0wCwYDVQQKEwR0 4 | ZXN0MQ0wCwYDVQQLEwR0ZXN0MQ0wCwYDVQQDEwR0ZXN0MRMwEQYJKoZIhvcNAQkB 5 | FgR0ZXN0MB4XDTE3MDQyNjIxMDk0NloXDTM3MDQyMTIxMDk0NlowbTELMAkGA1UE 6 | BhMCZ2IxDTALBgNVBAgTBHRlc3QxDTALBgNVBAcTBHRlc3QxDTALBgNVBAoTBHRl 7 | c3QxDTALBgNVBAsTBHRlc3QxDTALBgNVBAMTBHRlc3QxEzARBgkqhkiG9w0BCQEW 8 | BHRlc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCgOHQE2ylSb7xr 9 | LOkD1dTky+Wr8C8T0SSwDv4ozFCC9GrN+s8SeLkR6h5+Y/MyU4TFDk7YaZ1son8P 10 | iqOrQMc5sM26tyfANMp0nv8aOtQeS4VtjEmEwpwnk62QMiFUNIVNoMMMFc6jEfFo 11 | /gBFRAvr3K16KrRCp1rl9rXbXsrcgrdQM1qzzTSrF/+a0XKDCCTzHDeNF5kNQxEY 12 | NkF5vmD0S7OjEOY2ZVF9dD0nrSL6+YGlrTXmn4vLf2wBwdzGaF1fsQ/ibjTG1NdR 13 | 0W6ut9oELsih6OXLbjusTbILgsbJP2n5HC9uu+P15CTM1SrntRfq2mKlxgkMAD+U 14 | eI9Hi6jM2/SpVZKlldTC77lFaQWwhGKO5IApcqnB43RdJAgXy5o43A+iy7/LgzGd 15 | FJiTcisj+HBg07XzV48hSkNmhHO69t/bvMbIRHV9W4snvfYUTFspGwl30eRfOS68 16 | lPFO4QJOdw8zYweaooDApChBptQpUO6OKuyZfzNZKnhtVvqMJYGXaa98AvftZyWl 17 | l2X1Etxc1oX6hhhlnEPhtfzbyCcej//P40JT2sGCS/wb/2Eyi7YfLIg6vAw8Sn0Q 18 | 8UiASwhQOylinbdexqaOSkoAbGK6IX7Lj0kHInRkAf128YecOD6uYwvaHJ1AxOLF 19 | 5FBrnhwtGq78lp78+BTa9biWR7K8ywIDAQABo4HSMIHPMB0GA1UdDgQWBBQQDRQF 20 | HUXpwSKNZtgoamVDMqJpFjCBnwYDVR0jBIGXMIGUgBQQDRQFHUXpwSKNZtgoamVD 21 | MqJpFqFxpG8wbTELMAkGA1UEBhMCZ2IxDTALBgNVBAgTBHRlc3QxDTALBgNVBAcT 22 | BHRlc3QxDTALBgNVBAoTBHRlc3QxDTALBgNVBAsTBHRlc3QxDTALBgNVBAMTBHRl 23 | c3QxEzARBgkqhkiG9w0BCQEWBHRlc3SCCQC5mvcxp2yQ7DAMBgNVHRMEBTADAQH/ 24 | MA0GCSqGSIb3DQEBCwUAA4ICAQCIelvwSoa5N+PWuSwHYK5CDFV5t1sNI/UGSJ7h 25 | 1lnLyLiI9N9+ZcQ3kk0/bIQ/0KvKIKg7i2G+D30Lolb/mfY/bX6abUQN7oQCzR/n 26 | SjRUgKUyo7CxDh0lTItAyAk+3lz/Nc27d6BuMGYYgfemQVROUsrN7SAZg3Y9H73M 27 | +zcEuhj0cG5rnTQlrXcRJDMYa9q3kQ52KI8QtcuiKWa/oci13XltgphiqM3SkMYe 28 | htAXUAtbCIp+SU6XYS8BfbkvpejUCE/iIAHk0RyWWC+ddIYQLkiAudyj1uegE6wb 29 | PGtP8+TklHAYLmNOPy1Xva7CWVl3uCz6r5Wt+F3RnOQab1o/OcvLSr5LrBf22UiU 30 | GN3CioegRabDVauZ82lNWNyuLrgtPV5w7KJUvqAO5aIQd1gthh0LO9AgRzHSidBu 31 | iB3wunULLT5zohr0Qgj9hVOWbdEcfXmWH99IrS3NuOpx1HMPi01sQZoiIyebO7b8 32 | K7zvJl/zQRL32TGhObpfHcLgU+fPC2qs7H+9N7dORDZH6parU5dc8JOdMqdbmn+u 33 | hhHZ4uQuvtIy8p3vgJzf8TNVF8X1STiY6ToLcoDwXGTlSTSKwXiqqvHcIzrwTn72 34 | IZsVqScXqkPuobDlYSfPJGw8V0bBm9Nvp08imPwIBVOGSheJF0UOT4o52gPOPY1x 35 | 5f7kXQ== 36 | -----END CERTIFICATE----- 37 | -------------------------------------------------------------------------------- /models/fixtures/ca.key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | Proc-Type: 4,ENCRYPTED 3 | DEK-Info: AES-256-CBC,D3AAB6025CBAA4C58854C385D4F3BEC4 4 | 5 | X0DDFC3CI7xyw09HOIaEoo9MXZuBCWBLYxUhXQagqezPqbPZ4bc7cirYw4u8kY1G 6 | OjT+XPta4sNwby/Dg54e69njrtJ8XqMjUmvLZrSnfdqVo2UQJj5DhfjbOsVFMWQu 7 | l+VkuJWWUcJgD5hfc8ayIZgCczf3q/rzUfhdWSbmSRPD5LWK53oxn3ALge/l3nZi 8 | nGHGIF57HoR5ajqphZ3LqkuzkfJA2NXgE3FQmEXlOyq9URAMuxr/IdK3E+N3Rypz 9 | tF/ogVrL9hTMpcBn3ePUlJIxaEPnyI6+OXLIP3TKWooEZnRPknkAV8ZLYYPUQmGM 10 | 4OhFTBjSLMyPh9IB4xifurL9k1YGzMrby2G1VYMk2NPwog1ahsOGer1pfKvSmHfx 11 | gM9fxQ5X7y0CIYBekCs7XEpFuZlhDNZD/ahLoWgHGF5rKc/E4HJDcM+XiCPoCvqw 12 | /b3J7BJ8h+w5k62fjb1znaqCHEDTX2LtFrQlKyeUNGGLoVFQx6Og1KmW/WpkPxkX 13 | EROTWwwN4JvmZmdE1Y5hkfu7c1cIB7LnLcapMHz3Bm+QHQD2e10+qwGgnkMJCluu 14 | 60zemNDlZ/kwPEK8x9+QKi8IyvOhIwfe8lwvBSQaoScpit7CgP//O03POUMZmv0a 15 | bjSHkUGFJfAttyoFoLOdB6LLLlKVS3zbNYrJRRwWEH4M61COKK2lPhqAgk7m33p8 16 | xhJnH/xOAKnR+UNA1or1cvQYJo7+7buO89c+Ms8cF54MEqqnXsFLMoRFJmfxyMzO 17 | v2K0ILdEP7ocFar3sG+XePg66a999GbhqDkrkVoNdchchQlZUuS1eobQj7fpEW4Y 18 | hrLhU8EgFAf4cBjhz2IXBpYhgkz2GPkeHM7IuCFZUq5CmYNWmfLiytrMUMcxMmAS 19 | RFgLz+PdHlKfWvkjQ4pYQoaECYKcSyPoNQgva7larCkQtGJ5q+xPaFev/XTiSlId 20 | C7cap7boeJ2BBFdnwKagxJd5GPlivNMOk3WiV9HGOjTLPrj6JJzDANE2Vkme9+2n 21 | HaQHyr7KkyZIqXIE5dxUI/SaBBIt3iciQKPj3KI6oRVY21eLHfeC0Um2ZWekUe8c 22 | EgdxObJXZWceQvTjxPfv3YuO8dTsd27r5aegzDK05OKWtDjwQUmXhOcnbtKRzLrm 23 | LJOzt8S0Ux6vtcn+7Fi/5Pu8xDxi4KdDagSp0geP4/rp+B/6pyicUgS+S6I2icEq 24 | NAwlVMjWuVHs2zCKvDyvEbsIHl0YDVfBzDGvFJvUvBC3etqz50HIEbXlX7gvFoY7 25 | dZPIu+HK07yEbnOp2u5dyrWX/lg5DOhkwXugEocgO4mrNVRG7tJIP3KBZlOTtee1 26 | gA1qgGFGFVJmMN7KJuW/PDX+UXNWLPtUPJs9Z33MEeLTgErqmmZaiWjsjy4spi9j 27 | ddU6VQIaekxOVC+z5apngGmBpLJOOBmqPpiQ0wZz3YRnTl3SDJZULLmw+8koxo1t 28 | KI29QiPUY4Y0CvxuGxDg3ReYuUuvBg8IGM+5XepKqIaIQWYMnqTspvNEEM27yS6o 29 | QTjPIq85cFIKPIgFFyoWkcWdoCNzhQgGTy02Y6Wk9IYiWcPoCZFZOKXSm7TugM5w 30 | qKkqI4cwO5hgCuhSJZqKN1AQUrtrET/rZPgbYEpI1L/9AiHVRf3r4wHJHuE4Nesz 31 | 5IPHpbKgFNdSqbUrr8YhEBrAoRTRBIpQl/Yf1S5fIPzOq5XhhLEvVN/rU1KLCs4m 32 | Gs+OHT6vuQ+Lt1IUzM/AmJVNgwU+KHakcT3SReDU5hQhQUXYSiUZUePK67oRxFjj 33 | mKhuQTXDje/+Rbb5Hi63SciZVhENUSUK691A0mup7QaPsv4O0QNYQmJWSgjkJqie 34 | Pd5X6UJyrEjLQlCd9eMNn5UTKvDuUtlAQMKVbPO8QRs705hbpPpabmgseNzP8rSk 35 | kG1XV836EmmoQ7FkAxNwudlIIx3Nnz/GYbxL4pHX/q37xHbTpOUYilgH8u9kiEAZ 36 | rz1ZiuTAGH6bRv23K9pWHMhrZhej2cabb6DgTNhBS+FSP0w9zcTO/9NWPqTY7bXv 37 | /ALMsd9Ko/jedkQnK82fPCM19MENCv9iPq8yi0JKzr0dV9PIhNbF0Fb3CoCcwsh1 38 | liaGFOl5km5CVwERYb9fXYF8s6ptvCnrJDTwnex37RkEMAPgaXKM5iYpXVQRbwz3 39 | uXJyK+Nbax7b24Sk+xDTEacrGF0nPHxgcd6V0fj0ED+U/7G0+wvDSXGcnLTztVSU 40 | wprqVmKun3mHX1D1h4O+byqnRs3PRYLZYyezrBKRyyiecuwBptMnvcMZJuhASpxZ 41 | AYKzi3rAqaQggMX5JcY4Vnof6MO6WL1nlqVe0hWM78W5cUGtNAmaAXaVxAy8a3vH 42 | tHlVTq8dtJZSws8XRKZ8tAZxRJQDnQ5MaxEN4fdomEJvX6qKOuviTHtKzjVfJLw5 43 | CJ7LtW3rMl1vPUqXy8IEMbKtPEtsCu/iGCiPteOIhmdItarqMpFeuuEVRzBZHTUl 44 | /Ps1O1KA2qYxqtQN2lDTyzqXZLiUSnK9x6LEdaFpa8oyUKqYQTp75zrJVbIUIM7z 45 | bIOtLQaUmmyN/W1Jbn+0pMIvk6NdBA9X0qui2B6aYER6rAA4FHl8jZoXR2FMEdFI 46 | hSi4QrOmi+kHAhmFJBNLLhmbO9TiwXrF5qwVUICWpElKi186/CSddKUlY4XuYP+r 47 | 8MgPJaKRzH5ib7LwlJhKmeoQXtWMwiog71qJMkyqlgWzYmVIt6v3rwgabxTK/5jb 48 | XYA0mhHZY7UKfMgRb/KXaLhf6RCjoHAwEdodeneVuH5TRKUobIhnKJZgeg1qGrPi 49 | kGN33MbF1SGH/rI7GNGqGUhm2JJjK2rlfaMG5OGq7w20EWTZIkvNdXa/S9g8IFz+ 50 | BP5fkj3Y9D3tdRVnofXkJNhvdQue38JarXFVEA/sRjWbl7vlMGtt1bHShkJsNhoT 51 | samg/SUagkgcD83KVMyQawDp7Y/AZT4PLN+DaKJPwqaY2mQZXHIbsHfbC3bEmh/H 52 | RCiWniDzkLtyiKi0wjCZ0Ja7qVaQyaiI+bsE1qyb7sMa/Pm9QRTcBCo16sxgYQQe 53 | tSZzxlIR9zWJkITizukQUbW7RTgJ8/EuUuxFoshOeFzFNezCw1D5fRt8/3d9P1dr 54 | -----END RSA PRIVATE KEY----- 55 | -------------------------------------------------------------------------------- /models/fixtures/db.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geckoboard/sql-dataset/a8fcd336439a8a80f0564a705e179da483522fd1/models/fixtures/db.sqlite -------------------------------------------------------------------------------- /models/fixtures/invalid_config.yml: -------------------------------------------------------------------------------- 1 | geckoboard_api_key: '1234dsfd21322' -- 2 | database: 3 | - driver: postgres 4 | url: "postgres://fake" 5 | refresh_time_sec: 60 6 | datasets: 7 | - name: active.users.by.org.plan 8 | update_type: replace 9 | sql: SELECT o.plan_type, count(*) user_count FROM users u, organisation o where o.user_id = u.id AND o.plan_type <> 'trial' order by user_count DESC limit 10 10 | fields: 11 | -------------------------------------------------------------------------------- /models/fixtures/mssql.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS builds; 2 | CREATE TABLE builds(id INT PRIMARY KEY, build_cost FLOAT, percent_passed INT, run_time FLOAT, app_name VARCHAR(50), triggered_by VARCHAR(255), created_at DATETIME NOT NULL); 3 | 4 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(1, 0.54, 80, 0.31882276212, 'everdeen', 'maeve millay', '2017-03-21 11:12:00'); 5 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(2, 1.11, 95, 118.18382961212, 'react', 'dr.robert ford', '2017-04-23 12:32:00'); 6 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(3, 0.24, 24, 0.21882232124, 'geckoboard-ruby', 'maeve millay', '2017-04-23 13:42:00'); 7 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(4, 1.44, 100, 144.31838122382, 'everdeen', 'bernard', '2017-03-21 11:13:00'); 8 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(5, 0.92, 55, 77.21381276421, 'geckoboard-ruby', 'bernard', '2017-04-23 13:43:00'); 9 | 10 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(6, 2.64, NULL, 321.93774373, 'westworld', 'dolores', '2017-03-23 15:11:00'); 11 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(7, NULL, NULL, NULL, 'geckoboard-ruby', 'bernard', '2017-03-23 16:12:00'); 12 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(8, NULL, 1, 0.12349876543, '', 'dr.robert ford', '2017-03-23 16:22:00'); 13 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(9, 11.32, 34, 46.432763287, '', 'hector', '2017-03-23 16:44:00') 14 | -------------------------------------------------------------------------------- /models/fixtures/mysql.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS builds; 2 | CREATE TABLE IF NOT EXISTS builds (id INTEGER PRIMARY KEY, build_cost FLOAT, percent_passed INTEGER, run_time DOUBLE, app_name TEXT, triggered_by TEXT, created_at TIMESTAMP NOT NULL); 3 | 4 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(1, 0.54, 80, 0.31882276212, 'everdeen', 'maeve millay', '2017-03-21 11:12:00'); 5 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(2, 1.11, 95, 118.18382961212, 'react', 'dr.robert ford', '2017-04-23 12:32:00'); 6 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(3, 0.24, 24, 0.21882232124, 'geckoboard-ruby', 'maeve millay', '2017-04-23 13:42:00'); 7 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(4, 1.44, 100, 144.31838122382, 'everdeen', 'bernard', '2017-03-21 11:13:00'); 8 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(5, 0.92, 55, 77.21381276421, 'geckoboard-ruby', 'bernard', '2017-04-23 13:43:00'); 9 | 10 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(6, 2.64, NULL, 321.93774373, 'westworld', 'dolores', '2017-03-23 15:11:00'); 11 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(7, NULL, NULL, NULL, 'geckoboard-ruby', 'bernard', '2017-03-23 16:12:00'); 12 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(8, NULL, 1, 0.12349876543, '', 'dr.robert ford', '2017-03-23 16:22:00'); 13 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(9, 11.32, 34, 46.432763287, '', 'hector', '2017-03-23 16:44:00') 14 | -------------------------------------------------------------------------------- /models/fixtures/postgres.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS builds; 2 | CREATE TABLE IF NOT EXISTS builds (id INTEGER PRIMARY KEY, build_cost FLOAT, percent_passed INTEGER, run_time DOUBLE PRECISION, app_name TEXT, triggered_by TEXT, created_at TIMESTAMP NOT NULL); 3 | 4 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(1, 0.54, 80, 0.31882276212, 'everdeen', 'maeve millay', '2017-03-21 11:12:00') ON CONFLICT DO NOTHING; 5 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(2, 1.11, 95, 118.18382961212, 'react', 'dr.robert ford', '2017-04-23 12:32:00') ON CONFLICT DO NOTHING; 6 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(3, 0.24, 24, 0.21882232124, 'geckoboard-ruby', 'maeve millay', '2017-04-23 13:42:00') ON CONFLICT DO NOTHING; 7 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(4, 1.44, 100, 144.31838122382, 'everdeen', 'bernard', '2017-03-21 11:13:00') ON CONFLICT DO NOTHING; 8 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(5, 0.92, 55, 77.21381276421, 'geckoboard-ruby', 'bernard', '2017-04-23 13:43:00') ON CONFLICT DO NOTHING; 9 | 10 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(6, 2.64, NULL, 321.93774373, 'westworld', 'dolores', '2017-03-23 15:11:00') ON CONFLICT DO NOTHING; 11 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(7, NULL, NULL, NULL, 'geckoboard-ruby', 'bernard', '2017-03-23 16:12:00') ON CONFLICT DO NOTHING; 12 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(8, NULL, 1, 0.12349876543, '', 'dr.robert ford', '2017-03-23 16:22:00') ON CONFLICT DO NOTHING; 13 | INSERT INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(9, 11.32, 34, 46.432763287, '', 'hector', '2017-03-23 16:44:00') ON CONFLICT DO NOTHING 14 | -------------------------------------------------------------------------------- /models/fixtures/sqlite.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS builds (id INTEGER PRIMARY KEY, build_cost FLOAT, percent_passed INTEGER, run_time DOUBLE, app_name TEXT, triggered_by TEXT, created_at TIMESTAMP NOT NULL); 2 | 3 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(1, 0.54, 80, 0.31882276212, 'everdeen', 'maeve millay', '2017-03-21 11:12:00'); 4 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(2, 1.11, 95, 118.18382961212, 'react', 'dr.robert ford', '2017-04-23 12:32:00'); 5 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(3, 0.24, 24, 0.21882232124, 'geckoboard-ruby', 'maeve millay', '2017-04-23 13:42:00'); 6 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(4, 1.44, 100, 144.31838122382, 'everdeen', 'bernard', '2017-03-21 11:13:00'); 7 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(5, 0.92, 55, 77.21381276421, 'geckoboard-ruby', 'bernard', '2017-04-23 13:43:00'); 8 | 9 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(6, 2.64, NULL, 321.93774373, 'westworld', 'dolores', '2017-03-23 15:11:00'); 10 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(7, NULL, NULL, NULL, 'geckoboard-ruby', 'bernard', '2017-03-23 16:12:00'); 11 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(8, NULL, 1, 0.12349876543, '', 'dr.robert ford', '2017-03-23 16:22:00'); 12 | REPLACE INTO builds(id, build_cost, percent_passed, run_time, app_name, triggered_by, created_at) VALUES(9, 11.32, 34, 46.432763287, '', 'hector', '2017-03-23 16:44:00') 13 | -------------------------------------------------------------------------------- /models/fixtures/test.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDtTCCAp2gAwIBAgIJAM22Xb07wqIZMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV 3 | BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX 4 | aWRnaXRzIFB0eSBMdGQwHhcNMTcwNTA5MjIwOTE4WhcNMTgwNTA5MjIwOTE4WjBF 5 | MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 6 | ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB 7 | CgKCAQEAs/wub6gGXJVErkH4ILBalhiw2tm4jRyOhMoqeTCjlDLXAhknMbzgul+A 8 | SVUYeb4yOUc+NQwHyrODDDgihvk6WLCVCz2FLzOPB4+mshmi2ovkBW7PcsfTd2Zi 9 | SvqZIOq8+gatb3Oqe12Pw1dGi8rZZzO3nz1IxA8d8uYRflWFoe8ksfeT9CvYuVBx 10 | TkH/MCZU165dYCk4bDSrtN+uXzsiYnAvmDAgRAM4DcBR6IjoXv9E788vkcbVB1C9 11 | V9Vpipjt+ZfJdylWS9EIhBkjZa5TJ3LiGm2ntHSxOeR+S2P54BnCO//UIDtaenvy 12 | scpcXaQkmic4zsdP9IuCP+YBECg5AwIDAQABo4GnMIGkMB0GA1UdDgQWBBTn7UH5 13 | buAAZdywD+xEBCVygo6PJDB1BgNVHSMEbjBsgBTn7UH5buAAZdywD+xEBCVygo6P 14 | JKFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNV 15 | BAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAM22Xb07wqIZMAwGA1UdEwQF 16 | MAMBAf8wDQYJKoZIhvcNAQEFBQADggEBACE5yWrRc1YrPccsfVDfJrX94hsDhGeQ 17 | 5qctfvCq8cN3KH/wdS3G+CLvQjJ59fL1aEa3A40KNLpKl43gQqb76KoNXTtIohEK 18 | ht4cp0keJyCMi8APzuM8eNQrVWARSYWLejm+T9NhNhsF840T70iY0R9f5r4eZcRW 19 | ydo6MWKlFXtDHVJxNYYozXXIxsQ48TCw/8p4PHfbK0L6g0PtODT6r3aCMwVGqfU5 20 | dSOFEwKO6pUP4yre+vHkOLZOd+hcaz8FwqSn+JHI/UGDzDtgXz39eHCDtkhzlSK/ 21 | BJFoaFrb30AWO70J4RORqDaXCpnd40WatzVWQcp11a58PMZHJrvjTX4= 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /models/fixtures/test.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpAIBAAKCAQEAs/wub6gGXJVErkH4ILBalhiw2tm4jRyOhMoqeTCjlDLXAhkn 3 | Mbzgul+ASVUYeb4yOUc+NQwHyrODDDgihvk6WLCVCz2FLzOPB4+mshmi2ovkBW7P 4 | csfTd2ZiSvqZIOq8+gatb3Oqe12Pw1dGi8rZZzO3nz1IxA8d8uYRflWFoe8ksfeT 5 | 9CvYuVBxTkH/MCZU165dYCk4bDSrtN+uXzsiYnAvmDAgRAM4DcBR6IjoXv9E788v 6 | kcbVB1C9V9Vpipjt+ZfJdylWS9EIhBkjZa5TJ3LiGm2ntHSxOeR+S2P54BnCO//U 7 | IDtaenvyscpcXaQkmic4zsdP9IuCP+YBECg5AwIDAQABAoIBABp+YZuAAUe5lT2N 8 | amftbbgwdEAS0m67KGN1muDx/vI+tZWSfEl/AxmMG0cwJoUtMRlrWkXWuoLk/8JZ 9 | tQNnRmZtv9LCwIsdLM2xIJmQ2n8PHoaKNDEyJvepc4iT8Nx+kUjAmOESBqNYN2RK 10 | wZCsUGo3m6zuCXsKup7ZrPOKxTv7x5fBvb8Rf4o41qn0rNLloGtw336248qlgqbH 11 | wtNP+Qksd4LUKq1CkH9DAcUmiv3JDXlokWAd8tVjRjyl8NL2iy9g9fzdpTmjMigQ 12 | 18uFJ/IIA2fZrIYmDm+J4edyIENR7YKtY2z0/HP93g0P6vmfdQwAV51YRyb6sCOP 13 | RRqEUGECgYEA2mfzegA/iT9z6bHGdvsvMMKVLAHTUH2YWMIaY6M8SsZsOYbhcA/m 14 | 38t+pa8hTr8yRJz0J0XsJ9LWsQP27CfPiA+y6mV8+KHfi+KEN1x3V4128mArK/qD 15 | j8Xm9Q028g3yti52WWYlpvrsZ7WCKndIcW0KBqh/WHCiJ0CWlf2Q3FECgYEA0vc2 16 | ZtW3wGW1wzffbBbcW2IZrBXafyaYXLTFFxC7j/TSFJqCywVYuOkG0Dr9E+xiedK8 17 | LQgSHPgF7u1Mw5QAq5MdcB+dZZB09JIBMhVX5w6dPeHFU3mMYbWUrXu9h+bmMh33 18 | SSWnYIS6FIb9i8q/npkbuFPfxTioVljIApplLxMCgYEA1T9MDmHxp1tqHNJ0WjXV 19 | FMHYjrhVkDChMICM7Z4zPztP7jdRJG6SWQ7C4JkHZ3DtburkxPfTpeqJrxqU3G3e 20 | hxX09kITbFv4/gc0Wy2QZM7+RZc6b91Q2W88myXE8UBHLDRfX9iJiOlVK8mgh0Ai 21 | XoU9ldStSjfnS3YX0elbqJECgYAufhK8KP9c5E1hX2/al1MqxHzZ4tsLSIstax1A 22 | Twy21gJyTfbjHSOHZLt+qnFZsa+mH14fInczcSmFlUBknbpkFYDYU+9REbvkpkSj 23 | L/b2Uc8vcxEUq2XXprfEX8/OIoG6q8XyldzENZv4qCM6ao0+O4nTXpCjGzBmDxzN 24 | Dg+chwKBgQCc/WUpeWPCpcfQTdaV09KN2X5y9r73O0K/nx7jVhqbFlNFDUH8vzjB 25 | uNfX39amhQVeSDLSGE2dWpxh3+hOvHWm2PEJqsoXKa4Udebfbngc6WXcgv0sdwZf 26 | z+ASQo/jgY47QHwY/V+Pneuf+HayefXQv7mTwOsmSbqfnd0F/onw3A== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /models/fixtures/valid_config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | geckoboard_api_key: '1234dsfd21322' 3 | database: 4 | driver: postgres 5 | username: "root" 6 | password: "pass234" 7 | host: "/var/postgres/POSTGRES.5543" 8 | protocol: "unix" 9 | name: "someDB" 10 | tls_config: 11 | key_file: "path/test.key" 12 | cert_file: "path/test.crt" 13 | params: 14 | charset: "utf-8" 15 | refresh_time_sec: 60 16 | datasets: 17 | - name: active.users.by.org.plan 18 | update_type: replace 19 | sql: SELECT o.plan_type, count(*) user_count FROM users u, organisation o where o.user_id = u.id AND o.plan_type <> 'trial' order by user_count DESC limit 10 20 | fields: 21 | - type: 'number' 22 | name: "count" 23 | - type: 'string' 24 | name: "org" 25 | key: "custom_org" 26 | -------------------------------------------------------------------------------- /models/fixtures/valid_config2.yml: -------------------------------------------------------------------------------- 1 | --- 2 | geckoboard_api_key: '1234dsfd21322' 3 | database: 4 | driver: postgres 5 | host: "fake-host" 6 | port: "5433" 7 | name: "someDB" 8 | tls_config: 9 | ca_file: "path/cert.pem" 10 | ssl_mode: "verify-full" 11 | refresh_time_sec: 60 12 | datasets: 13 | - name: active.users.by.org.plan 14 | update_type: replace 15 | sql: SELECT o.plan_type, count(*) user_count FROM users u, organisation o where o.user_id = u.id AND o.plan_type <> 'trial' order by user_count DESC limit 10 16 | fields: 17 | - type: number 18 | name: count 19 | optional: true 20 | - type: string 21 | name: org 22 | - type: money 23 | name: Total Earnings 24 | currency_code: USD 25 | -------------------------------------------------------------------------------- /models/fixtures/valid_config_all_envs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | geckoboard_api_key: "{{ TEST_API_KEY }}" 3 | database: 4 | driver: postgres 5 | username: "{{TEST_DB_USER}}" 6 | password: "{{ TEST_DB_PASS }}" 7 | host: "{{TEST_DB_HOST }}" 8 | port: "{{ TEST_DB_PORT }}" 9 | name: "{{ TEST_DB_NAME}}" 10 | tls_config: 11 | ssl_mode: "verify-full" 12 | refresh_time_sec: 60 13 | datasets: 14 | - name: some.number 15 | update_type: replace 16 | sql: SELECT 124 17 | fields: 18 | - type: 'number' 19 | name: "count" 20 | -------------------------------------------------------------------------------- /models/fixtures/valid_config_with_missing_envs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | geckoboard_api_key: "{{ NOT_EXISING_KEY }}" 3 | database: 4 | driver: postgres 5 | username: "{{ AGAIN INVALID }}" 6 | password: "{{ NOT-VALID }}" 7 | host: "{{ }}" 8 | protocol: "unix" 9 | name: "{{ INVAL^&ID }}" 10 | refresh_time_sec: 60 11 | datasets: 12 | - name: some.number 13 | update_type: replace 14 | sql: SELECT 124 15 | fields: 16 | - type: 'number' 17 | name: "count" 18 | -------------------------------------------------------------------------------- /models/number.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | ) 7 | 8 | const ( 9 | intType = "int" 10 | float32Type = "float32" 11 | float64Type = "float64" 12 | ) 13 | 14 | type Number struct { 15 | Int64 int64 16 | Float32 float32 17 | Float64 float64 18 | 19 | Type string 20 | } 21 | 22 | func (n *Number) Value(optional bool) interface{} { 23 | switch n.Type { 24 | case intType: 25 | return n.Int64 26 | case float32Type: 27 | return n.Float32 28 | case float64Type: 29 | return n.Float64 30 | default: 31 | if optional { 32 | return nil 33 | } 34 | 35 | return 0 36 | } 37 | } 38 | 39 | func (n *Number) Scan(value interface{}) error { 40 | switch value.(type) { 41 | case string: 42 | return fmt.Errorf("can't convert string %#v to number", value.(string)) 43 | case float64: 44 | n.Type = float64Type 45 | n.Float64 = value.(float64) 46 | case float32: 47 | n.Type = float32Type 48 | n.Float32 = value.(float32) 49 | case int, int32, int64: 50 | n.Type = intType 51 | n.Int64 = value.(int64) 52 | case []byte: 53 | return n.pruneBytes(value.([]byte)) 54 | } 55 | 56 | return nil 57 | } 58 | 59 | func (n *Number) pruneBytes(value []byte) error { 60 | var floatPrecision uint8 61 | 62 | if len(value) == 0 { 63 | return nil 64 | } 65 | 66 | for i, b := range value { 67 | if b == 46 { 68 | floatPrecision = uint8(len(value[i:])) - 1 69 | break 70 | } 71 | } 72 | 73 | if floatPrecision == 0 { 74 | i, err := strconv.ParseInt(string(value), 10, 64) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | n.Type = intType 80 | n.Int64 = i 81 | } else { 82 | f, err := strconv.ParseFloat(string(value), 64) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | n.Type = float64Type 88 | n.Float64 = f 89 | } 90 | 91 | return nil 92 | } 93 | -------------------------------------------------------------------------------- /models/sql.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "time" 7 | 8 | "gopkg.in/guregu/null.v3" 9 | 10 | _ "github.com/denisenkom/go-mssqldb" 11 | _ "github.com/go-sql-driver/mysql" 12 | _ "github.com/lib/pq" 13 | _ "github.com/mattn/go-sqlite3" 14 | ) 15 | 16 | const dateFormat = "2006-01-02" 17 | 18 | // DatasetRows holds a slice of map[string]interface{} 19 | // which is used to send to a geckoboard dataset 20 | type DatasetRows []map[string]interface{} 21 | 22 | // BuildDataset calls queryDatasource to query the datasource for a 23 | // dataset entry and builds up a slice of rows ready for processing by the client 24 | func (ds Dataset) BuildDataset(dc *DatabaseConfig, db *sql.DB) (DatasetRows, error) { 25 | datasetRecs := DatasetRows{} 26 | recs, err := ds.queryDatasource(dc, db) 27 | 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | for _, row := range recs { 33 | data := make(map[string]interface{}) 34 | 35 | for i, col := range row.([]interface{}) { 36 | f := ds.Fields[i] 37 | k := f.KeyValue() 38 | 39 | switch f.Type { 40 | case NumberType, MoneyType, PercentageType, DurationType: 41 | data[k] = col.(*Number).Value(f.Optional) 42 | case StringType: 43 | data[k] = col.(*null.String).String 44 | case DateType: 45 | d := col.(*null.Time) 46 | if d.Valid { 47 | data[k] = d.Time.Format(dateFormat) 48 | } else { 49 | data[k] = nil 50 | } 51 | case DatetimeType: 52 | d := col.(*null.Time) 53 | if d.Valid { 54 | data[k] = d.Time.Format(time.RFC3339) 55 | } else { 56 | data[k] = nil 57 | } 58 | } 59 | } 60 | 61 | datasetRecs = append(datasetRecs, data) 62 | } 63 | 64 | return datasetRecs, nil 65 | } 66 | 67 | func (ds Dataset) queryDatasource(dc *DatabaseConfig, db *sql.DB) (records []interface{}, err error) { 68 | rows, err := db.Query(ds.SQL) 69 | 70 | if err != nil { 71 | return nil, fmt.Errorf(errFailedSQLQuery, err) 72 | } 73 | 74 | defer rows.Close() 75 | 76 | for rows.Next() { 77 | var rvp []interface{} 78 | for _, v := range ds.Fields { 79 | rvp = append(rvp, v.fieldTypeMapping()) 80 | } 81 | 82 | err = rows.Scan(rvp...) 83 | 84 | if err != nil { 85 | return nil, fmt.Errorf(errParseSQLResultSet, err) 86 | } 87 | 88 | records = append(records, rvp) 89 | } 90 | 91 | if err = rows.Err(); err != nil { 92 | return nil, err 93 | } 94 | 95 | return records, nil 96 | } 97 | 98 | func (f Field) fieldTypeMapping() interface{} { 99 | switch f.Type { 100 | case NumberType, MoneyType, PercentageType, DurationType: 101 | var x Number 102 | return &x 103 | case StringType: 104 | var x null.String 105 | return &x 106 | case DateType, DatetimeType: 107 | var x null.Time 108 | return &x 109 | } 110 | 111 | return nil 112 | } 113 | -------------------------------------------------------------------------------- /models/sql_test.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "strings" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func TestBuildDatasetSQLiteDriver(t *testing.T) { 14 | testCases := []struct { 15 | config Config 16 | out []map[string]interface{} 17 | err string 18 | }{ 19 | { 20 | config: Config{ 21 | GeckoboardAPIKey: "1234-12345", 22 | RefreshTimeSec: 120, 23 | DatabaseConfig: &DatabaseConfig{ 24 | Driver: SQLiteDriver, 25 | URL: "models/fixtures/nonexisting", 26 | }, 27 | Datasets: []Dataset{ 28 | { 29 | Name: "users.count", 30 | UpdateType: Replace, 31 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 32 | Fields: []Field{ 33 | {Name: "App", Type: StringType}, 34 | {Name: "Build Count", Type: MoneyType}, 35 | }, 36 | }, 37 | }, 38 | }, 39 | out: nil, 40 | err: fmt.Sprintf(errFailedSQLQuery, "unable to open database file: no such file or directory"), 41 | }, 42 | { 43 | config: Config{ 44 | DatabaseConfig: &DatabaseConfig{ 45 | Driver: SQLiteDriver, 46 | URL: "fixtures/db.sqlite", 47 | }, 48 | Datasets: []Dataset{ 49 | { 50 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 51 | Fields: []Field{ 52 | {Name: "App", Type: NumberType}, 53 | {Name: "Build Count", Type: NumberType}, 54 | }, 55 | }, 56 | }, 57 | }, 58 | out: nil, 59 | err: fmt.Sprintf(errParseSQLResultSet, `sql: Scan error on column index 0, name "app_name": can't convert string "" to number`), 60 | }, 61 | { 62 | config: Config{ 63 | DatabaseConfig: &DatabaseConfig{ 64 | Driver: SQLiteDriver, 65 | URL: "fixtures/db.sqlite", 66 | }, 67 | Datasets: []Dataset{ 68 | { 69 | SQL: "SELECT app_name, create_at FROM builds order by app_name", 70 | Fields: []Field{ 71 | {Name: "App", Type: StringType}, 72 | {Name: "Build Date", Type: DatetimeType}, 73 | }, 74 | }, 75 | }, 76 | }, 77 | out: nil, 78 | err: `Query failed. This is the error received: no such column: create_at`, 79 | }, 80 | { 81 | config: Config{ 82 | DatabaseConfig: &DatabaseConfig{ 83 | Driver: SQLiteDriver, 84 | URL: "fixtures/db.sqlite", 85 | }, 86 | Datasets: []Dataset{ 87 | { 88 | SQL: "SELECT app_name, build_cost, created_at FROM builds GROUP BY app_name order by app_name", 89 | Fields: []Field{ 90 | {Name: "App", Type: StringType}, 91 | {Name: "Build Count", Type: NumberType}, 92 | }, 93 | }, 94 | }, 95 | }, 96 | out: nil, 97 | err: fmt.Sprintf(errParseSQLResultSet, "sql: expected 3 destination arguments in Scan, not 2"), 98 | }, 99 | { 100 | // StringType and Number as an int64 101 | config: Config{ 102 | DatabaseConfig: &DatabaseConfig{ 103 | Driver: SQLiteDriver, 104 | URL: "fixtures/db.sqlite", 105 | }, 106 | Datasets: []Dataset{ 107 | { 108 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 109 | Fields: []Field{ 110 | {Name: "App", Type: StringType}, 111 | {Name: "Build Count", Type: NumberType}, 112 | }, 113 | }, 114 | }, 115 | }, 116 | out: []map[string]interface{}{ 117 | { 118 | "app": "", 119 | "build_count": int64(2), 120 | }, 121 | { 122 | "app": "everdeen", 123 | "build_count": int64(2), 124 | }, 125 | { 126 | "app": "geckoboard-ruby", 127 | "build_count": int64(3), 128 | }, 129 | { 130 | "app": "react", 131 | "build_count": int64(1), 132 | }, 133 | { 134 | "app": "westworld", 135 | "build_count": int64(1), 136 | }, 137 | }, 138 | err: "", 139 | }, 140 | { 141 | // Date only with money type (sqlite lib doesn't support DATE(col) in select as time.Time) 142 | config: Config{ 143 | DatabaseConfig: &DatabaseConfig{ 144 | Driver: SQLiteDriver, 145 | URL: "fixtures/db.sqlite", 146 | }, 147 | Datasets: []Dataset{ 148 | { 149 | SQL: "SELECT created_at, CAST(build_cost*100 AS INTEGER) FROM builds order by created_at", 150 | Fields: []Field{ 151 | {Name: "Day", Type: DateType}, 152 | {Name: "Build Cost", Type: MoneyType}, 153 | }, 154 | }, 155 | }, 156 | }, 157 | out: []map[string]interface{}{ 158 | { 159 | "day": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 160 | "build_cost": int64(54), 161 | }, 162 | { 163 | "day": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 164 | "build_cost": int64(144), 165 | }, 166 | { 167 | "day": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 168 | "build_cost": int64(264), 169 | }, 170 | { 171 | "day": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 172 | "build_cost": 0, 173 | }, 174 | { 175 | "day": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 176 | "build_cost": 0, 177 | }, 178 | { 179 | "day": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 180 | "build_cost": int64(1132), 181 | }, 182 | { 183 | "day": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 184 | "build_cost": int64(111), 185 | }, 186 | { 187 | "day": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 188 | "build_cost": int64(24), 189 | }, 190 | { 191 | "day": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 192 | "build_cost": int64(92), 193 | }, 194 | }, 195 | err: "", 196 | }, 197 | { 198 | // Datetime type example 199 | config: Config{ 200 | DatabaseConfig: &DatabaseConfig{ 201 | Driver: SQLiteDriver, 202 | URL: "fixtures/db.sqlite", 203 | }, 204 | Datasets: []Dataset{ 205 | { 206 | SQL: "SELECT app_name, created_at FROM builds order by created_at;", 207 | Fields: []Field{ 208 | {Name: "App", Type: StringType}, 209 | {Name: "Day", Type: DatetimeType}, 210 | }, 211 | }, 212 | }, 213 | }, 214 | out: []map[string]interface{}{ 215 | { 216 | "app": "everdeen", 217 | "day": parseTime("2017-03-21T11:12:00Z", t).Format(time.RFC3339), 218 | }, 219 | { 220 | "app": "everdeen", 221 | "day": parseTime("2017-03-21T11:13:00Z", t).Format(time.RFC3339), 222 | }, 223 | { 224 | "app": "westworld", 225 | "day": parseTime("2017-03-23T15:11:00Z", t).Format(time.RFC3339), 226 | }, 227 | { 228 | "app": "geckoboard-ruby", 229 | "day": parseTime("2017-03-23T16:12:00Z", t).Format(time.RFC3339), 230 | }, 231 | { 232 | "app": "", 233 | "day": parseTime("2017-03-23T16:22:00Z", t).Format(time.RFC3339), 234 | }, 235 | { 236 | "app": "", 237 | "day": parseTime("2017-03-23T16:44:00Z", t).Format(time.RFC3339), 238 | }, 239 | { 240 | "app": "react", 241 | "day": parseTime("2017-04-23T12:32:00Z", t).Format(time.RFC3339), 242 | }, 243 | { 244 | "app": "geckoboard-ruby", 245 | "day": parseTime("2017-04-23T13:42:00Z", t).Format(time.RFC3339), 246 | }, 247 | { 248 | "app": "geckoboard-ruby", 249 | "day": parseTime("2017-04-23T13:43:00Z", t).Format(time.RFC3339), 250 | }, 251 | }, 252 | err: "", 253 | }, 254 | { 255 | // PercentageType with stringType 256 | config: Config{ 257 | DatabaseConfig: &DatabaseConfig{ 258 | Driver: SQLiteDriver, 259 | URL: "fixtures/db.sqlite", 260 | }, 261 | Datasets: []Dataset{ 262 | { 263 | SQL: "SELECT app_name, CAST(percent_passed/100.00 AS FLOAT) FROM builds order by app_name, created_at", 264 | Fields: []Field{ 265 | {Name: "App", Type: StringType}, 266 | {Name: "Percentage Completed", Type: PercentageType}, 267 | }, 268 | }, 269 | }, 270 | }, 271 | out: []map[string]interface{}{ 272 | { 273 | "app": "", 274 | "percentage_completed": 0.01, 275 | }, 276 | { 277 | "app": "", 278 | "percentage_completed": 0.34, 279 | }, 280 | { 281 | "app": "everdeen", 282 | "percentage_completed": 0.8, 283 | }, 284 | { 285 | "app": "everdeen", 286 | "percentage_completed": 1.0, 287 | }, 288 | { 289 | "app": "geckoboard-ruby", 290 | "percentage_completed": 0, 291 | }, 292 | { 293 | "app": "geckoboard-ruby", 294 | "percentage_completed": 0.24, 295 | }, 296 | { 297 | "app": "geckoboard-ruby", 298 | "percentage_completed": 0.55, 299 | }, 300 | { 301 | "app": "react", 302 | "percentage_completed": 0.95, 303 | }, 304 | { 305 | "app": "westworld", 306 | "percentage_completed": 0, 307 | }, 308 | }, 309 | err: "", 310 | }, 311 | { 312 | // NumberType as float64 and date only type 313 | config: Config{ 314 | DatabaseConfig: &DatabaseConfig{ 315 | Driver: SQLiteDriver, 316 | URL: "fixtures/db.sqlite", 317 | }, 318 | Datasets: []Dataset{ 319 | { 320 | SQL: "SELECT app_name, created_at, run_time FROM builds where app_name <> '' order by app_name, created_at", 321 | Fields: []Field{ 322 | {Name: "App", Type: StringType}, 323 | {Name: "Date", Type: DateType}, 324 | {Name: "Run time", Type: NumberType}, 325 | }, 326 | }, 327 | }, 328 | }, 329 | out: []map[string]interface{}{ 330 | { 331 | "app": "everdeen", 332 | "date": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 333 | "run_time": 0.31882276212, 334 | }, 335 | { 336 | "app": "everdeen", 337 | "date": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 338 | "run_time": 144.31838122382, 339 | }, 340 | { 341 | "app": "geckoboard-ruby", 342 | "date": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 343 | "run_time": 0, 344 | }, 345 | { 346 | "app": "geckoboard-ruby", 347 | "date": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 348 | "run_time": 0.21882232124, 349 | }, 350 | { 351 | "app": "geckoboard-ruby", 352 | "date": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 353 | "run_time": 77.21381276421, 354 | }, 355 | { 356 | "app": "react", 357 | "date": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 358 | "run_time": 118.18382961212, 359 | }, 360 | { 361 | "app": "westworld", 362 | "date": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 363 | "run_time": 321.93774373, 364 | }, 365 | }, 366 | err: "", 367 | }, 368 | { 369 | config: Config{ 370 | DatabaseConfig: &DatabaseConfig{ 371 | Driver: SQLiteDriver, 372 | URL: "fixtures/db.sqlite", 373 | }, 374 | Datasets: []Dataset{ 375 | { 376 | SQL: "SELECT app_name, null, ROUND(run_time, 7) FROM builds order by created_at limit 1", 377 | Fields: []Field{ 378 | {Name: "App", Type: StringType}, 379 | {Name: "Date", Type: DateType}, 380 | {Name: "Run time", Type: NumberType}, 381 | }, 382 | }, 383 | }, 384 | }, 385 | out: []map[string]interface{}{ 386 | { 387 | "app": "everdeen", 388 | "date": nil, 389 | "run_time": 0.3188228, 390 | }, 391 | }, 392 | err: "", 393 | }, 394 | { 395 | //NumberType as optional and is null returns null 396 | config: Config{ 397 | DatabaseConfig: &DatabaseConfig{ 398 | Driver: SQLiteDriver, 399 | URL: "fixtures/db.sqlite", 400 | }, 401 | Datasets: []Dataset{ 402 | { 403 | SQL: `SELECT "test", null FROM builds limit 1`, 404 | Fields: []Field{ 405 | {Name: "App", Type: StringType}, 406 | {Name: "Run time", Type: NumberType, Optional: true}, 407 | }, 408 | }, 409 | }, 410 | }, 411 | out: []map[string]interface{}{ 412 | { 413 | "app": "test", 414 | "run_time": nil, 415 | }, 416 | }, 417 | }, 418 | { 419 | // No rows returns empty slice 420 | config: Config{ 421 | DatabaseConfig: &DatabaseConfig{ 422 | Driver: SQLiteDriver, 423 | URL: "fixtures/db.sqlite", 424 | }, 425 | Datasets: []Dataset{ 426 | { 427 | SQL: `SELECT "test", null FROM builds WHERE id < 0`, 428 | Fields: []Field{ 429 | {Name: "App", Type: StringType}, 430 | {Name: "Run time", Type: NumberType, Optional: true}, 431 | }, 432 | }, 433 | }, 434 | }, 435 | out: DatasetRows{}, 436 | }, 437 | } 438 | 439 | for idx, tc := range testCases { 440 | db := NewDBConnection(t, tc.config.DatabaseConfig.Driver, tc.config.DatabaseConfig.URL) 441 | out, err := tc.config.Datasets[0].BuildDataset(tc.config.DatabaseConfig, db) 442 | 443 | if tc.err == "" && err != nil { 444 | t.Errorf("[%d] Expected no error but got %s", idx, err) 445 | } 446 | 447 | if err != nil && tc.err != err.Error() { 448 | t.Errorf("[%d] Expected error %s but got %s", idx, tc.err, err) 449 | } 450 | 451 | if err == nil && out == nil { 452 | t.Errorf("expected slice to be initialized when no error but wasn't") 453 | } 454 | 455 | if len(out) != len(tc.out) { 456 | t.Errorf("[%d] Expected slice size %d but got %d", idx, len(tc.out), len(out)) 457 | continue 458 | } 459 | 460 | for i, mp := range out { 461 | for k, v := range mp { 462 | if tc.out[i][k] != v { 463 | t.Errorf("[%d-%d] Expected key '%s' to have value %v but got %v", idx, i, k, tc.out[i][k], v) 464 | } 465 | } 466 | } 467 | } 468 | } 469 | 470 | func TestBuildDatasetPostgresDriver(t *testing.T) { 471 | // Setup the postgres and run the insert 472 | env, ok := os.LookupEnv("POSTGRES_URL") 473 | if !ok { 474 | t.Errorf("This test requires real postgres db using env:POSTGRES_URL ensure the db exists" + 475 | " eg. postgres://postgres:postgres@localhost:5432/testDbName?sslmode=disable") 476 | 477 | return 478 | } 479 | 480 | db, err := sql.Open("postgres", env) 481 | contents, err := ioutil.ReadFile("fixtures/postgres.sql") 482 | if err != nil { 483 | t.Fatal(err) 484 | } 485 | 486 | queries := strings.Split(string(contents), ";") 487 | 488 | for _, query := range queries { 489 | if _, err = db.Exec(query); err != nil { 490 | t.Fatal(err) 491 | } 492 | } 493 | 494 | testCases := []struct { 495 | config Config 496 | out []map[string]interface{} 497 | err string 498 | }{ 499 | { 500 | config: Config{ 501 | GeckoboardAPIKey: "1234-12345", 502 | RefreshTimeSec: 120, 503 | DatabaseConfig: &DatabaseConfig{ 504 | Driver: PostgresDriver, 505 | URL: "postgres://postgresql:postgres@127.0.0.1:9999", 506 | }, 507 | Datasets: []Dataset{ 508 | { 509 | Name: "users.count", 510 | UpdateType: Replace, 511 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 512 | Fields: []Field{ 513 | {Name: "App", Type: StringType}, 514 | {Name: "Build Count", Type: MoneyType}, 515 | }, 516 | }, 517 | }, 518 | }, 519 | out: nil, 520 | err: fmt.Sprintf(errFailedSQLQuery, "dial tcp 127.0.0.1:9999: connect: connection refused"), 521 | }, 522 | { 523 | config: Config{ 524 | DatabaseConfig: &DatabaseConfig{ 525 | Driver: PostgresDriver, 526 | URL: env, 527 | }, 528 | Datasets: []Dataset{ 529 | { 530 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 531 | Fields: []Field{ 532 | {Name: "App", Type: NumberType}, 533 | {Name: "Build Count", Type: NumberType}, 534 | }, 535 | }, 536 | }, 537 | }, 538 | out: nil, 539 | err: fmt.Sprintf(errParseSQLResultSet, `sql: Scan error on column index 0, name "app_name": can't convert string "" to number`), 540 | }, 541 | { 542 | config: Config{ 543 | DatabaseConfig: &DatabaseConfig{ 544 | Driver: PostgresDriver, 545 | URL: env, 546 | }, 547 | Datasets: []Dataset{ 548 | { 549 | SQL: "SELECT app_name, create_at FROM builds order by app_name", 550 | Fields: []Field{ 551 | {Name: "App", Type: StringType}, 552 | {Name: "Build Date", Type: DatetimeType}, 553 | }, 554 | }, 555 | }, 556 | }, 557 | out: nil, 558 | err: fmt.Sprintf(errFailedSQLQuery, `pq: column "create_at" does not exist`), 559 | }, 560 | { 561 | config: Config{ 562 | DatabaseConfig: &DatabaseConfig{ 563 | Driver: PostgresDriver, 564 | URL: env, 565 | }, 566 | Datasets: []Dataset{ 567 | { 568 | SQL: "SELECT app_name, build_cost, created_at FROM builds GROUP BY app_name order by app_name", 569 | Fields: []Field{ 570 | {Name: "App", Type: StringType}, 571 | {Name: "Build Count", Type: NumberType}, 572 | }, 573 | }, 574 | }, 575 | }, 576 | out: nil, 577 | err: fmt.Sprintf(errFailedSQLQuery, `pq: column "builds.build_cost" must appear in the GROUP BY clause or be used in an aggregate function`), 578 | }, 579 | { 580 | // StringType and Number as an int64 581 | config: Config{ 582 | DatabaseConfig: &DatabaseConfig{ 583 | Driver: PostgresDriver, 584 | URL: env, 585 | }, 586 | Datasets: []Dataset{ 587 | { 588 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 589 | Fields: []Field{ 590 | {Name: "App", Type: StringType}, 591 | {Name: "Build Count", Type: NumberType}, 592 | }, 593 | }, 594 | }, 595 | }, 596 | out: []map[string]interface{}{ 597 | { 598 | "app": "", 599 | "build_count": int64(2), 600 | }, 601 | { 602 | "app": "everdeen", 603 | "build_count": int64(2), 604 | }, 605 | { 606 | "app": "geckoboard-ruby", 607 | "build_count": int64(3), 608 | }, 609 | { 610 | "app": "react", 611 | "build_count": int64(1), 612 | }, 613 | { 614 | "app": "westworld", 615 | "build_count": int64(1), 616 | }, 617 | }, 618 | err: "", 619 | }, 620 | { 621 | // Date only with money type grouping by date in sql 622 | config: Config{ 623 | DatabaseConfig: &DatabaseConfig{ 624 | Driver: PostgresDriver, 625 | URL: env, 626 | }, 627 | Datasets: []Dataset{ 628 | { 629 | SQL: "SELECT DATE(created_at) dte, SUM(CAST(build_cost*100 AS INTEGER)) FROM builds GROUP BY DATE(created_at) order by DATE(created_at)", 630 | Fields: []Field{ 631 | {Name: "Day", Type: DateType}, 632 | {Name: "Build Cost", Type: MoneyType}, 633 | }, 634 | }, 635 | }, 636 | }, 637 | out: []map[string]interface{}{ 638 | { 639 | "day": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 640 | "build_cost": int64(198), 641 | }, 642 | { 643 | "day": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 644 | "build_cost": int64(1396), 645 | }, 646 | { 647 | "day": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 648 | "build_cost": int64(227), 649 | }, 650 | }, 651 | err: "", 652 | }, 653 | { 654 | // Datetime type example 655 | config: Config{ 656 | DatabaseConfig: &DatabaseConfig{ 657 | Driver: PostgresDriver, 658 | URL: env, 659 | }, 660 | Datasets: []Dataset{ 661 | { 662 | SQL: "SELECT app_name, created_at FROM builds order by created_at;", 663 | Fields: []Field{ 664 | {Name: "App", Type: StringType}, 665 | {Name: "Day", Type: DatetimeType}, 666 | }, 667 | }, 668 | }, 669 | }, 670 | out: []map[string]interface{}{ 671 | { 672 | "app": "everdeen", 673 | "day": parseTime("2017-03-21T11:12:00Z", t).Format(time.RFC3339), 674 | }, 675 | { 676 | "app": "everdeen", 677 | "day": parseTime("2017-03-21T11:13:00Z", t).Format(time.RFC3339), 678 | }, 679 | { 680 | "app": "westworld", 681 | "day": parseTime("2017-03-23T15:11:00Z", t).Format(time.RFC3339), 682 | }, 683 | { 684 | "app": "geckoboard-ruby", 685 | "day": parseTime("2017-03-23T16:12:00Z", t).Format(time.RFC3339), 686 | }, 687 | { 688 | "app": "", 689 | "day": parseTime("2017-03-23T16:22:00Z", t).Format(time.RFC3339), 690 | }, 691 | { 692 | "app": "", 693 | "day": parseTime("2017-03-23T16:44:00Z", t).Format(time.RFC3339), 694 | }, 695 | { 696 | "app": "react", 697 | "day": parseTime("2017-04-23T12:32:00Z", t).Format(time.RFC3339), 698 | }, 699 | { 700 | "app": "geckoboard-ruby", 701 | "day": parseTime("2017-04-23T13:42:00Z", t).Format(time.RFC3339), 702 | }, 703 | { 704 | "app": "geckoboard-ruby", 705 | "day": parseTime("2017-04-23T13:43:00Z", t).Format(time.RFC3339), 706 | }, 707 | }, 708 | err: "", 709 | }, 710 | { 711 | // PercentageType with stringType 712 | config: Config{ 713 | DatabaseConfig: &DatabaseConfig{ 714 | Driver: PostgresDriver, 715 | URL: env, 716 | }, 717 | Datasets: []Dataset{ 718 | { 719 | SQL: "SELECT app_name, CAST(percent_passed/100.00 AS FLOAT) FROM builds order by app_name", 720 | Fields: []Field{ 721 | {Name: "App", Type: StringType}, 722 | {Name: "Percentage Completed", Type: PercentageType}, 723 | }, 724 | }, 725 | }, 726 | }, 727 | out: []map[string]interface{}{ 728 | { 729 | "app": "", 730 | "percentage_completed": 0.01, 731 | }, 732 | { 733 | "app": "", 734 | "percentage_completed": 0.34, 735 | }, 736 | { 737 | "app": "everdeen", 738 | "percentage_completed": 0.8, 739 | }, 740 | { 741 | "app": "everdeen", 742 | "percentage_completed": 1.0, 743 | }, 744 | { 745 | "app": "geckoboard-ruby", 746 | "percentage_completed": 0.55, 747 | }, 748 | { 749 | "app": "geckoboard-ruby", 750 | "percentage_completed": 0, 751 | }, 752 | { 753 | "app": "geckoboard-ruby", 754 | "percentage_completed": 0.24, 755 | }, 756 | { 757 | "app": "react", 758 | "percentage_completed": 0.95, 759 | }, 760 | { 761 | "app": "westworld", 762 | "percentage_completed": 0, 763 | }, 764 | }, 765 | err: "", 766 | }, 767 | { 768 | // NumberType as float64 and date only type 769 | config: Config{ 770 | DatabaseConfig: &DatabaseConfig{ 771 | Driver: PostgresDriver, 772 | URL: env, 773 | }, 774 | Datasets: []Dataset{ 775 | { 776 | SQL: "SELECT app_name, created_at, run_time FROM builds where app_name <> '' order by app_name, created_at", 777 | Fields: []Field{ 778 | {Name: "App", Type: StringType}, 779 | {Name: "Date", Type: DateType}, 780 | {Name: "Run time", Type: NumberType}, 781 | }, 782 | }, 783 | }, 784 | }, 785 | out: []map[string]interface{}{ 786 | { 787 | "app": "everdeen", 788 | "date": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 789 | "run_time": 0.31882276212, 790 | }, 791 | { 792 | "app": "everdeen", 793 | "date": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 794 | "run_time": 144.31838122382, 795 | }, 796 | { 797 | "app": "geckoboard-ruby", 798 | "date": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 799 | "run_time": 0, 800 | }, 801 | { 802 | "app": "geckoboard-ruby", 803 | "date": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 804 | "run_time": 0.21882232124, 805 | }, 806 | { 807 | "app": "geckoboard-ruby", 808 | "date": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 809 | "run_time": 77.21381276421, 810 | }, 811 | { 812 | "app": "react", 813 | "date": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 814 | "run_time": 118.18382961212, 815 | }, 816 | { 817 | "app": "westworld", 818 | "date": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 819 | "run_time": 321.93774373, 820 | }, 821 | }, 822 | err: "", 823 | }, 824 | { 825 | config: Config{ 826 | DatabaseConfig: &DatabaseConfig{ 827 | Driver: PostgresDriver, 828 | URL: env, 829 | }, 830 | Datasets: []Dataset{ 831 | { 832 | SQL: "SELECT app_name, null, ROUND(CAST(run_time AS NUMERIC), 7) FROM builds order by created_at limit 1", 833 | Fields: []Field{ 834 | {Name: "App", Type: StringType}, 835 | {Name: "Date", Type: DateType}, 836 | {Name: "Run time", Type: NumberType}, 837 | }, 838 | }, 839 | }, 840 | }, 841 | out: []map[string]interface{}{ 842 | { 843 | "app": "everdeen", 844 | "date": nil, 845 | "run_time": 0.3188228, 846 | }, 847 | }, 848 | err: "", 849 | }, 850 | { 851 | //NumberType as optional and is null returns null 852 | config: Config{ 853 | DatabaseConfig: &DatabaseConfig{ 854 | Driver: PostgresDriver, 855 | URL: env, 856 | }, 857 | Datasets: []Dataset{ 858 | { 859 | SQL: `SELECT 'test', null FROM builds limit 1`, 860 | Fields: []Field{ 861 | {Name: "App", Type: StringType}, 862 | {Name: "Run time", Type: NumberType, Optional: true}, 863 | }, 864 | }, 865 | }, 866 | }, 867 | out: []map[string]interface{}{ 868 | { 869 | "app": "test", 870 | "run_time": nil, 871 | }, 872 | }, 873 | }, 874 | { 875 | // No rows returns empty slice 876 | config: Config{ 877 | DatabaseConfig: &DatabaseConfig{ 878 | Driver: PostgresDriver, 879 | URL: env, 880 | }, 881 | Datasets: []Dataset{ 882 | { 883 | SQL: `SELECT 'test', null FROM builds WHERE id < 0`, 884 | Fields: []Field{ 885 | {Name: "App", Type: StringType}, 886 | {Name: "Run time", Type: NumberType, Optional: true}, 887 | }, 888 | }, 889 | }, 890 | }, 891 | out: DatasetRows{}, 892 | }, 893 | } 894 | 895 | for idx, tc := range testCases { 896 | db := NewDBConnection(t, tc.config.DatabaseConfig.Driver, tc.config.DatabaseConfig.URL) 897 | out, err := tc.config.Datasets[0].BuildDataset(tc.config.DatabaseConfig, db) 898 | 899 | if tc.err == "" && err != nil { 900 | t.Errorf("[%d] Expected no error but got %s", idx, err) 901 | } 902 | 903 | if err != nil && tc.err != err.Error() { 904 | t.Errorf("[%d] Expected error %s but got %s", idx, tc.err, err) 905 | } 906 | 907 | if err == nil && out == nil { 908 | t.Errorf("expected slice to be initialized when no error but wasn't") 909 | } 910 | 911 | if len(out) != len(tc.out) { 912 | t.Errorf("[%d] Expected slice size %d but got %#v", idx, len(tc.out), out) 913 | continue 914 | } 915 | 916 | for i, mp := range out { 917 | for k, v := range mp { 918 | if tc.out[i][k] != v { 919 | t.Errorf("[%d-%d] Expected key '%s' to have value %v but got %v", idx, i, k, tc.out[i][k], v) 920 | } 921 | } 922 | } 923 | } 924 | } 925 | 926 | func TestBuildDatasetMySQLDriver(t *testing.T) { 927 | // Setup the postgres and run the insert 928 | env, ok := os.LookupEnv("MYSQL_URL") 929 | if !ok { 930 | t.Errorf("This test requires real mysql db using env:MYSQL_URL ensure the db exists" + 931 | " eg. [username[:password]@][protocol[(address)]]/dbname[?parseTime=true]") 932 | 933 | return 934 | } 935 | 936 | db, err := sql.Open("mysql", env) 937 | contents, err := ioutil.ReadFile("fixtures/mysql.sql") 938 | if err != nil { 939 | t.Fatal(err) 940 | } 941 | 942 | queries := strings.Split(string(contents), ";") 943 | 944 | for _, query := range queries { 945 | if _, err = db.Exec(query); err != nil { 946 | t.Fatal(err) 947 | } 948 | } 949 | 950 | testCases := []struct { 951 | config Config 952 | out []map[string]interface{} 953 | err string 954 | }{ 955 | { 956 | config: Config{ 957 | GeckoboardAPIKey: "1234-12345", 958 | RefreshTimeSec: 120, 959 | DatabaseConfig: &DatabaseConfig{ 960 | Driver: MySQLDriver, 961 | URL: "root@tcp(127.0.0.1:9999)/testdb", 962 | }, 963 | Datasets: []Dataset{ 964 | { 965 | Name: "users.count", 966 | UpdateType: Replace, 967 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 968 | Fields: []Field{ 969 | {Name: "App", Type: StringType}, 970 | {Name: "Build Count", Type: MoneyType}, 971 | }, 972 | }, 973 | }, 974 | }, 975 | out: nil, 976 | err: fmt.Sprintf(errFailedSQLQuery, "dial tcp 127.0.0.1:9999: connect: connection refused"), 977 | }, 978 | { 979 | config: Config{ 980 | DatabaseConfig: &DatabaseConfig{ 981 | Driver: MySQLDriver, 982 | URL: env, 983 | }, 984 | Datasets: []Dataset{ 985 | { 986 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 987 | Fields: []Field{ 988 | {Name: "App", Type: NumberType}, 989 | {Name: "Build Count", Type: NumberType}, 990 | }, 991 | }, 992 | }, 993 | }, 994 | out: nil, 995 | err: fmt.Sprintf(errParseSQLResultSet, `sql: Scan error on column index 0, name "app_name": strconv.ParseInt: parsing "everdeen": invalid syntax`), 996 | }, 997 | { 998 | config: Config{ 999 | DatabaseConfig: &DatabaseConfig{ 1000 | Driver: MySQLDriver, 1001 | URL: env, 1002 | }, 1003 | Datasets: []Dataset{ 1004 | { 1005 | SQL: "SELECT app_name, create_at FROM builds order by app_name", 1006 | Fields: []Field{ 1007 | {Name: "App", Type: StringType}, 1008 | {Name: "Build Date", Type: DatetimeType}, 1009 | }, 1010 | }, 1011 | }, 1012 | }, 1013 | out: nil, 1014 | err: fmt.Sprintf(errFailedSQLQuery, `Error 1054: Unknown column 'create_at' in 'field list'`), 1015 | }, 1016 | { 1017 | config: Config{ 1018 | DatabaseConfig: &DatabaseConfig{ 1019 | Driver: MySQLDriver, 1020 | URL: env, 1021 | }, 1022 | Datasets: []Dataset{ 1023 | { 1024 | SQL: "SELECT app_name, build_cost, created_at FROM builds GROUP BY app_name order by app_name", 1025 | Fields: []Field{ 1026 | {Name: "App", Type: StringType}, 1027 | {Name: "Build Count", Type: NumberType}, 1028 | }, 1029 | }, 1030 | }, 1031 | }, 1032 | out: nil, 1033 | err: fmt.Sprintf(errFailedSQLQuery, `Error 1055: Expression #2 of SELECT list is not in GROUP BY clause and contains nonaggregated column 'testdb.builds.build_cost' which is not functionally dependent on columns in GROUP BY clause; this is incompatible with sql_mode=only_full_group_by`), 1034 | }, 1035 | { 1036 | // StringType and Number as an int64 1037 | config: Config{ 1038 | DatabaseConfig: &DatabaseConfig{ 1039 | Driver: MySQLDriver, 1040 | URL: env, 1041 | }, 1042 | Datasets: []Dataset{ 1043 | { 1044 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 1045 | Fields: []Field{ 1046 | {Name: "App", Type: StringType}, 1047 | {Name: "Build Count", Type: NumberType}, 1048 | }, 1049 | }, 1050 | }, 1051 | }, 1052 | out: []map[string]interface{}{ 1053 | { 1054 | "app": "", 1055 | "build_count": int64(2), 1056 | }, 1057 | { 1058 | "app": "everdeen", 1059 | "build_count": int64(2), 1060 | }, 1061 | { 1062 | "app": "geckoboard-ruby", 1063 | "build_count": int64(3), 1064 | }, 1065 | { 1066 | "app": "react", 1067 | "build_count": int64(1), 1068 | }, 1069 | { 1070 | "app": "westworld", 1071 | "build_count": int64(1), 1072 | }, 1073 | }, 1074 | err: "", 1075 | }, 1076 | { 1077 | // Date only with money type grouping by date in sql 1078 | config: Config{ 1079 | DatabaseConfig: &DatabaseConfig{ 1080 | Driver: MySQLDriver, 1081 | URL: env, 1082 | }, 1083 | Datasets: []Dataset{ 1084 | { 1085 | SQL: "SELECT DATE(created_at) dte, SUM(CAST(build_cost*100 AS SIGNED INTEGER)) FROM builds GROUP BY DATE(created_at) order by DATE(created_at)", 1086 | Fields: []Field{ 1087 | {Name: "Day", Type: DateType}, 1088 | {Name: "Build Cost", Type: MoneyType}, 1089 | }, 1090 | }, 1091 | }, 1092 | }, 1093 | out: []map[string]interface{}{ 1094 | { 1095 | "day": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 1096 | "build_cost": int64(198), 1097 | }, 1098 | { 1099 | "day": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 1100 | "build_cost": int64(1396), 1101 | }, 1102 | { 1103 | "day": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 1104 | "build_cost": int64(227), 1105 | }, 1106 | }, 1107 | err: "", 1108 | }, 1109 | { 1110 | // Datetime type example 1111 | config: Config{ 1112 | DatabaseConfig: &DatabaseConfig{ 1113 | Driver: MySQLDriver, 1114 | URL: env, 1115 | }, 1116 | Datasets: []Dataset{ 1117 | { 1118 | SQL: "SELECT app_name, created_at FROM builds order by created_at;", 1119 | Fields: []Field{ 1120 | {Name: "App", Type: StringType}, 1121 | {Name: "Day", Type: DatetimeType}, 1122 | }, 1123 | }, 1124 | }, 1125 | }, 1126 | out: []map[string]interface{}{ 1127 | { 1128 | "app": "everdeen", 1129 | "day": parseTime("2017-03-21T11:12:00Z", t).Format(time.RFC3339), 1130 | }, 1131 | { 1132 | "app": "everdeen", 1133 | "day": parseTime("2017-03-21T11:13:00Z", t).Format(time.RFC3339), 1134 | }, 1135 | { 1136 | "app": "westworld", 1137 | "day": parseTime("2017-03-23T15:11:00Z", t).Format(time.RFC3339), 1138 | }, 1139 | { 1140 | "app": "geckoboard-ruby", 1141 | "day": parseTime("2017-03-23T16:12:00Z", t).Format(time.RFC3339), 1142 | }, 1143 | { 1144 | "app": "", 1145 | "day": parseTime("2017-03-23T16:22:00Z", t).Format(time.RFC3339), 1146 | }, 1147 | { 1148 | "app": "", 1149 | "day": parseTime("2017-03-23T16:44:00Z", t).Format(time.RFC3339), 1150 | }, 1151 | { 1152 | "app": "react", 1153 | "day": parseTime("2017-04-23T12:32:00Z", t).Format(time.RFC3339), 1154 | }, 1155 | { 1156 | "app": "geckoboard-ruby", 1157 | "day": parseTime("2017-04-23T13:42:00Z", t).Format(time.RFC3339), 1158 | }, 1159 | { 1160 | "app": "geckoboard-ruby", 1161 | "day": parseTime("2017-04-23T13:43:00Z", t).Format(time.RFC3339), 1162 | }, 1163 | }, 1164 | err: "", 1165 | }, 1166 | { 1167 | // PercentageType with stringType 1168 | config: Config{ 1169 | DatabaseConfig: &DatabaseConfig{ 1170 | Driver: MySQLDriver, 1171 | URL: env, 1172 | }, 1173 | Datasets: []Dataset{ 1174 | { 1175 | SQL: "SELECT app_name, CAST(percent_passed/100.00 AS DECIMAL(3,2)) FROM builds order by app_name, created_at", 1176 | Fields: []Field{ 1177 | {Name: "App", Type: StringType}, 1178 | {Name: "Percentage Completed", Type: PercentageType}, 1179 | }, 1180 | }, 1181 | }, 1182 | }, 1183 | out: []map[string]interface{}{ 1184 | { 1185 | "app": "", 1186 | "percentage_completed": 0.01, 1187 | }, 1188 | { 1189 | "app": "", 1190 | "percentage_completed": 0.34, 1191 | }, 1192 | { 1193 | "app": "everdeen", 1194 | "percentage_completed": 0.8, 1195 | }, 1196 | { 1197 | "app": "everdeen", 1198 | "percentage_completed": 1.0, 1199 | }, 1200 | { 1201 | "app": "geckoboard-ruby", 1202 | "percentage_completed": 0, 1203 | }, 1204 | { 1205 | "app": "geckoboard-ruby", 1206 | "percentage_completed": 0.24, 1207 | }, 1208 | { 1209 | "app": "geckoboard-ruby", 1210 | "percentage_completed": 0.55, 1211 | }, 1212 | { 1213 | "app": "react", 1214 | "percentage_completed": 0.95, 1215 | }, 1216 | { 1217 | "app": "westworld", 1218 | "percentage_completed": 0, 1219 | }, 1220 | }, 1221 | err: "", 1222 | }, 1223 | { 1224 | // NumberType as float64 and date only type 1225 | config: Config{ 1226 | DatabaseConfig: &DatabaseConfig{ 1227 | Driver: MySQLDriver, 1228 | URL: env, 1229 | }, 1230 | Datasets: []Dataset{ 1231 | { 1232 | SQL: "SELECT app_name, created_at, run_time FROM builds where app_name <> '' order by app_name, created_at", 1233 | Fields: []Field{ 1234 | {Name: "App", Type: StringType}, 1235 | {Name: "Date", Type: DateType}, 1236 | {Name: "Run time", Type: NumberType}, 1237 | }, 1238 | }, 1239 | }, 1240 | }, 1241 | out: []map[string]interface{}{ 1242 | { 1243 | "app": "everdeen", 1244 | "date": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 1245 | "run_time": 0.31882276212, 1246 | }, 1247 | { 1248 | "app": "everdeen", 1249 | "date": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 1250 | "run_time": 144.31838122382, 1251 | }, 1252 | { 1253 | "app": "geckoboard-ruby", 1254 | "date": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 1255 | "run_time": 0, 1256 | }, 1257 | { 1258 | "app": "geckoboard-ruby", 1259 | "date": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 1260 | "run_time": 0.21882232124, 1261 | }, 1262 | { 1263 | "app": "geckoboard-ruby", 1264 | "date": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 1265 | "run_time": 77.21381276421, 1266 | }, 1267 | { 1268 | "app": "react", 1269 | "date": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 1270 | "run_time": 118.18382961212, 1271 | }, 1272 | { 1273 | "app": "westworld", 1274 | "date": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 1275 | "run_time": 321.93774373, 1276 | }, 1277 | }, 1278 | err: "", 1279 | }, 1280 | { 1281 | config: Config{ 1282 | DatabaseConfig: &DatabaseConfig{ 1283 | Driver: MySQLDriver, 1284 | URL: env, 1285 | }, 1286 | Datasets: []Dataset{ 1287 | { 1288 | SQL: "SELECT app_name, null, ROUND(run_time, 7) FROM builds order by created_at limit 1", 1289 | Fields: []Field{ 1290 | {Name: "App", Type: StringType}, 1291 | {Name: "Date", Type: DateType}, 1292 | {Name: "Run time", Type: NumberType}, 1293 | }, 1294 | }, 1295 | }, 1296 | }, 1297 | out: []map[string]interface{}{ 1298 | { 1299 | "app": "everdeen", 1300 | "date": nil, 1301 | "run_time": 0.3188228, 1302 | }, 1303 | }, 1304 | err: "", 1305 | }, 1306 | { 1307 | //NumberType as optional and is null returns null 1308 | config: Config{ 1309 | DatabaseConfig: &DatabaseConfig{ 1310 | Driver: MySQLDriver, 1311 | URL: env, 1312 | }, 1313 | Datasets: []Dataset{ 1314 | { 1315 | SQL: `SELECT "test", null FROM builds limit 1`, 1316 | Fields: []Field{ 1317 | {Name: "App", Type: StringType}, 1318 | {Name: "Run time", Type: NumberType, Optional: true}, 1319 | }, 1320 | }, 1321 | }, 1322 | }, 1323 | out: []map[string]interface{}{ 1324 | { 1325 | "app": "test", 1326 | "run_time": nil, 1327 | }, 1328 | }, 1329 | }, 1330 | { 1331 | // No rows returns empty slice 1332 | config: Config{ 1333 | DatabaseConfig: &DatabaseConfig{ 1334 | Driver: MySQLDriver, 1335 | URL: env, 1336 | }, 1337 | Datasets: []Dataset{ 1338 | { 1339 | SQL: `SELECT "test", null FROM builds WHERE id < 0`, 1340 | Fields: []Field{ 1341 | {Name: "App", Type: StringType}, 1342 | {Name: "Run time", Type: NumberType, Optional: true}, 1343 | }, 1344 | }, 1345 | }, 1346 | }, 1347 | out: DatasetRows{}, 1348 | }, 1349 | } 1350 | 1351 | for idx, tc := range testCases { 1352 | db := NewDBConnection(t, tc.config.DatabaseConfig.Driver, tc.config.DatabaseConfig.URL) 1353 | out, err := tc.config.Datasets[0].BuildDataset(tc.config.DatabaseConfig, db) 1354 | 1355 | if tc.err == "" && err != nil { 1356 | t.Errorf("[%d] Expected no error but got %s", idx, err) 1357 | } 1358 | 1359 | if err != nil && tc.err != err.Error() { 1360 | t.Errorf("[%d] Expected error %s but got %s", idx, tc.err, err) 1361 | } 1362 | 1363 | if err == nil && out == nil { 1364 | t.Errorf("expected slice to be initialized when no error but wasn't") 1365 | } 1366 | 1367 | if len(out) != len(tc.out) { 1368 | fmt.Printf("%#v\n", out) 1369 | t.Errorf("[%d] Expected slice size %d but got %d", idx, len(tc.out), len(out)) 1370 | continue 1371 | } 1372 | 1373 | for i, mp := range out { 1374 | for k, v := range mp { 1375 | if tc.out[i][k] != v { 1376 | t.Errorf("[%d-%d] Expected key '%s' to have value %v but got %v", idx, i, k, tc.out[i][k], v) 1377 | } 1378 | } 1379 | } 1380 | } 1381 | } 1382 | 1383 | func TestBuildDatasetMSSQLDriver(t *testing.T) { 1384 | env, ok := os.LookupEnv("MSSQL_URL") 1385 | 1386 | if !ok { 1387 | t.Errorf("This test requires real mssql db using env:MSSQL_URL ensure the db exists") 1388 | return 1389 | } 1390 | 1391 | db, err := sql.Open("mssql", env) 1392 | contents, err := ioutil.ReadFile("fixtures/mssql.sql") 1393 | if err != nil { 1394 | t.Fatal(err) 1395 | } 1396 | 1397 | queries := strings.Split(string(contents), ";") 1398 | 1399 | for _, query := range queries { 1400 | if _, err = db.Exec(query); err != nil { 1401 | t.Fatal(err) 1402 | } 1403 | } 1404 | 1405 | testCases := []struct { 1406 | config Config 1407 | out []map[string]interface{} 1408 | err string 1409 | }{ 1410 | { 1411 | config: Config{ 1412 | GeckoboardAPIKey: "1234-12345", 1413 | RefreshTimeSec: 120, 1414 | DatabaseConfig: &DatabaseConfig{ 1415 | Driver: MSSQLDriver, 1416 | URL: "odbc:server=localhost;user id=userx;password=Passw3d;database=testdb", 1417 | }, 1418 | Datasets: []Dataset{ 1419 | { 1420 | Name: "users.count", 1421 | UpdateType: Replace, 1422 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 1423 | Fields: []Field{ 1424 | {Name: "App", Type: StringType}, 1425 | {Name: "Build Count", Type: MoneyType}, 1426 | }, 1427 | }, 1428 | }, 1429 | }, 1430 | out: nil, 1431 | err: fmt.Sprintf(errFailedSQLQuery, "login error: mssql: Login failed for user 'userx'."), 1432 | }, 1433 | { 1434 | config: Config{ 1435 | DatabaseConfig: &DatabaseConfig{ 1436 | Driver: MSSQLDriver, 1437 | URL: env, 1438 | }, 1439 | Datasets: []Dataset{ 1440 | { 1441 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 1442 | Fields: []Field{ 1443 | {Name: "App", Type: NumberType}, 1444 | {Name: "Build Count", Type: NumberType}, 1445 | }, 1446 | }, 1447 | }, 1448 | }, 1449 | out: nil, 1450 | err: fmt.Sprintf(errParseSQLResultSet, `sql: Scan error on column index 0, name "app_name": can't convert string "" to number`), 1451 | }, 1452 | { 1453 | config: Config{ 1454 | DatabaseConfig: &DatabaseConfig{ 1455 | Driver: MSSQLDriver, 1456 | URL: env, 1457 | }, 1458 | Datasets: []Dataset{ 1459 | { 1460 | SQL: "SELECT app_name, create_at FROM builds order by app_name", 1461 | Fields: []Field{ 1462 | {Name: "App", Type: StringType}, 1463 | {Name: "Build Date", Type: DatetimeType}, 1464 | }, 1465 | }, 1466 | }, 1467 | }, 1468 | out: nil, 1469 | err: fmt.Sprintf(errFailedSQLQuery, `mssql: Invalid column name 'create_at'.`), 1470 | }, 1471 | { 1472 | config: Config{ 1473 | DatabaseConfig: &DatabaseConfig{ 1474 | Driver: MSSQLDriver, 1475 | URL: env, 1476 | }, 1477 | Datasets: []Dataset{ 1478 | { 1479 | SQL: "SELECT app_name, build_cost, created_at FROM builds GROUP BY app_name order by app_name", 1480 | Fields: []Field{ 1481 | {Name: "App", Type: StringType}, 1482 | {Name: "Build Count", Type: NumberType}, 1483 | }, 1484 | }, 1485 | }, 1486 | }, 1487 | out: nil, 1488 | err: fmt.Sprintf(errFailedSQLQuery, `mssql: Column 'builds.build_cost' is invalid in the select list because it is not contained in either an aggregate function or the GROUP BY clause.`), 1489 | }, 1490 | { 1491 | // StringType and Number as an int64 1492 | config: Config{ 1493 | DatabaseConfig: &DatabaseConfig{ 1494 | Driver: MSSQLDriver, 1495 | URL: env, 1496 | }, 1497 | Datasets: []Dataset{ 1498 | { 1499 | SQL: "SELECT app_name, count(*) FROM builds GROUP BY app_name order by app_name", 1500 | Fields: []Field{ 1501 | {Name: "App", Type: StringType}, 1502 | {Name: "Build Count", Type: NumberType}, 1503 | }, 1504 | }, 1505 | }, 1506 | }, 1507 | out: []map[string]interface{}{ 1508 | { 1509 | "app": "", 1510 | "build_count": int64(2), 1511 | }, 1512 | { 1513 | "app": "everdeen", 1514 | "build_count": int64(2), 1515 | }, 1516 | { 1517 | "app": "geckoboard-ruby", 1518 | "build_count": int64(3), 1519 | }, 1520 | { 1521 | "app": "react", 1522 | "build_count": int64(1), 1523 | }, 1524 | { 1525 | "app": "westworld", 1526 | "build_count": int64(1), 1527 | }, 1528 | }, 1529 | err: "", 1530 | }, 1531 | { 1532 | // Date only with money type grouping by date in sql 1533 | config: Config{ 1534 | DatabaseConfig: &DatabaseConfig{ 1535 | Driver: MSSQLDriver, 1536 | URL: env, 1537 | }, 1538 | Datasets: []Dataset{ 1539 | { 1540 | SQL: "SELECT CONVERT(date, created_at) dte, SUM(CAST(build_cost*100 AS INT)) FROM builds GROUP BY CONVERT(date, created_at) order by CONVERT(date, created_at)", 1541 | Fields: []Field{ 1542 | {Name: "Day", Type: DateType}, 1543 | {Name: "Build Cost", Type: MoneyType}, 1544 | }, 1545 | }, 1546 | }, 1547 | }, 1548 | out: []map[string]interface{}{ 1549 | { 1550 | "day": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 1551 | "build_cost": int64(198), 1552 | }, 1553 | { 1554 | "day": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 1555 | "build_cost": int64(1396), 1556 | }, 1557 | { 1558 | "day": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 1559 | "build_cost": int64(227), 1560 | }, 1561 | }, 1562 | err: "", 1563 | }, 1564 | { 1565 | // Datetime type example 1566 | config: Config{ 1567 | DatabaseConfig: &DatabaseConfig{ 1568 | Driver: MSSQLDriver, 1569 | URL: env, 1570 | }, 1571 | Datasets: []Dataset{ 1572 | { 1573 | SQL: "SELECT app_name, created_at FROM builds order by created_at;", 1574 | Fields: []Field{ 1575 | {Name: "App", Type: StringType}, 1576 | {Name: "Day", Type: DatetimeType}, 1577 | }, 1578 | }, 1579 | }, 1580 | }, 1581 | out: []map[string]interface{}{ 1582 | { 1583 | "app": "everdeen", 1584 | "day": parseTime("2017-03-21T11:12:00Z", t).Format(time.RFC3339), 1585 | }, 1586 | { 1587 | "app": "everdeen", 1588 | "day": parseTime("2017-03-21T11:13:00Z", t).Format(time.RFC3339), 1589 | }, 1590 | { 1591 | "app": "westworld", 1592 | "day": parseTime("2017-03-23T15:11:00Z", t).Format(time.RFC3339), 1593 | }, 1594 | { 1595 | "app": "geckoboard-ruby", 1596 | "day": parseTime("2017-03-23T16:12:00Z", t).Format(time.RFC3339), 1597 | }, 1598 | { 1599 | "app": "", 1600 | "day": parseTime("2017-03-23T16:22:00Z", t).Format(time.RFC3339), 1601 | }, 1602 | { 1603 | "app": "", 1604 | "day": parseTime("2017-03-23T16:44:00Z", t).Format(time.RFC3339), 1605 | }, 1606 | { 1607 | "app": "react", 1608 | "day": parseTime("2017-04-23T12:32:00Z", t).Format(time.RFC3339), 1609 | }, 1610 | { 1611 | "app": "geckoboard-ruby", 1612 | "day": parseTime("2017-04-23T13:42:00Z", t).Format(time.RFC3339), 1613 | }, 1614 | { 1615 | "app": "geckoboard-ruby", 1616 | "day": parseTime("2017-04-23T13:43:00Z", t).Format(time.RFC3339), 1617 | }, 1618 | }, 1619 | err: "", 1620 | }, 1621 | { 1622 | // PercentageType with stringType 1623 | config: Config{ 1624 | DatabaseConfig: &DatabaseConfig{ 1625 | Driver: MSSQLDriver, 1626 | URL: env, 1627 | }, 1628 | Datasets: []Dataset{ 1629 | { 1630 | SQL: "SELECT app_name, CAST(percent_passed/100.00 AS DECIMAL(3,2)) FROM builds order by app_name, created_at", 1631 | Fields: []Field{ 1632 | {Name: "App", Type: StringType}, 1633 | {Name: "Percentage Completed", Type: PercentageType}, 1634 | }, 1635 | }, 1636 | }, 1637 | }, 1638 | out: []map[string]interface{}{ 1639 | { 1640 | "app": "", 1641 | "percentage_completed": 0.01, 1642 | }, 1643 | { 1644 | "app": "", 1645 | "percentage_completed": 0.34, 1646 | }, 1647 | { 1648 | "app": "everdeen", 1649 | "percentage_completed": 0.8, 1650 | }, 1651 | { 1652 | "app": "everdeen", 1653 | "percentage_completed": 1.0, 1654 | }, 1655 | { 1656 | "app": "geckoboard-ruby", 1657 | "percentage_completed": 0, 1658 | }, 1659 | { 1660 | "app": "geckoboard-ruby", 1661 | "percentage_completed": 0.24, 1662 | }, 1663 | { 1664 | "app": "geckoboard-ruby", 1665 | "percentage_completed": 0.55, 1666 | }, 1667 | { 1668 | "app": "react", 1669 | "percentage_completed": 0.95, 1670 | }, 1671 | { 1672 | "app": "westworld", 1673 | "percentage_completed": 0, 1674 | }, 1675 | }, 1676 | err: "", 1677 | }, 1678 | { 1679 | // NumberType as float64 and date only type 1680 | config: Config{ 1681 | DatabaseConfig: &DatabaseConfig{ 1682 | Driver: MSSQLDriver, 1683 | URL: env, 1684 | }, 1685 | Datasets: []Dataset{ 1686 | { 1687 | SQL: "SELECT app_name, created_at, run_time FROM builds where app_name <> '' order by app_name, created_at", 1688 | Fields: []Field{ 1689 | {Name: "App", Type: StringType}, 1690 | {Name: "Date", Type: DateType}, 1691 | {Name: "Run time", Type: NumberType}, 1692 | }, 1693 | }, 1694 | }, 1695 | }, 1696 | out: []map[string]interface{}{ 1697 | { 1698 | "app": "everdeen", 1699 | "date": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 1700 | "run_time": 0.31882276212, 1701 | }, 1702 | { 1703 | "app": "everdeen", 1704 | "date": parseTime("2017-03-21T00:00:00Z", t).Format(dateFormat), 1705 | "run_time": 144.31838122382, 1706 | }, 1707 | { 1708 | "app": "geckoboard-ruby", 1709 | "date": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 1710 | "run_time": 0, 1711 | }, 1712 | { 1713 | "app": "geckoboard-ruby", 1714 | "date": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 1715 | "run_time": 0.21882232124, 1716 | }, 1717 | { 1718 | "app": "geckoboard-ruby", 1719 | "date": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 1720 | "run_time": 77.21381276421, 1721 | }, 1722 | { 1723 | "app": "react", 1724 | "date": parseTime("2017-04-23T00:00:00Z", t).Format(dateFormat), 1725 | "run_time": 118.18382961212, 1726 | }, 1727 | { 1728 | "app": "westworld", 1729 | "date": parseTime("2017-03-23T00:00:00Z", t).Format(dateFormat), 1730 | "run_time": 321.93774373, 1731 | }, 1732 | }, 1733 | err: "", 1734 | }, 1735 | { 1736 | config: Config{ 1737 | DatabaseConfig: &DatabaseConfig{ 1738 | Driver: MSSQLDriver, 1739 | URL: env, 1740 | }, 1741 | Datasets: []Dataset{ 1742 | { 1743 | SQL: "SELECT TOP 1 app_name, null, ROUND(run_time, 7) FROM builds order by created_at", 1744 | Fields: []Field{ 1745 | {Name: "App", Type: StringType}, 1746 | {Name: "Date", Type: DateType}, 1747 | {Name: "Run time", Type: NumberType}, 1748 | }, 1749 | }, 1750 | }, 1751 | }, 1752 | out: []map[string]interface{}{ 1753 | { 1754 | "app": "everdeen", 1755 | "date": nil, 1756 | "run_time": 0.3188228, 1757 | }, 1758 | }, 1759 | err: "", 1760 | }, 1761 | { 1762 | //NumberType as optional and is null returns null 1763 | config: Config{ 1764 | DatabaseConfig: &DatabaseConfig{ 1765 | Driver: MSSQLDriver, 1766 | URL: env, 1767 | }, 1768 | Datasets: []Dataset{ 1769 | { 1770 | SQL: `SELECT 'test', NULL`, 1771 | Fields: []Field{ 1772 | {Name: "App", Type: StringType}, 1773 | {Name: "Run time", Type: NumberType, Optional: true}, 1774 | }, 1775 | }, 1776 | }, 1777 | }, 1778 | out: []map[string]interface{}{ 1779 | { 1780 | "app": "test", 1781 | "run_time": nil, 1782 | }, 1783 | }, 1784 | }, 1785 | { 1786 | // No rows returns empty slice 1787 | config: Config{ 1788 | DatabaseConfig: &DatabaseConfig{ 1789 | Driver: MSSQLDriver, 1790 | URL: env, 1791 | }, 1792 | Datasets: []Dataset{ 1793 | { 1794 | SQL: `SELECT 'test', null FROM builds WHERE id < 0`, 1795 | Fields: []Field{ 1796 | {Name: "App", Type: StringType}, 1797 | {Name: "Run time", Type: NumberType, Optional: true}, 1798 | }, 1799 | }, 1800 | }, 1801 | }, 1802 | out: DatasetRows{}, 1803 | }, 1804 | } 1805 | 1806 | for idx, tc := range testCases { 1807 | db := NewDBConnection(t, tc.config.DatabaseConfig.Driver, tc.config.DatabaseConfig.URL) 1808 | out, err := tc.config.Datasets[0].BuildDataset(tc.config.DatabaseConfig, db) 1809 | 1810 | if tc.err == "" && err != nil { 1811 | t.Errorf("[%d] Expected no error but got %s", idx, err) 1812 | } 1813 | 1814 | if err != nil && tc.err != err.Error() { 1815 | t.Errorf("[%d] Expected error %s but got %s", idx, tc.err, err) 1816 | } 1817 | 1818 | if err != nil && tc.err != err.Error() { 1819 | t.Errorf("[%d] Expected error %s but got %s", idx, tc.err, err) 1820 | } 1821 | 1822 | if len(out) != len(tc.out) { 1823 | t.Errorf("[%d] Expected slice size %d but got %d", idx, len(tc.out), len(out)) 1824 | continue 1825 | } 1826 | 1827 | for i, mp := range out { 1828 | for k, v := range mp { 1829 | if tc.out[i][k] != v { 1830 | t.Errorf("[%d-%d] Expected key '%s' to have value %v but got %v", idx, i, k, tc.out[i][k], v) 1831 | } 1832 | } 1833 | } 1834 | } 1835 | } 1836 | 1837 | func parseTime(str string, t *testing.T) time.Time { 1838 | tme, err := time.Parse(time.RFC3339, str) 1839 | 1840 | if err != nil { 1841 | t.Fatal(err) 1842 | } 1843 | 1844 | return tme 1845 | 1846 | } 1847 | 1848 | func NewDBConnection(t *testing.T, driver, url string) *sql.DB { 1849 | pool, err := sql.Open(driver, url) 1850 | 1851 | if err != nil { 1852 | t.Fatalf("Database open failed: %s", err) 1853 | } 1854 | 1855 | return pool 1856 | } 1857 | -------------------------------------------------------------------------------- /scripts/wait_for_mysql: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ATTEMPTS=0 4 | 5 | until docker exec -it $1 mysql -uroot -proot -e "show databases;" -P 3306 > /dev/null 2>&1; 6 | do 7 | COUNTER=$((COUNTER + 1)) 8 | echo "Waiting for mysql in docker container" 9 | 10 | if [[ "$COUNTER" == 20 ]]; then 11 | echo "MySQL still not responding after 20 attempts... Stopping" 12 | exit 1 13 | fi 14 | 15 | sleep 1; 16 | done 17 | -------------------------------------------------------------------------------- /scripts/wait_for_postgres: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Taken from https://starkandwayne.com/blog/how-to-know-when-your-postgres-service-is-ready/ 4 | 5 | ATTEMPTS=0 6 | until docker exec -it $1 /usr/bin/pg_isready -h localhost -p 5432 -U postgres 7 | do 8 | COUNTER=$((COUNTER + 1)) 9 | echo "Waiting for postgres in docker container" 10 | 11 | if [[ "$COUNTER" == 20 ]]; then 12 | echo "PSQL still not responding after 20 attempts... Stopping" 13 | exit 1 14 | fi 15 | 16 | sleep 1; 17 | done 18 | --------------------------------------------------------------------------------