├── .circleci └── config.yml ├── .gitignore ├── LICENSE ├── README.md ├── build.sh ├── go.mod ├── go.sum ├── main.go ├── main_test.go └── test_db └── ip_dns.db /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Golang CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-go/ for more details 4 | version: 2 5 | jobs: 6 | build: 7 | docker: 8 | - image: circleci/golang:latest 9 | working_directory: /go/src/github.com/assafmo/SQLiteQueryServer 10 | steps: 11 | - checkout 12 | - run: go version 13 | - run: go get -v -t -d ./... 14 | - run: go get github.com/mattn/goveralls 15 | - run: go test -v -cover -race -coverprofile=./coverage.out ./... 16 | - run: $GOPATH/bin/goveralls -coverprofile=./coverage.out -service=circle-ci -repotoken=$COVERALLS_TOKEN 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | debug 2 | debug.test 3 | SQLiteQueryServer* 4 | release 5 | .vscode 6 | *wal 7 | *shm -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Assaf Morami 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SQLiteQueryServer 2 | 3 | Bulk query SQLite database over the network. 4 | Way faster than [SQLiteProxy](https://github.com/assafmo/SQLiteProxy)! 5 | 6 | [![CircleCI](https://circleci.com/gh/assafmo/SQLiteQueryServer.svg?style=shield&circle-token=cda4af2f2b6cc0035287b25086c596d2ef44d9ce)](https://circleci.com/gh/assafmo/SQLiteQueryServer) 7 | [![Coverage Status](https://coveralls.io/repos/github/assafmo/SQLiteQueryServer/badge.svg?branch=master)](https://coveralls.io/github/assafmo/SQLiteQueryServer?branch=master) 8 | [![Go Report Card](https://goreportcard.com/badge/github.com/assafmo/SQLiteQueryServer)](https://goreportcard.com/report/github.com/assafmo/SQLiteQueryServer) 9 | 10 | # Installation 11 | 12 | - Download a precompiled binary from https://github.com/assafmo/SQLiteQueryServer/releases 13 | - Or use `go get`: 14 | 15 | ```bash 16 | go get -u github.com/assafmo/SQLiteQueryServer 17 | ``` 18 | 19 | This package uses `github.com/mattn/go-sqlite3`. Compilation errors might be resolved by reading https://github.com/mattn/go-sqlite3#compilation. 20 | 21 | - Or use Ubuntu PPA: 22 | 23 | ```bash 24 | curl -SsL https://assafmo.github.io/ppa/ubuntu/KEY.gpg | sudo apt-key add - 25 | sudo curl -SsL -o /etc/apt/sources.list.d/assafmo.list https://assafmo.github.io/ppa/ubuntu/assafmo.list 26 | sudo apt update 27 | sudo apt install sqlitequeryserver 28 | ``` 29 | 30 | # Usage 31 | 32 | ``` 33 | Usage of SQLiteQueryServer: 34 | -db string 35 | Filesystem path of the SQLite database 36 | -port uint 37 | HTTP port to listen on (default 80) 38 | -query string 39 | SQL query to prepare for 40 | ``` 41 | 42 | Note: SQLiteQueryServer is optimized for the SELECT command. Other commands such as INSERT, UPDATE, DELETE, CREATE, etc might be slow because SQLiteQueryServer doesn't use transactions (yet). Also, the response format and error messages from these commands may be odd or unexpected. 43 | 44 | # Examples 45 | 46 | ## Creating a server 47 | 48 | ```bash 49 | SQLiteQueryServer --db "$DB_PATH" --query "$PARAMETERIZED_SQL_QUERY" --port "$PORT" 50 | ``` 51 | 52 | ```bash 53 | SQLiteQueryServer --db ./test_db/ip_dns.db --query "SELECT * FROM ip_dns WHERE dns = ?" --port 8080 54 | ``` 55 | 56 | This will expose the `./test_db/ip_dns.db` database with the query `SELECT * FROM ip_dns WHERE dns = ?` on port `8080`. 57 | Requests will need to provide the query parameters. 58 | 59 | ## Querying the server 60 | 61 | ```bash 62 | echo -e "github.com\none.one.one.one\ngoogle-public-dns-a.google.com" | curl "http://localhost:8080/query" --data-binary @- 63 | ``` 64 | 65 | ```bash 66 | echo -e "$QUERY1_PARAM1,$QUERY1_PARAM2\n$QUERY2_PARAM1,$QUERY2_PARAM2" | curl "http://$ADDRESS:$PORT/query" --data-binary @- 67 | ``` 68 | 69 | ```bash 70 | curl "http://$ADDRESS:$PORT/query" -d "$PARAM_1,$PARAM_2,...,$PARAM_N" 71 | ``` 72 | 73 | - Request must be a HTTP POST to "http://$ADDRESS:$PORT/query". 74 | - Request body must be a valid CSV. 75 | - Request body must not have a CSV header. 76 | - Each request body line is a different query. 77 | - Each param in a line corresponds to a query param (a question mark in the query string). 78 | - Static query (without any query params): 79 | - The request must be a HTTP GET to "http://$ADDRESS:$PORT/query". 80 | - The query executes only once. 81 | 82 | ## Getting a response 83 | 84 | ```bash 85 | echo -e "github.com\none.one.one.one\ngoogle-public-dns-a.google.com" | curl "http://localhost:8080/query" --data-binary @- 86 | ``` 87 | 88 | ```json 89 | [ 90 | { 91 | "in": ["github.com"], 92 | "headers": ["ip", "dns"], 93 | "out": [["192.30.253.112", "github.com"], ["192.30.253.113", "github.com"]] 94 | }, 95 | { 96 | "in": ["one.one.one.one"], 97 | "headers": ["ip", "dns"], 98 | "out": [["1.1.1.1", "one.one.one.one"]] 99 | }, 100 | { 101 | "in": ["google-public-dns-a.google.com"], 102 | "headers": ["ip", "dns"], 103 | "out": [["8.8.8.8", "google-public-dns-a.google.com"]] 104 | } 105 | ] 106 | ``` 107 | 108 | - If response status is 200 (OK), response is a JSON array (`Content-Type: application/json`). 109 | - Each element in the array: 110 | - Is a result of a query 111 | - Has an "in" field which is an array of the input params (a request body line). 112 | - Has an "headers" field which is an array of headers of the SQL query result. 113 | - Has an "out" field which is an array of arrays of results. Each inner array is a result row. 114 | - Element #1 is the result of query #1, Element #2 is the result of query #2, and so forth. 115 | - Static query (without any query params): 116 | - The response JSON has only one element. 117 | 118 | ## Static query 119 | 120 | ```bash 121 | SQLiteQueryServer --db ./test_db/ip_dns.db --query "SELECT * FROM ip_dns" --port 8080 122 | ``` 123 | 124 | ```bash 125 | curl "http://localhost:8080/query" 126 | ``` 127 | 128 | ```json 129 | [ 130 | { 131 | "in": [], 132 | "headers": ["ip", "dns"], 133 | "out": [ 134 | ["1.1.1.1", "one.one.one.one"], 135 | ["8.8.8.8", "google-public-dns-a.google.com"], 136 | ["192.30.253.112", "github.com"], 137 | ["192.30.253.113", "github.com"] 138 | ] 139 | } 140 | ] 141 | ``` 142 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # build into ./release/ 4 | 5 | set -e 6 | set -v 7 | 8 | go test -race -cover ./... 9 | 10 | rm -rf release 11 | mkdir -p release 12 | 13 | VERSION=$(git describe --tags $(git rev-list --tags --max-count=1)) 14 | 15 | xgo --targets windows/amd64 --dest release --out SQLiteQueryServer-"${VERSION}" --ldflags "-s -w" . 16 | xgo --targets linux/amd64 --dest release --out SQLiteQueryServer-"${VERSION}" --tags linux --ldflags "-s -w -extldflags -static" . 17 | xgo --targets darwin/amd64 --dest release --out SQLiteQueryServer-"${VERSION}" --tags darwin --ldflags "-s -w" . 18 | 19 | ( 20 | # zip 21 | cd release 22 | find -type f | 23 | parallel --bar 'zip "$(echo "{}" | sed "s/.exe//").zip" "{}" && rm -f "{}"' 24 | 25 | # deb 26 | mkdir -p ./deb/bin 27 | unzip -o -d ./deb/bin *-linux-amd64.zip 28 | mv -f ./deb/bin/*-linux-amd64 ./deb/bin/SQLiteQueryServer 29 | 30 | mkdir -p ./deb/DEBIAN 31 | cat > ./deb/DEBIAN/control < 37 | Homepage: https://github.com/assafmo/SQLiteQueryServer 38 | Installed-Size: $(ls -l --block-size=KB ./deb/bin/SQLiteQueryServer | awk '{print $5}' | tr -d 'kB') 39 | Description: Bulk query SQLite database over the network. 40 | EOF 41 | 42 | dpkg-deb --build ./deb/ . 43 | rm -rf ./deb/ 44 | ) 45 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/assafmo/SQLiteQueryServer 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/json-iterator/go v1.1.9 7 | github.com/mattn/go-sqlite3 v2.0.3+incompatible 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 5 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 6 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 7 | github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= 8 | github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 9 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 10 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 11 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 12 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 13 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 14 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 15 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 16 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 17 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 18 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/csv" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "log" 10 | "net/http" 11 | "os" 12 | "regexp" 13 | "strconv" 14 | "strings" 15 | 16 | json "github.com/json-iterator/go" 17 | _ "github.com/mattn/go-sqlite3" 18 | ) 19 | 20 | const version = "1.4.0" 21 | 22 | func main() { 23 | if err := cmd(os.Args[1:]); err != nil { 24 | log.Fatal(err) 25 | } 26 | } 27 | 28 | func cmd(cmdArgs []string) error { 29 | log.Printf("SQLiteQueryServer v%s\n", version) 30 | log.Println("https://github.com/assafmo/SQLiteQueryServer") 31 | 32 | // Parse cmd args 33 | var flagSet = flag.NewFlagSet("cmd flags", flag.ContinueOnError) 34 | 35 | var dbPath string 36 | var queryString string 37 | var serverPort uint 38 | 39 | flagSet.StringVar(&dbPath, "db", "", "Filesystem path of the SQLite database") 40 | flagSet.StringVar(&queryString, "query", "", "SQL query to prepare for") 41 | flagSet.UintVar(&serverPort, "port", 80, "HTTP port to listen on") 42 | 43 | err := flagSet.Parse(cmdArgs) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | // Init db and query 49 | queryHandler, err := initQueryHandler(dbPath, queryString, serverPort) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | // Start the server 55 | log.Printf("Starting server on port %d...\n", serverPort) 56 | log.Printf("Starting server with query '%s'...\n", queryString) 57 | 58 | http.HandleFunc("/query", queryHandler) 59 | err = http.ListenAndServe(fmt.Sprintf(":%d", serverPort), nil) 60 | 61 | return err 62 | } 63 | 64 | type queryResult struct { 65 | In []string `json:"in"` 66 | Headers []string `json:"headers"` 67 | Out [][]interface{} `json:"out"` 68 | } 69 | 70 | func initQueryHandler(dbPath string, queryString string, serverPort uint) (func(w http.ResponseWriter, r *http.Request), error) { 71 | // Init db and query 72 | 73 | if dbPath == "" { 74 | return nil, fmt.Errorf("Must provide --db param") 75 | } 76 | if queryString == "" { 77 | return nil, fmt.Errorf("Must provide --query param") 78 | } 79 | if _, err := os.Stat(dbPath); os.IsNotExist(err) { 80 | return nil, fmt.Errorf("Database file '%s' doesn't exist", dbPath) 81 | } 82 | 83 | db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?mode=rw&cache=shared&_journal_mode=WAL", dbPath)) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | db.SetMaxOpenConns(1) 89 | 90 | queryStmt, err := db.Prepare(queryString) 91 | if err != nil { 92 | db.Close() 93 | return nil, err 94 | } 95 | 96 | helpMessage := buildHelpMessage("", queryString, queryStmt, serverPort) 97 | 98 | return func(w http.ResponseWriter, r *http.Request) { 99 | w.Header().Set("Server", "SQLiteQueryServer v"+version) 100 | w.Header().Set("Access-Control-Allow-Origin", "*") 101 | w.Header().Set("X-Content-Type-Options", "nosniff") 102 | 103 | if r.URL.Path != "/query" { 104 | http.Error(w, helpMessage, http.StatusNotFound) 105 | return 106 | } 107 | if r.Method != "POST" && r.Method != "GET" { 108 | http.Error(w, helpMessage, http.StatusMethodNotAllowed) 109 | return 110 | } 111 | 112 | // Init fullResponse 113 | fullResponse := []queryResult{} 114 | 115 | var reqCsvReader *csv.Reader 116 | if r.Method == "GET" { 117 | // Static query 118 | reqCsvReader = csv.NewReader(strings.NewReader("")) 119 | } else { 120 | // Parameterized query 121 | reqCsvReader = csv.NewReader(r.Body) 122 | } 123 | reqCsvReader.FieldsPerRecord = -1 124 | 125 | // Iterate over each query 126 | for { 127 | csvRecord, err := reqCsvReader.Read() 128 | if r.Method == "POST" { 129 | // Parameterized query 130 | if err == io.EOF || err == http.ErrBodyReadAfterClose { 131 | // EOF || last line is without \n 132 | break 133 | } else if err != nil { 134 | http.Error(w, fmt.Sprintf("\n\nError reading request body: %v\n\n%s", err, helpMessage), http.StatusInternalServerError) 135 | return 136 | } 137 | } else { 138 | csvRecord = make([]string, 0) 139 | } 140 | 141 | // Init queryResponse 142 | // Set queryResponse.Headers to the query's params (the fields of the csv record) 143 | var queryResponse queryResult 144 | queryResponse.In = csvRecord 145 | 146 | queryParams := make([]interface{}, len(csvRecord)) 147 | for i := range csvRecord { 148 | queryParams[i] = csvRecord[i] 149 | } 150 | 151 | rows, err := queryStmt.Query(queryParams...) 152 | if err != nil { 153 | http.Error(w, fmt.Sprintf("\n\nError executing query for params %#v: %v\n\n%s", csvRecord, err, helpMessage), http.StatusInternalServerError) 154 | return 155 | } 156 | defer rows.Close() 157 | 158 | // Set queryResponse.Headers to the query's columns 159 | // Init queryResponse.Out 160 | cols, err := rows.Columns() 161 | if err != nil { 162 | http.Error(w, fmt.Sprintf("\n\nError reading columns for query with params %#v: %v\n\n%s", csvRecord, err, helpMessage), http.StatusInternalServerError) 163 | return 164 | } 165 | 166 | queryResponse.Headers = cols 167 | queryResponse.Out = make([][]interface{}, 0) 168 | 169 | // Iterate over returned rows for this query 170 | // Append each row to queryResponse.Out 171 | for rows.Next() { 172 | row := make([]interface{}, len(cols)) 173 | pointers := make([]interface{}, len(row)) 174 | 175 | for i := range row { 176 | pointers[i] = &row[i] 177 | } 178 | 179 | err = rows.Scan(pointers...) 180 | if err != nil { 181 | http.Error(w, fmt.Sprintf("\n\nError reading query results for params %#v: %v\n\n%s", csvRecord, err, helpMessage), http.StatusInternalServerError) 182 | return 183 | } 184 | 185 | queryResponse.Out = append(queryResponse.Out, row) 186 | } 187 | err = rows.Err() 188 | if err != nil { 189 | http.Error(w, fmt.Sprintf("\n\nError executing query: %v\n\n%s", err, helpMessage), http.StatusInternalServerError) 190 | return 191 | } 192 | 193 | fullResponse = append(fullResponse, queryResponse) 194 | 195 | if r.Method == "GET" { 196 | // Static query - execute only once 197 | break 198 | } 199 | } 200 | 201 | // Return json 202 | w.Header().Add("Content-Type", "application/json") 203 | 204 | answerJSON, err := json.Marshal(fullResponse) 205 | if err != nil { 206 | http.Error(w, fmt.Sprintf("\n\nError encoding json: %v\n\n%s", err, helpMessage), http.StatusInternalServerError) 207 | return 208 | } 209 | 210 | _, err = w.Write(answerJSON) 211 | if err != nil { 212 | http.Error(w, fmt.Sprintf("\n\nError sending json to client: %v\n\n%s", err, helpMessage), http.StatusInternalServerError) 213 | return 214 | } 215 | }, nil 216 | } 217 | 218 | func buildHelpMessage(helpMessage string, queryString string, queryStmt *sql.Stmt, serverPort uint) string { 219 | helpMessage += fmt.Sprintf(`Query: 220 | %s 221 | 222 | `, queryString) 223 | 224 | queryParamsCount, err := countParams(queryStmt) 225 | if err != nil { 226 | log.Printf("Error extracting params count from query: %v\n", err) 227 | } else { 228 | helpMessage += fmt.Sprintf(`Params count (question marks in query): 229 | %d 230 | 231 | `, queryParamsCount) 232 | } 233 | 234 | helpMessage += fmt.Sprintf(`Request examples: 235 | $ echo -e "$QUERY1_PARAM1,$QUERY1_PARAM2\n$QUERY2_PARAM1,$QUERY2_PARAM2" curl "http://$ADDRESS:%d/query" --data-binary @- 236 | $ curl "http://$ADDRESS:%d/query" -d "$PARAM_1,$PARAM_2,...,$PARAM_N" 237 | 238 | - Request must be a HTTP POST to "http://$ADDRESS:%d/query". 239 | - Request body must be a valid CSV. 240 | - Request body must not have a CSV header. 241 | - Each request body line is a different query. 242 | - Each param in a line corresponds to a query param (a question mark in the query string). 243 | - Static query (without any query params): 244 | - The request must be a HTTP GET to "http://$ADDRESS:%d/query". 245 | - The query executes only once. 246 | 247 | `, serverPort, serverPort, serverPort, serverPort) 248 | 249 | helpMessage += fmt.Sprintf(`Response example: 250 | $ echo -e "github.com\none.one.one.one\ngoogle-public-dns-a.google.com" | curl "http://$ADDRESS:%d/query" --data-binary @- 251 | [ 252 | { 253 | "in": ["github.com"], 254 | "headers": ["ip","dns"], 255 | "out": [ 256 | ["192.30.253.112","github.com"], 257 | ["192.30.253.113","github.com"] 258 | ] 259 | }, 260 | { 261 | "in": ["one.one.one.one"], 262 | "headers": ["ip","dns"], 263 | "out": [ 264 | ["1.1.1.1","one.one.one.one"] 265 | ] 266 | }, 267 | { 268 | "in": ["google-public-dns-a.google.com"], 269 | "headers": ["ip","dns"], 270 | "out": [ 271 | ["8.8.8.8","google-public-dns-a.google.com"] 272 | ] 273 | } 274 | ] 275 | 276 | - Response is a JSON array (Content-Type: application/json). 277 | - Each element in the array: 278 | - Is a result of a query 279 | - Has an "in" fields which is an array of the input params (a request body line). 280 | - Has an "headers" fields which is an array of headers of the SQL query result. 281 | - Has an "out" field which is an array of arrays of results. Each inner array is a result row. 282 | - Element #1 is the result of query #1, Element #2 is the result of query #2, and so forth. 283 | - Static query (without any query params): 284 | - The response JSON has only one element. 285 | 286 | For more info visit https://github.com/assafmo/SQLiteQueryServer 287 | `, serverPort) 288 | 289 | return helpMessage 290 | } 291 | 292 | func countParams(queryStmt *sql.Stmt) (int, error) { 293 | // Query with 0 params 294 | rows, err := queryStmt.Query() 295 | if err == nil { 296 | // Query went fine, this means it has 0 params 297 | rows.Close() 298 | return 0, nil 299 | } 300 | 301 | // Query returned an error 302 | // Parse the error to get the expected params count 303 | regex := regexp.MustCompile(`sql: expected (\p{N}+) arguments, got 0`) 304 | regexSubmatches := regex.FindAllStringSubmatch(err.Error(), 1) 305 | if len(regexSubmatches) != 1 || len(regexSubmatches[0]) != 2 { 306 | // This is weird 307 | // queryStmt is prepared (compiled) so it is valid 308 | // but yet there was an error executing queryStmt 309 | return -1, fmt.Errorf("Cannot extract params count from query error: %v", err) 310 | } 311 | 312 | countString := regexSubmatches[0][1] 313 | count, err := strconv.Atoi(countString) 314 | if err != nil { 315 | // This is even weirder 316 | // The regex is \p{N}+ (unicode number sequence) and there was a match, 317 | // but converting it from string to int returned an error 318 | return -1, fmt.Errorf(`Cannot convert \p{N}+ regex to int: %v`, err) 319 | } 320 | return count, nil 321 | } 322 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | "net/http/httptest" 12 | "os" 13 | "strings" 14 | "testing" 15 | 16 | json "github.com/json-iterator/go" 17 | ) 18 | 19 | var testDbPath = "./test_db/ip_dns.db" 20 | 21 | func TestResultCount(t *testing.T) { 22 | log.SetOutput(&bytes.Buffer{}) 23 | 24 | reqString := "github.com\none.one.one.one\ngoogle-public-dns-a.google.com" 25 | 26 | req := httptest.NewRequest("POST", 27 | "http://example.org/query", 28 | strings.NewReader(reqString)) 29 | w := httptest.NewRecorder() 30 | queryHandler, err := initQueryHandler(testDbPath, "SELECT * FROM ip_dns WHERE dns = ?", 0) 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | queryHandler(w, req) 35 | 36 | resp := w.Result() 37 | defer resp.Body.Close() 38 | 39 | if resp.StatusCode != http.StatusOK { 40 | t.Fatalf(`resp.StatusCode (%d) != http.StatusOK (%d)`, resp.StatusCode, http.StatusOK) 41 | } 42 | 43 | if resp.Header.Get("Content-Type") != "application/json" { 44 | t.Fatalf(`resp.Header.Get("Content-Type") (%s) != "application/json"`, resp.Header.Get("Content-Type")) 45 | } 46 | 47 | var answer []interface{} 48 | decoder := json.NewDecoder(resp.Body) 49 | err = decoder.Decode(&answer) 50 | if err != nil { 51 | t.Fatal(err) 52 | } 53 | 54 | if len(answer) != 3 { 55 | t.Fatal(`len(answer) != 3`) 56 | } 57 | } 58 | 59 | func TestAnswersOrder(t *testing.T) { 60 | log.SetOutput(&bytes.Buffer{}) 61 | 62 | reqString := "github.com\none.one.one.one\ngoogle-public-dns-a.google.com" 63 | 64 | req := httptest.NewRequest("POST", 65 | "http://example.org/query", 66 | strings.NewReader(reqString)) 67 | w := httptest.NewRecorder() 68 | queryHandler, err := initQueryHandler(testDbPath, "SELECT * FROM ip_dns WHERE dns = ?", 0) 69 | if err != nil { 70 | t.Fatal(err) 71 | } 72 | queryHandler(w, req) 73 | 74 | resp := w.Result() 75 | defer resp.Body.Close() 76 | 77 | if resp.StatusCode != http.StatusOK { 78 | t.Fatalf(`resp.StatusCode (%d) != http.StatusOK (%d)`, resp.StatusCode, http.StatusOK) 79 | } 80 | 81 | if resp.Header.Get("Content-Type") != "application/json" { 82 | t.Fatalf(`resp.Header.Get("Content-Type") (%s) != "application/json"`, resp.Header.Get("Content-Type")) 83 | } 84 | 85 | var fullResponse []queryResult 86 | decoder := json.NewDecoder(resp.Body) 87 | err = decoder.Decode(&fullResponse) 88 | if err != nil { 89 | t.Fatal(err) 90 | } 91 | 92 | for i := 0; i < 3; i++ { 93 | if len(fullResponse[i].In) != 1 { 94 | t.Fatalf(`len(answer[%d].In) != 1`, i) 95 | } 96 | } 97 | 98 | for i, v := range []string{"github.com", "one.one.one.one", "google-public-dns-a.google.com"} { 99 | if fullResponse[i].In[0] != v { 100 | t.Fatalf(`answer[%d].In[0] != "%s"`, i, v) 101 | } 102 | } 103 | } 104 | 105 | func TestAnswersHeaders(t *testing.T) { 106 | log.SetOutput(&bytes.Buffer{}) 107 | 108 | reqString := "github.com\none.one.one.one\ngoogle-public-dns-a.google.com" 109 | 110 | req := httptest.NewRequest("POST", 111 | "http://example.org/query", 112 | strings.NewReader(reqString)) 113 | w := httptest.NewRecorder() 114 | queryHandler, err := initQueryHandler(testDbPath, "SELECT * FROM ip_dns WHERE dns = ?", 0) 115 | if err != nil { 116 | t.Fatal(err) 117 | } 118 | queryHandler(w, req) 119 | 120 | resp := w.Result() 121 | defer resp.Body.Close() 122 | 123 | if resp.StatusCode != http.StatusOK { 124 | t.Fatalf(`resp.StatusCode (%d) != http.StatusOK (%d)`, resp.StatusCode, http.StatusOK) 125 | } 126 | 127 | if resp.Header.Get("Content-Type") != "application/json" { 128 | t.Fatalf(`resp.Header.Get("Content-Type") (%s) != "application/json"`, resp.Header.Get("Content-Type")) 129 | } 130 | 131 | var fullResponse []queryResult 132 | decoder := json.NewDecoder(resp.Body) 133 | err = decoder.Decode(&fullResponse) 134 | if err != nil { 135 | t.Fatal(err) 136 | } 137 | 138 | for i := 0; i < 3; i++ { 139 | if len(fullResponse[i].Headers) != 2 { 140 | t.Fatalf(`len(answer[%d].Headers) != 2`, i) 141 | } 142 | } 143 | 144 | for i := 0; i < 3; i++ { 145 | if fullResponse[i].Headers[0] != "ip" { 146 | t.Fatalf(`answer[%d].In[0] != "ip"`, i) 147 | } 148 | if fullResponse[i].Headers[1] != "dns" { 149 | t.Fatalf(`answer[%d].In[1] != "dns"`, i) 150 | } 151 | } 152 | } 153 | 154 | func TestAnswersRows(t *testing.T) { 155 | log.SetOutput(&bytes.Buffer{}) 156 | 157 | reqString := "github.com\none.one.one.one\ngoogle-public-dns-a.google.com" 158 | 159 | req := httptest.NewRequest("POST", 160 | "http://example.org/query", 161 | strings.NewReader(reqString)) 162 | w := httptest.NewRecorder() 163 | queryHandler, err := initQueryHandler(testDbPath, "SELECT * FROM ip_dns WHERE dns = ?", 0) 164 | if err != nil { 165 | t.Fatal(err) 166 | } 167 | queryHandler(w, req) 168 | 169 | resp := w.Result() 170 | defer resp.Body.Close() 171 | 172 | if resp.StatusCode != http.StatusOK { 173 | t.Fatalf(`resp.StatusCode (%d) != http.StatusOK (%d)`, resp.StatusCode, http.StatusOK) 174 | } 175 | 176 | if resp.Header.Get("Content-Type") != "application/json" { 177 | t.Fatalf(`resp.Header.Get("Content-Type") (%s) != "application/json"`, resp.Header.Get("Content-Type")) 178 | } 179 | 180 | var fullResponse []queryResult 181 | decoder := json.NewDecoder(resp.Body) 182 | err = decoder.Decode(&fullResponse) 183 | if err != nil { 184 | t.Fatal(err) 185 | } 186 | 187 | expectedResponse := []queryResult{ 188 | { 189 | Out: [][]interface{}{ 190 | {"192.30.253.112", "github.com"}, 191 | {"192.30.253.113", "github.com"}, 192 | }}, 193 | { 194 | Out: [][]interface{}{ 195 | {"1.1.1.1", "one.one.one.one"}, 196 | }}, 197 | { 198 | Out: [][]interface{}{ 199 | {"8.8.8.8", "google-public-dns-a.google.com"}, 200 | }}, 201 | } 202 | 203 | compare(t, fullResponse, expectedResponse) 204 | } 205 | 206 | func TestMoreThanOneParam(t *testing.T) { 207 | log.SetOutput(&bytes.Buffer{}) 208 | 209 | reqString := "github.com,192.30.253.112\none.one.one.one,1.1.1.1" 210 | 211 | req := httptest.NewRequest("POST", 212 | "http://example.org/query", 213 | strings.NewReader(reqString)) 214 | w := httptest.NewRecorder() 215 | queryHandler, err := initQueryHandler(testDbPath, 216 | "SELECT * FROM ip_dns WHERE dns = ? AND ip = ?", 217 | 0) 218 | if err != nil { 219 | t.Fatal(err) 220 | } 221 | queryHandler(w, req) 222 | 223 | resp := w.Result() 224 | defer resp.Body.Close() 225 | 226 | if resp.StatusCode != http.StatusOK { 227 | t.Fatalf(`resp.StatusCode (%d) != http.StatusOK (%d)`, resp.StatusCode, http.StatusOK) 228 | } 229 | 230 | if resp.Header.Get("Content-Type") != "application/json" { 231 | t.Fatalf(`resp.Header.Get("Content-Type") (%s) != "application/json"`, resp.Header.Get("Content-Type")) 232 | } 233 | 234 | var fullResponse []queryResult 235 | decoder := json.NewDecoder(resp.Body) 236 | err = decoder.Decode(&fullResponse) 237 | if err != nil { 238 | t.Fatal(err) 239 | } 240 | 241 | expectedResponse := []queryResult{ 242 | { 243 | Out: [][]interface{}{ 244 | {"192.30.253.112", "github.com"}, 245 | }}, 246 | { 247 | Out: [][]interface{}{ 248 | {"1.1.1.1", "one.one.one.one"}, 249 | }}, 250 | } 251 | 252 | compare(t, fullResponse, expectedResponse) 253 | } 254 | 255 | func TestZeroParams(t *testing.T) { 256 | log.SetOutput(&bytes.Buffer{}) 257 | 258 | req := httptest.NewRequest("GET", 259 | "http://example.org/query", 260 | nil) 261 | w := httptest.NewRecorder() 262 | queryHandler, err := initQueryHandler(testDbPath, "SELECT * FROM ip_dns", 0) 263 | if err != nil { 264 | t.Fatal(err) 265 | } 266 | queryHandler(w, req) 267 | 268 | resp := w.Result() 269 | defer resp.Body.Close() 270 | 271 | if resp.StatusCode != http.StatusOK { 272 | t.Fatalf(`resp.StatusCode (%d) != http.StatusOK (%d)`, resp.StatusCode, http.StatusOK) 273 | } 274 | 275 | if resp.Header.Get("Content-Type") != "application/json" { 276 | t.Fatalf(`resp.Header.Get("Content-Type") (%s) != "application/json"`, resp.Header.Get("Content-Type")) 277 | } 278 | 279 | var answer []queryResult 280 | decoder := json.NewDecoder(resp.Body) 281 | err = decoder.Decode(&answer) 282 | if err != nil { 283 | t.Fatal(err) 284 | } 285 | 286 | expectedResponse := []queryResult{ 287 | { 288 | Out: [][]interface{}{ 289 | {"1.1.1.1", "one.one.one.one"}, 290 | {"8.8.8.8", "google-public-dns-a.google.com"}, 291 | {"192.30.253.112", "github.com"}, 292 | {"192.30.253.113", "github.com"}, 293 | }}, 294 | } 295 | 296 | compare(t, answer, expectedResponse) 297 | } 298 | 299 | func compare(t *testing.T, answer []queryResult, expectedResponse []queryResult) { 300 | for i, v := range expectedResponse { 301 | if len(v.Out) != len(answer[i].Out) { 302 | t.Fatalf(`len(v.Out) (%v) != len(answer[%d].Out) (%v)`, len(v.Out), i, len(answer[i].Out)) 303 | } 304 | 305 | for rowI, rowV := range v.Out { 306 | if len(rowV) != len(answer[i].Out[rowI]) { 307 | t.Fatalf(`len(rowV) (%v) != len(answer[%d].Out[%d]) (%v)`, len(rowV), i, rowI, len(answer[i].Out[rowI])) 308 | } 309 | 310 | for cellI, cellV := range rowV { 311 | if cellV != answer[i].Out[rowI][cellI] { 312 | t.Fatalf(`cellV (%v) != answer[%d].Out[%d][%d] (%v)`, cellV, i, rowI, cellI, answer[i].Out[rowI][cellI]) 313 | } 314 | } 315 | } 316 | } 317 | } 318 | 319 | func TestBadParamsCount(t *testing.T) { 320 | log.SetOutput(&bytes.Buffer{}) 321 | 322 | reqString := "github.com,1" 323 | 324 | req := httptest.NewRequest("POST", 325 | "http://example.org/query", 326 | strings.NewReader(reqString)) 327 | w := httptest.NewRecorder() 328 | queryHandler, err := initQueryHandler(testDbPath, "SELECT * FROM ip_dns WHERE dns = ?", 0) 329 | if err != nil { 330 | t.Fatal(err) 331 | } 332 | queryHandler(w, req) 333 | 334 | resp := w.Result() 335 | defer resp.Body.Close() 336 | 337 | if resp.StatusCode != http.StatusInternalServerError { 338 | t.Fatalf(`resp.StatusCode (%d) != http.StatusInternalServerError (%d)`, resp.StatusCode, http.StatusInternalServerError) 339 | } 340 | 341 | respBytes, err := ioutil.ReadAll(resp.Body) 342 | if err != nil { 343 | t.Fatal(err) 344 | } 345 | respString := string(respBytes) 346 | 347 | if !strings.Contains(respString, "sql: expected 1 arguments, got 2") { 348 | t.Fatal(`Error string should contain "sql: expected 1 arguments, got 2"`) 349 | } 350 | } 351 | 352 | func TestBadPathRequest(t *testing.T) { 353 | log.SetOutput(&bytes.Buffer{}) 354 | 355 | reqString := `github.com` 356 | 357 | req := httptest.NewRequest("POST", 358 | "http://example.org/queri", 359 | strings.NewReader(reqString)) 360 | w := httptest.NewRecorder() 361 | queryHandler, err := initQueryHandler(testDbPath, "SELECT * FROM ip_dns WHERE dns = ?", 0) 362 | if err != nil { 363 | t.Fatal(err) 364 | } 365 | queryHandler(w, req) 366 | 367 | resp := w.Result() 368 | defer resp.Body.Close() 369 | 370 | if resp.StatusCode != http.StatusNotFound { 371 | t.Fatalf(`resp.StatusCode (%d) != http.StatusNotFound (%d)`, resp.StatusCode, http.StatusNotFound) 372 | } 373 | } 374 | 375 | func TestBadMethodRequest(t *testing.T) { 376 | log.SetOutput(&bytes.Buffer{}) 377 | 378 | req := httptest.NewRequest("PUT", 379 | "http://example.org/query", 380 | nil) 381 | w := httptest.NewRecorder() 382 | queryHandler, err := initQueryHandler(testDbPath, "SELECT * FROM ip_dns WHERE dns = ?", 0) 383 | if err != nil { 384 | t.Fatal(err) 385 | } 386 | queryHandler(w, req) 387 | 388 | resp := w.Result() 389 | defer resp.Body.Close() 390 | 391 | if resp.StatusCode != http.StatusMethodNotAllowed { 392 | t.Fatalf(`resp.StatusCode (%d) != http.StatusMethodNotAllowed (%d)`, resp.StatusCode, http.StatusMethodNotAllowed) 393 | } 394 | } 395 | 396 | type errReader int 397 | 398 | func (errReader) Read(p []byte) (int, error) { 399 | return 0, errors.New("test error") 400 | } 401 | 402 | func TestBadBodySendRequest(t *testing.T) { 403 | log.SetOutput(&bytes.Buffer{}) 404 | 405 | var reqBody errReader 406 | 407 | req := httptest.NewRequest("POST", 408 | "http://example.org/query", 409 | reqBody) 410 | 411 | w := httptest.NewRecorder() 412 | queryHandler, err := initQueryHandler(testDbPath, "SELECT * FROM ip_dns WHERE dns = ?", 0) 413 | if err != nil { 414 | t.Fatal(err) 415 | } 416 | queryHandler(w, req) 417 | 418 | resp := w.Result() 419 | defer resp.Body.Close() 420 | 421 | if resp.StatusCode != http.StatusInternalServerError { 422 | t.Fatalf(`resp.StatusCode (%d) != http.StatusInternalServerError (%d)`, resp.StatusCode, http.StatusInternalServerError) 423 | } 424 | 425 | respBytes, err := ioutil.ReadAll(resp.Body) 426 | if err != nil { 427 | t.Fatal(err) 428 | } 429 | respString := string(respBytes) 430 | 431 | if !strings.Contains(respString, "Error reading request body: test error") { 432 | t.Fatal(`Error string should contain "Error reading request body: test error"`) 433 | } 434 | } 435 | 436 | func TestMainDbFileNotThere(t *testing.T) { 437 | err := cmd([]string{ 438 | "--db", 439 | "blabla", 440 | "--query", 441 | "SELECT * FROM ip_dns WHERE dns = ?", 442 | }) 443 | if err == nil { 444 | t.Fatal(`Should throw an error`) 445 | } 446 | if !strings.Contains(err.Error(), "Database file 'blabla' doesn't exist") { 447 | t.Fatalf(`Should throw a file doesn't exist error: %v`, err) 448 | } 449 | } 450 | 451 | func TestMainEmptyDbParam(t *testing.T) { 452 | log.SetOutput(&bytes.Buffer{}) 453 | 454 | err := cmd([]string{ 455 | "--query", 456 | "SELECT * FROM ip_dns WHERE dns = ?", 457 | }) 458 | if err == nil { 459 | t.Fatal(`Should throw an error`) 460 | } 461 | if !strings.Contains(err.Error(), "Must provide --db param") { 462 | t.Fatalf(`Should throw a "Must provide --db param" error: %v`, err) 463 | } 464 | } 465 | 466 | func TestMainEmptyQueryParam(t *testing.T) { 467 | log.SetOutput(&bytes.Buffer{}) 468 | 469 | err := cmd([]string{ 470 | "--db", 471 | testDbPath, 472 | }) 473 | if err == nil { 474 | t.Fatal(`Should throw an error`) 475 | } 476 | if err == nil || !strings.Contains(err.Error(), "Must provide --query param") { 477 | t.Fatalf(`Should throw a "Must provide --query param" error: %v`, err) 478 | } 479 | } 480 | 481 | func TestMainPortOutOfRange(t *testing.T) { 482 | log.SetOutput(&bytes.Buffer{}) 483 | 484 | err := cmd([]string{ 485 | "--db", 486 | testDbPath, 487 | "--query", 488 | "SELECT * FROM ip_dns WHERE dns = ?", 489 | "--port", 490 | "66000", 491 | }) 492 | if err == nil { 493 | t.Fatal(`Should throw an error`) 494 | } 495 | if err == nil || !strings.Contains(err.Error(), "invalid port") { 496 | t.Fatalf(`Should throw a "invalid port" error: %v`, err) 497 | } 498 | } 499 | 500 | func TestMainInvalidPort(t *testing.T) { 501 | os.Stderr = nil 502 | 503 | err := cmd([]string{ 504 | "--db", 505 | testDbPath, 506 | "--query", 507 | "SELECT * FROM ip_dns WHERE dns = ?", 508 | "--port", 509 | "-1", 510 | }) 511 | if err == nil { 512 | t.Fatal(`Should throw an error`) 513 | } 514 | if err == nil || !strings.Contains(err.Error(), `invalid value "-1" for flag -port: parse error`) { 515 | t.Fatalf(`Should throw a 'invalid value "-1" for flag -port: parse error' error: %v`, err) 516 | } 517 | } 518 | 519 | func TestMainInvalidQuery(t *testing.T) { 520 | os.Stderr = nil 521 | 522 | err := cmd([]string{ 523 | "--db", 524 | testDbPath, 525 | "--query", 526 | "BANANA * FROM ip_dns WHERE dns = ?", 527 | }) 528 | if err == nil { 529 | t.Fatal(`Should throw an error`) 530 | } 531 | if err == nil || !strings.Contains(err.Error(), "syntax error") { 532 | t.Fatalf(`Should throw a 'syntax error' error: %v`, err) 533 | } 534 | } 535 | 536 | func TestCountParamsZero(t *testing.T) { 537 | db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?mode=rw&cache=shared&_journal_mode=WAL", testDbPath)) 538 | if err != nil { 539 | t.Fatalf("Should open testDbPath (%s) just fine", testDbPath) 540 | } 541 | defer db.Close() 542 | 543 | db.SetMaxOpenConns(1) 544 | 545 | queryStmt, err := db.Prepare("select * from ip_dns") 546 | if err != nil { 547 | t.Fatal("Should prepare query just fine") 548 | } 549 | 550 | count, err := countParams(queryStmt) 551 | if err != nil { 552 | t.Fatal("Shouldn't throw an error") 553 | } 554 | if count != 0 { 555 | t.Fatal("should return 0") 556 | } 557 | } 558 | 559 | func TestCountParamsNotZero(t *testing.T) { 560 | db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?mode=rw&cache=shared&_journal_mode=WAL", testDbPath)) 561 | if err != nil { 562 | t.Fatalf("Should open testDbPath (%s) just fine", testDbPath) 563 | } 564 | defer db.Close() 565 | 566 | db.SetMaxOpenConns(1) 567 | 568 | for i := 1; i <= 10; i++ { 569 | t.Run(fmt.Sprintf("TestCount%dParams", i), func(t *testing.T) { 570 | where := make([]string, i) 571 | for j := 0; j < i; j++ { 572 | where[j] = "ip = ?" 573 | } 574 | 575 | queryStmt, err := db.Prepare(fmt.Sprintf("select * from ip_dns where %s", strings.Join(where, " AND "))) 576 | if err != nil { 577 | t.Fatal("Should prepare query just fine") 578 | } 579 | 580 | count, err := countParams(queryStmt) 581 | if err != nil { 582 | t.Fatal("Shouldn't throw an error") 583 | } 584 | if count != i { 585 | t.Fatalf("Should return %d", i) 586 | } 587 | }) 588 | } 589 | } 590 | 591 | func TestCountParamsHandleDBError(t *testing.T) { 592 | db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?mode=rw&cache=shared&_journal_mode=WAL", testDbPath)) 593 | if err != nil { 594 | t.Fatalf("Should open testDbPath (%s) just fine", testDbPath) 595 | } 596 | 597 | db.SetMaxOpenConns(1) 598 | 599 | queryStmt, err := db.Prepare("select * from ip_dns") 600 | if err != nil { 601 | t.Fatal("Should prepare query just fine") 602 | } 603 | 604 | db.Close() 605 | 606 | count, err := countParams(queryStmt) 607 | if count != -1 { 608 | t.Fatal("Count should equal -1") 609 | } 610 | if err == nil || !strings.Contains(err.Error(), "Cannot extract params count from query error") { 611 | t.Fatalf(`Should throw a 'syntax error' error: %v`, err) 612 | } 613 | } 614 | -------------------------------------------------------------------------------- /test_db/ip_dns.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/assafmo/SQLiteQueryServer/f4dae58c15bf9552ed12cb80e053bfc78ea17d38/test_db/ip_dns.db --------------------------------------------------------------------------------