├── script └── docker-entrypoint.sh ├── cluster ├── message.proto ├── join.go ├── service.go └── message.pb.go ├── cmd ├── version.go ├── tqlite │ ├── remove.go │ ├── execute.go │ ├── query.go │ ├── backup.go │ └── main.go └── tqlited │ └── main.go ├── store ├── config.go ├── server.go └── transport.go ├── .gitignore ├── Dockerfile ├── LICENSE.md ├── command ├── command.proto ├── marshal.go └── command.pb.go ├── go.mod ├── log └── log.go ├── http ├── request.go └── endpoint.go ├── tcp └── conn.go ├── README.md ├── db └── db.go └── go.sum /script/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # User wants to override options, so merge with defaults. 4 | if [ "${1:0:1}" = '-' ]; then 5 | set -- tqlited -http-addr 0.0.0.0:4001 -raft-addr 0.0.0.0:4002 $@ /tqlite/file/data 6 | fi 7 | 8 | exec "$@" -------------------------------------------------------------------------------- /cluster/message.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option go_package = "github.com/minghsu0107/tqlite/cluster"; 4 | 5 | message Address { 6 | string url = 1; 7 | } 8 | 9 | message Command { 10 | enum Type { 11 | COMMAND_TYPE_UNKNOWN = 0; 12 | COMMAND_TYPE_GET_NODE_API_URL = 1; 13 | } 14 | Type type = 1; 15 | } 16 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | // These variables are populated via the Go linker. 4 | var ( 5 | // tqlite version 6 | Version = "1" 7 | 8 | // Commit this code was built at. 9 | Commit = "unknown" 10 | 11 | // Branch the code was built from. 12 | Branch = "unknown" 13 | 14 | // Timestamp of build. 15 | Buildtime = "unknown" 16 | ) 17 | -------------------------------------------------------------------------------- /store/config.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | // DBConfig represents the configuration of the underlying SQLite database. 4 | type DBConfig struct { 5 | DSN string // Any custom DSN 6 | Memory bool // Whether the database is in-memory only. 7 | } 8 | 9 | // NewDBConfig returns a new DB config instance. 10 | func NewDBConfig(dsn string, memory bool) *DBConfig { 11 | return &DBConfig{DSN: dsn, Memory: memory} 12 | } 13 | -------------------------------------------------------------------------------- /store/server.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | // Server represents another node in the cluster. 4 | type Server struct { 5 | ID string `json:"id,omitempty"` 6 | Addr string `json:"addr,omitempty"` 7 | Suffrage string `json:"suffrage,omitempty"` 8 | } 9 | 10 | // Servers is a set of Servers. 11 | type Servers []*Server 12 | 13 | func (s Servers) Less(i, j int) bool { return s[i].ID < s[j].ID } 14 | func (s Servers) Len() int { return len(s) } 15 | func (s Servers) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | # Executables 4 | tqlited 5 | **/tqlited 6 | !**/tqlited/ 7 | 8 | tqlite 9 | **/tqlite 10 | !**/tqlite/ 11 | 12 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 13 | *.o 14 | *.a 15 | *.so 16 | 17 | # Folders 18 | _obj 19 | _test 20 | .vagrant 21 | 22 | # Logs 23 | *.log 24 | 25 | # Temporary vim files 26 | *.swp 27 | 28 | # Notes 29 | TODO 30 | 31 | # Architecture specific extensions/prefixes 32 | *.[568vq] 33 | [568vq].out 34 | 35 | *.cgo1.go 36 | *.cgo2.c 37 | _cgo_defun.c 38 | _cgo_gotypes.go 39 | _cgo_export.* 40 | 41 | _testmain.go 42 | 43 | *.exe 44 | *.test 45 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20 AS builder 2 | 3 | RUN mkdir -p /src 4 | WORKDIR /src 5 | 6 | COPY go.mod go.sum ./ 7 | RUN go mod download 8 | 9 | COPY . . 10 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o tqlite -v ./cmd/tqlite 11 | RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -ldflags "-extldflags -static" -o tqlited -v ./cmd/tqlited 12 | 13 | FROM alpine:3.14 14 | 15 | COPY --from=builder /src/tqlite /src/tqlited /usr/local/bin/ 16 | RUN apk add --no-cache bash 17 | 18 | ENV TQLITE_VERSION=1.0.0 19 | RUN mkdir -p /tqlite/file 20 | VOLUME /tqlite/file 21 | 22 | EXPOSE 4001 4002 23 | COPY ./script/docker-entrypoint.sh /bin/docker-entrypoint.sh 24 | 25 | ENTRYPOINT ["docker-entrypoint.sh"] 26 | CMD ["tqlited", "-http-addr", "0.0.0.0:4001", "-raft-addr", "0.0.0.0:4002", "/tqlite/file/data"] 27 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Hao-Ming, Hsu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /store/transport.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "net" 5 | "time" 6 | 7 | "github.com/hashicorp/raft" 8 | ) 9 | 10 | // Listener is the interface expected by the Store for Transports. 11 | type Listener interface { 12 | net.Listener 13 | Dial(address string, timeout time.Duration) (net.Conn, error) 14 | } 15 | 16 | // Transport is the network service provided to Raft, and wraps a Listener. 17 | type Transport struct { 18 | ln Listener 19 | } 20 | 21 | // NewTransport returns an initialized Transport. 22 | func NewTransport(ln Listener) *Transport { 23 | return &Transport{ 24 | ln: ln, 25 | } 26 | } 27 | 28 | // Dial creates a new network connection. 29 | func (t *Transport) Dial(addr raft.ServerAddress, timeout time.Duration) (net.Conn, error) { 30 | return t.ln.Dial(string(addr), timeout) 31 | } 32 | 33 | // Accept waits for the next connection. 34 | func (t *Transport) Accept() (net.Conn, error) { 35 | return t.ln.Accept() 36 | } 37 | 38 | // Close closes the transport 39 | func (t *Transport) Close() error { 40 | return t.ln.Close() 41 | } 42 | 43 | // Addr returns the binding address of the transport. 44 | func (t *Transport) Addr() net.Addr { 45 | return t.ln.Addr() 46 | } 47 | -------------------------------------------------------------------------------- /command/command.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package command; 3 | 4 | option go_package = "github.com/minghsu0107/tqlite/command"; 5 | 6 | message Parameter { 7 | oneof value { 8 | sint64 i = 1; 9 | double d = 2; 10 | bool b = 3; 11 | bytes y = 4; 12 | string s = 5; 13 | } 14 | } 15 | 16 | message Statement { 17 | string sql = 1; 18 | repeated Parameter parameters = 2; 19 | } 20 | 21 | message Request { 22 | bool transaction = 1; 23 | repeated Statement statements = 2; 24 | } 25 | 26 | message QueryRequest { 27 | Request request = 1; 28 | bool timings = 2; 29 | enum Level { 30 | QUERY_REQUEST_LEVEL_NONE = 0; 31 | QUERY_REQUEST_LEVEL_WEAK = 1; 32 | QUERY_REQUEST_LEVEL_STRONG = 2; 33 | } 34 | Level level = 3; 35 | int64 freshness = 4; 36 | } 37 | 38 | message ExecuteRequest { 39 | Request request = 1; 40 | bool timings = 2; 41 | } 42 | 43 | message Noop { 44 | string id = 1; 45 | } 46 | 47 | message Command { 48 | enum Type { 49 | COMMAND_TYPE_UNKNOWN = 0; 50 | COMMAND_TYPE_QUERY = 1; 51 | COMMAND_TYPE_EXECUTE = 2; 52 | COMMAND_TYPE_NOOP = 3; 53 | } 54 | Type type = 1; 55 | bytes sub_command = 2; 56 | bool compressed = 3; 57 | } -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/minghsu0107/tqlite 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/Bowery/prompt v0.0.0-20190916142128-fa8279994f75 7 | github.com/golang/protobuf v1.5.2 8 | github.com/hashicorp/raft v1.3.1 9 | github.com/hashicorp/raft-boltdb v0.0.0-20210422161416-485fa74b0b01 10 | github.com/mkideal/cli v0.2.5 11 | github.com/mkideal/pkg v0.1.2 12 | github.com/rqlite/go-sqlite3 v1.20.2 13 | google.golang.org/protobuf v1.26.0 14 | ) 15 | 16 | require ( 17 | github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect 18 | github.com/boltdb/bolt v1.3.1 // indirect 19 | github.com/hashicorp/go-hclog v0.9.1 // indirect 20 | github.com/hashicorp/go-immutable-radix v1.0.0 // indirect 21 | github.com/hashicorp/go-msgpack v0.5.5 // indirect 22 | github.com/hashicorp/golang-lru v0.5.0 // indirect 23 | github.com/labstack/gommon v0.3.0 // indirect 24 | github.com/mattn/go-colorable v0.1.7 // indirect 25 | github.com/mattn/go-isatty v0.0.12 // indirect 26 | github.com/mkideal/expr v0.1.0 // indirect 27 | golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect 28 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 // indirect 29 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect 30 | ) 31 | -------------------------------------------------------------------------------- /cmd/tqlite/remove.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | ) 10 | 11 | func removeNode(client *http.Client, id string, argv *argT, timer bool) error { 12 | u := url.URL{ 13 | Scheme: argv.Protocol, 14 | Host: fmt.Sprintf("%s:%d", argv.Host, argv.Port), 15 | Path: fmt.Sprintf("%sremove", argv.Prefix), 16 | } 17 | urlStr := u.String() 18 | 19 | b, err := json.Marshal(map[string]string{ 20 | "id": id, 21 | }) 22 | if err != nil { 23 | return err 24 | } 25 | 26 | nRedirect := 0 27 | for { 28 | req, err := http.NewRequest("DELETE", urlStr, bytes.NewReader(b)) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | resp, err := client.Do(req) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | if resp.StatusCode == http.StatusUnauthorized { 39 | return fmt.Errorf("unauthorized") 40 | } 41 | 42 | if resp.StatusCode == http.StatusMovedPermanently { 43 | nRedirect++ 44 | if nRedirect > maxRedirect { 45 | return fmt.Errorf("maximum leader redirect limit exceeded") 46 | } 47 | urlStr = resp.Header["Location"][0] 48 | continue 49 | } 50 | 51 | if resp.StatusCode != http.StatusOK { 52 | return fmt.Errorf("server responded with: %s", resp.Status) 53 | } 54 | 55 | return nil 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/hashicorp/raft" 7 | "github.com/hashicorp/raft-boltdb" 8 | ) 9 | 10 | // Log is an object that can return information about the Raft log. 11 | type Log struct { 12 | *raftboltdb.BoltStore 13 | } 14 | 15 | // NewLog returns an instantiated Log object. 16 | func NewLog(path string) (*Log, error) { 17 | bs, err := raftboltdb.NewBoltStore(path) 18 | if err != nil { 19 | return nil, fmt.Errorf("new bolt store: %s", err) 20 | } 21 | return &Log{bs}, nil 22 | } 23 | 24 | // Indexes returns the first and last indexes. 25 | func (l *Log) Indexes() (uint64, uint64, error) { 26 | fi, err := l.FirstIndex() 27 | if err != nil { 28 | return 0, 0, fmt.Errorf("failed to get first index: %s", err) 29 | } 30 | li, err := l.LastIndex() 31 | if err != nil { 32 | return 0, 0, fmt.Errorf("failed to get last index: %s", err) 33 | } 34 | return fi, li, nil 35 | } 36 | 37 | // LastCommandIndex returns the index of the last Command 38 | // log entry written to the Raft log. Returns an index of 39 | // zero if no such log exists. 40 | func (l *Log) LastCommandIndex() (uint64, error) { 41 | fi, li, err := l.Indexes() 42 | if err != nil { 43 | return 0, fmt.Errorf("get indexes: %s", err) 44 | } 45 | 46 | // Check for empty log. 47 | if li == 0 { 48 | return 0, nil 49 | } 50 | 51 | var rl raft.Log 52 | for i := li; i >= fi; i-- { 53 | if err := l.GetLog(i, &rl); err != nil { 54 | return 0, fmt.Errorf("get log at index %d: %s", i, err) 55 | } 56 | if rl.Type == raft.LogCommand { 57 | return i, nil 58 | } 59 | } 60 | return 0, nil 61 | } 62 | -------------------------------------------------------------------------------- /cmd/tqlite/execute.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "net/http" 8 | "net/url" 9 | "strings" 10 | 11 | "github.com/mkideal/cli" 12 | ) 13 | 14 | // Result represents execute result 15 | type Result struct { 16 | LastInsertID int `json:"last_insert_id,omitempty"` 17 | RowsAffected int `json:"rows_affected,omitempty"` 18 | Time float64 `json:"time,omitempty"` 19 | Error string `json:"error,omitempty"` 20 | } 21 | 22 | type executeResponse struct { 23 | Results []*Result `json:"results,omitempty"` 24 | Error string `json:"error,omitempty"` 25 | Time float64 `json:"time,omitempty"` 26 | } 27 | 28 | func executeWithClient(ctx *cli.Context, client *http.Client, argv *argT, timer bool, stmt string) error { 29 | queryStr := url.Values{} 30 | if timer { 31 | queryStr.Set("timings", "") 32 | } 33 | u := url.URL{ 34 | Scheme: argv.Protocol, 35 | Host: fmt.Sprintf("%s:%d", argv.Host, argv.Port), 36 | Path: fmt.Sprintf("%sdb/execute", argv.Prefix), 37 | } 38 | urlStr := u.String() 39 | 40 | requestData := strings.NewReader(makeJSONBody(stmt)) 41 | 42 | nRedirect := 0 43 | for { 44 | if _, err := requestData.Seek(0, io.SeekStart); err != nil { 45 | return err 46 | } 47 | 48 | req, err := http.NewRequest("POST", urlStr, requestData) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | resp, err := client.Do(req) 54 | if err != nil { 55 | return err 56 | } 57 | response, err := ioutil.ReadAll(resp.Body) 58 | if err != nil { 59 | return err 60 | } 61 | resp.Body.Close() 62 | 63 | if resp.StatusCode == http.StatusUnauthorized { 64 | return fmt.Errorf("unauthorized") 65 | } 66 | 67 | if resp.StatusCode == http.StatusMovedPermanently { 68 | nRedirect++ 69 | if nRedirect > maxRedirect { 70 | return fmt.Errorf("maximum leader redirect limit exceeded") 71 | } 72 | urlStr = resp.Header["Location"][0] 73 | continue 74 | } 75 | 76 | if resp.StatusCode != http.StatusOK { 77 | return fmt.Errorf("server responded with %s: %s", resp.Status, response) 78 | } 79 | 80 | // Parse response and write results 81 | ret := &executeResponse{} 82 | if err := parseResponse(&response, &ret); err != nil { 83 | return err 84 | } 85 | if ret.Error != "" { 86 | return fmt.Errorf(ret.Error) 87 | } 88 | if len(ret.Results) != 1 { 89 | return fmt.Errorf("unexpected results length: %d", len(ret.Results)) 90 | } 91 | 92 | result := ret.Results[0] 93 | if result.Error != "" { 94 | ctx.String("Error: %s\n", result.Error) 95 | return nil 96 | } 97 | 98 | rowString := "row" 99 | if result.RowsAffected > 1 { 100 | rowString = "rows" 101 | } 102 | if timer { 103 | ctx.String("%d %s affected (%f sec)\n", result.RowsAffected, rowString, result.Time) 104 | } else { 105 | ctx.String("%d %s affected\n", result.RowsAffected, rowString) 106 | } 107 | 108 | if timer { 109 | fmt.Printf("Run Time: %f seconds\n", result.Time) 110 | } 111 | return nil 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /http/request.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | 7 | "github.com/minghsu0107/tqlite/command" 8 | ) 9 | 10 | var ( 11 | // ErrNoStatements is returned when a request is empty 12 | ErrNoStatements = errors.New("no statements") 13 | 14 | // ErrInvalidRequest is returned when a request cannot be parsed. 15 | ErrInvalidRequest = errors.New("invalid request") 16 | 17 | // ErrUnsupportedType is returned when a request contains an unsupported type. 18 | ErrUnsupportedType = errors.New("unsupported type") 19 | ) 20 | 21 | // ParseRequest generates a set of Statements for a given byte slice. 22 | func ParseRequest(b []byte) ([]*command.Statement, error) { 23 | if b == nil { 24 | return nil, ErrNoStatements 25 | } 26 | 27 | simple := []string{} // Represents a set of unparameterized queries 28 | parameterized := [][]interface{}{} // Represents a set of parameterized queries 29 | 30 | // Try simple form first. 31 | err := json.Unmarshal(b, &simple) 32 | if err == nil { 33 | if len(simple) == 0 { 34 | return nil, ErrNoStatements 35 | } 36 | 37 | stmts := make([]*command.Statement, len(simple)) 38 | for i := range simple { 39 | stmts[i] = &command.Statement{ 40 | Sql: simple[i], 41 | } 42 | } 43 | return stmts, nil 44 | } 45 | 46 | // Next try parameterized form. 47 | if err := json.Unmarshal(b, ¶meterized); err != nil { 48 | return nil, ErrInvalidRequest 49 | } 50 | stmts := make([]*command.Statement, len(parameterized)) 51 | 52 | for i := range parameterized { 53 | if len(parameterized[i]) == 0 { 54 | return nil, ErrNoStatements 55 | } 56 | 57 | sql, ok := parameterized[i][0].(string) 58 | if !ok { 59 | return nil, ErrInvalidRequest 60 | } 61 | stmts[i] = &command.Statement{ 62 | Sql: sql, 63 | Parameters: nil, 64 | } 65 | if len(parameterized[i]) == 1 { 66 | // No actual parameters after the SQL string 67 | continue 68 | } 69 | 70 | stmts[i].Parameters = make([]*command.Parameter, len(parameterized[i])-1) 71 | 72 | for j := range parameterized[i][1:] { 73 | switch v := parameterized[i][j+1].(type) { 74 | case int: 75 | case int64: 76 | stmts[i].Parameters[j] = &command.Parameter{ 77 | Value: &command.Parameter_I{ 78 | I: v, 79 | }, 80 | } 81 | case float64: 82 | stmts[i].Parameters[j] = &command.Parameter{ 83 | Value: &command.Parameter_D{ 84 | D: v, 85 | }, 86 | } 87 | case bool: 88 | stmts[i].Parameters[j] = &command.Parameter{ 89 | Value: &command.Parameter_B{ 90 | B: v, 91 | }, 92 | } 93 | case []byte: 94 | stmts[i].Parameters[j] = &command.Parameter{ 95 | Value: &command.Parameter_Y{ 96 | Y: v, 97 | }, 98 | } 99 | case string: 100 | stmts[i].Parameters[j] = &command.Parameter{ 101 | Value: &command.Parameter_S{ 102 | S: v, 103 | }, 104 | } 105 | default: 106 | return nil, ErrUnsupportedType 107 | } 108 | } 109 | } 110 | return stmts, nil 111 | 112 | } 113 | -------------------------------------------------------------------------------- /cluster/join.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "net" 11 | "net/http" 12 | "os" 13 | "time" 14 | 15 | httpd "github.com/minghsu0107/tqlite/http" 16 | ) 17 | 18 | var ( 19 | // ErrJoinFailed is returned when a node fails to join a cluster 20 | ErrJoinFailed = errors.New("failed to join cluster") 21 | ) 22 | 23 | // Join attempts to join the cluster at one of the addresses given in joinAddr. 24 | // It walks through joinAddr in order, and sets the node ID and Raft address of 25 | // the joining node as id addr respectively. It returns the endpoint successfully 26 | // used to join the cluster. 27 | func Join(srcIP string, joinAddr []string, id, addr string, voter bool, numAttempts int, 28 | attemptInterval time.Duration) (string, error) { 29 | var err error 30 | var j string 31 | logger := log.New(os.Stderr, "[cluster-join] ", log.LstdFlags) 32 | 33 | for i := 0; i < numAttempts; i++ { 34 | for _, a := range joinAddr { 35 | j, err = join(srcIP, a, id, addr, voter, logger) 36 | if err == nil { 37 | // Success! 38 | return j, nil 39 | } 40 | } 41 | logger.Printf("failed to join cluster at %s: %s, sleeping %s before retry", joinAddr, err.Error(), attemptInterval) 42 | time.Sleep(attemptInterval) 43 | } 44 | logger.Printf("failed to join cluster at %s, after %d attempts", joinAddr, numAttempts) 45 | return "", ErrJoinFailed 46 | } 47 | 48 | func join(srcIP, joinAddr, id, addr string, voter bool, logger *log.Logger) (string, error) { 49 | if id == "" { 50 | return "", fmt.Errorf("node ID not set") 51 | } 52 | // The specified source IP is optional 53 | var dialer *net.Dialer 54 | dialer = &net.Dialer{} 55 | if srcIP != "" { 56 | netAddr := &net.TCPAddr{ 57 | IP: net.ParseIP(srcIP), 58 | Port: 0, 59 | } 60 | dialer = &net.Dialer{LocalAddr: netAddr} 61 | } 62 | // Join using IP address, as that is what Hashicorp Raft works in. 63 | resv, err := net.ResolveTCPAddr("tcp", addr) 64 | if err != nil { 65 | return "", err 66 | } 67 | 68 | // Check for protocol scheme, and insert default if necessary. 69 | fullAddr := httpd.NormalizeAddr(fmt.Sprintf("%s/join", joinAddr)) 70 | 71 | // Create and configure the client to connect to the other node. 72 | tr := &http.Transport{ 73 | Dial: dialer.Dial, 74 | } 75 | client := &http.Client{Transport: tr} 76 | client.CheckRedirect = func(req *http.Request, via []*http.Request) error { 77 | return http.ErrUseLastResponse 78 | } 79 | 80 | for { 81 | b, err := json.Marshal(map[string]interface{}{ 82 | "id": id, 83 | "addr": resv.String(), 84 | "voter": voter, 85 | }) 86 | if err != nil { 87 | return "", err 88 | } 89 | 90 | // Attempt to join. 91 | resp, err := client.Post(fullAddr, "application-type/json", bytes.NewReader(b)) 92 | if err != nil { 93 | return "", err 94 | } 95 | defer resp.Body.Close() 96 | 97 | b, err = ioutil.ReadAll(resp.Body) 98 | if err != nil { 99 | return "", err 100 | } 101 | 102 | switch resp.StatusCode { 103 | case http.StatusOK: 104 | return fullAddr, nil 105 | case http.StatusMovedPermanently: 106 | fullAddr = resp.Header.Get("location") 107 | if fullAddr == "" { 108 | return "", fmt.Errorf("failed to join, invalid redirect received") 109 | } 110 | continue 111 | default: 112 | return "", fmt.Errorf("failed to join, node returned: %s: (%s)", resp.Status, string(b)) 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /cmd/tqlite/query.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "net/url" 8 | 9 | "github.com/mkideal/cli" 10 | "github.com/mkideal/pkg/textutil" 11 | ) 12 | 13 | // Rows represents query result 14 | type Rows struct { 15 | Columns []string `json:"columns"` 16 | Types []string `json:"types"` 17 | Values [][]interface{} `json:"values"` 18 | Time float64 `json:"time"` 19 | Error string `json:"error,omitempty"` 20 | } 21 | 22 | // RowCount implements textutil.Table interface 23 | func (r *Rows) RowCount() int { 24 | return len(r.Values) + 1 25 | } 26 | 27 | // ColCount implements textutil.Table interface 28 | func (r *Rows) ColCount() int { 29 | return len(r.Columns) 30 | } 31 | 32 | // Get implements textutil.Table interface 33 | func (r *Rows) Get(i, j int) string { 34 | if i == 0 { 35 | if j >= len(r.Columns) { 36 | return "" 37 | } 38 | return r.Columns[j] 39 | } 40 | 41 | if r.Values == nil { 42 | return "NULL" 43 | } 44 | 45 | if i-1 >= len(r.Values) { 46 | return "NULL" 47 | } 48 | if j >= len(r.Values[i-1]) { 49 | return "NULL" 50 | } 51 | return fmt.Sprintf("%v", r.Values[i-1][j]) 52 | } 53 | 54 | func (r *Rows) validate() error { 55 | if r.Error != "" { 56 | return fmt.Errorf(r.Error) 57 | } 58 | if r.Columns == nil || r.Types == nil { 59 | return fmt.Errorf("unexpected result") 60 | } 61 | return nil 62 | } 63 | 64 | // headerRenderStyle render the header of result 65 | type headerRenderStyle struct { 66 | textutil.DefaultStyle 67 | } 68 | 69 | func (render headerRenderStyle) CellRender(row, col int, cell string, cw *textutil.ColorWriter) { 70 | if row != 0 { 71 | fmt.Fprintf(cw, cell) 72 | } else { 73 | fmt.Fprintf(cw, cw.Color.Cyan(cell)) 74 | } 75 | } 76 | 77 | var headerRender = &headerRenderStyle{} 78 | 79 | type queryResponse struct { 80 | Results []*Rows `json:"results"` 81 | Error string `json:"error,omitempty"` 82 | Time float64 `json:"time"` 83 | } 84 | 85 | func queryWithClient(ctx *cli.Context, client *http.Client, argv *argT, timer bool, query string) error { 86 | queryStr := url.Values{} 87 | queryStr.Set("q", query) 88 | if timer { 89 | queryStr.Set("timings", "") 90 | } 91 | u := url.URL{ 92 | Scheme: argv.Protocol, 93 | Host: fmt.Sprintf("%s:%d", argv.Host, argv.Port), 94 | Path: fmt.Sprintf("%sdb/query", argv.Prefix), 95 | RawQuery: queryStr.Encode(), 96 | } 97 | urlStr := u.String() 98 | 99 | nRedirect := 0 100 | for { 101 | req, err := http.NewRequest("GET", urlStr, nil) 102 | if err != nil { 103 | return err 104 | } 105 | 106 | resp, err := client.Do(req) 107 | if err != nil { 108 | return err 109 | } 110 | response, err := ioutil.ReadAll(resp.Body) 111 | if err != nil { 112 | return err 113 | } 114 | resp.Body.Close() 115 | 116 | if resp.StatusCode == http.StatusUnauthorized { 117 | return fmt.Errorf("unauthorized") 118 | } 119 | 120 | if resp.StatusCode == http.StatusMovedPermanently { 121 | nRedirect++ 122 | if nRedirect > maxRedirect { 123 | return fmt.Errorf("maximum leader redirect limit exceeded") 124 | } 125 | urlStr = resp.Header["Location"][0] 126 | continue 127 | } 128 | 129 | if resp.StatusCode != http.StatusOK { 130 | return fmt.Errorf("server responded with %s: %s", resp.Status, response) 131 | } 132 | 133 | // Parse response and write results 134 | ret := &queryResponse{} 135 | if err := parseResponse(&response, &ret); err != nil { 136 | return err 137 | } 138 | if ret.Error != "" { 139 | return fmt.Errorf(ret.Error) 140 | } 141 | if len(ret.Results) != 1 { 142 | return fmt.Errorf("unexpected results length: %d", len(ret.Results)) 143 | } 144 | 145 | result := ret.Results[0] 146 | if err := result.validate(); err != nil { 147 | return err 148 | } 149 | textutil.WriteTable(ctx, result, headerRender) 150 | 151 | if timer { 152 | fmt.Printf("Run Time: %f seconds\n", result.Time) 153 | } 154 | return nil 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /cmd/tqlite/backup.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "net/url" 9 | 10 | "github.com/mkideal/cli" 11 | ) 12 | 13 | type backupResponse struct { 14 | BackupFile []byte 15 | } 16 | 17 | type restoreResponse struct { 18 | Results []*Result `json:"results"` 19 | } 20 | 21 | type sqliteStatus struct { 22 | FkConstraint string `json:"fk_constraints"` 23 | } 24 | 25 | type store struct { 26 | SqliteStatus sqliteStatus `json:"sqlite3"` 27 | } 28 | 29 | type statusResponse struct { 30 | Store *store `json:"store"` 31 | } 32 | 33 | func makeBackupRequest(urlStr string) (*http.Request, error) { 34 | req, err := http.NewRequest("GET", urlStr, nil) 35 | if err != nil { 36 | return nil, err 37 | } 38 | return req, nil 39 | } 40 | 41 | func backup(ctx *cli.Context, filename string, argv *argT) error { 42 | queryStr := url.Values{} 43 | u := url.URL{ 44 | Scheme: argv.Protocol, 45 | Host: fmt.Sprintf("%s:%d", argv.Host, argv.Port), 46 | Path: fmt.Sprintf("%sdb/backup", argv.Prefix), 47 | RawQuery: queryStr.Encode(), 48 | } 49 | response, err := sendRequest(ctx, makeBackupRequest, u.String(), argv) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | err = ioutil.WriteFile(filename, *response, 0644) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | ctx.String("backup file written successfully\n") 60 | return nil 61 | } 62 | 63 | func dump(ctx *cli.Context, filename string, argv *argT) error { 64 | queryStr := url.Values{} 65 | queryStr.Set("fmt", "sql") 66 | u := url.URL{ 67 | Scheme: argv.Protocol, 68 | Host: fmt.Sprintf("%s:%d", argv.Host, argv.Port), 69 | Path: fmt.Sprintf("%sdb/backup", argv.Prefix), 70 | RawQuery: queryStr.Encode(), 71 | } 72 | response, err := sendRequest(ctx, makeBackupRequest, u.String(), argv) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | err = ioutil.WriteFile(filename, *response, 0644) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | ctx.String("SQL text file written successfully\n") 83 | return nil 84 | } 85 | 86 | func makeRestoreRequest(b []byte) func(string) (*http.Request, error) { 87 | return func(urlStr string) (*http.Request, error) { 88 | req, err := http.NewRequest("POST", urlStr, bytes.NewReader(b)) 89 | req.Header["Content-type"] = []string{"text/plain"} 90 | if err != nil { 91 | return nil, err 92 | } 93 | return req, nil 94 | } 95 | } 96 | 97 | func restore(ctx *cli.Context, filename string, argv *argT) error { 98 | statusURL := fmt.Sprintf("%s://%s:%d/status", argv.Protocol, argv.Host, argv.Port) 99 | client := http.Client{Transport: &http.Transport{ 100 | Proxy: http.ProxyFromEnvironment, 101 | }} 102 | 103 | req, err := http.NewRequest("GET", statusURL, nil) 104 | if err != nil { 105 | return err 106 | } 107 | 108 | statusResp, err := client.Do(req) 109 | if err != nil { 110 | return err 111 | } 112 | defer statusResp.Body.Close() 113 | 114 | if statusResp.StatusCode == http.StatusUnauthorized { 115 | return fmt.Errorf("unauthorized") 116 | } 117 | 118 | body, err := ioutil.ReadAll(statusResp.Body) 119 | if err != nil { 120 | return err 121 | } 122 | statusRet := &statusResponse{} 123 | if err := parseResponse(&body, &statusRet); err != nil { 124 | return err 125 | } 126 | if statusRet.Store == nil { 127 | return fmt.Errorf("unexpected server response: store status not found") 128 | } 129 | 130 | restoreFile, err := ioutil.ReadFile(filename) 131 | if err != nil { 132 | return err 133 | } 134 | 135 | // It is cheaper to append the actual pragma command to the restore file 136 | fkEnabled := statusRet.Store.SqliteStatus.FkConstraint == "enabled" 137 | if fkEnabled { 138 | restoreFile = append(restoreFile, []byte("PRAGMA foreign_keys=ON;")...) 139 | } else { 140 | restoreFile = append(restoreFile, []byte("PRAGMA foreign_keys=OFF;")...) 141 | } 142 | 143 | queryStr := url.Values{} 144 | restoreURL := url.URL{ 145 | Scheme: argv.Protocol, 146 | Host: fmt.Sprintf("%s:%d", argv.Host, argv.Port), 147 | Path: fmt.Sprintf("%sdb/load", argv.Prefix), 148 | RawQuery: queryStr.Encode(), 149 | } 150 | response, err := sendRequest(ctx, makeRestoreRequest(restoreFile), restoreURL.String(), argv) 151 | if err != nil { 152 | return err 153 | } 154 | 155 | restoreRet := &restoreResponse{} 156 | if err := parseResponse(response, &restoreRet); err != nil { 157 | return err 158 | } 159 | if len(restoreRet.Results) < 1 { 160 | return fmt.Errorf("unexpected results length: %d", len(restoreRet.Results)) 161 | } 162 | if resultError := restoreRet.Results[0].Error; resultError != "" { 163 | ctx.String("Error: %s\n", resultError) 164 | return nil 165 | } 166 | 167 | ctx.String("last inserted ID: %d\n", restoreRet.Results[0].LastInsertID) 168 | ctx.String("rows affected: %d\n", restoreRet.Results[0].RowsAffected) 169 | ctx.String("database restored successfully\n") 170 | return nil 171 | } 172 | -------------------------------------------------------------------------------- /command/marshal.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "expvar" 7 | "fmt" 8 | "io/ioutil" 9 | 10 | "github.com/golang/protobuf/proto" 11 | ) 12 | 13 | const ( 14 | defaultBatchThreshold = 5 15 | defaultSizeThreshold = 150 16 | ) 17 | 18 | // Requester is the interface objects must support to be marshaled 19 | // successfully. 20 | type Requester interface { 21 | proto.Message 22 | GetRequest() *Request 23 | } 24 | 25 | // RequestMarshaler marshals Request objects, potentially performing 26 | // gzip compression. 27 | type RequestMarshaler struct { 28 | BatchThreshold int 29 | SizeThreshold int 30 | ForceCompression bool 31 | } 32 | 33 | const ( 34 | numRequests = "num_requests" 35 | numCompressedRequests = "num_compressed_requests" 36 | numUncompressedRequests = "num_uncompressed_requests" 37 | numCompressedBytes = "num_compressed_bytes" 38 | numPrecompressedBytes = "num_precompressed_bytes" 39 | numUncompressedBytes = "num_uncompressed_bytes" 40 | numCompressionMisses = "num_compression_misses" 41 | ) 42 | 43 | // stats captures stats for the Proto marshaler. 44 | var stats *expvar.Map 45 | 46 | func init() { 47 | stats = expvar.NewMap("proto") 48 | stats.Add(numRequests, 0) 49 | stats.Add(numCompressedRequests, 0) 50 | stats.Add(numUncompressedRequests, 0) 51 | stats.Add(numCompressedBytes, 0) 52 | stats.Add(numUncompressedBytes, 0) 53 | stats.Add(numCompressionMisses, 0) 54 | stats.Add(numPrecompressedBytes, 0) 55 | } 56 | 57 | // NewRequestMarshaler returns an initialized RequestMarshaler. 58 | func NewRequestMarshaler() *RequestMarshaler { 59 | return &RequestMarshaler{ 60 | BatchThreshold: defaultBatchThreshold, 61 | SizeThreshold: defaultSizeThreshold, 62 | } 63 | } 64 | 65 | // Marshal marshals a Requester object, returning a byte slice, a bool 66 | // indicating whether the contents are compressed, or an error. 67 | func (m *RequestMarshaler) Marshal(r Requester) ([]byte, bool, error) { 68 | stats.Add(numRequests, 0) 69 | compress := false 70 | 71 | stmts := r.GetRequest().GetStatements() 72 | if len(stmts) >= m.BatchThreshold { 73 | compress = true 74 | } else { 75 | for i := range stmts { 76 | if len(stmts[i].Sql) >= m.SizeThreshold { 77 | compress = true 78 | break 79 | } 80 | } 81 | } 82 | 83 | b, err := proto.Marshal(r) 84 | if err != nil { 85 | return nil, false, err 86 | } 87 | ubz := len(b) 88 | stats.Add(numPrecompressedBytes, int64(ubz)) 89 | 90 | if compress { 91 | // Let's try compression. 92 | var buf bytes.Buffer 93 | gzw, err := gzip.NewWriterLevel(&buf, gzip.BestCompression) 94 | if err != nil { 95 | return nil, false, fmt.Errorf("gzip new writer: %s", err) 96 | } 97 | 98 | if _, err := gzw.Write(b); err != nil { 99 | return nil, false, fmt.Errorf("gzip Write: %s", err) 100 | } 101 | if err := gzw.Close(); err != nil { 102 | return nil, false, fmt.Errorf("gzip Close: %s", err) 103 | } 104 | 105 | // Is compression better? 106 | if ubz > len(buf.Bytes()) || m.ForceCompression { 107 | // Yes! Let's keep it. 108 | b = buf.Bytes() 109 | stats.Add(numCompressedRequests, 1) 110 | stats.Add(numCompressedBytes, int64(len(b))) 111 | } else { 112 | // No. :-( Dump it. 113 | compress = false 114 | stats.Add(numCompressionMisses, 1) 115 | } 116 | } else { 117 | stats.Add(numUncompressedRequests, 1) 118 | stats.Add(numUncompressedBytes, int64(len(b))) 119 | } 120 | 121 | return b, compress, nil 122 | } 123 | 124 | // Stats returns status and diagnostic information about 125 | // the RequestMarshaler. 126 | func (m *RequestMarshaler) Stats() map[string]interface{} { 127 | return map[string]interface{}{ 128 | "compression_size": m.SizeThreshold, 129 | "compression_batch": m.BatchThreshold, 130 | "force_compression": m.ForceCompression, 131 | } 132 | } 133 | 134 | // Marshal marshals a Command. 135 | func Marshal(c *Command) ([]byte, error) { 136 | return proto.Marshal(c) 137 | } 138 | 139 | // Unmarshal unmarshals a Command 140 | func Unmarshal(b []byte, c *Command) error { 141 | return proto.Unmarshal(b, c) 142 | } 143 | 144 | // MarshalNoop marshals a Noop command 145 | func MarshalNoop(c *Noop) ([]byte, error) { 146 | return proto.Marshal(c) 147 | } 148 | 149 | // UnmarshalNoop unmarshals a Noop command 150 | func UnmarshalNoop(b []byte, c *Noop) error { 151 | return proto.Unmarshal(b, c) 152 | } 153 | 154 | // UnmarshalSubCommand unmarshalls a sub command m. It assumes that 155 | // m is the correct type. 156 | func UnmarshalSubCommand(c *Command, m proto.Message) error { 157 | b := c.SubCommand 158 | if c.Compressed { 159 | gz, err := gzip.NewReader(bytes.NewReader(b)) 160 | if err != nil { 161 | fmt.Errorf("unmarshal sub gzip NewReader: %s", err) 162 | } 163 | 164 | ub, err := ioutil.ReadAll(gz) 165 | if err != nil { 166 | fmt.Errorf("unmarshal sub gzip ReadAll: %s", err) 167 | } 168 | 169 | if err := gz.Close(); err != nil { 170 | fmt.Errorf("unmarshal sub gzip Close: %s", err) 171 | } 172 | b = ub 173 | } 174 | 175 | if err := proto.Unmarshal(b, m); err != nil { 176 | return fmt.Errorf("proto unmarshal: %s", err) 177 | } 178 | return nil 179 | } 180 | -------------------------------------------------------------------------------- /cluster/service.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | import ( 4 | "encoding/binary" 5 | "expvar" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "net" 11 | "os" 12 | "sync" 13 | "time" 14 | 15 | "google.golang.org/protobuf/proto" 16 | ) 17 | 18 | // stats captures stats for the Cluster service. 19 | var stats *expvar.Map 20 | 21 | const ( 22 | numGetNodeAPI = "num_get_node_api" 23 | numGetNodeAPIRequest = "num_get_node_api_req" 24 | numGetNodeAPIResponse = "num_get_node_api_resp" 25 | ) 26 | 27 | const ( 28 | // MuxRaftHeader is the byte used to indicate internode Raft communications. 29 | MuxRaftHeader = 1 30 | 31 | // MuxClusterHeader is the byte used to request internode cluster state information. 32 | MuxClusterHeader = 2 // Cluster state communications 33 | ) 34 | 35 | func init() { 36 | stats = expvar.NewMap("cluster") 37 | stats.Add(numGetNodeAPI, 0) 38 | stats.Add(numGetNodeAPIRequest, 0) 39 | stats.Add(numGetNodeAPIResponse, 0) 40 | } 41 | 42 | // Transport is the interface the network layer must provide. 43 | type Transport interface { 44 | net.Listener 45 | 46 | // Dial is used to create a connection to a service listening 47 | // on an address. 48 | Dial(address string, timeout time.Duration) (net.Conn, error) 49 | } 50 | 51 | // Service provides information about the node and cluster. 52 | type Service struct { 53 | tn Transport // Network layer this service uses 54 | addr net.Addr // Address on which this service is listening 55 | timeout time.Duration 56 | 57 | mu sync.RWMutex 58 | apiAddr string // host:port this node serves the HTTP API. 59 | 60 | logger *log.Logger 61 | } 62 | 63 | // New returns a new instance of the cluster service 64 | func New(tn Transport) *Service { 65 | return &Service{ 66 | tn: tn, 67 | addr: tn.Addr(), 68 | timeout: 10 * time.Second, 69 | logger: log.New(os.Stderr, "[cluster] ", log.LstdFlags), 70 | } 71 | } 72 | 73 | // Open opens the Service. 74 | func (s *Service) Open() error { 75 | go s.serve() 76 | s.logger.Println("service listening on", s.tn.Addr()) 77 | return nil 78 | } 79 | 80 | // Close closes the service. 81 | func (s *Service) Close() error { 82 | s.tn.Close() 83 | return nil 84 | } 85 | 86 | // Addr returns the address the service is listening on. 87 | func (s *Service) Addr() string { 88 | return s.addr.String() 89 | } 90 | 91 | // SetAPIAddr sets the API address the cluster service returns. 92 | func (s *Service) SetAPIAddr(addr string) { 93 | s.mu.Lock() 94 | defer s.mu.Unlock() 95 | s.apiAddr = addr 96 | } 97 | 98 | // GetAPIAddr returns the previously-set API address 99 | func (s *Service) GetAPIAddr() string { 100 | s.mu.RLock() 101 | defer s.mu.RUnlock() 102 | return s.apiAddr 103 | } 104 | 105 | // GetNodeAPIAddr retrieves the API Address for the node at nodeAddr 106 | func (s *Service) GetNodeAPIAddr(nodeAddr string) (string, error) { 107 | stats.Add(numGetNodeAPI, 1) 108 | 109 | conn, err := s.tn.Dial(nodeAddr, s.timeout) 110 | if err != nil { 111 | return "", fmt.Errorf("dial connection: %s", err) 112 | } 113 | defer conn.Close() 114 | 115 | // Send the request 116 | c := &Command{ 117 | Type: Command_COMMAND_TYPE_GET_NODE_API_URL, 118 | } 119 | p, err := proto.Marshal(c) 120 | if err != nil { 121 | return "", fmt.Errorf("command marshal: %s", err) 122 | } 123 | 124 | // Write length of Protobuf, the Protobuf 125 | b := make([]byte, 4) 126 | binary.LittleEndian.PutUint16(b[0:], uint16(len(p))) 127 | 128 | _, err = conn.Write(b) 129 | if err != nil { 130 | return "", fmt.Errorf("write protobuf length: %s", err) 131 | } 132 | _, err = conn.Write(p) 133 | if err != nil { 134 | return "", fmt.Errorf("write protobuf: %s", err) 135 | } 136 | 137 | b, err = ioutil.ReadAll(conn) 138 | if err != nil { 139 | return "", fmt.Errorf("read protobuf bytes: %s", err) 140 | } 141 | 142 | a := &Address{} 143 | err = proto.Unmarshal(b, a) 144 | if err != nil { 145 | return "", fmt.Errorf("protobuf unmarshal: %s", err) 146 | } 147 | 148 | return a.Url, nil 149 | } 150 | 151 | // Stats returns status of the Service. 152 | func (s *Service) Stats() (map[string]interface{}, error) { 153 | st := map[string]interface{}{ 154 | "addr": s.addr.String(), 155 | "timeout": s.timeout.String(), 156 | "api_addr": s.apiAddr, 157 | } 158 | 159 | return st, nil 160 | } 161 | 162 | func (s *Service) serve() error { 163 | for { 164 | conn, err := s.tn.Accept() 165 | if err != nil { 166 | return err 167 | } 168 | 169 | go s.handleConn(conn) 170 | } 171 | } 172 | 173 | func (s *Service) handleConn(conn net.Conn) { 174 | defer conn.Close() 175 | 176 | b := make([]byte, 4) 177 | _, err := io.ReadFull(conn, b) 178 | if err != nil { 179 | return 180 | } 181 | sz := binary.LittleEndian.Uint16(b[0:]) 182 | 183 | b = make([]byte, sz) 184 | _, err = io.ReadFull(conn, b) 185 | if err != nil { 186 | return 187 | } 188 | 189 | c := &Command{} 190 | err = proto.Unmarshal(b, c) 191 | if err != nil { 192 | conn.Close() 193 | } 194 | 195 | switch c.Type { 196 | case Command_COMMAND_TYPE_GET_NODE_API_URL: 197 | stats.Add(numGetNodeAPIRequest, 1) 198 | s.mu.RLock() 199 | defer s.mu.RUnlock() 200 | 201 | a := &Address{} 202 | scheme := "http" 203 | a.Url = fmt.Sprintf("%s://%s", scheme, s.apiAddr) 204 | 205 | b, err = proto.Marshal(a) 206 | if err != nil { 207 | conn.Close() 208 | } 209 | conn.Write(b) 210 | stats.Add(numGetNodeAPIResponse, 1) 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /tcp/conn.go: -------------------------------------------------------------------------------- 1 | package tcp 2 | 3 | import ( 4 | "errors" 5 | "expvar" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net" 10 | "os" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | const ( 16 | // DefaultTimeout is the default length of time to wait for first byte. 17 | DefaultTimeout = 30 * time.Second 18 | ) 19 | 20 | // stats captures stats for the mux system. 21 | var stats *expvar.Map 22 | 23 | const ( 24 | numConnectionsHandled = "num_connections_handled" 25 | numUnregisteredHandlers = "num_unregistered_handlers" 26 | ) 27 | 28 | func init() { 29 | stats = expvar.NewMap("mux") 30 | stats.Add(numConnectionsHandled, 0) 31 | stats.Add(numUnregisteredHandlers, 0) 32 | } 33 | 34 | // Layer represents the connection between nodes. It can be both used to 35 | // make connections to other nodes, and receive connections from other 36 | // nodes. 37 | type Layer struct { 38 | ln net.Listener 39 | header byte 40 | addr net.Addr 41 | } 42 | 43 | // Dial creates a new network connection. 44 | func (l *Layer) Dial(addr string, timeout time.Duration) (net.Conn, error) { 45 | dialer := &net.Dialer{Timeout: timeout} 46 | 47 | var err error 48 | var conn net.Conn 49 | conn, err = dialer.Dial("tcp", addr) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | // Write a marker byte to indicate message type. 55 | _, err = conn.Write([]byte{l.header}) 56 | if err != nil { 57 | conn.Close() 58 | return nil, err 59 | } 60 | return conn, err 61 | } 62 | 63 | // Accept waits for the next connection. 64 | func (l *Layer) Accept() (net.Conn, error) { return l.ln.Accept() } 65 | 66 | // Close closes the layer. 67 | func (l *Layer) Close() error { return l.ln.Close() } 68 | 69 | // Addr returns the local address for the layer. 70 | func (l *Layer) Addr() net.Addr { 71 | return l.addr 72 | } 73 | 74 | // Mux multiplexes a network connection. 75 | type Mux struct { 76 | ln net.Listener 77 | addr net.Addr 78 | m map[byte]*listener 79 | 80 | wg sync.WaitGroup 81 | 82 | // The amount of time to wait for the first header byte. 83 | Timeout time.Duration 84 | 85 | // Out-of-band error logger 86 | Logger *log.Logger 87 | } 88 | 89 | // NewMux returns a new instance of Mux for ln. If adv is nil, 90 | // then the addr of ln is used. 91 | func NewMux(ln net.Listener, adv net.Addr) (*Mux, error) { 92 | addr := adv 93 | if addr == nil { 94 | addr = ln.Addr() 95 | } 96 | 97 | return &Mux{ 98 | ln: ln, 99 | addr: addr, 100 | m: make(map[byte]*listener), 101 | Timeout: DefaultTimeout, 102 | Logger: log.New(os.Stderr, "[mux] ", log.LstdFlags), 103 | }, nil 104 | } 105 | 106 | // Serve handles connections from ln and multiplexes then across registered listener. 107 | func (mux *Mux) Serve() error { 108 | mux.Logger.Printf("mux serving on %s, advertising %s", mux.ln.Addr().String(), mux.addr) 109 | 110 | for { 111 | // Wait for the next connection. 112 | // If it returns a temporary error then simply retry. 113 | // If it returns any other error then exit immediately. 114 | conn, err := mux.ln.Accept() 115 | if err, ok := err.(interface { 116 | Temporary() bool 117 | }); ok && err.Temporary() { 118 | continue 119 | } 120 | if err != nil { 121 | // Wait for all connections to be demuxed 122 | mux.wg.Wait() 123 | for _, ln := range mux.m { 124 | close(ln.c) 125 | } 126 | return err 127 | } 128 | 129 | // Demux in a goroutine to 130 | mux.wg.Add(1) 131 | go mux.handleConn(conn) 132 | } 133 | } 134 | 135 | // Stats returns status of the mux. 136 | func (mux *Mux) Stats() (interface{}, error) { 137 | s := map[string]string{ 138 | "addr": mux.addr.String(), 139 | "timeout": mux.Timeout.String(), 140 | } 141 | 142 | return s, nil 143 | } 144 | 145 | func (mux *Mux) handleConn(conn net.Conn) { 146 | stats.Add(numConnectionsHandled, 1) 147 | 148 | defer mux.wg.Done() 149 | // Set a read deadline so connections with no data don't timeout. 150 | if err := conn.SetReadDeadline(time.Now().Add(mux.Timeout)); err != nil { 151 | conn.Close() 152 | mux.Logger.Printf("tcp.Mux: cannot set read deadline: %s", err) 153 | return 154 | } 155 | 156 | // Read first byte from connection to determine handler. 157 | var typ [1]byte 158 | if _, err := io.ReadFull(conn, typ[:]); err != nil { 159 | conn.Close() 160 | mux.Logger.Printf("tcp.Mux: cannot read header byte: %s", err) 161 | return 162 | } 163 | 164 | // Reset read deadline and let the listener handle that. 165 | if err := conn.SetReadDeadline(time.Time{}); err != nil { 166 | conn.Close() 167 | mux.Logger.Printf("tcp.Mux: cannot reset set read deadline: %s", err) 168 | return 169 | } 170 | 171 | // Retrieve handler based on first byte. 172 | handler := mux.m[typ[0]] 173 | if handler == nil { 174 | conn.Close() 175 | stats.Add(numUnregisteredHandlers, 1) 176 | mux.Logger.Printf("tcp.Mux: handler not registered: %d (unsupported protocol?)", typ[0]) 177 | return 178 | } 179 | 180 | // Send connection to handler. The handler is responsible for closing the connection. 181 | handler.c <- conn 182 | } 183 | 184 | // Listen returns a Layer associated with the given header. Any connection 185 | // accepted by mux is multiplexed based on the initial header byte. 186 | func (mux *Mux) Listen(header byte) *Layer { 187 | // Ensure two listeners are not created for the same header byte. 188 | if _, ok := mux.m[header]; ok { 189 | panic(fmt.Sprintf("listener already registered under header byte: %d", header)) 190 | } 191 | // mux.Logger.Printf("received handler registration request for header %d", header) 192 | 193 | // Create a new listener and assign it. 194 | ln := &listener{ 195 | c: make(chan net.Conn), 196 | } 197 | mux.m[header] = ln 198 | 199 | layer := &Layer{ 200 | ln: ln, 201 | header: header, 202 | addr: mux.addr, 203 | } 204 | 205 | return layer 206 | } 207 | 208 | // listener is a receiver for connections received by Mux. 209 | type listener struct { 210 | c chan net.Conn 211 | } 212 | 213 | // Accept waits for and returns the next connection to the listener. 214 | func (ln *listener) Accept() (c net.Conn, err error) { 215 | conn, ok := <-ln.c 216 | if !ok { 217 | return nil, errors.New("network connection closed") 218 | } 219 | return conn, nil 220 | } 221 | 222 | // Close is a no-op. The mux's listener should be closed instead. 223 | func (ln *listener) Close() error { return nil } 224 | 225 | // Addr always returns nil 226 | func (ln *listener) Addr() net.Addr { return nil } 227 | -------------------------------------------------------------------------------- /cluster/message.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.26.0 4 | // protoc v3.13.0 5 | // source: message.proto 6 | 7 | package cluster 8 | 9 | import ( 10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 | reflect "reflect" 13 | sync "sync" 14 | ) 15 | 16 | const ( 17 | // Verify that this generated code is sufficiently up-to-date. 18 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 19 | // Verify that runtime/protoimpl is sufficiently up-to-date. 20 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 21 | ) 22 | 23 | type Command_Type int32 24 | 25 | const ( 26 | Command_COMMAND_TYPE_UNKNOWN Command_Type = 0 27 | Command_COMMAND_TYPE_GET_NODE_API_URL Command_Type = 1 28 | ) 29 | 30 | // Enum value maps for Command_Type. 31 | var ( 32 | Command_Type_name = map[int32]string{ 33 | 0: "COMMAND_TYPE_UNKNOWN", 34 | 1: "COMMAND_TYPE_GET_NODE_API_URL", 35 | } 36 | Command_Type_value = map[string]int32{ 37 | "COMMAND_TYPE_UNKNOWN": 0, 38 | "COMMAND_TYPE_GET_NODE_API_URL": 1, 39 | } 40 | ) 41 | 42 | func (x Command_Type) Enum() *Command_Type { 43 | p := new(Command_Type) 44 | *p = x 45 | return p 46 | } 47 | 48 | func (x Command_Type) String() string { 49 | return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) 50 | } 51 | 52 | func (Command_Type) Descriptor() protoreflect.EnumDescriptor { 53 | return file_message_proto_enumTypes[0].Descriptor() 54 | } 55 | 56 | func (Command_Type) Type() protoreflect.EnumType { 57 | return &file_message_proto_enumTypes[0] 58 | } 59 | 60 | func (x Command_Type) Number() protoreflect.EnumNumber { 61 | return protoreflect.EnumNumber(x) 62 | } 63 | 64 | // Deprecated: Use Command_Type.Descriptor instead. 65 | func (Command_Type) EnumDescriptor() ([]byte, []int) { 66 | return file_message_proto_rawDescGZIP(), []int{1, 0} 67 | } 68 | 69 | type Address struct { 70 | state protoimpl.MessageState 71 | sizeCache protoimpl.SizeCache 72 | unknownFields protoimpl.UnknownFields 73 | 74 | Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` 75 | } 76 | 77 | func (x *Address) Reset() { 78 | *x = Address{} 79 | if protoimpl.UnsafeEnabled { 80 | mi := &file_message_proto_msgTypes[0] 81 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 82 | ms.StoreMessageInfo(mi) 83 | } 84 | } 85 | 86 | func (x *Address) String() string { 87 | return protoimpl.X.MessageStringOf(x) 88 | } 89 | 90 | func (*Address) ProtoMessage() {} 91 | 92 | func (x *Address) ProtoReflect() protoreflect.Message { 93 | mi := &file_message_proto_msgTypes[0] 94 | if protoimpl.UnsafeEnabled && x != nil { 95 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 96 | if ms.LoadMessageInfo() == nil { 97 | ms.StoreMessageInfo(mi) 98 | } 99 | return ms 100 | } 101 | return mi.MessageOf(x) 102 | } 103 | 104 | // Deprecated: Use Address.ProtoReflect.Descriptor instead. 105 | func (*Address) Descriptor() ([]byte, []int) { 106 | return file_message_proto_rawDescGZIP(), []int{0} 107 | } 108 | 109 | func (x *Address) GetUrl() string { 110 | if x != nil { 111 | return x.Url 112 | } 113 | return "" 114 | } 115 | 116 | type Command struct { 117 | state protoimpl.MessageState 118 | sizeCache protoimpl.SizeCache 119 | unknownFields protoimpl.UnknownFields 120 | 121 | Type Command_Type `protobuf:"varint,1,opt,name=type,proto3,enum=Command_Type" json:"type,omitempty"` 122 | } 123 | 124 | func (x *Command) Reset() { 125 | *x = Command{} 126 | if protoimpl.UnsafeEnabled { 127 | mi := &file_message_proto_msgTypes[1] 128 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 129 | ms.StoreMessageInfo(mi) 130 | } 131 | } 132 | 133 | func (x *Command) String() string { 134 | return protoimpl.X.MessageStringOf(x) 135 | } 136 | 137 | func (*Command) ProtoMessage() {} 138 | 139 | func (x *Command) ProtoReflect() protoreflect.Message { 140 | mi := &file_message_proto_msgTypes[1] 141 | if protoimpl.UnsafeEnabled && x != nil { 142 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 143 | if ms.LoadMessageInfo() == nil { 144 | ms.StoreMessageInfo(mi) 145 | } 146 | return ms 147 | } 148 | return mi.MessageOf(x) 149 | } 150 | 151 | // Deprecated: Use Command.ProtoReflect.Descriptor instead. 152 | func (*Command) Descriptor() ([]byte, []int) { 153 | return file_message_proto_rawDescGZIP(), []int{1} 154 | } 155 | 156 | func (x *Command) GetType() Command_Type { 157 | if x != nil { 158 | return x.Type 159 | } 160 | return Command_COMMAND_TYPE_UNKNOWN 161 | } 162 | 163 | var File_message_proto protoreflect.FileDescriptor 164 | 165 | var file_message_proto_rawDesc = []byte{ 166 | 0x0a, 0x0d, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 167 | 0x1b, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 168 | 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x71, 0x0a, 0x07, 169 | 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x21, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 170 | 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 171 | 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x43, 0x0a, 0x04, 0x54, 0x79, 172 | 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x54, 0x59, 173 | 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x21, 0x0a, 0x1d, 174 | 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45, 0x54, 175 | 0x5f, 0x4e, 0x4f, 0x44, 0x45, 0x5f, 0x41, 0x50, 0x49, 0x5f, 0x55, 0x52, 0x4c, 0x10, 0x01, 0x42, 176 | 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x71, 177 | 0x6c, 0x69, 0x74, 0x65, 0x2f, 0x72, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x2f, 0x63, 0x6c, 0x75, 0x73, 178 | 0x74, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 179 | } 180 | 181 | var ( 182 | file_message_proto_rawDescOnce sync.Once 183 | file_message_proto_rawDescData = file_message_proto_rawDesc 184 | ) 185 | 186 | func file_message_proto_rawDescGZIP() []byte { 187 | file_message_proto_rawDescOnce.Do(func() { 188 | file_message_proto_rawDescData = protoimpl.X.CompressGZIP(file_message_proto_rawDescData) 189 | }) 190 | return file_message_proto_rawDescData 191 | } 192 | 193 | var file_message_proto_enumTypes = make([]protoimpl.EnumInfo, 1) 194 | var file_message_proto_msgTypes = make([]protoimpl.MessageInfo, 2) 195 | var file_message_proto_goTypes = []interface{}{ 196 | (Command_Type)(0), // 0: Command.Type 197 | (*Address)(nil), // 1: Address 198 | (*Command)(nil), // 2: Command 199 | } 200 | var file_message_proto_depIdxs = []int32{ 201 | 0, // 0: Command.type:type_name -> Command.Type 202 | 1, // [1:1] is the sub-list for method output_type 203 | 1, // [1:1] is the sub-list for method input_type 204 | 1, // [1:1] is the sub-list for extension type_name 205 | 1, // [1:1] is the sub-list for extension extendee 206 | 0, // [0:1] is the sub-list for field type_name 207 | } 208 | 209 | func init() { file_message_proto_init() } 210 | func file_message_proto_init() { 211 | if File_message_proto != nil { 212 | return 213 | } 214 | if !protoimpl.UnsafeEnabled { 215 | file_message_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 216 | switch v := v.(*Address); i { 217 | case 0: 218 | return &v.state 219 | case 1: 220 | return &v.sizeCache 221 | case 2: 222 | return &v.unknownFields 223 | default: 224 | return nil 225 | } 226 | } 227 | file_message_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 228 | switch v := v.(*Command); i { 229 | case 0: 230 | return &v.state 231 | case 1: 232 | return &v.sizeCache 233 | case 2: 234 | return &v.unknownFields 235 | default: 236 | return nil 237 | } 238 | } 239 | } 240 | type x struct{} 241 | out := protoimpl.TypeBuilder{ 242 | File: protoimpl.DescBuilder{ 243 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 244 | RawDescriptor: file_message_proto_rawDesc, 245 | NumEnums: 1, 246 | NumMessages: 2, 247 | NumExtensions: 0, 248 | NumServices: 0, 249 | }, 250 | GoTypes: file_message_proto_goTypes, 251 | DependencyIndexes: file_message_proto_depIdxs, 252 | EnumInfos: file_message_proto_enumTypes, 253 | MessageInfos: file_message_proto_msgTypes, 254 | }.Build() 255 | File_message_proto = out.File 256 | file_message_proto_rawDesc = nil 257 | file_message_proto_goTypes = nil 258 | file_message_proto_depIdxs = nil 259 | } 260 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TQLite 2 | *tqlite* is a distributed SQL database with replication, fault-tolerance, tunable consistency and leader election. It uses [SQLite](https://www.sqlite.org/index.html), a small, fast and self-contained SQL engine, as the basic unit in the cluster. 3 | ## Motivation 4 | SQLite is a popular embedded SQL database. It is lightweight, full-featured, and easy to use. However, it is prone to single-point-of-failure due to its single-file-based nature. 5 | 6 | tqlite provides you a lightweight, reliable and highly available SQL cluster, with **easy deployment, and operation**. Think of tqlite as a SQL version of [etcd](https://github.com/coreos/etcd/) or [Consul](https://github.com/hashicorp/consul). 7 | 8 | ## How it works 9 | tqlite ensures the system state is in accordance with a quorum of nodes in the cluster using [Raft](https://raft.github.io/), a well-kown concensus algorithm in a distributed system. 10 | ## Key features 11 | - Lightweight deployment with a single binary 12 | - Support dumping, backing up, and restoring database 13 | - Straightforward HTTP data API 14 | - Distributed consensus system 15 | - Tunable read consistency 16 | ## Quick start 17 | ### Installation 18 | Docker container is available: 19 | ```bash 20 | docker pull minghsu0107/tqlite:v1 21 | ``` 22 | Or you could build from source: 23 | ```bash 24 | git clone https://github.com/minghsu0107/tqlite.git 25 | go build -o tqlite -v ./cmd/tqlite 26 | go build -o tqlited -v ./cmd/tqlited 27 | ``` 28 | ### Running first node 29 | You can start a single tqlite node first: 30 | ```bash 31 | docker network create tqlite-net 32 | docker run --name node1 -p 4001:4001 --network tqlite-net minghsu0107/tqlite:v1 -node-id 1 -http-addr 0.0.0.0:4001 -http-adv-addr localhost:4001 -raft-addr 0.0.0.0:4002 -raft-adv-addr node1:4002 33 | ``` 34 | 35 | This single node becomes the leader automatically. You can pass `-h` to `tqlited` to list all configuration options. 36 | ### Joining a cluster 37 | To be fault-tolerant, we could run tqlite in the cluster mode. For example, we could join the second and third node to the cluster by simply running: 38 | ```bash 39 | docker run --name node2 -p 4011:4001 --network tqlite-net minghsu0107/tqlite:v1 -node-id 2 -http-addr 0.0.0.0:4001 -http-adv-addr localhost:4011 -raft-addr 0.0.0.0:4002 -raft-adv-addr node2:4002 -join http://node1:4001 40 | 41 | docker run --name node3 -p 4021:4001 --network tqlite-net minghsu0107/tqlite:v1 -node-id 3 -http-addr 0.0.0.0:4001 -http-adv-addr localhost:4021 -raft-addr 0.0.0.0:4002 -raft-adv-addr node3:4002 -join http://node1:4001 42 | ``` 43 | Now you have a fully replicated cluster where a majority, or a quorum, of nodes are required to reach conensus on any change to the cluster state. A quorum is is defined as `(N/2)+1` where N is the number of nodes in the cluster. In this example, a 3-node cluster is able to tolerate a single node failure. 44 | ### Using client CLI 45 | Now, we are going to use tqlite client CLI to insert some data to the leader node. The leader will then replicate data to all followers within the cluster. 46 | ```bash 47 | docker exec -it node1 bash 48 | tqlite 49 | ``` 50 | ``` 51 | $ tqlite 52 | 127.0.0.1:4001> CREATE TABLE students (id INTEGER NOT NULL PRIMARY KEY, name TEXT); 53 | 0 row affected 54 | 127.0.0.1:4001> .schema 55 | +--------------------------------------------------------------------+ 56 | | sql | 57 | +--------------------------------------------------------------------+ 58 | | CREATE TABLE students (id INTEGER NOT NULL PRIMARY KEY, name TEXT) | 59 | +--------------------------------------------------------------------+ 60 | 127.0.0.1:4001> INSERT INTO students(name) VALUES("ming"); 61 | 1 row affected 62 | 127.0.0.1:4001> SELECT * FROM students; 63 | +----+------+ 64 | | id | name | 65 | +----+------+ 66 | | 1 | ming | 67 | +----+------+ 68 | ``` 69 | You can see that tqlite client CLI is compatible with SQLite, minimizing the operation costs. 70 | ## Data API 71 | Inspired by Elasticsearch, tqlite exposes data by a rich HTTP API, allowing full control over nodes to query from or write to. We could use HTTP API to do CRUD operations with tunable consistency. Take above `students` table as an example: 72 | ```bash 73 | # query 74 | curl -XPOST 'localhost:4001/db/query?pretty&timings' -H "Content-Type: application/json" -d '[ 75 | "SELECT * FROM students" 76 | ]' 77 | ``` 78 | Query result: 79 | ``` 80 | { 81 | "results": [ 82 | { 83 | "columns": [ 84 | "id", 85 | "name" 86 | ], 87 | "types": [ 88 | "integer", 89 | "text" 90 | ], 91 | "values": [ 92 | [ 93 | 1, 94 | "ming" 95 | ] 96 | ], 97 | "time": 0.000053034 98 | } 99 | ], 100 | "time": 0.000098828 101 | } 102 | ``` 103 | 104 | In addition, you could pass parameterized statements to avoid SQL injections: 105 | ```bash 106 | # write 107 | curl -XPOST 'localhost:4001/db/execute?pretty&timings' -H "Content-Type: application/json" -d '[ 108 | ["INSERT INTO students(name) VALUES(?)", "alice"] 109 | ]' 110 | # read 111 | curl -XPOST 'localhost:4001/db/query?pretty&timings' -H "Content-Type: application/json" -d '[ 112 | ["SELECT * FROM students WHERE name=?", "alice"] 113 | ]' 114 | ``` 115 | You could start a transaction by adding `transaction` query parameter: 116 | ```bash 117 | curl -XPOST 'localhost:4001/db/execute?pretty&transaction' -H "Content-Type: application/json" -d "[ 118 | \"INSERT INTO students(name) VALUES('alan')\", 119 | \"INSERT INTO students(name) VALUES('monica')\" 120 | ]" 121 | ``` 122 | Multiple insertions or updates in a transaction are contained within a single Raft log entry and will not be interleaved with other requests. 123 | ### Write Consistency 124 | Any write request received by followers will be fowarded to the leader. A write request received by the leader is accepted once it replicates the data to a quorum of nodes through Raft successfully. In the below command, we send a write request to `node2`, a follower. Thus the request will be redirected to the leader: 125 | ```bash 126 | curl -i -XPOST 'localhost:4021/db/execute?pretty&timings' -H "Content-Type: application/json" -d '[ 127 | ["INSERT INTO students(name) VALUES(?)", "bob"] 128 | ]' 129 | ``` 130 | Result: 131 | ``` 132 | HTTP/1.1 301 Moved Permanently 133 | Content-Type: application/json; charset=utf-8 134 | Location: http://localhost:4001/db/execute?pretty&timings 135 | X-Tqlite-Version: 1 136 | Date: Mon, 07 Jun 2021 17:25:13 GMT 137 | Content-Length: 0 138 | ``` 139 | Then it is up the clients to re-issue the query command to the leader. 140 | ### Read Consistency 141 | As for read operations, query with consistency level `none` will result in a local read. That is, the node simply queries its local SQLite database directly. In HTTP data API, We should set the query string parameter `level` to `none` to enable it: 142 | ```bash 143 | curl -i -XPOST 'localhost:4021/db/query?pretty&timings&level=none' -H "Content-Type: application/json" -d '[ 144 | ["SELECT * FROM students WHERE name=?", "alice"] 145 | ]' 146 | ``` 147 | In the above query, we send read request to `node2` and will receive an instant response from `node2` without checking its leadership with other peers in the cluster. 148 | 149 | If we send read request to the leader node with consistency level set to `weak`, which is the **default** consistency level, tqlite will instruct the leader to check its local state that it is the leader before querying local SQLite database. If the node receiving the read request is a follower, the request will be redirected to the leader. However, a very small window of time (milliseconds by default) during which the node may return stale data. This is because after the leader check, but before querying local SQLite database, another node could be elected Leader and make changes to the cluster. 150 | 151 | If we send read request to the leader node with consistency level set to `strong`, tqlite will read from leader node and send the request through Raft consensus system, ensuring that the **node remains the leader at all times during query processing**. If the node receiving the read request is a follower, the request will be redirected to the leader. The `strong` consistency solves the issue associated with `weak` consistency. However, this will involve the leader contacting at least a quorum of nodes and will therefore increase query response times. 152 | 153 | Redirection example: 154 | ```bash 155 | curl -i -XPOST 'localhost:4021/db/query?pretty&timings&level=strong' -H "Content-Type: application/json" -d '[ 156 | ["SELECT * FROM students WHERE name=?", "alice"] 157 | ]' 158 | ``` 159 | Result: 160 | ``` 161 | HTTP/1.1 301 Moved Permanently 162 | Content-Type: application/json; charset=utf-8 163 | Location: http://localhost:4001/db/query?pretty&timings&level=strong 164 | X-Tqlite-Version: 1 165 | Date: Mon, 07 Jun 2021 17:25:57 GMT 166 | Content-Length: 0 167 | ``` 168 | ## In-memory store 169 | To enhance the performance, tqlite runs SQLite [in-memory](https://www.sqlite.org/inmemorydb.html) by default, meaning that there is no actual file created on disk. The data durability is guaranteed by the Raft journal, so the database could be recreated in the memory on restart. However, you could still enable the disk mode by adding flag `-on-disk` to `tqlited`. 170 | -------------------------------------------------------------------------------- /cmd/tqlite/main.go: -------------------------------------------------------------------------------- 1 | // Command tqlite is the command-line interface for tqlite. 2 | package main 3 | 4 | import ( 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | "sort" 13 | "strings" 14 | "time" 15 | 16 | "github.com/Bowery/prompt" 17 | "github.com/minghsu0107/tqlite/cmd" 18 | "github.com/mkideal/cli" 19 | ) 20 | 21 | const maxRedirect = 21 22 | 23 | type argT struct { 24 | cli.Helper 25 | Protocol string `cli:"s,scheme" usage:"protocol scheme" dft:"http"` 26 | Host string `cli:"H,host" usage:"tqlited host address" dft:"127.0.0.1"` 27 | Port uint16 `cli:"p,port" usage:"tqlited host port" dft:"4001"` 28 | Prefix string `cli:"P,prefix" usage:"tqlited HTTP URL prefix" dft:"/"` 29 | Version bool `cli:"v,version" usage:"display CLI version"` 30 | } 31 | 32 | var cliHelp = []string{ 33 | `.backup Write database backup to SQLite file`, 34 | `.dump Dump the database in SQL text format to a file`, 35 | `.expvar Show expvar (Go runtime) information for connected node`, 36 | `.help Show this message`, 37 | `.indexes Show names of all indexes`, 38 | `.restore Restore the database from a SQLite dump file`, 39 | `.nodes Show connection status of all nodes in cluster`, 40 | `.schema Show CREATE statements for all tables`, 41 | `.status Show status and diagnostic information for connected node`, 42 | `.sysdump Dump system diagnostics to a file for offline analysis`, 43 | `.tables List names of tables`, 44 | `.timer on|off Turn query timer on or off`, 45 | `.remove Remove a node from the cluster`, 46 | } 47 | 48 | func main() { 49 | cli.SetUsageStyle(cli.ManualStyle) 50 | cli.Run(new(argT), func(ctx *cli.Context) error { 51 | argv := ctx.Argv().(*argT) 52 | if argv.Help { 53 | ctx.WriteUsage() 54 | return nil 55 | } 56 | 57 | if argv.Version { 58 | ctx.String("Version %s, commmit %s, branch %s, built on %s\n", cmd.Version, 59 | cmd.Commit, cmd.Branch, cmd.Buildtime) 60 | return nil 61 | } 62 | 63 | client, err := getHTTPClient(argv) 64 | if err != nil { 65 | ctx.String("%s %v\n", ctx.Color().Red("ERR!"), err) 66 | return nil 67 | } 68 | 69 | version, err := getVersionWithClient(client, argv) 70 | if err != nil { 71 | ctx.String("%s %v\n", ctx.Color().Red("ERR!"), err) 72 | return nil 73 | } 74 | 75 | fmt.Println("Welcome to the tqlite CLI. Enter \".help\" for usage hints.") 76 | fmt.Printf("Version %s, commit %s, branch %s\n", cmd.Version, cmd.Commit, cmd.Branch) 77 | fmt.Printf("Connected to tqlited version %s\n", version) 78 | 79 | timer := false 80 | prefix := fmt.Sprintf("%s:%d>", argv.Host, argv.Port) 81 | term, err := prompt.NewTerminal() 82 | if err != nil { 83 | ctx.String("%s %v\n", ctx.Color().Red("ERR!"), err) 84 | return nil 85 | } 86 | term.Close() 87 | 88 | FOR_READ: 89 | for { 90 | term.Reopen() 91 | line, err := term.Basic(prefix, false) 92 | term.Close() 93 | if err != nil { 94 | return err 95 | } 96 | 97 | line = strings.TrimSpace(line) 98 | if line == "" { 99 | continue 100 | } 101 | var ( 102 | index = strings.Index(line, " ") 103 | cmd = line 104 | ) 105 | if index >= 0 { 106 | cmd = line[:index] 107 | } 108 | cmd = strings.ToUpper(cmd) 109 | switch cmd { 110 | case ".TABLES": 111 | err = queryWithClient(ctx, client, argv, timer, `SELECT name FROM sqlite_master WHERE type="table"`) 112 | case ".INDEXES": 113 | err = queryWithClient(ctx, client, argv, timer, `SELECT sql FROM sqlite_master WHERE type="index"`) 114 | case ".SCHEMA": 115 | err = queryWithClient(ctx, client, argv, timer, "SELECT sql FROM sqlite_master") 116 | case ".TIMER": 117 | err = toggleTimer(line[index+1:], &timer) 118 | case ".STATUS": 119 | err = status(ctx, cmd, line, argv) 120 | case ".NODES": 121 | err = nodes(ctx, cmd, line, argv) 122 | case ".EXPVAR": 123 | err = expvar(ctx, cmd, line, argv) 124 | case ".REMOVE": 125 | err = removeNode(client, line[index+1:], argv, timer) 126 | case ".BACKUP": 127 | if index == -1 || index == len(line)-1 { 128 | err = fmt.Errorf("Please specify an output file for the backup") 129 | break 130 | } 131 | err = backup(ctx, line[index+1:], argv) 132 | case ".RESTORE": 133 | if index == -1 || index == len(line)-1 { 134 | err = fmt.Errorf("Please specify an input file to restore from") 135 | break 136 | } 137 | err = restore(ctx, line[index+1:], argv) 138 | case ".SYSDUMP": 139 | if index == -1 || index == len(line)-1 { 140 | err = fmt.Errorf("Please specify an output file for the sysdump") 141 | break 142 | } 143 | err = sysdump(ctx, line[index+1:], argv) 144 | case ".DUMP": 145 | if index == -1 || index == len(line)-1 { 146 | err = fmt.Errorf("Please specify an output file for the SQL text") 147 | break 148 | } 149 | err = dump(ctx, line[index+1:], argv) 150 | case ".HELP": 151 | err = help(ctx, cmd, line, argv) 152 | case ".QUIT", "QUIT", "EXIT": 153 | break FOR_READ 154 | case "SELECT": 155 | err = queryWithClient(ctx, client, argv, timer, line) 156 | default: 157 | err = executeWithClient(ctx, client, argv, timer, line) 158 | } 159 | if err != nil { 160 | ctx.String("%s %v\n", ctx.Color().Red("ERR!"), err) 161 | } 162 | } 163 | ctx.String("bye~\n") 164 | return nil 165 | }) 166 | } 167 | 168 | func toggleTimer(op string, flag *bool) error { 169 | if op != "on" && op != "off" { 170 | return fmt.Errorf("invalid option '%s'. Use 'on' or 'off' (default)", op) 171 | } 172 | *flag = (op == "on") 173 | return nil 174 | } 175 | 176 | func makeJSONBody(line string) string { 177 | data, err := json.Marshal([]string{line}) 178 | if err != nil { 179 | return "" 180 | } 181 | return string(data) 182 | } 183 | 184 | func help(ctx *cli.Context, cmd, line string, argv *argT) error { 185 | sort.Strings(cliHelp) 186 | fmt.Printf(strings.Join(cliHelp, "\n")) 187 | return nil 188 | } 189 | 190 | func status(ctx *cli.Context, cmd, line string, argv *argT) error { 191 | url := fmt.Sprintf("%s://%s:%d/status", argv.Protocol, argv.Host, argv.Port) 192 | return cliJSON(ctx, cmd, line, url, argv) 193 | } 194 | 195 | func nodes(ctx *cli.Context, cmd, line string, argv *argT) error { 196 | url := fmt.Sprintf("%s://%s:%d/nodes", argv.Protocol, argv.Host, argv.Port) 197 | return cliJSON(ctx, cmd, line, url, argv) 198 | } 199 | 200 | func expvar(ctx *cli.Context, cmd, line string, argv *argT) error { 201 | url := fmt.Sprintf("%s://%s:%d/debug/vars", argv.Protocol, argv.Host, argv.Port) 202 | return cliJSON(ctx, cmd, line, url, argv) 203 | } 204 | 205 | func sysdump(ctx *cli.Context, filename string, argv *argT) error { 206 | f, err := os.Create(filename) 207 | if err != nil { 208 | return err 209 | } 210 | 211 | urls := []string{ 212 | fmt.Sprintf("%s://%s:%d/status?pretty", argv.Protocol, argv.Host, argv.Port), 213 | fmt.Sprintf("%s://%s:%d/nodes?pretty", argv.Protocol, argv.Host, argv.Port), 214 | fmt.Sprintf("%s://%s:%d/debug/vars", argv.Protocol, argv.Host, argv.Port), 215 | } 216 | 217 | if err := urlsToWriter(urls, f, argv); err != nil { 218 | return err 219 | } 220 | return f.Close() 221 | } 222 | 223 | func getHTTPClient(argv *argT) (*http.Client, error) { 224 | client := http.Client{Transport: &http.Transport{ 225 | Proxy: http.ProxyFromEnvironment, 226 | }} 227 | 228 | // Explicitly handle redirects. 229 | client.CheckRedirect = func(req *http.Request, via []*http.Request) error { 230 | return http.ErrUseLastResponse 231 | } 232 | 233 | return &client, nil 234 | } 235 | 236 | func getVersionWithClient(client *http.Client, argv *argT) (string, error) { 237 | u := url.URL{ 238 | Scheme: argv.Protocol, 239 | Host: fmt.Sprintf("%s:%d", argv.Host, argv.Port), 240 | Path: fmt.Sprintf("%s/status", argv.Prefix), 241 | } 242 | urlStr := u.String() 243 | 244 | req, err := http.NewRequest("GET", urlStr, nil) 245 | if err != nil { 246 | return "", err 247 | } 248 | 249 | resp, err := client.Do(req) 250 | if err != nil { 251 | return "", err 252 | } 253 | 254 | version, ok := resp.Header["X-Tqlite-Version"] 255 | if !ok || len(version) != 1 { 256 | return "unknown", nil 257 | } 258 | return version[0], nil 259 | } 260 | 261 | func sendRequest(ctx *cli.Context, makeNewRequest func(string) (*http.Request, error), urlStr string, argv *argT) (*[]byte, error) { 262 | url := urlStr 263 | 264 | client := http.Client{Transport: &http.Transport{ 265 | Proxy: http.ProxyFromEnvironment, 266 | }} 267 | 268 | // Explicitly handle redirects. 269 | client.CheckRedirect = func(req *http.Request, via []*http.Request) error { 270 | return http.ErrUseLastResponse 271 | } 272 | 273 | nRedirect := 0 274 | for { 275 | req, err := makeNewRequest(url) 276 | if err != nil { 277 | return nil, err 278 | } 279 | 280 | resp, err := client.Do(req) 281 | if err != nil { 282 | return nil, err 283 | } 284 | response, err := ioutil.ReadAll(resp.Body) 285 | if err != nil { 286 | return nil, err 287 | } 288 | resp.Body.Close() 289 | 290 | if resp.StatusCode == http.StatusUnauthorized { 291 | return nil, fmt.Errorf("unauthorized") 292 | } 293 | 294 | if resp.StatusCode == http.StatusMovedPermanently { 295 | nRedirect++ 296 | if nRedirect > maxRedirect { 297 | return nil, fmt.Errorf("maximum leader redirect limit exceeded") 298 | } 299 | url = resp.Header["Location"][0] 300 | continue 301 | } 302 | 303 | if resp.StatusCode != http.StatusOK { 304 | return nil, fmt.Errorf("server responded with: %s", resp.Status) 305 | } 306 | 307 | return &response, nil 308 | } 309 | } 310 | 311 | func parseResponse(response *[]byte, ret interface{}) error { 312 | return json.Unmarshal(*response, ret) 313 | } 314 | 315 | // cliJSON fetches JSON from a URL, and displays it at the CLI. 316 | func cliJSON(ctx *cli.Context, cmd, line, url string, argv *argT) error { 317 | // Recursive JSON printer. 318 | var pprint func(indent int, m map[string]interface{}) 319 | pprint = func(indent int, m map[string]interface{}) { 320 | indentation := " " 321 | for k, v := range m { 322 | if v == nil { 323 | continue 324 | } 325 | switch v.(type) { 326 | case map[string]interface{}: 327 | for i := 0; i < indent; i++ { 328 | fmt.Print(indentation) 329 | } 330 | fmt.Printf("%s:\n", k) 331 | pprint(indent+1, v.(map[string]interface{})) 332 | default: 333 | for i := 0; i < indent; i++ { 334 | fmt.Print(indentation) 335 | } 336 | fmt.Printf("%s: %v\n", k, v) 337 | } 338 | } 339 | } 340 | 341 | client := http.Client{Transport: &http.Transport{ 342 | Proxy: http.ProxyFromEnvironment, 343 | }} 344 | 345 | req, err := http.NewRequest("GET", url, nil) 346 | if err != nil { 347 | return err 348 | } 349 | 350 | resp, err := client.Do(req) 351 | if err != nil { 352 | return err 353 | } 354 | defer resp.Body.Close() 355 | 356 | if resp.StatusCode == http.StatusUnauthorized { 357 | return fmt.Errorf("unauthorized") 358 | } 359 | 360 | body, err := ioutil.ReadAll(resp.Body) 361 | if err != nil { 362 | return err 363 | } 364 | 365 | ret := make(map[string]interface{}) 366 | if err := json.Unmarshal(body, &ret); err != nil { 367 | return err 368 | } 369 | 370 | // Specific key requested? 371 | parts := strings.Split(line, " ") 372 | if len(parts) >= 2 { 373 | ret = map[string]interface{}{parts[1]: ret[parts[1]]} 374 | } 375 | pprint(0, ret) 376 | 377 | return nil 378 | } 379 | 380 | func urlsToWriter(urls []string, w io.Writer, argv *argT) error { 381 | client := http.Client{ 382 | Transport: &http.Transport{ 383 | Proxy: http.ProxyFromEnvironment, 384 | }, 385 | Timeout: 10 * time.Second, 386 | } 387 | 388 | for i := range urls { 389 | err := func() error { 390 | w.Write([]byte("\n=========================================\n")) 391 | w.Write([]byte(fmt.Sprintf("URL: %s\n", urls[i]))) 392 | 393 | req, err := http.NewRequest("GET", urls[i], nil) 394 | if err != nil { 395 | return err 396 | } 397 | 398 | resp, err := client.Do(req) 399 | if err != nil { 400 | if _, err := w.Write([]byte(fmt.Sprintf("Status: %s\n\n", err))); err != nil { 401 | return err 402 | } 403 | return nil 404 | } 405 | defer resp.Body.Close() 406 | 407 | if _, err := w.Write([]byte(fmt.Sprintf("Status: %s\n\n", resp.Status))); err != nil { 408 | return err 409 | } 410 | 411 | if resp.StatusCode != http.StatusOK { 412 | return nil 413 | } 414 | 415 | body, err := ioutil.ReadAll(resp.Body) 416 | if err != nil { 417 | return err 418 | } 419 | 420 | if _, err := w.Write(body); err != nil { 421 | return err 422 | } 423 | return nil 424 | }() 425 | if err != nil { 426 | return err 427 | } 428 | } 429 | 430 | return nil 431 | } 432 | -------------------------------------------------------------------------------- /cmd/tqlited/main.go: -------------------------------------------------------------------------------- 1 | // Command tqlited is the tqlite server. 2 | package main 3 | 4 | import ( 5 | "flag" 6 | "fmt" 7 | "log" 8 | "net" 9 | "os" 10 | "os/signal" 11 | "path/filepath" 12 | "runtime" 13 | "runtime/pprof" 14 | "strings" 15 | "time" 16 | 17 | "github.com/minghsu0107/tqlite/cluster" 18 | "github.com/minghsu0107/tqlite/cmd" 19 | httpd "github.com/minghsu0107/tqlite/http" 20 | "github.com/minghsu0107/tqlite/store" 21 | "github.com/minghsu0107/tqlite/tcp" 22 | ) 23 | 24 | var httpAddr string 25 | var httpAdv string 26 | var joinSrcIP string 27 | var nodeID string 28 | var raftAddr string 29 | var raftAdv string 30 | var joinAddr string 31 | var joinAttempts int 32 | var joinInterval string 33 | var expvar bool 34 | var pprofEnabled bool 35 | var dsn string 36 | var onDisk bool 37 | var raftLogLevel string 38 | var raftNonVoter bool 39 | var raftSnapThreshold uint64 40 | var raftSnapInterval string 41 | var raftLeaderLeaseTimeout string 42 | var raftHeartbeatTimeout string 43 | var raftElectionTimeout string 44 | var raftApplyTimeout string 45 | var raftOpenTimeout string 46 | var raftWaitForLeader bool 47 | var raftShutdownOnRemove bool 48 | var compressionSize int 49 | var compressionBatch int 50 | var showVersion bool 51 | var cpuProfile string 52 | var memProfile string 53 | 54 | const name = `tqlited` 55 | const desc = `tqlite is a lightweight, distributed relational database, which uses SQLite as its 56 | storage engine. It provides an easy-to-use, fault-tolerant store for relational data.` 57 | 58 | func init() { 59 | flag.StringVar(&nodeID, "node-id", "", "Unique name for node. If not set, set to Raft address") 60 | flag.StringVar(&httpAddr, "http-addr", "localhost:4001", "HTTP server bind address") 61 | flag.StringVar(&httpAdv, "http-adv-addr", "", "Advertised HTTP address. If not set, same as HTTP server") 62 | flag.StringVar(&joinSrcIP, "join-source-ip", "", "Set source IP address during Join request") 63 | flag.StringVar(&raftAddr, "raft-addr", "localhost:4002", "Raft communication bind address") 64 | flag.StringVar(&raftAdv, "raft-adv-addr", "", "Advertised Raft communication address. If not set, same as Raft bind") 65 | flag.StringVar(&joinAddr, "join", "", "Comma-delimited list of nodes, through which a cluster can be joined (proto://host:port)") 66 | flag.IntVar(&joinAttempts, "join-attempts", 5, "Number of join attempts to make") 67 | flag.StringVar(&joinInterval, "join-interval", "5s", "Period between join attempts") 68 | flag.BoolVar(&expvar, "expvar", true, "Serve expvar data on HTTP server") 69 | flag.BoolVar(&pprofEnabled, "pprof", true, "Serve pprof data on HTTP server") 70 | flag.StringVar(&dsn, "dsn", "", `SQLite DSN parameters. E.g. "cache=shared&mode=memory"`) 71 | flag.BoolVar(&onDisk, "on-disk", false, "Use an on-disk SQLite database") 72 | flag.BoolVar(&showVersion, "version", false, "Show version information and exit") 73 | flag.BoolVar(&raftNonVoter, "raft-non-voter", false, "Configure as non-voting node") 74 | flag.StringVar(&raftHeartbeatTimeout, "raft-timeout", "1s", "Raft heartbeat timeout") 75 | flag.StringVar(&raftElectionTimeout, "raft-election-timeout", "1s", "Raft election timeout") 76 | flag.StringVar(&raftApplyTimeout, "raft-apply-timeout", "10s", "Raft apply timeout") 77 | flag.StringVar(&raftOpenTimeout, "raft-open-timeout", "120s", "Time for initial Raft logs to be applied. Use 0s duration to skip wait") 78 | flag.BoolVar(&raftWaitForLeader, "raft-leader-wait", true, "Node waits for a leader before answering requests") 79 | flag.Uint64Var(&raftSnapThreshold, "raft-snap", 8192, "Number of outstanding log entries that trigger snapshot") 80 | flag.StringVar(&raftSnapInterval, "raft-snap-int", "30s", "Snapshot threshold check interval") 81 | flag.StringVar(&raftLeaderLeaseTimeout, "raft-leader-lease-timeout", "0s", "Raft leader lease timeout. Use 0s for Raft default") 82 | flag.BoolVar(&raftShutdownOnRemove, "raft-remove-shutdown", false, "Shutdown Raft if node removed") 83 | flag.StringVar(&raftLogLevel, "raft-log-level", "INFO", "Minimum log level for Raft module") 84 | flag.IntVar(&compressionSize, "compression-size", 150, "Request query size for compression attempt") 85 | flag.IntVar(&compressionBatch, "compression-batch", 5, "Request batch threshold for compression attempt") 86 | flag.StringVar(&cpuProfile, "cpu-profile", "", "Path to file for CPU profiling information") 87 | flag.StringVar(&memProfile, "mem-profile", "", "Path to file for memory profiling information") 88 | 89 | flag.Usage = func() { 90 | fmt.Fprintf(os.Stderr, "\n%s\n\n", desc) 91 | fmt.Fprintf(os.Stderr, "Usage: %s [flags] \n", name) 92 | flag.PrintDefaults() 93 | } 94 | } 95 | 96 | func main() { 97 | flag.Parse() 98 | 99 | if showVersion { 100 | fmt.Printf("%s %s %s %s %s (commit %s, branch %s)\n", 101 | name, cmd.Version, runtime.GOOS, runtime.GOARCH, runtime.Version(), cmd.Commit, cmd.Branch) 102 | os.Exit(0) 103 | } 104 | 105 | // Ensure the data path is set. 106 | if flag.NArg() < 1 { 107 | fmt.Fprintf(os.Stderr, "fatal: no data directory set\n") 108 | os.Exit(1) 109 | } 110 | 111 | // Ensure no args come after the data directory. 112 | if flag.NArg() > 1 { 113 | fmt.Fprintf(os.Stderr, "fatal: arguments after data directory are not accepted\n") 114 | os.Exit(1) 115 | } 116 | 117 | dataPath := flag.Arg(0) 118 | 119 | // Configure logging and pump out initial message. 120 | log.SetFlags(log.LstdFlags) 121 | log.SetOutput(os.Stderr) 122 | log.SetPrefix(fmt.Sprintf("[%s] ", name)) 123 | // log.Printf("%s starting, version %s, commit %s, branch %s", name, cmd.Version, cmd.Commit, cmd.Branch) 124 | // log.Printf("%s, target architecture is %s, operating system target is %s", runtime.Version(), runtime.GOARCH, runtime.GOOS) 125 | log.Printf("launch command: %s", strings.Join(os.Args, " ")) 126 | 127 | // Start requested profiling. 128 | startProfile(cpuProfile, memProfile) 129 | 130 | // Create internode network mux and configure. 131 | muxLn, err := net.Listen("tcp", raftAddr) 132 | if err != nil { 133 | log.Fatalf("failed to listen on %s: %s", raftAddr, err.Error()) 134 | } 135 | mux, err := startNodeMux(muxLn) 136 | if err != nil { 137 | log.Fatalf("failed to start node mux: %s", err.Error()) 138 | } 139 | raftTn := mux.Listen(cluster.MuxRaftHeader) 140 | 141 | // Create cluster service, so nodes can learn information about each other. This can be started 142 | // now since it doesn't require a functioning Store yet. 143 | clstr, err := clusterService(mux.Listen(cluster.MuxClusterHeader)) 144 | if err != nil { 145 | log.Fatalf("failed to create cluster service: %s", err.Error()) 146 | } 147 | 148 | // Create and open the store. 149 | dataPath, err = filepath.Abs(dataPath) 150 | if err != nil { 151 | log.Fatalf("failed to determine absolute data path: %s", err.Error()) 152 | } 153 | dbConf := store.NewDBConfig(dsn, !onDisk) 154 | 155 | str := store.New(raftTn, &store.StoreConfig{ 156 | DBConf: dbConf, 157 | Dir: dataPath, 158 | ID: idOrRaftAddr(), 159 | }) 160 | 161 | // Set optional parameters on store. 162 | str.SetRequestCompression(compressionBatch, compressionSize) 163 | str.RaftLogLevel = raftLogLevel 164 | str.ShutdownOnRemove = raftShutdownOnRemove 165 | str.SnapshotThreshold = raftSnapThreshold 166 | str.SnapshotInterval, err = time.ParseDuration(raftSnapInterval) 167 | if err != nil { 168 | log.Fatalf("failed to parse Raft Snapsnot interval %s: %s", raftSnapInterval, err.Error()) 169 | } 170 | str.LeaderLeaseTimeout, err = time.ParseDuration(raftLeaderLeaseTimeout) 171 | if err != nil { 172 | log.Fatalf("failed to parse Raft Leader lease timeout %s: %s", raftLeaderLeaseTimeout, err.Error()) 173 | } 174 | str.HeartbeatTimeout, err = time.ParseDuration(raftHeartbeatTimeout) 175 | if err != nil { 176 | log.Fatalf("failed to parse Raft heartbeat timeout %s: %s", raftHeartbeatTimeout, err.Error()) 177 | } 178 | str.ElectionTimeout, err = time.ParseDuration(raftElectionTimeout) 179 | if err != nil { 180 | log.Fatalf("failed to parse Raft election timeout %s: %s", raftElectionTimeout, err.Error()) 181 | } 182 | str.ApplyTimeout, err = time.ParseDuration(raftApplyTimeout) 183 | if err != nil { 184 | log.Fatalf("failed to parse Raft apply timeout %s: %s", raftApplyTimeout, err.Error()) 185 | } 186 | 187 | // Any prexisting node state? 188 | var enableBootstrap bool 189 | isNew := store.IsNewNode(dataPath) 190 | if isNew { 191 | log.Printf("no preexisting node state detected in %s, node may be bootstrapping", dataPath) 192 | enableBootstrap = true // New node, so we may be bootstrapping 193 | } else { 194 | log.Printf("preexisting node state detected in %s", dataPath) 195 | } 196 | 197 | // Determine join addresses 198 | var joins []string 199 | joins, err = determineJoinAddresses() 200 | if err != nil { 201 | log.Fatalf("unable to determine join addresses: %s", err.Error()) 202 | } 203 | 204 | // Supplying join addresses means bootstrapping a new cluster won't 205 | // be required. 206 | if len(joins) > 0 { 207 | enableBootstrap = false 208 | log.Println("join addresses specified, node is not bootstrapping") 209 | } else { 210 | log.Println("no join addresses set") 211 | } 212 | 213 | // Join address supplied, but we don't need them! 214 | if !isNew && len(joins) > 0 { 215 | log.Println("node is already member of cluster, ignoring join addresses") 216 | } 217 | 218 | // Now, open store. 219 | if err := str.Open(enableBootstrap); err != nil { 220 | log.Fatalf("failed to open store: %s", err.Error()) 221 | } 222 | 223 | // Execute any requested join operation. 224 | if len(joins) > 0 && isNew { 225 | log.Println("join addresses are:", joins) 226 | advAddr := raftAddr 227 | if raftAdv != "" { 228 | advAddr = raftAdv 229 | } 230 | 231 | joinDur, err := time.ParseDuration(joinInterval) 232 | if err != nil { 233 | log.Fatalf("failed to parse Join interval %s: %s", joinInterval, err.Error()) 234 | } 235 | 236 | if j, err := cluster.Join(joinSrcIP, joins, str.ID(), advAddr, !raftNonVoter, 237 | joinAttempts, joinDur); err != nil { 238 | log.Fatalf("failed to join cluster at %s: %s", joins, err.Error()) 239 | } else { 240 | log.Println("successfully joined cluster at", j) 241 | } 242 | 243 | } 244 | 245 | // Wait until the store is in full consensus. 246 | if err := waitForConsensus(str); err != nil { 247 | log.Fatalf(err.Error()) 248 | } 249 | log.Println("store has reached consensus") 250 | 251 | // Start the HTTP API server. 252 | if err := startHTTPService(str, clstr); err != nil { 253 | log.Fatalf("failed to start HTTP server: %s", err.Error()) 254 | } 255 | log.Println("node is ready") 256 | 257 | // Block until signalled. 258 | terminate := make(chan os.Signal, 1) 259 | signal.Notify(terminate, os.Interrupt) 260 | <-terminate 261 | if err := str.Close(true); err != nil { 262 | log.Printf("failed to close store: %s", err.Error()) 263 | } 264 | clstr.Close() 265 | muxLn.Close() 266 | stopProfile() 267 | log.Println("tqlite server stopped") 268 | } 269 | 270 | func determineJoinAddresses() ([]string, error) { 271 | var addrs []string 272 | if joinAddr != "" { 273 | // Explicit join addresses are first priority. 274 | addrs = strings.Split(joinAddr, ",") 275 | } 276 | 277 | return addrs, nil 278 | } 279 | 280 | func waitForConsensus(str *store.Store) error { 281 | openTimeout, err := time.ParseDuration(raftOpenTimeout) 282 | if err != nil { 283 | return fmt.Errorf("failed to parse Raft open timeout %s: %s", raftOpenTimeout, err.Error()) 284 | } 285 | if _, err := str.WaitForLeader(openTimeout); err != nil { 286 | if raftWaitForLeader { 287 | return fmt.Errorf("leader did not appear within timeout: %s", err.Error()) 288 | } 289 | log.Println("ignoring error while waiting for leader") 290 | } 291 | if openTimeout != 0 { 292 | if err := str.WaitForApplied(openTimeout); err != nil { 293 | return fmt.Errorf("log was not fully applied within timeout: %s", err.Error()) 294 | } 295 | } else { 296 | log.Println("not waiting for logs to be applied") 297 | } 298 | return nil 299 | } 300 | 301 | func startHTTPService(str *store.Store, cltr *cluster.Service) error { 302 | // Create HTTP server 303 | var s *httpd.Service 304 | s = httpd.New(httpAddr, str, cltr) 305 | 306 | s.Expvar = expvar 307 | s.Pprof = pprofEnabled 308 | s.BuildInfo = map[string]interface{}{ 309 | "commit": cmd.Commit, 310 | "branch": cmd.Branch, 311 | "version": cmd.Version, 312 | "build_time": cmd.Buildtime, 313 | } 314 | return s.Start() 315 | } 316 | 317 | func startNodeMux(ln net.Listener) (*tcp.Mux, error) { 318 | var adv net.Addr 319 | var err error 320 | if raftAdv != "" { 321 | adv, err = net.ResolveTCPAddr("tcp", raftAdv) 322 | if err != nil { 323 | return nil, fmt.Errorf("failed to resolve advertise address %s: %s", raftAdv, err.Error()) 324 | } 325 | } 326 | 327 | var mux *tcp.Mux 328 | mux, err = tcp.NewMux(ln, adv) 329 | if err != nil { 330 | return nil, fmt.Errorf("failed to create node-to-node mux: %s", err.Error()) 331 | } 332 | 333 | go mux.Serve() 334 | 335 | return mux, nil 336 | } 337 | 338 | func clusterService(tn cluster.Transport) (*cluster.Service, error) { 339 | c := cluster.New(tn) 340 | apiAddr := httpAddr 341 | if httpAdv != "" { 342 | apiAddr = httpAdv 343 | } 344 | c.SetAPIAddr(apiAddr) 345 | 346 | if err := c.Open(); err != nil { 347 | return nil, err 348 | } 349 | return c, nil 350 | } 351 | 352 | func idOrRaftAddr() string { 353 | if nodeID != "" { 354 | return nodeID 355 | } 356 | if raftAdv == "" { 357 | return raftAddr 358 | } 359 | return raftAdv 360 | } 361 | 362 | // prof stores the file locations of active profiles. 363 | var prof struct { 364 | cpu *os.File 365 | mem *os.File 366 | } 367 | 368 | // startProfile initializes the CPU and memory profile, if specified. 369 | func startProfile(cpuprofile, memprofile string) { 370 | if cpuprofile != "" { 371 | f, err := os.Create(cpuprofile) 372 | if err != nil { 373 | log.Fatalf("failed to create CPU profile file at %s: %s", cpuprofile, err.Error()) 374 | } 375 | log.Printf("writing CPU profile to: %s\n", cpuprofile) 376 | prof.cpu = f 377 | pprof.StartCPUProfile(prof.cpu) 378 | } 379 | 380 | if memprofile != "" { 381 | f, err := os.Create(memprofile) 382 | if err != nil { 383 | log.Fatalf("failed to create memory profile file at %s: %s", cpuprofile, err.Error()) 384 | } 385 | log.Printf("writing memory profile to: %s\n", memprofile) 386 | prof.mem = f 387 | runtime.MemProfileRate = 4096 388 | } 389 | } 390 | 391 | // stopProfile closes the CPU and memory profiles if they are running. 392 | func stopProfile() { 393 | if prof.cpu != nil { 394 | pprof.StopCPUProfile() 395 | prof.cpu.Close() 396 | log.Println("CPU profiling stopped") 397 | } 398 | if prof.mem != nil { 399 | pprof.Lookup("heap").WriteTo(prof.mem, 0) 400 | prof.mem.Close() 401 | log.Println("memory profiling stopped") 402 | } 403 | } 404 | -------------------------------------------------------------------------------- /db/db.go: -------------------------------------------------------------------------------- 1 | // Package db exposes a lightweight abstraction over the SQLite code. 2 | // It performs some basic mapping of lower-level types to tqlite types. 3 | package db 4 | 5 | import ( 6 | "database/sql/driver" 7 | "expvar" 8 | "fmt" 9 | "io" 10 | "os" 11 | "strings" 12 | "time" 13 | 14 | "github.com/minghsu0107/tqlite/command" 15 | "github.com/rqlite/go-sqlite3" 16 | ) 17 | 18 | const bkDelay = 250 19 | 20 | const ( 21 | fkChecks = "PRAGMA foreign_keys" 22 | fkChecksEnabled = "PRAGMA foreign_keys=ON" 23 | fkChecksDisabled = "PRAGMA foreign_keys=OFF" 24 | 25 | numExecutions = "executions" 26 | numExecutionErrors = "execution_errors" 27 | numQueries = "queries" 28 | numETx = "execute_transactions" 29 | numQTx = "query_transactions" 30 | ) 31 | 32 | // DBVersion is the SQLite version. 33 | var DBVersion string 34 | 35 | // stats captures stats for the DB layer. 36 | var stats *expvar.Map 37 | 38 | func init() { 39 | DBVersion, _, _ = sqlite3.Version() 40 | stats = expvar.NewMap("db") 41 | stats.Add(numExecutions, 0) 42 | stats.Add(numExecutionErrors, 0) 43 | stats.Add(numQueries, 0) 44 | stats.Add(numETx, 0) 45 | stats.Add(numQTx, 0) 46 | 47 | } 48 | 49 | // DB is the SQL database. 50 | type DB struct { 51 | sqlite3conn *sqlite3.SQLiteConn // Driver connection to database. 52 | path string // Path to database file. 53 | dsn string // DSN, if any. 54 | memory bool // In-memory only. 55 | } 56 | 57 | // Result represents the outcome of an operation that changes rows. 58 | type Result struct { 59 | LastInsertID int64 `json:"last_insert_id,omitempty"` 60 | RowsAffected int64 `json:"rows_affected,omitempty"` 61 | Error string `json:"error,omitempty"` 62 | Time float64 `json:"time,omitempty"` 63 | } 64 | 65 | // Rows represents the outcome of an operation that returns query data. 66 | type Rows struct { 67 | Columns []string `json:"columns,omitempty"` 68 | Types []string `json:"types,omitempty"` 69 | Values [][]interface{} `json:"values,omitempty"` 70 | Error string `json:"error,omitempty"` 71 | Time float64 `json:"time,omitempty"` 72 | } 73 | 74 | // Open opens a file-based database, creating it if it does not exist. 75 | func Open(dbPath string) (*DB, error) { 76 | return open(fqdsn(dbPath, "")) 77 | } 78 | 79 | // OpenWithDSN opens a file-based database, creating it if it does not exist. 80 | func OpenWithDSN(dbPath, dsn string) (*DB, error) { 81 | return open(fqdsn(dbPath, dsn)) 82 | } 83 | 84 | // OpenInMemory opens an in-memory database. 85 | func OpenInMemory() (*DB, error) { 86 | return open(fqdsn(":memory:", "")) 87 | } 88 | 89 | // OpenInMemoryWithDSN opens an in-memory database with a specific DSN. 90 | func OpenInMemoryWithDSN(dsn string) (*DB, error) { 91 | return open(fqdsn(":memory:", dsn)) 92 | } 93 | 94 | // LoadInMemoryWithDSN loads an in-memory database with that at the path, 95 | // with the specified DSN 96 | func LoadInMemoryWithDSN(dbPath, dsn string) (*DB, error) { 97 | db, err := OpenInMemoryWithDSN(dsn) 98 | if err != nil { 99 | return nil, err 100 | } 101 | 102 | srcDB, err := Open(dbPath) 103 | if err != nil { 104 | return nil, err 105 | } 106 | 107 | if err := copyDatabase(db.sqlite3conn, srcDB.sqlite3conn); err != nil { 108 | return nil, err 109 | } 110 | 111 | if err := srcDB.Close(); err != nil { 112 | return nil, err 113 | } 114 | 115 | return db, nil 116 | } 117 | 118 | // DeserializeInMemoryWithDSN loads an in-memory database with that contained 119 | // in the byte slide, with the specified DSN. The byte slice must not be changed 120 | // or garbage-collected until after this function returns. 121 | func DeserializeInMemoryWithDSN(b []byte, dsn string) (*DB, error) { 122 | tmpDB, err := OpenInMemoryWithDSN(dsn) 123 | if err != nil { 124 | return nil, fmt.Errorf("DeserializeInMemoryWithDSN: %s", err.Error()) 125 | } 126 | defer tmpDB.Close() 127 | 128 | if err := tmpDB.sqlite3conn.Deserialize(b, ""); err != nil { 129 | return nil, fmt.Errorf("DeserializeInMemoryWithDSN: %s", err.Error()) 130 | } 131 | 132 | // tmpDB is still using memory in Go space, so it needs to be explicitly 133 | // copied to a new database. 134 | db, err := OpenInMemoryWithDSN(dsn) 135 | if err != nil { 136 | return nil, fmt.Errorf("DeserializeInMemoryWithDSN: %s", err.Error()) 137 | } 138 | 139 | if err := copyDatabase(db.sqlite3conn, tmpDB.sqlite3conn); err != nil { 140 | return nil, fmt.Errorf("DeserializeInMemoryWithDSN: %s", err.Error()) 141 | } 142 | 143 | return db, nil 144 | } 145 | 146 | // Close closes the underlying database connection. 147 | func (db *DB) Close() error { 148 | return db.sqlite3conn.Close() 149 | } 150 | 151 | func open(dbPath string) (*DB, error) { 152 | d := sqlite3.SQLiteDriver{} 153 | dbc, err := d.Open(dbPath) 154 | if err != nil { 155 | return nil, err 156 | } 157 | 158 | return &DB{ 159 | sqlite3conn: dbc.(*sqlite3.SQLiteConn), 160 | path: dbPath, 161 | }, nil 162 | } 163 | 164 | // EnableFKConstraints allows control of foreign key constraint checks. 165 | func (db *DB) EnableFKConstraints(e bool) error { 166 | q := fkChecksEnabled 167 | if !e { 168 | q = fkChecksDisabled 169 | } 170 | _, err := db.sqlite3conn.Exec(q, nil) 171 | return err 172 | } 173 | 174 | // FKConstraints returns whether FK constraints are set or not. 175 | func (db *DB) FKConstraints() (bool, error) { 176 | r, err := db.sqlite3conn.Query(fkChecks, nil) 177 | if err != nil { 178 | return false, err 179 | } 180 | 181 | dest := make([]driver.Value, len(r.Columns())) 182 | types := r.(*sqlite3.SQLiteRows).DeclTypes() 183 | if err := r.Next(dest); err != nil { 184 | return false, err 185 | } 186 | 187 | values := normalizeRowValues(dest, types) 188 | if values[0] == int64(1) { 189 | return true, nil 190 | } 191 | return false, nil 192 | } 193 | 194 | // Size returns the size of the database in bytes. "Size" is defined as 195 | // page_count * schema.page_size. 196 | func (db *DB) Size() (int64, error) { 197 | query := `SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size()` 198 | r, err := db.QueryStringStmt(query) 199 | if err != nil { 200 | return 0, err 201 | } 202 | 203 | return r[0].Values[0][0].(int64), nil 204 | } 205 | 206 | // FileSize returns the size of the SQLite file on disk. If running in 207 | // on-memory mode, this function returns 0. 208 | func (db *DB) FileSize() (int64, error) { 209 | if db.memory { 210 | return 0, nil 211 | } 212 | fi, err := os.Stat(db.path) 213 | if err != nil { 214 | return 0, err 215 | } 216 | return fi.Size(), nil 217 | } 218 | 219 | // TransactionActive returns whether a transaction is currently active 220 | // i.e. if the database is NOT in autocommit mode. 221 | func (db *DB) TransactionActive() bool { 222 | return !db.sqlite3conn.AutoCommit() 223 | } 224 | 225 | // AbortTransaction aborts -- rolls back -- any active transaction. Calling code 226 | // should know exactly what it is doing if it decides to call this function. It 227 | // can be used to clean up any dangling state that may result from certain 228 | // error scenarios. 229 | func (db *DB) AbortTransaction() error { 230 | _, err := db.ExecuteStringStmt("ROLLBACK") 231 | return err 232 | } 233 | 234 | // ExecuteStringStmt executes a single query that modifies the database. This is 235 | // primarily a convenience function. 236 | func (db *DB) ExecuteStringStmt(query string) ([]*Result, error) { 237 | r := &command.Request{ 238 | Statements: []*command.Statement{ 239 | { 240 | Sql: query, 241 | }, 242 | }, 243 | } 244 | return db.Execute(r, false) 245 | } 246 | 247 | // Execute executes queries that modify the database. 248 | func (db *DB) Execute(req *command.Request, xTime bool) ([]*Result, error) { 249 | stats.Add(numExecutions, int64(len(req.Statements))) 250 | 251 | tx := req.Transaction 252 | if tx { 253 | stats.Add(numETx, 1) 254 | } 255 | 256 | type Execer interface { 257 | Exec(query string, args []driver.Value) (driver.Result, error) 258 | } 259 | 260 | var allResults []*Result 261 | err := func() error { 262 | var execer Execer 263 | var rollback bool 264 | var t driver.Tx 265 | var err error 266 | 267 | // Check for the err, if set rollback. 268 | defer func() { 269 | if t != nil { 270 | if rollback { 271 | t.Rollback() 272 | return 273 | } 274 | t.Commit() 275 | } 276 | }() 277 | 278 | // handleError sets the error field on the given result. It returns 279 | // whether the caller should continue processing or break. 280 | handleError := func(result *Result, err error) bool { 281 | stats.Add(numExecutionErrors, 1) 282 | 283 | result.Error = err.Error() 284 | allResults = append(allResults, result) 285 | if tx { 286 | rollback = true // Will trigger the rollback. 287 | return false 288 | } 289 | return true 290 | } 291 | 292 | execer = db.sqlite3conn 293 | 294 | // Create the correct execution object, depending on whether a 295 | // transaction was requested. 296 | if tx { 297 | t, err = db.sqlite3conn.Begin() 298 | if err != nil { 299 | return err 300 | } 301 | } 302 | 303 | // Execute each statement. 304 | for _, stmt := range req.Statements { 305 | sql := stmt.Sql 306 | if sql == "" { 307 | continue 308 | } 309 | 310 | result := &Result{} 311 | start := time.Now() 312 | 313 | parameters, err := parametersToValues(stmt.Parameters) 314 | if err != nil { 315 | if handleError(result, err) { 316 | continue 317 | } 318 | break 319 | } 320 | 321 | r, err := execer.Exec(sql, parameters) 322 | if err != nil { 323 | if handleError(result, err) { 324 | continue 325 | } 326 | break 327 | } 328 | if r == nil { 329 | continue 330 | } 331 | 332 | lid, err := r.LastInsertId() 333 | if err != nil { 334 | if handleError(result, err) { 335 | continue 336 | } 337 | break 338 | } 339 | result.LastInsertID = lid 340 | 341 | ra, err := r.RowsAffected() 342 | if err != nil { 343 | if handleError(result, err) { 344 | continue 345 | } 346 | break 347 | } 348 | result.RowsAffected = ra 349 | if xTime { 350 | result.Time = time.Now().Sub(start).Seconds() 351 | } 352 | allResults = append(allResults, result) 353 | } 354 | 355 | return nil 356 | }() 357 | 358 | return allResults, err 359 | } 360 | 361 | // QueryStringStmt executes a single query that return rows, but don't modify database. 362 | func (db *DB) QueryStringStmt(query string) ([]*Rows, error) { 363 | r := &command.Request{ 364 | Statements: []*command.Statement{ 365 | { 366 | Sql: query, 367 | }, 368 | }, 369 | } 370 | return db.Query(r, false) 371 | } 372 | 373 | // Query executes queries that return rows, but don't modify the database. 374 | func (db *DB) Query(req *command.Request, xTime bool) ([]*Rows, error) { 375 | stats.Add(numQueries, int64(len(req.Statements))) 376 | 377 | tx := req.Transaction 378 | if tx { 379 | stats.Add(numQTx, 1) 380 | } 381 | 382 | type Queryer interface { 383 | Query(query string, args []driver.Value) (driver.Rows, error) 384 | } 385 | 386 | var allRows []*Rows 387 | err := func() (err error) { 388 | var queryer Queryer 389 | var t driver.Tx 390 | defer func() { 391 | // XXX THIS DOESN'T ACTUALLY WORK! Might as WELL JUST COMMIT? 392 | if t != nil { 393 | if err != nil { 394 | t.Rollback() 395 | return 396 | } 397 | t.Commit() 398 | } 399 | }() 400 | 401 | queryer = db.sqlite3conn 402 | 403 | // Create the correct query object, depending on whether a 404 | // transaction was requested. 405 | if tx { 406 | t, err = db.sqlite3conn.Begin() 407 | if err != nil { 408 | return err 409 | } 410 | } 411 | 412 | for _, stmt := range req.Statements { 413 | sql := stmt.Sql 414 | if sql == "" { 415 | continue 416 | } 417 | 418 | rows := &Rows{} 419 | start := time.Now() 420 | 421 | parameters, err := parametersToValues(stmt.Parameters) 422 | if err != nil { 423 | rows.Error = err.Error() 424 | allRows = append(allRows, rows) 425 | continue 426 | } 427 | 428 | rs, err := queryer.Query(sql, parameters) 429 | if err != nil { 430 | rows.Error = err.Error() 431 | allRows = append(allRows, rows) 432 | continue 433 | } 434 | defer rs.Close() 435 | columns := rs.Columns() 436 | 437 | rows.Columns = columns 438 | rows.Types = rs.(*sqlite3.SQLiteRows).DeclTypes() 439 | dest := make([]driver.Value, len(rows.Columns)) 440 | for { 441 | err := rs.Next(dest) 442 | if err != nil { 443 | if err != io.EOF { 444 | rows.Error = err.Error() 445 | } 446 | break 447 | } 448 | 449 | values := normalizeRowValues(dest, rows.Types) 450 | rows.Values = append(rows.Values, values) 451 | } 452 | if xTime { 453 | rows.Time = time.Now().Sub(start).Seconds() 454 | } 455 | allRows = append(allRows, rows) 456 | } 457 | 458 | return nil 459 | }() 460 | 461 | return allRows, err 462 | } 463 | 464 | // Backup writes a consistent snapshot of the database to the given file. 465 | // This function can be called when changes to the database are in flight. 466 | func (db *DB) Backup(path string) error { 467 | dstDB, err := Open(path) 468 | if err != nil { 469 | return err 470 | } 471 | 472 | defer func(db *DB, err *error) { 473 | cerr := db.Close() 474 | if *err == nil { 475 | *err = cerr 476 | } 477 | }(dstDB, &err) 478 | 479 | if err := copyDatabase(dstDB.sqlite3conn, db.sqlite3conn); err != nil { 480 | return fmt.Errorf("backup database: %s", err) 481 | } 482 | return nil 483 | } 484 | 485 | // Copy copies the contents of the database to the given database. All other 486 | // attributes of the given database remain untouched e.g. whether it's an 487 | // on-disk database. This function can be called when changes to the source 488 | // database are in flight. 489 | func (db *DB) Copy(dstDB *DB) error { 490 | if err := copyDatabase(dstDB.sqlite3conn, db.sqlite3conn); err != nil { 491 | return fmt.Errorf("copy database: %s", err) 492 | } 493 | return nil 494 | } 495 | 496 | // Serialize returns a byte slice representation of the SQLite database. For 497 | // an ordinary on-disk database file, the serialization is just a copy of the 498 | // disk file. For an in-memory database or a "TEMP" database, the serialization 499 | // is the same sequence of bytes which would be written to disk if that database 500 | // were backed up to disk. 501 | // 502 | // It is up to the caller to ensure no changes or transactions are in progress 503 | // when this function is called. 504 | func (db *DB) Serialize() ([]byte, error) { 505 | b := db.sqlite3conn.Serialize("") 506 | if b == nil { 507 | return nil, fmt.Errorf("failed to serialize database") 508 | } 509 | return b, nil 510 | } 511 | 512 | // Dump writes a consistent snapshot of the database in SQL text format. 513 | // This function can be called when changes to the database are in flight. 514 | func (db *DB) Dump(w io.Writer) error { 515 | if _, err := w.Write([]byte("PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n")); err != nil { 516 | return err 517 | } 518 | 519 | // Get a new connection, so the dump creation is isolated from other activity. 520 | dstDB, err := OpenInMemory() 521 | if err != nil { 522 | return err 523 | } 524 | defer func(db *DB, err *error) { 525 | cerr := db.Close() 526 | if *err == nil { 527 | *err = cerr 528 | } 529 | }(dstDB, &err) 530 | 531 | if err := copyDatabase(dstDB.sqlite3conn, db.sqlite3conn); err != nil { 532 | return err 533 | } 534 | 535 | // Get the schema. 536 | query := `SELECT "name", "type", "sql" FROM "sqlite_master" 537 | WHERE "sql" NOT NULL AND "type" == 'table' ORDER BY "name"` 538 | rows, err := dstDB.QueryStringStmt(query) 539 | if err != nil { 540 | return err 541 | } 542 | row := rows[0] 543 | for _, v := range row.Values { 544 | table := v[0].(string) 545 | var stmt string 546 | 547 | if table == "sqlite_sequence" { 548 | stmt = `DELETE FROM "sqlite_sequence";` 549 | } else if table == "sqlite_stat1" { 550 | stmt = `ANALYZE "sqlite_master";` 551 | } else if strings.HasPrefix(table, "sqlite_") { 552 | continue 553 | } else { 554 | stmt = v[2].(string) 555 | } 556 | 557 | if _, err := w.Write([]byte(fmt.Sprintf("%s;\n", stmt))); err != nil { 558 | return err 559 | } 560 | 561 | tableIndent := strings.Replace(table, `"`, `""`, -1) 562 | r, err := dstDB.QueryStringStmt(fmt.Sprintf(`PRAGMA table_info("%s")`, tableIndent)) 563 | if err != nil { 564 | return err 565 | } 566 | var columnNames []string 567 | for _, w := range r[0].Values { 568 | columnNames = append(columnNames, fmt.Sprintf(`'||quote("%s")||'`, w[1].(string))) 569 | } 570 | 571 | query = fmt.Sprintf(`SELECT 'INSERT INTO "%s" VALUES(%s)' FROM "%s";`, 572 | tableIndent, 573 | strings.Join(columnNames, ","), 574 | tableIndent) 575 | r, err = dstDB.QueryStringStmt(query) 576 | if err != nil { 577 | return err 578 | } 579 | for _, x := range r[0].Values { 580 | y := fmt.Sprintf("%s;\n", x[0].(string)) 581 | if _, err := w.Write([]byte(y)); err != nil { 582 | return err 583 | } 584 | } 585 | } 586 | 587 | // Do indexes, triggers, and views. 588 | query = `SELECT "name", "type", "sql" FROM "sqlite_master" 589 | WHERE "sql" NOT NULL AND "type" IN ('index', 'trigger', 'view')` 590 | rows, err = db.QueryStringStmt(query) 591 | if err != nil { 592 | return err 593 | } 594 | row = rows[0] 595 | for _, v := range row.Values { 596 | if _, err := w.Write([]byte(fmt.Sprintf("%s;\n", v[2]))); err != nil { 597 | return err 598 | } 599 | } 600 | 601 | if _, err := w.Write([]byte("COMMIT;\n")); err != nil { 602 | return err 603 | } 604 | 605 | return nil 606 | } 607 | 608 | func copyDatabase(dst *sqlite3.SQLiteConn, src *sqlite3.SQLiteConn) error { 609 | bk, err := dst.Backup("main", src, "main") 610 | if err != nil { 611 | return err 612 | } 613 | 614 | for { 615 | done, err := bk.Step(-1) 616 | if err != nil { 617 | bk.Finish() 618 | return err 619 | } 620 | if done { 621 | break 622 | } 623 | time.Sleep(bkDelay * time.Millisecond) 624 | } 625 | 626 | if err := bk.Finish(); err != nil { 627 | return err 628 | } 629 | 630 | return nil 631 | } 632 | 633 | // parametersToValues maps values in the proto params to SQL driver values. 634 | func parametersToValues(parameters []*command.Parameter) ([]driver.Value, error) { 635 | if parameters == nil { 636 | return nil, nil 637 | } 638 | 639 | values := make([]driver.Value, len(parameters)) 640 | for i := range parameters { 641 | switch w := parameters[i].GetValue().(type) { 642 | case *command.Parameter_I: 643 | values[i] = w.I 644 | case *command.Parameter_D: 645 | values[i] = w.D 646 | case *command.Parameter_B: 647 | values[i] = w.B 648 | case *command.Parameter_Y: 649 | values[i] = w.Y 650 | case *command.Parameter_S: 651 | values[i] = w.S 652 | default: 653 | return nil, fmt.Errorf("unsupported type: %T", w) 654 | } 655 | } 656 | return values, nil 657 | } 658 | 659 | // normalizeRowValues performs some normalization of values in the returned rows. 660 | // Text values come over (from sqlite-go) as []byte instead of strings 661 | // for some reason, so we have explicitly convert (but only when type 662 | // is "text" so we don't affect BLOB types) 663 | func normalizeRowValues(row []driver.Value, types []string) []interface{} { 664 | values := make([]interface{}, len(types)) 665 | for i, v := range row { 666 | if isTextType(types[i]) { 667 | switch val := v.(type) { 668 | case []byte: 669 | values[i] = string(val) 670 | default: 671 | values[i] = val 672 | } 673 | } else { 674 | values[i] = v 675 | } 676 | } 677 | return values 678 | } 679 | 680 | // isTextType returns whether the given type has a SQLite text affinity. 681 | // http://www.sqlite.org/datatype3.html 682 | func isTextType(t string) bool { 683 | return t == "text" || 684 | t == "json" || 685 | t == "" || 686 | strings.HasPrefix(t, "varchar") || 687 | strings.HasPrefix(t, "varying character") || 688 | strings.HasPrefix(t, "nchar") || 689 | strings.HasPrefix(t, "native character") || 690 | strings.HasPrefix(t, "nvarchar") || 691 | strings.HasPrefix(t, "clob") 692 | } 693 | 694 | // fqdsn returns the fully-qualified datasource name. 695 | func fqdsn(path, dsn string) string { 696 | if dsn != "" { 697 | return fmt.Sprintf("file:%s?%s", path, dsn) 698 | } 699 | return path 700 | } 701 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= 4 | github.com/Bowery/prompt v0.0.0-20190916142128-fa8279994f75 h1:xGHheKK44eC6K0u5X+DZW/fRaR1LnDdqPHMZMWx5fv8= 5 | github.com/Bowery/prompt v0.0.0-20190916142128-fa8279994f75/go.mod h1:4/6eNcqZ09BZ9wLK3tZOjBA1nDj+B0728nlX5YRlSmQ= 6 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 7 | github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= 8 | github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= 9 | github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= 10 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 11 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 12 | github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= 13 | github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= 14 | github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= 15 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 16 | github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= 17 | github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= 18 | github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= 19 | github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= 20 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 21 | github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= 22 | github.com/comail/colog v0.0.0-20160416085026-fba8e7b1f46c/go.mod h1:1WwgAwMKQLYG5I2FBhpVx94YTOAuB2W59IZ7REjSE6Y= 23 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 24 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 25 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 26 | github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= 27 | github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= 28 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 29 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 30 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 31 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 32 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 33 | github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 34 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 35 | github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:56xuuqnHyryaerycW3BfssRdxQstACi0Epw/yC5E2xM= 36 | github.com/go-xorm/xorm v0.7.9/go.mod h1:XiVxrMMIhFkwSkh96BW7PACl7UhLtx2iJIHMdmjh5sQ= 37 | github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= 38 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 39 | github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 40 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 41 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 42 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 43 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 44 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 45 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 46 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 47 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 48 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 49 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 50 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 51 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 52 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 53 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 54 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 55 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 56 | github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 57 | github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 58 | github.com/hashicorp/go-hclog v0.9.1 h1:9PZfAcVEvez4yhLH2TBU64/h/z4xlFI80cWXRrxuKuM= 59 | github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= 60 | github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= 61 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 62 | github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= 63 | github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 64 | github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= 65 | github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= 66 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 67 | github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= 68 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 69 | github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= 70 | github.com/hashicorp/raft v1.3.1 h1:zDT8ke8y2aP4wf9zPTB2uSIeavJ3Hx/ceY4jxI2JxuY= 71 | github.com/hashicorp/raft v1.3.1/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= 72 | github.com/hashicorp/raft-boltdb v0.0.0-20210422161416-485fa74b0b01 h1:EfDtu7qY4bD9hNY9sIryn1L/Ycvo+/WPEFT2Crwdclg= 73 | github.com/hashicorp/raft-boltdb v0.0.0-20210422161416-485fa74b0b01/go.mod h1:L6EUYfWjwPIkX9uqJBsGb3fppuOcRx3t7z2joJnIf/g= 74 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 75 | github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= 76 | github.com/jackc/pgx v3.6.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= 77 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 78 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 79 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 80 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 81 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 82 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 83 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 84 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 85 | github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= 86 | github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= 87 | github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 88 | github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 89 | github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= 90 | github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 91 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 92 | github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= 93 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 94 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 95 | github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 96 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 97 | github.com/mkideal/cli v0.2.5 h1:Lc2v2bijpYbl4TUAbstLx9X7o90agDWk7Z3I8AZTJ1Y= 98 | github.com/mkideal/cli v0.2.5/go.mod h1:XaQYNUpBxFxm15Gs9HILpG6bRuTKMWvuW3bSc+M8p0g= 99 | github.com/mkideal/expr v0.1.0 h1:fzborV9TeSUmLm0aEQWTWcexDURFFo4v5gHSc818Kl8= 100 | github.com/mkideal/expr v0.1.0/go.mod h1:vL1DsSb87ZtU6IEjOtUfxw98z0FQbzS8xlGtnPkKdzg= 101 | github.com/mkideal/log v1.0.0/go.mod h1:UHY5EOk5+/f2z2bQ6BGspFw5gl4F1SKjZI7Bqetm724= 102 | github.com/mkideal/pkg v0.1.2 h1:w4CGlOIp9exb1Ypo+XrbIR75ZRruzScNNcX840DSCQ8= 103 | github.com/mkideal/pkg v0.1.2/go.mod h1:4iVkIRF6ThYaNZtD6J/p9gHdy5nXv7EJ+cKuzmmFPAY= 104 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 105 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 106 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 107 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 108 | github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= 109 | github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= 110 | github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 111 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 112 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 113 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 114 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 115 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 116 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 117 | github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= 118 | github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= 119 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 120 | github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 121 | github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 122 | github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 123 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 124 | github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 125 | github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 126 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 127 | github.com/rqlite/go-sqlite3 v1.20.2 h1:mHVEJARM9aF2B9wsKr7L5JlQIuqs2VpB+8leK/qZNSI= 128 | github.com/rqlite/go-sqlite3 v1.20.2/go.mod h1:ml55MVv28UP7V8zrxILd2EsrI6Wfsz76YSskpg08Ut4= 129 | github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= 130 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 131 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 132 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 133 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 134 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 135 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 136 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 137 | github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= 138 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 139 | github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= 140 | github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= 141 | go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= 142 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 143 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 144 | golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 145 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 146 | golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= 147 | golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= 148 | golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= 149 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 150 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 151 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 152 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 153 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 154 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 155 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 156 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 157 | golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 158 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 159 | golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 160 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 161 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 162 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 163 | golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 164 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 165 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 166 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 167 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 168 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 169 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 170 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 171 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 172 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 173 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 174 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 175 | golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 176 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 177 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 178 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 179 | golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 180 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 181 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 182 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 183 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 184 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= 185 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 186 | golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= 187 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= 188 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 189 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 190 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 191 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 192 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 193 | golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 194 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 195 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 196 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 197 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 198 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 199 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 200 | google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= 201 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 202 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 203 | google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 204 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 205 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 206 | google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 207 | google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= 208 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 209 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 210 | google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= 211 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 212 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 213 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 214 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 215 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 216 | gopkg.in/redis.v5 v5.2.9/go.mod h1:6gtv0/+A4iM08kdRfocWYB3bLX2tebpNtfKlFT6H4mY= 217 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 218 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 219 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 220 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 221 | honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 222 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 223 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 224 | xorm.io/builder v0.3.6/go.mod h1:LEFAPISnRzG+zxaxj2vPicRwz67BdhFreKg8yv8/TgU= 225 | xorm.io/core v0.7.2-0.20190928055935-90aeac8d08eb/go.mod h1:jJfd0UAEzZ4t87nbQYtVjmqpIODugN6PD2D9E+dJvdM= 226 | -------------------------------------------------------------------------------- /command/command.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.26.0 4 | // protoc v3.13.0 5 | // source: command.proto 6 | 7 | package command 8 | 9 | import ( 10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 | reflect "reflect" 13 | sync "sync" 14 | ) 15 | 16 | const ( 17 | // Verify that this generated code is sufficiently up-to-date. 18 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 19 | // Verify that runtime/protoimpl is sufficiently up-to-date. 20 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 21 | ) 22 | 23 | type QueryRequest_Level int32 24 | 25 | const ( 26 | QueryRequest_QUERY_REQUEST_LEVEL_NONE QueryRequest_Level = 0 27 | QueryRequest_QUERY_REQUEST_LEVEL_WEAK QueryRequest_Level = 1 28 | QueryRequest_QUERY_REQUEST_LEVEL_STRONG QueryRequest_Level = 2 29 | ) 30 | 31 | // Enum value maps for QueryRequest_Level. 32 | var ( 33 | QueryRequest_Level_name = map[int32]string{ 34 | 0: "QUERY_REQUEST_LEVEL_NONE", 35 | 1: "QUERY_REQUEST_LEVEL_WEAK", 36 | 2: "QUERY_REQUEST_LEVEL_STRONG", 37 | } 38 | QueryRequest_Level_value = map[string]int32{ 39 | "QUERY_REQUEST_LEVEL_NONE": 0, 40 | "QUERY_REQUEST_LEVEL_WEAK": 1, 41 | "QUERY_REQUEST_LEVEL_STRONG": 2, 42 | } 43 | ) 44 | 45 | func (x QueryRequest_Level) Enum() *QueryRequest_Level { 46 | p := new(QueryRequest_Level) 47 | *p = x 48 | return p 49 | } 50 | 51 | func (x QueryRequest_Level) String() string { 52 | return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) 53 | } 54 | 55 | func (QueryRequest_Level) Descriptor() protoreflect.EnumDescriptor { 56 | return file_command_proto_enumTypes[0].Descriptor() 57 | } 58 | 59 | func (QueryRequest_Level) Type() protoreflect.EnumType { 60 | return &file_command_proto_enumTypes[0] 61 | } 62 | 63 | func (x QueryRequest_Level) Number() protoreflect.EnumNumber { 64 | return protoreflect.EnumNumber(x) 65 | } 66 | 67 | // Deprecated: Use QueryRequest_Level.Descriptor instead. 68 | func (QueryRequest_Level) EnumDescriptor() ([]byte, []int) { 69 | return file_command_proto_rawDescGZIP(), []int{3, 0} 70 | } 71 | 72 | type Command_Type int32 73 | 74 | const ( 75 | Command_COMMAND_TYPE_UNKNOWN Command_Type = 0 76 | Command_COMMAND_TYPE_QUERY Command_Type = 1 77 | Command_COMMAND_TYPE_EXECUTE Command_Type = 2 78 | Command_COMMAND_TYPE_NOOP Command_Type = 3 79 | ) 80 | 81 | // Enum value maps for Command_Type. 82 | var ( 83 | Command_Type_name = map[int32]string{ 84 | 0: "COMMAND_TYPE_UNKNOWN", 85 | 1: "COMMAND_TYPE_QUERY", 86 | 2: "COMMAND_TYPE_EXECUTE", 87 | 3: "COMMAND_TYPE_NOOP", 88 | } 89 | Command_Type_value = map[string]int32{ 90 | "COMMAND_TYPE_UNKNOWN": 0, 91 | "COMMAND_TYPE_QUERY": 1, 92 | "COMMAND_TYPE_EXECUTE": 2, 93 | "COMMAND_TYPE_NOOP": 3, 94 | } 95 | ) 96 | 97 | func (x Command_Type) Enum() *Command_Type { 98 | p := new(Command_Type) 99 | *p = x 100 | return p 101 | } 102 | 103 | func (x Command_Type) String() string { 104 | return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) 105 | } 106 | 107 | func (Command_Type) Descriptor() protoreflect.EnumDescriptor { 108 | return file_command_proto_enumTypes[1].Descriptor() 109 | } 110 | 111 | func (Command_Type) Type() protoreflect.EnumType { 112 | return &file_command_proto_enumTypes[1] 113 | } 114 | 115 | func (x Command_Type) Number() protoreflect.EnumNumber { 116 | return protoreflect.EnumNumber(x) 117 | } 118 | 119 | // Deprecated: Use Command_Type.Descriptor instead. 120 | func (Command_Type) EnumDescriptor() ([]byte, []int) { 121 | return file_command_proto_rawDescGZIP(), []int{6, 0} 122 | } 123 | 124 | type Parameter struct { 125 | state protoimpl.MessageState 126 | sizeCache protoimpl.SizeCache 127 | unknownFields protoimpl.UnknownFields 128 | 129 | // Types that are assignable to Value: 130 | // *Parameter_I 131 | // *Parameter_D 132 | // *Parameter_B 133 | // *Parameter_Y 134 | // *Parameter_S 135 | Value isParameter_Value `protobuf_oneof:"value"` 136 | } 137 | 138 | func (x *Parameter) Reset() { 139 | *x = Parameter{} 140 | if protoimpl.UnsafeEnabled { 141 | mi := &file_command_proto_msgTypes[0] 142 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 143 | ms.StoreMessageInfo(mi) 144 | } 145 | } 146 | 147 | func (x *Parameter) String() string { 148 | return protoimpl.X.MessageStringOf(x) 149 | } 150 | 151 | func (*Parameter) ProtoMessage() {} 152 | 153 | func (x *Parameter) ProtoReflect() protoreflect.Message { 154 | mi := &file_command_proto_msgTypes[0] 155 | if protoimpl.UnsafeEnabled && x != nil { 156 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 157 | if ms.LoadMessageInfo() == nil { 158 | ms.StoreMessageInfo(mi) 159 | } 160 | return ms 161 | } 162 | return mi.MessageOf(x) 163 | } 164 | 165 | // Deprecated: Use Parameter.ProtoReflect.Descriptor instead. 166 | func (*Parameter) Descriptor() ([]byte, []int) { 167 | return file_command_proto_rawDescGZIP(), []int{0} 168 | } 169 | 170 | func (m *Parameter) GetValue() isParameter_Value { 171 | if m != nil { 172 | return m.Value 173 | } 174 | return nil 175 | } 176 | 177 | func (x *Parameter) GetI() int64 { 178 | if x, ok := x.GetValue().(*Parameter_I); ok { 179 | return x.I 180 | } 181 | return 0 182 | } 183 | 184 | func (x *Parameter) GetD() float64 { 185 | if x, ok := x.GetValue().(*Parameter_D); ok { 186 | return x.D 187 | } 188 | return 0 189 | } 190 | 191 | func (x *Parameter) GetB() bool { 192 | if x, ok := x.GetValue().(*Parameter_B); ok { 193 | return x.B 194 | } 195 | return false 196 | } 197 | 198 | func (x *Parameter) GetY() []byte { 199 | if x, ok := x.GetValue().(*Parameter_Y); ok { 200 | return x.Y 201 | } 202 | return nil 203 | } 204 | 205 | func (x *Parameter) GetS() string { 206 | if x, ok := x.GetValue().(*Parameter_S); ok { 207 | return x.S 208 | } 209 | return "" 210 | } 211 | 212 | type isParameter_Value interface { 213 | isParameter_Value() 214 | } 215 | 216 | type Parameter_I struct { 217 | I int64 `protobuf:"zigzag64,1,opt,name=i,proto3,oneof"` 218 | } 219 | 220 | type Parameter_D struct { 221 | D float64 `protobuf:"fixed64,2,opt,name=d,proto3,oneof"` 222 | } 223 | 224 | type Parameter_B struct { 225 | B bool `protobuf:"varint,3,opt,name=b,proto3,oneof"` 226 | } 227 | 228 | type Parameter_Y struct { 229 | Y []byte `protobuf:"bytes,4,opt,name=y,proto3,oneof"` 230 | } 231 | 232 | type Parameter_S struct { 233 | S string `protobuf:"bytes,5,opt,name=s,proto3,oneof"` 234 | } 235 | 236 | func (*Parameter_I) isParameter_Value() {} 237 | 238 | func (*Parameter_D) isParameter_Value() {} 239 | 240 | func (*Parameter_B) isParameter_Value() {} 241 | 242 | func (*Parameter_Y) isParameter_Value() {} 243 | 244 | func (*Parameter_S) isParameter_Value() {} 245 | 246 | type Statement struct { 247 | state protoimpl.MessageState 248 | sizeCache protoimpl.SizeCache 249 | unknownFields protoimpl.UnknownFields 250 | 251 | Sql string `protobuf:"bytes,1,opt,name=sql,proto3" json:"sql,omitempty"` 252 | Parameters []*Parameter `protobuf:"bytes,2,rep,name=parameters,proto3" json:"parameters,omitempty"` 253 | } 254 | 255 | func (x *Statement) Reset() { 256 | *x = Statement{} 257 | if protoimpl.UnsafeEnabled { 258 | mi := &file_command_proto_msgTypes[1] 259 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 260 | ms.StoreMessageInfo(mi) 261 | } 262 | } 263 | 264 | func (x *Statement) String() string { 265 | return protoimpl.X.MessageStringOf(x) 266 | } 267 | 268 | func (*Statement) ProtoMessage() {} 269 | 270 | func (x *Statement) ProtoReflect() protoreflect.Message { 271 | mi := &file_command_proto_msgTypes[1] 272 | if protoimpl.UnsafeEnabled && x != nil { 273 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 274 | if ms.LoadMessageInfo() == nil { 275 | ms.StoreMessageInfo(mi) 276 | } 277 | return ms 278 | } 279 | return mi.MessageOf(x) 280 | } 281 | 282 | // Deprecated: Use Statement.ProtoReflect.Descriptor instead. 283 | func (*Statement) Descriptor() ([]byte, []int) { 284 | return file_command_proto_rawDescGZIP(), []int{1} 285 | } 286 | 287 | func (x *Statement) GetSql() string { 288 | if x != nil { 289 | return x.Sql 290 | } 291 | return "" 292 | } 293 | 294 | func (x *Statement) GetParameters() []*Parameter { 295 | if x != nil { 296 | return x.Parameters 297 | } 298 | return nil 299 | } 300 | 301 | type Request struct { 302 | state protoimpl.MessageState 303 | sizeCache protoimpl.SizeCache 304 | unknownFields protoimpl.UnknownFields 305 | 306 | Transaction bool `protobuf:"varint,1,opt,name=transaction,proto3" json:"transaction,omitempty"` 307 | Statements []*Statement `protobuf:"bytes,2,rep,name=statements,proto3" json:"statements,omitempty"` 308 | } 309 | 310 | func (x *Request) Reset() { 311 | *x = Request{} 312 | if protoimpl.UnsafeEnabled { 313 | mi := &file_command_proto_msgTypes[2] 314 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 315 | ms.StoreMessageInfo(mi) 316 | } 317 | } 318 | 319 | func (x *Request) String() string { 320 | return protoimpl.X.MessageStringOf(x) 321 | } 322 | 323 | func (*Request) ProtoMessage() {} 324 | 325 | func (x *Request) ProtoReflect() protoreflect.Message { 326 | mi := &file_command_proto_msgTypes[2] 327 | if protoimpl.UnsafeEnabled && x != nil { 328 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 329 | if ms.LoadMessageInfo() == nil { 330 | ms.StoreMessageInfo(mi) 331 | } 332 | return ms 333 | } 334 | return mi.MessageOf(x) 335 | } 336 | 337 | // Deprecated: Use Request.ProtoReflect.Descriptor instead. 338 | func (*Request) Descriptor() ([]byte, []int) { 339 | return file_command_proto_rawDescGZIP(), []int{2} 340 | } 341 | 342 | func (x *Request) GetTransaction() bool { 343 | if x != nil { 344 | return x.Transaction 345 | } 346 | return false 347 | } 348 | 349 | func (x *Request) GetStatements() []*Statement { 350 | if x != nil { 351 | return x.Statements 352 | } 353 | return nil 354 | } 355 | 356 | type QueryRequest struct { 357 | state protoimpl.MessageState 358 | sizeCache protoimpl.SizeCache 359 | unknownFields protoimpl.UnknownFields 360 | 361 | Request *Request `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` 362 | Timings bool `protobuf:"varint,2,opt,name=timings,proto3" json:"timings,omitempty"` 363 | Level QueryRequest_Level `protobuf:"varint,3,opt,name=level,proto3,enum=command.QueryRequest_Level" json:"level,omitempty"` 364 | Freshness int64 `protobuf:"varint,4,opt,name=freshness,proto3" json:"freshness,omitempty"` 365 | } 366 | 367 | func (x *QueryRequest) Reset() { 368 | *x = QueryRequest{} 369 | if protoimpl.UnsafeEnabled { 370 | mi := &file_command_proto_msgTypes[3] 371 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 372 | ms.StoreMessageInfo(mi) 373 | } 374 | } 375 | 376 | func (x *QueryRequest) String() string { 377 | return protoimpl.X.MessageStringOf(x) 378 | } 379 | 380 | func (*QueryRequest) ProtoMessage() {} 381 | 382 | func (x *QueryRequest) ProtoReflect() protoreflect.Message { 383 | mi := &file_command_proto_msgTypes[3] 384 | if protoimpl.UnsafeEnabled && x != nil { 385 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 386 | if ms.LoadMessageInfo() == nil { 387 | ms.StoreMessageInfo(mi) 388 | } 389 | return ms 390 | } 391 | return mi.MessageOf(x) 392 | } 393 | 394 | // Deprecated: Use QueryRequest.ProtoReflect.Descriptor instead. 395 | func (*QueryRequest) Descriptor() ([]byte, []int) { 396 | return file_command_proto_rawDescGZIP(), []int{3} 397 | } 398 | 399 | func (x *QueryRequest) GetRequest() *Request { 400 | if x != nil { 401 | return x.Request 402 | } 403 | return nil 404 | } 405 | 406 | func (x *QueryRequest) GetTimings() bool { 407 | if x != nil { 408 | return x.Timings 409 | } 410 | return false 411 | } 412 | 413 | func (x *QueryRequest) GetLevel() QueryRequest_Level { 414 | if x != nil { 415 | return x.Level 416 | } 417 | return QueryRequest_QUERY_REQUEST_LEVEL_NONE 418 | } 419 | 420 | func (x *QueryRequest) GetFreshness() int64 { 421 | if x != nil { 422 | return x.Freshness 423 | } 424 | return 0 425 | } 426 | 427 | type ExecuteRequest struct { 428 | state protoimpl.MessageState 429 | sizeCache protoimpl.SizeCache 430 | unknownFields protoimpl.UnknownFields 431 | 432 | Request *Request `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` 433 | Timings bool `protobuf:"varint,2,opt,name=timings,proto3" json:"timings,omitempty"` 434 | } 435 | 436 | func (x *ExecuteRequest) Reset() { 437 | *x = ExecuteRequest{} 438 | if protoimpl.UnsafeEnabled { 439 | mi := &file_command_proto_msgTypes[4] 440 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 441 | ms.StoreMessageInfo(mi) 442 | } 443 | } 444 | 445 | func (x *ExecuteRequest) String() string { 446 | return protoimpl.X.MessageStringOf(x) 447 | } 448 | 449 | func (*ExecuteRequest) ProtoMessage() {} 450 | 451 | func (x *ExecuteRequest) ProtoReflect() protoreflect.Message { 452 | mi := &file_command_proto_msgTypes[4] 453 | if protoimpl.UnsafeEnabled && x != nil { 454 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 455 | if ms.LoadMessageInfo() == nil { 456 | ms.StoreMessageInfo(mi) 457 | } 458 | return ms 459 | } 460 | return mi.MessageOf(x) 461 | } 462 | 463 | // Deprecated: Use ExecuteRequest.ProtoReflect.Descriptor instead. 464 | func (*ExecuteRequest) Descriptor() ([]byte, []int) { 465 | return file_command_proto_rawDescGZIP(), []int{4} 466 | } 467 | 468 | func (x *ExecuteRequest) GetRequest() *Request { 469 | if x != nil { 470 | return x.Request 471 | } 472 | return nil 473 | } 474 | 475 | func (x *ExecuteRequest) GetTimings() bool { 476 | if x != nil { 477 | return x.Timings 478 | } 479 | return false 480 | } 481 | 482 | type Noop struct { 483 | state protoimpl.MessageState 484 | sizeCache protoimpl.SizeCache 485 | unknownFields protoimpl.UnknownFields 486 | 487 | Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` 488 | } 489 | 490 | func (x *Noop) Reset() { 491 | *x = Noop{} 492 | if protoimpl.UnsafeEnabled { 493 | mi := &file_command_proto_msgTypes[5] 494 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 495 | ms.StoreMessageInfo(mi) 496 | } 497 | } 498 | 499 | func (x *Noop) String() string { 500 | return protoimpl.X.MessageStringOf(x) 501 | } 502 | 503 | func (*Noop) ProtoMessage() {} 504 | 505 | func (x *Noop) ProtoReflect() protoreflect.Message { 506 | mi := &file_command_proto_msgTypes[5] 507 | if protoimpl.UnsafeEnabled && x != nil { 508 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 509 | if ms.LoadMessageInfo() == nil { 510 | ms.StoreMessageInfo(mi) 511 | } 512 | return ms 513 | } 514 | return mi.MessageOf(x) 515 | } 516 | 517 | // Deprecated: Use Noop.ProtoReflect.Descriptor instead. 518 | func (*Noop) Descriptor() ([]byte, []int) { 519 | return file_command_proto_rawDescGZIP(), []int{5} 520 | } 521 | 522 | func (x *Noop) GetId() string { 523 | if x != nil { 524 | return x.Id 525 | } 526 | return "" 527 | } 528 | 529 | type Command struct { 530 | state protoimpl.MessageState 531 | sizeCache protoimpl.SizeCache 532 | unknownFields protoimpl.UnknownFields 533 | 534 | Type Command_Type `protobuf:"varint,1,opt,name=type,proto3,enum=command.Command_Type" json:"type,omitempty"` 535 | SubCommand []byte `protobuf:"bytes,2,opt,name=sub_command,json=subCommand,proto3" json:"sub_command,omitempty"` 536 | Compressed bool `protobuf:"varint,3,opt,name=compressed,proto3" json:"compressed,omitempty"` 537 | } 538 | 539 | func (x *Command) Reset() { 540 | *x = Command{} 541 | if protoimpl.UnsafeEnabled { 542 | mi := &file_command_proto_msgTypes[6] 543 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 544 | ms.StoreMessageInfo(mi) 545 | } 546 | } 547 | 548 | func (x *Command) String() string { 549 | return protoimpl.X.MessageStringOf(x) 550 | } 551 | 552 | func (*Command) ProtoMessage() {} 553 | 554 | func (x *Command) ProtoReflect() protoreflect.Message { 555 | mi := &file_command_proto_msgTypes[6] 556 | if protoimpl.UnsafeEnabled && x != nil { 557 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 558 | if ms.LoadMessageInfo() == nil { 559 | ms.StoreMessageInfo(mi) 560 | } 561 | return ms 562 | } 563 | return mi.MessageOf(x) 564 | } 565 | 566 | // Deprecated: Use Command.ProtoReflect.Descriptor instead. 567 | func (*Command) Descriptor() ([]byte, []int) { 568 | return file_command_proto_rawDescGZIP(), []int{6} 569 | } 570 | 571 | func (x *Command) GetType() Command_Type { 572 | if x != nil { 573 | return x.Type 574 | } 575 | return Command_COMMAND_TYPE_UNKNOWN 576 | } 577 | 578 | func (x *Command) GetSubCommand() []byte { 579 | if x != nil { 580 | return x.SubCommand 581 | } 582 | return nil 583 | } 584 | 585 | func (x *Command) GetCompressed() bool { 586 | if x != nil { 587 | return x.Compressed 588 | } 589 | return false 590 | } 591 | 592 | var File_command_proto protoreflect.FileDescriptor 593 | 594 | var file_command_proto_rawDesc = []byte{ 595 | 0x0a, 0x0d, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 596 | 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x22, 0x64, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x61, 597 | 0x6d, 0x65, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x01, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x12, 598 | 0x48, 0x00, 0x52, 0x01, 0x69, 0x12, 0x0e, 0x0a, 0x01, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 599 | 0x48, 0x00, 0x52, 0x01, 0x64, 0x12, 0x0e, 0x0a, 0x01, 0x62, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 600 | 0x48, 0x00, 0x52, 0x01, 0x62, 0x12, 0x0e, 0x0a, 0x01, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 601 | 0x48, 0x00, 0x52, 0x01, 0x79, 0x12, 0x0e, 0x0a, 0x01, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 602 | 0x48, 0x00, 0x52, 0x01, 0x73, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x51, 603 | 0x0a, 0x09, 0x53, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x73, 604 | 0x71, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x71, 0x6c, 0x12, 0x32, 0x0a, 605 | 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 606 | 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x50, 0x61, 0x72, 0x61, 607 | 0x6d, 0x65, 0x74, 0x65, 0x72, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 608 | 0x73, 0x22, 0x5f, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x0b, 609 | 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 610 | 0x08, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x32, 611 | 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 612 | 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x53, 0x74, 0x61, 613 | 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 614 | 0x74, 0x73, 0x22, 0x8a, 0x02, 0x0a, 0x0c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 615 | 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 616 | 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x52, 617 | 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 618 | 0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 619 | 0x52, 0x07, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x12, 0x31, 0x0a, 0x05, 0x6c, 0x65, 0x76, 620 | 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 621 | 0x6e, 0x64, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 622 | 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1c, 0x0a, 0x09, 623 | 0x66, 0x72, 0x65, 0x73, 0x68, 0x6e, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 624 | 0x09, 0x66, 0x72, 0x65, 0x73, 0x68, 0x6e, 0x65, 0x73, 0x73, 0x22, 0x63, 0x0a, 0x05, 0x4c, 0x65, 625 | 0x76, 0x65, 0x6c, 0x12, 0x1c, 0x0a, 0x18, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x52, 0x45, 0x51, 626 | 0x55, 0x45, 0x53, 0x54, 0x5f, 0x4c, 0x45, 0x56, 0x45, 0x4c, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 627 | 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 628 | 0x53, 0x54, 0x5f, 0x4c, 0x45, 0x56, 0x45, 0x4c, 0x5f, 0x57, 0x45, 0x41, 0x4b, 0x10, 0x01, 0x12, 629 | 0x1e, 0x0a, 0x1a, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 630 | 0x5f, 0x4c, 0x45, 0x56, 0x45, 0x4c, 0x5f, 0x53, 0x54, 0x52, 0x4f, 0x4e, 0x47, 0x10, 0x02, 0x22, 631 | 0x56, 0x0a, 0x0e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 632 | 0x74, 0x12, 0x2a, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 633 | 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x52, 0x65, 0x71, 634 | 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 635 | 0x07, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 636 | 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x22, 0x16, 0x0a, 0x04, 0x4e, 0x6f, 0x6f, 0x70, 0x12, 637 | 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 638 | 0xe0, 0x01, 0x0a, 0x07, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x29, 0x0a, 0x04, 0x74, 639 | 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 640 | 0x61, 0x6e, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x54, 0x79, 0x70, 0x65, 641 | 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x75, 0x62, 0x5f, 0x63, 0x6f, 642 | 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x73, 0x75, 0x62, 643 | 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x72, 644 | 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 645 | 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x22, 0x69, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 646 | 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 647 | 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x4f, 0x4d, 648 | 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x51, 0x55, 0x45, 0x52, 0x59, 0x10, 649 | 0x01, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x54, 0x59, 0x50, 650 | 0x45, 0x5f, 0x45, 0x58, 0x45, 0x43, 0x55, 0x54, 0x45, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x43, 651 | 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x4f, 0x4f, 0x50, 652 | 0x10, 0x03, 0x42, 0x22, 0x5a, 0x20, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 653 | 0x2f, 0x72, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x2f, 0x72, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x2f, 0x63, 654 | 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 655 | } 656 | 657 | var ( 658 | file_command_proto_rawDescOnce sync.Once 659 | file_command_proto_rawDescData = file_command_proto_rawDesc 660 | ) 661 | 662 | func file_command_proto_rawDescGZIP() []byte { 663 | file_command_proto_rawDescOnce.Do(func() { 664 | file_command_proto_rawDescData = protoimpl.X.CompressGZIP(file_command_proto_rawDescData) 665 | }) 666 | return file_command_proto_rawDescData 667 | } 668 | 669 | var file_command_proto_enumTypes = make([]protoimpl.EnumInfo, 2) 670 | var file_command_proto_msgTypes = make([]protoimpl.MessageInfo, 7) 671 | var file_command_proto_goTypes = []interface{}{ 672 | (QueryRequest_Level)(0), // 0: command.QueryRequest.Level 673 | (Command_Type)(0), // 1: command.Command.Type 674 | (*Parameter)(nil), // 2: command.Parameter 675 | (*Statement)(nil), // 3: command.Statement 676 | (*Request)(nil), // 4: command.Request 677 | (*QueryRequest)(nil), // 5: command.QueryRequest 678 | (*ExecuteRequest)(nil), // 6: command.ExecuteRequest 679 | (*Noop)(nil), // 7: command.Noop 680 | (*Command)(nil), // 8: command.Command 681 | } 682 | var file_command_proto_depIdxs = []int32{ 683 | 2, // 0: command.Statement.parameters:type_name -> command.Parameter 684 | 3, // 1: command.Request.statements:type_name -> command.Statement 685 | 4, // 2: command.QueryRequest.request:type_name -> command.Request 686 | 0, // 3: command.QueryRequest.level:type_name -> command.QueryRequest.Level 687 | 4, // 4: command.ExecuteRequest.request:type_name -> command.Request 688 | 1, // 5: command.Command.type:type_name -> command.Command.Type 689 | 6, // [6:6] is the sub-list for method output_type 690 | 6, // [6:6] is the sub-list for method input_type 691 | 6, // [6:6] is the sub-list for extension type_name 692 | 6, // [6:6] is the sub-list for extension extendee 693 | 0, // [0:6] is the sub-list for field type_name 694 | } 695 | 696 | func init() { file_command_proto_init() } 697 | func file_command_proto_init() { 698 | if File_command_proto != nil { 699 | return 700 | } 701 | if !protoimpl.UnsafeEnabled { 702 | file_command_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 703 | switch v := v.(*Parameter); i { 704 | case 0: 705 | return &v.state 706 | case 1: 707 | return &v.sizeCache 708 | case 2: 709 | return &v.unknownFields 710 | default: 711 | return nil 712 | } 713 | } 714 | file_command_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 715 | switch v := v.(*Statement); i { 716 | case 0: 717 | return &v.state 718 | case 1: 719 | return &v.sizeCache 720 | case 2: 721 | return &v.unknownFields 722 | default: 723 | return nil 724 | } 725 | } 726 | file_command_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { 727 | switch v := v.(*Request); i { 728 | case 0: 729 | return &v.state 730 | case 1: 731 | return &v.sizeCache 732 | case 2: 733 | return &v.unknownFields 734 | default: 735 | return nil 736 | } 737 | } 738 | file_command_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { 739 | switch v := v.(*QueryRequest); i { 740 | case 0: 741 | return &v.state 742 | case 1: 743 | return &v.sizeCache 744 | case 2: 745 | return &v.unknownFields 746 | default: 747 | return nil 748 | } 749 | } 750 | file_command_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { 751 | switch v := v.(*ExecuteRequest); i { 752 | case 0: 753 | return &v.state 754 | case 1: 755 | return &v.sizeCache 756 | case 2: 757 | return &v.unknownFields 758 | default: 759 | return nil 760 | } 761 | } 762 | file_command_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { 763 | switch v := v.(*Noop); i { 764 | case 0: 765 | return &v.state 766 | case 1: 767 | return &v.sizeCache 768 | case 2: 769 | return &v.unknownFields 770 | default: 771 | return nil 772 | } 773 | } 774 | file_command_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { 775 | switch v := v.(*Command); i { 776 | case 0: 777 | return &v.state 778 | case 1: 779 | return &v.sizeCache 780 | case 2: 781 | return &v.unknownFields 782 | default: 783 | return nil 784 | } 785 | } 786 | } 787 | file_command_proto_msgTypes[0].OneofWrappers = []interface{}{ 788 | (*Parameter_I)(nil), 789 | (*Parameter_D)(nil), 790 | (*Parameter_B)(nil), 791 | (*Parameter_Y)(nil), 792 | (*Parameter_S)(nil), 793 | } 794 | type x struct{} 795 | out := protoimpl.TypeBuilder{ 796 | File: protoimpl.DescBuilder{ 797 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 798 | RawDescriptor: file_command_proto_rawDesc, 799 | NumEnums: 2, 800 | NumMessages: 7, 801 | NumExtensions: 0, 802 | NumServices: 0, 803 | }, 804 | GoTypes: file_command_proto_goTypes, 805 | DependencyIndexes: file_command_proto_depIdxs, 806 | EnumInfos: file_command_proto_enumTypes, 807 | MessageInfos: file_command_proto_msgTypes, 808 | }.Build() 809 | File_command_proto = out.File 810 | file_command_proto_rawDesc = nil 811 | file_command_proto_goTypes = nil 812 | file_command_proto_depIdxs = nil 813 | } 814 | -------------------------------------------------------------------------------- /http/endpoint.go: -------------------------------------------------------------------------------- 1 | // Package http provides the HTTP server for accessing the distributed database. 2 | // It also provides the endpoint for other nodes to join an existing cluster. 3 | package http 4 | 5 | import ( 6 | "encoding/json" 7 | "errors" 8 | "expvar" 9 | "fmt" 10 | "io" 11 | "io/ioutil" 12 | "log" 13 | "net" 14 | "net/http" 15 | "net/http/pprof" 16 | "os" 17 | "runtime" 18 | "strings" 19 | "sync" 20 | "time" 21 | 22 | "github.com/minghsu0107/tqlite/command" 23 | sql "github.com/minghsu0107/tqlite/db" 24 | "github.com/minghsu0107/tqlite/store" 25 | ) 26 | 27 | // Database is the interface any queryable system must implement 28 | type Database interface { 29 | // Execute executes a slice of queries, each of which is not expected 30 | // to return rows. If timings is true, then timing information will 31 | // be return. If tx is true, then either all queries will be executed 32 | // successfully or it will as though none executed. 33 | Execute(er *command.ExecuteRequest) ([]*sql.Result, error) 34 | 35 | // ExecuteOrAbort performs the same function as Execute(), but ensures 36 | // any transactions are aborted in case of any error. 37 | ExecuteOrAbort(er *command.ExecuteRequest) ([]*sql.Result, error) 38 | 39 | // Query executes a slice of queries, each of which returns rows. If 40 | // timings is true, then timing information will be returned. If tx 41 | // is true, then all queries will take place while a read transaction 42 | // is held on the database. 43 | Query(qr *command.QueryRequest) ([]*sql.Rows, error) 44 | } 45 | 46 | // Store is the interface the Raft-based database must implement. 47 | type Store interface { 48 | Database 49 | 50 | // Join joins the node with the given ID, reachable at addr, to this node. 51 | Join(id, addr string, voter bool) error 52 | 53 | // Remove removes the node, specified by id, from the cluster. 54 | Remove(id string) error 55 | 56 | // LeaderAddr returns the Raft address of the leader of the cluster. 57 | LeaderAddr() (string, error) 58 | 59 | // Stats returns stats on the Store. 60 | Stats() (map[string]interface{}, error) 61 | 62 | // Nodes returns the slice of store.Servers in the cluster 63 | Nodes() ([]*store.Server, error) 64 | 65 | // Backup wites backup of the node state to dst 66 | Backup(leader bool, f store.BackupFormat, dst io.Writer) error 67 | } 68 | 69 | // Cluster is the interface node API services must provide 70 | type Cluster interface { 71 | // GetNodeAPIAddr returns the HTTP API URL for the node at the given Raft address. 72 | GetNodeAPIAddr(nodeAddr string) (string, error) 73 | 74 | // Stats returns stats on the Cluster. 75 | Stats() (map[string]interface{}, error) 76 | } 77 | 78 | // Statuser is the interface status providers must implement. 79 | type Statuser interface { 80 | Stats() (interface{}, error) 81 | } 82 | 83 | // Response represents a response from the HTTP service. 84 | type Response struct { 85 | Results interface{} `json:"results,omitempty"` 86 | Error string `json:"error,omitempty"` 87 | Time float64 `json:"time,omitempty"` 88 | 89 | start time.Time 90 | end time.Time 91 | } 92 | 93 | // stats captures stats for the HTTP service. 94 | var stats *expvar.Map 95 | 96 | const ( 97 | numExecutions = "executions" 98 | numQueries = "queries" 99 | numBackups = "backups" 100 | numLoad = "loads" 101 | numJoins = "joins" 102 | 103 | // VersionHTTPHeader is the HTTP header key for the version. 104 | VersionHTTPHeader = "X-TQLITE-VERSION" 105 | ) 106 | 107 | func init() { 108 | stats = expvar.NewMap("http") 109 | stats.Add(numExecutions, 0) 110 | stats.Add(numQueries, 0) 111 | stats.Add(numBackups, 0) 112 | stats.Add(numLoad, 0) 113 | stats.Add(numJoins, 0) 114 | } 115 | 116 | // SetTime sets the Time attribute of the response. This way it will be present 117 | // in the serialized JSON version. 118 | func (r *Response) SetTime() { 119 | r.Time = r.end.Sub(r.start).Seconds() 120 | } 121 | 122 | // NewResponse returns a new instance of response. 123 | func NewResponse() *Response { 124 | return &Response{ 125 | start: time.Now(), 126 | } 127 | } 128 | 129 | // Service provides HTTP service. 130 | type Service struct { 131 | addr string // Bind address of the HTTP service. 132 | ln net.Listener // Service listener 133 | 134 | store Store // The Raft-backed database store. 135 | 136 | cluster Cluster // The Cluster service. 137 | 138 | start time.Time // Start up time. 139 | lastBackup time.Time // Time of last successful backup. 140 | 141 | statusMu sync.RWMutex 142 | statuses map[string]Statuser 143 | 144 | Expvar bool 145 | Pprof bool 146 | 147 | BuildInfo map[string]interface{} 148 | 149 | logger *log.Logger 150 | } 151 | 152 | // New returns an uninitialized HTTP service. 153 | func New(addr string, store Store, cluster Cluster) *Service { 154 | return &Service{ 155 | addr: addr, 156 | store: store, 157 | cluster: cluster, 158 | start: time.Now(), 159 | statuses: make(map[string]Statuser), 160 | logger: log.New(os.Stderr, "[http] ", log.LstdFlags), 161 | } 162 | } 163 | 164 | // Start starts the service. 165 | func (s *Service) Start() error { 166 | server := http.Server{ 167 | Handler: s, 168 | } 169 | 170 | var ln net.Listener 171 | var err error 172 | ln, err = net.Listen("tcp", s.addr) 173 | if err != nil { 174 | return err 175 | } 176 | 177 | s.ln = ln 178 | 179 | go func() { 180 | err := server.Serve(s.ln) 181 | if err != nil { 182 | s.logger.Println("HTTP service Serve() returned:", err.Error()) 183 | } 184 | }() 185 | s.logger.Println("service listening on", s.Addr()) 186 | 187 | return nil 188 | } 189 | 190 | // Close closes the service. 191 | func (s *Service) Close() { 192 | s.ln.Close() 193 | return 194 | } 195 | 196 | // ServeHTTP allows Service to serve HTTP requests. 197 | func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { 198 | s.addBuildVersion(w) 199 | 200 | switch { 201 | case strings.HasPrefix(r.URL.Path, "/db/execute"): 202 | stats.Add(numExecutions, 1) 203 | s.handleExecute(w, r) 204 | case strings.HasPrefix(r.URL.Path, "/db/query"): 205 | stats.Add(numQueries, 1) 206 | s.handleQuery(w, r) 207 | case strings.HasPrefix(r.URL.Path, "/db/backup"): 208 | stats.Add(numBackups, 1) 209 | s.handleBackup(w, r) 210 | case strings.HasPrefix(r.URL.Path, "/db/load"): 211 | stats.Add(numLoad, 1) 212 | s.handleLoad(w, r) 213 | case strings.HasPrefix(r.URL.Path, "/join"): 214 | stats.Add(numJoins, 1) 215 | s.handleJoin(w, r) 216 | case strings.HasPrefix(r.URL.Path, "/remove"): 217 | s.handleRemove(w, r) 218 | case strings.HasPrefix(r.URL.Path, "/status"): 219 | s.handleStatus(w, r) 220 | case strings.HasPrefix(r.URL.Path, "/nodes"): 221 | s.handleNodes(w, r) 222 | case r.URL.Path == "/debug/vars" && s.Expvar: 223 | s.handleExpvar(w, r) 224 | case strings.HasPrefix(r.URL.Path, "/debug/pprof") && s.Pprof: 225 | s.handlePprof(w, r) 226 | default: 227 | w.WriteHeader(http.StatusNotFound) 228 | } 229 | } 230 | 231 | // RegisterStatus allows other modules to register status for serving over HTTP. 232 | func (s *Service) RegisterStatus(key string, stat Statuser) error { 233 | s.statusMu.Lock() 234 | defer s.statusMu.Unlock() 235 | 236 | if _, ok := s.statuses[key]; ok { 237 | return fmt.Errorf("status already registered with key %s", key) 238 | } 239 | s.statuses[key] = stat 240 | 241 | return nil 242 | } 243 | 244 | // handleJoin handles cluster-join requests from other nodes. 245 | func (s *Service) handleJoin(w http.ResponseWriter, r *http.Request) { 246 | if r.Method != "POST" { 247 | w.WriteHeader(http.StatusMethodNotAllowed) 248 | return 249 | } 250 | 251 | b, err := ioutil.ReadAll(r.Body) 252 | if err != nil { 253 | w.WriteHeader(http.StatusBadRequest) 254 | return 255 | } 256 | md := map[string]interface{}{} 257 | if err := json.Unmarshal(b, &md); err != nil { 258 | w.WriteHeader(http.StatusBadRequest) 259 | return 260 | } 261 | 262 | remoteID, ok := md["id"] 263 | if !ok { 264 | w.WriteHeader(http.StatusBadRequest) 265 | return 266 | } 267 | 268 | remoteAddr, ok := md["addr"] 269 | if !ok { 270 | w.WriteHeader(http.StatusBadRequest) 271 | return 272 | } 273 | 274 | voter, ok := md["voter"] 275 | if !ok { 276 | voter = true 277 | } 278 | 279 | if err := s.store.Join(remoteID.(string), remoteAddr.(string), voter.(bool)); err != nil { 280 | if err == store.ErrNotLeader { 281 | leaderAPIAddr := s.LeaderAPIAddr() 282 | if leaderAPIAddr == "" { 283 | http.Error(w, err.Error(), http.StatusServiceUnavailable) 284 | return 285 | } 286 | 287 | redirect := s.FormRedirect(r, leaderAPIAddr) 288 | http.Redirect(w, r, redirect, http.StatusMovedPermanently) 289 | return 290 | } 291 | 292 | http.Error(w, err.Error(), http.StatusInternalServerError) 293 | return 294 | } 295 | } 296 | 297 | // handleRemove handles cluster-remove requests. 298 | func (s *Service) handleRemove(w http.ResponseWriter, r *http.Request) { 299 | if r.Method != "DELETE" { 300 | w.WriteHeader(http.StatusMethodNotAllowed) 301 | return 302 | } 303 | 304 | b, err := ioutil.ReadAll(r.Body) 305 | if err != nil { 306 | w.WriteHeader(http.StatusBadRequest) 307 | return 308 | } 309 | m := map[string]string{} 310 | if err := json.Unmarshal(b, &m); err != nil { 311 | w.WriteHeader(http.StatusBadRequest) 312 | return 313 | } 314 | 315 | if len(m) != 1 { 316 | w.WriteHeader(http.StatusBadRequest) 317 | return 318 | } 319 | 320 | remoteID, ok := m["id"] 321 | if !ok { 322 | w.WriteHeader(http.StatusBadRequest) 323 | return 324 | } 325 | 326 | if err := s.store.Remove(remoteID); err != nil { 327 | if err == store.ErrNotLeader { 328 | leaderAPIAddr := s.LeaderAPIAddr() 329 | if leaderAPIAddr == "" { 330 | http.Error(w, err.Error(), http.StatusServiceUnavailable) 331 | return 332 | } 333 | 334 | redirect := s.FormRedirect(r, leaderAPIAddr) 335 | http.Redirect(w, r, redirect, http.StatusMovedPermanently) 336 | return 337 | } 338 | 339 | http.Error(w, err.Error(), http.StatusInternalServerError) 340 | return 341 | } 342 | } 343 | 344 | // handleBackup returns the consistent database snapshot. 345 | func (s *Service) handleBackup(w http.ResponseWriter, r *http.Request) { 346 | if r.Method != "GET" { 347 | w.WriteHeader(http.StatusMethodNotAllowed) 348 | return 349 | } 350 | 351 | noLeader, err := noLeader(r) 352 | if err != nil { 353 | http.Error(w, err.Error(), http.StatusInternalServerError) 354 | return 355 | } 356 | 357 | bf, err := backupFormat(w, r) 358 | if err != nil { 359 | http.Error(w, err.Error(), http.StatusInternalServerError) 360 | return 361 | } 362 | 363 | err = s.store.Backup(!noLeader, bf, w) 364 | if err != nil { 365 | if err == store.ErrNotLeader { 366 | leaderAPIAddr := s.LeaderAPIAddr() 367 | if leaderAPIAddr == "" { 368 | http.Error(w, err.Error(), http.StatusServiceUnavailable) 369 | return 370 | } 371 | 372 | redirect := s.FormRedirect(r, leaderAPIAddr) 373 | http.Redirect(w, r, redirect, http.StatusMovedPermanently) 374 | return 375 | } 376 | http.Error(w, err.Error(), http.StatusInternalServerError) 377 | return 378 | } 379 | 380 | s.lastBackup = time.Now() 381 | } 382 | 383 | // handleLoad loads the state contained in a .dump output. This API is different 384 | // from others in that it expects a raw file, not wrapped in any kind of JSON. 385 | func (s *Service) handleLoad(w http.ResponseWriter, r *http.Request) { 386 | if r.Method != "POST" { 387 | w.WriteHeader(http.StatusMethodNotAllowed) 388 | return 389 | } 390 | 391 | resp := NewResponse() 392 | 393 | timings, err := timings(r) 394 | if err != nil { 395 | http.Error(w, err.Error(), http.StatusInternalServerError) 396 | return 397 | } 398 | 399 | b, err := ioutil.ReadAll(r.Body) 400 | if err != nil { 401 | http.Error(w, err.Error(), http.StatusBadRequest) 402 | return 403 | } 404 | r.Body.Close() 405 | 406 | // No JSON structure expected for this API. 407 | queries := []string{string(b)} 408 | er := executeRequestFromStrings(queries, timings, false) 409 | 410 | results, err := s.store.ExecuteOrAbort(er) 411 | if err != nil { 412 | if err == store.ErrNotLeader { 413 | leaderAPIAddr := s.LeaderAPIAddr() 414 | if leaderAPIAddr == "" { 415 | http.Error(w, err.Error(), http.StatusServiceUnavailable) 416 | return 417 | } 418 | 419 | redirect := s.FormRedirect(r, leaderAPIAddr) 420 | http.Redirect(w, r, redirect, http.StatusMovedPermanently) 421 | return 422 | } 423 | resp.Error = err.Error() 424 | } else { 425 | resp.Results = results 426 | } 427 | resp.end = time.Now() 428 | s.writeResponse(w, r, resp) 429 | } 430 | 431 | // handleStatus returns status on the system. 432 | func (s *Service) handleStatus(w http.ResponseWriter, r *http.Request) { 433 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 434 | 435 | if r.Method != "GET" { 436 | w.WriteHeader(http.StatusMethodNotAllowed) 437 | return 438 | } 439 | 440 | storeStatus, err := s.store.Stats() 441 | if err != nil { 442 | http.Error(w, err.Error(), http.StatusInternalServerError) 443 | return 444 | } 445 | 446 | rt := map[string]interface{}{ 447 | "GOARCH": runtime.GOARCH, 448 | "GOOS": runtime.GOOS, 449 | "GOMAXPROCS": runtime.GOMAXPROCS(0), 450 | "num_cpu": runtime.NumCPU(), 451 | "num_goroutine": runtime.NumGoroutine(), 452 | "version": runtime.Version(), 453 | } 454 | 455 | httpStatus := map[string]interface{}{ 456 | "addr": s.Addr().String(), 457 | } 458 | 459 | nodeStatus := map[string]interface{}{ 460 | "start_time": s.start, 461 | "uptime": time.Since(s.start).String(), 462 | } 463 | 464 | clusterStatus, err := s.cluster.Stats() 465 | if err != nil { 466 | http.Error(w, err.Error(), http.StatusInternalServerError) 467 | return 468 | } 469 | 470 | // Build the status response. 471 | status := map[string]interface{}{ 472 | "runtime": rt, 473 | "cluster": clusterStatus, 474 | "store": storeStatus, 475 | "http": httpStatus, 476 | "node": nodeStatus, 477 | } 478 | if !s.lastBackup.IsZero() { 479 | status["last_backup_time"] = s.lastBackup 480 | } 481 | if s.BuildInfo != nil { 482 | status["build"] = s.BuildInfo 483 | } 484 | 485 | // Add any registered statusers. 486 | func() { 487 | s.statusMu.RLock() 488 | defer s.statusMu.RUnlock() 489 | for k, v := range s.statuses { 490 | stat, err := v.Stats() 491 | if err != nil { 492 | http.Error(w, err.Error(), http.StatusInternalServerError) 493 | return 494 | } 495 | status[k] = stat 496 | } 497 | }() 498 | 499 | pretty, _ := isPretty(r) 500 | var b []byte 501 | if pretty { 502 | b, err = json.MarshalIndent(status, "", " ") 503 | } else { 504 | b, err = json.Marshal(status) 505 | } 506 | if err != nil { 507 | http.Error(w, err.Error(), http.StatusInternalServerError) 508 | } else { 509 | _, err = w.Write([]byte(b)) 510 | if err != nil { 511 | http.Error(w, err.Error(), http.StatusInternalServerError) 512 | return 513 | } 514 | } 515 | } 516 | 517 | // handleNodes returns status on the other voting nodes in the system. 518 | // This attempts to contact all the nodes in the cluster, so may take 519 | // some time to return. 520 | func (s *Service) handleNodes(w http.ResponseWriter, r *http.Request) { 521 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 522 | 523 | if r.Method != "GET" { 524 | w.WriteHeader(http.StatusMethodNotAllowed) 525 | return 526 | } 527 | 528 | t, err := timeout(r, time.Duration(1)) 529 | if err != nil { 530 | http.Error(w, err.Error(), http.StatusInternalServerError) 531 | return 532 | } 533 | 534 | includeNonVoters, err := nonVoters(r) 535 | if err != nil { 536 | http.Error(w, err.Error(), http.StatusInternalServerError) 537 | return 538 | } 539 | 540 | // Get nodes in the cluster, and possibly filter out non-voters. 541 | nodes, err := s.store.Nodes() 542 | if err != nil { 543 | http.Error(w, err.Error(), http.StatusInternalServerError) 544 | return 545 | } 546 | 547 | filteredNodes := make([]*store.Server, 0) 548 | for _, n := range nodes { 549 | if n.Suffrage != "Voter" && !includeNonVoters { 550 | continue 551 | } 552 | filteredNodes = append(filteredNodes, n) 553 | } 554 | 555 | lAddr, err := s.store.LeaderAddr() 556 | if err != nil { 557 | http.Error(w, err.Error(), http.StatusInternalServerError) 558 | return 559 | } 560 | 561 | apiAddrs, err := s.checkNodesAPIAddr(filteredNodes, t) 562 | if err != nil { 563 | http.Error(w, err.Error(), http.StatusInternalServerError) 564 | return 565 | } 566 | 567 | resp := make(map[string]struct { 568 | APIAddr string `json:"api_addr,omitempty"` 569 | Addr string `json:"addr,omitempty"` 570 | Reachable bool `json:"reachable"` 571 | Leader bool `json:"leader"` 572 | }) 573 | 574 | for _, n := range filteredNodes { 575 | nn := resp[n.ID] 576 | nn.Addr = n.Addr 577 | nn.Leader = nn.Addr == lAddr 578 | nn.APIAddr = apiAddrs[n.ID] 579 | nn.Reachable = apiAddrs[n.ID] != "" 580 | resp[n.ID] = nn 581 | } 582 | 583 | pretty, _ := isPretty(r) 584 | var b []byte 585 | if pretty { 586 | b, err = json.MarshalIndent(resp, "", " ") 587 | } else { 588 | b, err = json.Marshal(resp) 589 | } 590 | if err != nil { 591 | http.Error(w, err.Error(), http.StatusInternalServerError) 592 | } else { 593 | _, err = w.Write([]byte(b)) 594 | if err != nil { 595 | http.Error(w, err.Error(), http.StatusInternalServerError) 596 | return 597 | } 598 | } 599 | } 600 | 601 | // handleExecute handles queries that modify the database. 602 | func (s *Service) handleExecute(w http.ResponseWriter, r *http.Request) { 603 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 604 | 605 | if r.Method != "POST" { 606 | w.WriteHeader(http.StatusMethodNotAllowed) 607 | return 608 | } 609 | 610 | resp := NewResponse() 611 | 612 | isTx, err := isTx(r) 613 | if err != nil { 614 | http.Error(w, err.Error(), http.StatusInternalServerError) 615 | return 616 | } 617 | 618 | timings, err := timings(r) 619 | if err != nil { 620 | http.Error(w, err.Error(), http.StatusInternalServerError) 621 | return 622 | } 623 | 624 | b, err := ioutil.ReadAll(r.Body) 625 | if err != nil { 626 | http.Error(w, err.Error(), http.StatusBadRequest) 627 | return 628 | } 629 | r.Body.Close() 630 | 631 | stmts, err := ParseRequest(b) 632 | if err != nil { 633 | http.Error(w, err.Error(), http.StatusBadRequest) 634 | return 635 | } 636 | 637 | er := &command.ExecuteRequest{ 638 | Request: &command.Request{ 639 | Transaction: isTx, 640 | Statements: stmts, 641 | }, 642 | Timings: timings, 643 | } 644 | 645 | results, err := s.store.Execute(er) 646 | if err != nil { 647 | if err == store.ErrNotLeader { 648 | leaderAPIAddr := s.LeaderAPIAddr() 649 | if leaderAPIAddr == "" { 650 | http.Error(w, err.Error(), http.StatusServiceUnavailable) 651 | return 652 | } 653 | 654 | redirect := s.FormRedirect(r, leaderAPIAddr) 655 | http.Redirect(w, r, redirect, http.StatusMovedPermanently) 656 | return 657 | } 658 | resp.Error = err.Error() 659 | } else { 660 | resp.Results = results 661 | } 662 | resp.end = time.Now() 663 | s.writeResponse(w, r, resp) 664 | } 665 | 666 | // handleQuery handles queries that do not modify the database. 667 | func (s *Service) handleQuery(w http.ResponseWriter, r *http.Request) { 668 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 669 | 670 | if r.Method != "GET" && r.Method != "POST" { 671 | w.WriteHeader(http.StatusMethodNotAllowed) 672 | return 673 | } 674 | 675 | resp := NewResponse() 676 | 677 | isTx, err := isTx(r) 678 | if err != nil { 679 | http.Error(w, err.Error(), http.StatusBadRequest) 680 | return 681 | } 682 | 683 | timings, err := timings(r) 684 | if err != nil { 685 | http.Error(w, err.Error(), http.StatusBadRequest) 686 | return 687 | } 688 | 689 | lvl, err := level(r) 690 | if err != nil { 691 | http.Error(w, err.Error(), http.StatusBadRequest) 692 | return 693 | } 694 | 695 | frsh, err := freshness(r) 696 | if err != nil { 697 | http.Error(w, err.Error(), http.StatusBadRequest) 698 | return 699 | } 700 | 701 | // Get the query statement(s), and do tx if necessary. 702 | queries, err := requestQueries(r) 703 | if err != nil { 704 | w.WriteHeader(http.StatusBadRequest) 705 | return 706 | } 707 | 708 | qr := &command.QueryRequest{ 709 | Request: &command.Request{ 710 | Transaction: isTx, 711 | Statements: queries, 712 | }, 713 | Timings: timings, 714 | Level: lvl, 715 | Freshness: frsh.Nanoseconds(), 716 | } 717 | 718 | results, err := s.store.Query(qr) 719 | if err != nil { 720 | if err == store.ErrNotLeader { 721 | leaderAPIAddr := s.LeaderAPIAddr() 722 | if leaderAPIAddr == "" { 723 | http.Error(w, err.Error(), http.StatusServiceUnavailable) 724 | return 725 | } 726 | 727 | redirect := s.FormRedirect(r, leaderAPIAddr) 728 | http.Redirect(w, r, redirect, http.StatusMovedPermanently) 729 | return 730 | } 731 | resp.Error = err.Error() 732 | } else { 733 | resp.Results = results 734 | } 735 | resp.end = time.Now() 736 | s.writeResponse(w, r, resp) 737 | } 738 | 739 | // handleExpvar serves registered expvar information over HTTP. 740 | func (s *Service) handleExpvar(w http.ResponseWriter, r *http.Request) { 741 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 742 | 743 | fmt.Fprintf(w, "{\n") 744 | first := true 745 | expvar.Do(func(kv expvar.KeyValue) { 746 | if !first { 747 | fmt.Fprintf(w, ",\n") 748 | } 749 | first = false 750 | fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value) 751 | }) 752 | fmt.Fprintf(w, "\n}\n") 753 | } 754 | 755 | // handlePprof serves pprof information over HTTP. 756 | func (s *Service) handlePprof(w http.ResponseWriter, r *http.Request) { 757 | switch r.URL.Path { 758 | case "/debug/pprof/cmdline": 759 | pprof.Cmdline(w, r) 760 | case "/debug/pprof/profile": 761 | pprof.Profile(w, r) 762 | case "/debug/pprof/symbol": 763 | pprof.Symbol(w, r) 764 | default: 765 | pprof.Index(w, r) 766 | } 767 | } 768 | 769 | // Addr returns the address on which the Service is listening 770 | func (s *Service) Addr() net.Addr { 771 | return s.ln.Addr() 772 | } 773 | 774 | // FormRedirect returns the value for the "Location" header for a 301 response. 775 | func (s *Service) FormRedirect(r *http.Request, url string) string { 776 | rq := r.URL.RawQuery 777 | if rq != "" { 778 | rq = fmt.Sprintf("?%s", rq) 779 | } 780 | return fmt.Sprintf("%s%s%s", url, r.URL.Path, rq) 781 | } 782 | 783 | // LeaderAPIAddr returns the API address of the leader, as known by this node. 784 | func (s *Service) LeaderAPIAddr() string { 785 | nodeAddr, err := s.store.LeaderAddr() 786 | if err != nil { 787 | return "" 788 | } 789 | 790 | apiAddr, err := s.cluster.GetNodeAPIAddr(nodeAddr) 791 | 792 | if err != nil { 793 | return "" 794 | } 795 | return apiAddr 796 | } 797 | 798 | // checkNodesAPIAddr returns a map of node ID to API addresses, reachable 799 | // being defined as node responds to a simple request over the network. 800 | func (s *Service) checkNodesAPIAddr(nodes []*store.Server, timeout time.Duration) (map[string]string, error) { 801 | var wg sync.WaitGroup 802 | var mu sync.Mutex 803 | apiAddrs := make(map[string]string) 804 | 805 | // Assume unreachable 806 | for _, n := range nodes { 807 | apiAddrs[n.ID] = "" 808 | } 809 | 810 | // Now confirm. 811 | for _, n := range nodes { 812 | wg.Add(1) 813 | go func(id, raftAddr string) { 814 | defer wg.Done() 815 | apiAddr, err := s.cluster.GetNodeAPIAddr(raftAddr) 816 | if err == nil { 817 | mu.Lock() 818 | apiAddrs[id] = apiAddr 819 | mu.Unlock() 820 | } 821 | }(n.ID, n.Addr) 822 | } 823 | wg.Wait() 824 | 825 | return apiAddrs, nil 826 | } 827 | 828 | // addBuildVersion adds the build version to the HTTP response. 829 | func (s *Service) addBuildVersion(w http.ResponseWriter) { 830 | // Add version header to every response, if available. 831 | version := "unknown" 832 | if v, ok := s.BuildInfo["version"].(string); ok { 833 | version = v 834 | } 835 | w.Header().Add(VersionHTTPHeader, version) 836 | } 837 | 838 | // writeResponse writes the given response to the given writer. 839 | func (s *Service) writeResponse(w http.ResponseWriter, r *http.Request, j *Response) { 840 | var b []byte 841 | var err error 842 | pretty, _ := isPretty(r) 843 | timings, _ := timings(r) 844 | 845 | if timings { 846 | j.SetTime() 847 | } 848 | 849 | if pretty { 850 | b, err = json.MarshalIndent(j, "", " ") 851 | } else { 852 | b, err = json.Marshal(j) 853 | } 854 | 855 | if err != nil { 856 | http.Error(w, err.Error(), http.StatusInternalServerError) 857 | return 858 | } 859 | _, err = w.Write(b) 860 | if err != nil { 861 | s.logger.Println("writing response failed:", err.Error()) 862 | } 863 | } 864 | 865 | func requestQueries(r *http.Request) ([]*command.Statement, error) { 866 | if r.Method == "GET" { 867 | query, err := stmtParam(r) 868 | if err != nil || query == "" { 869 | return nil, errors.New("bad query GET request") 870 | } 871 | return []*command.Statement{ 872 | { 873 | Sql: query, 874 | }, 875 | }, nil 876 | } 877 | 878 | b, err := ioutil.ReadAll(r.Body) 879 | if err != nil { 880 | return nil, errors.New("bad query POST request") 881 | } 882 | r.Body.Close() 883 | 884 | return ParseRequest(b) 885 | } 886 | 887 | // queryParam returns whether the given query param is set to true. 888 | func queryParam(req *http.Request, param string) (bool, error) { 889 | err := req.ParseForm() 890 | if err != nil { 891 | return false, err 892 | } 893 | if _, ok := req.Form[param]; ok { 894 | return true, nil 895 | } 896 | return false, nil 897 | } 898 | 899 | // stmtParam returns the value for URL param 'q', if present. 900 | func stmtParam(req *http.Request) (string, error) { 901 | q := req.URL.Query() 902 | stmt := strings.TrimSpace(q.Get("q")) 903 | return stmt, nil 904 | } 905 | 906 | // fmtParam returns the value for URL param 'fmt', if present. 907 | func fmtParam(req *http.Request) (string, error) { 908 | q := req.URL.Query() 909 | return strings.TrimSpace(q.Get("fmt")), nil 910 | } 911 | 912 | // isPretty returns whether the HTTP response body should be pretty-printed. 913 | func isPretty(req *http.Request) (bool, error) { 914 | return queryParam(req, "pretty") 915 | } 916 | 917 | // isTx returns whether the HTTP request is requesting a transaction. 918 | func isTx(req *http.Request) (bool, error) { 919 | return queryParam(req, "transaction") 920 | } 921 | 922 | // noLeader returns whether processing should skip the leader check. 923 | func noLeader(req *http.Request) (bool, error) { 924 | return queryParam(req, "noleader") 925 | } 926 | 927 | // nonVoters returns whether a query is requesting to include non-voter results 928 | func nonVoters(req *http.Request) (bool, error) { 929 | return queryParam(req, "nonvoters") 930 | } 931 | 932 | // timings returns whether timings are requested. 933 | func timings(req *http.Request) (bool, error) { 934 | return queryParam(req, "timings") 935 | } 936 | 937 | // timeout returns the timeout included in the query, or the given default 938 | func timeout(req *http.Request, d time.Duration) (time.Duration, error) { 939 | q := req.URL.Query() 940 | tStr := q.Get("timeout") 941 | if tStr == "" { 942 | return d, nil 943 | } 944 | 945 | t, err := time.ParseDuration(tStr) 946 | if err != nil { 947 | return d, nil 948 | } 949 | return t, nil 950 | } 951 | 952 | // level returns the requested consistency level for a query 953 | func level(req *http.Request) (command.QueryRequest_Level, error) { 954 | q := req.URL.Query() 955 | lvl := strings.TrimSpace(q.Get("level")) 956 | 957 | switch strings.ToLower(lvl) { 958 | case "none": 959 | return command.QueryRequest_QUERY_REQUEST_LEVEL_NONE, nil 960 | case "weak": 961 | return command.QueryRequest_QUERY_REQUEST_LEVEL_WEAK, nil 962 | case "strong": 963 | return command.QueryRequest_QUERY_REQUEST_LEVEL_STRONG, nil 964 | default: 965 | return command.QueryRequest_QUERY_REQUEST_LEVEL_WEAK, nil 966 | } 967 | } 968 | 969 | // freshness returns any freshness requested with a query. 970 | func freshness(req *http.Request) (time.Duration, error) { 971 | q := req.URL.Query() 972 | f := strings.TrimSpace(q.Get("freshness")) 973 | if f == "" { 974 | return 0, nil 975 | } 976 | 977 | d, err := time.ParseDuration(f) 978 | if err != nil { 979 | return 0, err 980 | } 981 | return d, nil 982 | } 983 | 984 | // backupFormat returns the request backup format, setting the response header 985 | // accordingly. 986 | func backupFormat(w http.ResponseWriter, r *http.Request) (store.BackupFormat, error) { 987 | fmt, err := fmtParam(r) 988 | if err != nil { 989 | return store.BackupBinary, err 990 | } 991 | if fmt == "sql" { 992 | w.Header().Set("Content-Type", "application/sql") 993 | return store.BackupSQL, nil 994 | } 995 | w.Header().Set("Content-Type", "application/octet-stream") 996 | return store.BackupBinary, nil 997 | } 998 | 999 | func prettyEnabled(e bool) string { 1000 | if e { 1001 | return "enabled" 1002 | } 1003 | return "disabled" 1004 | } 1005 | 1006 | // NormalizeAddr ensures that the given URL has a HTTP protocol prefix. 1007 | // If none is supplied, it prefixes the URL with "http://". 1008 | func NormalizeAddr(addr string) string { 1009 | if !strings.HasPrefix(addr, "http://") { 1010 | return fmt.Sprintf("http://%s", addr) 1011 | } 1012 | return addr 1013 | } 1014 | 1015 | // queryRequestFromStrings converts a slice of strings into a command.QueryRequest 1016 | func executeRequestFromStrings(s []string, timings, tx bool) *command.ExecuteRequest { 1017 | stmts := make([]*command.Statement, len(s)) 1018 | for i := range s { 1019 | stmts[i] = &command.Statement{ 1020 | Sql: s[i], 1021 | } 1022 | 1023 | } 1024 | return &command.ExecuteRequest{ 1025 | Request: &command.Request{ 1026 | Statements: stmts, 1027 | Transaction: tx, 1028 | }, 1029 | Timings: timings, 1030 | } 1031 | } 1032 | 1033 | // queryRequestFromStrings converts a slice of strings into a command.QueryRequest 1034 | func queryRequestFromStrings(s []string, timings, tx bool) *command.QueryRequest { 1035 | stmts := make([]*command.Statement, len(s)) 1036 | for i := range s { 1037 | stmts[i] = &command.Statement{ 1038 | Sql: s[i], 1039 | } 1040 | 1041 | } 1042 | return &command.QueryRequest{ 1043 | Request: &command.Request{ 1044 | Statements: stmts, 1045 | Transaction: tx, 1046 | }, 1047 | Timings: timings, 1048 | } 1049 | } 1050 | --------------------------------------------------------------------------------