├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── TODO ├── db.go ├── db_test.go ├── scatter.go ├── scatter_test.go └── stmt.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: false 3 | go: 4 | - tip 5 | install: 6 | - go get -v golang.org/x/lint/golint 7 | - go get -d -t -v ./... 8 | - go build -v ./... 9 | script: 10 | - go vet ./... 11 | - $HOME/gopath/bin/golint . 12 | - go test -v ./... 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013-2016 Tomás Senart 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nap 2 | 3 | Nap is a library that abstracts access to master-slave physical SQL servers topologies as a single logical database mimicking the standard `sql.DB` APIs. 4 | 5 | Nap requires Go version 1.8 or greater. 6 | 7 | ## Install 8 | ```shell 9 | $ go get github.com/tsenart/nap 10 | ``` 11 | 12 | ## Usage 13 | ```go 14 | package main 15 | 16 | import ( 17 | "log" 18 | 19 | "github.com/tsenart/nap" 20 | _ "github.com/go-sql-driver/mysql" // Any sql.DB works 21 | ) 22 | 23 | func main() { 24 | // The first DSN is assumed to be the master and all 25 | // other to be slaves 26 | dsns := "tcp://user:password@master/dbname;" 27 | dsns += "tcp://user:password@slave01/dbname;" 28 | dsns += "tcp://user:password@slave02/dbname" 29 | 30 | db, err := nap.Open("mysql", dsns) 31 | if err != nil { 32 | log.Fatal(err) 33 | } 34 | 35 | if err := db.Ping(); err != nil { 36 | log.Fatalf("Some physical database is unreachable: %s", err) 37 | } 38 | 39 | // Read queries are directed to slaves with Query and QueryRow. 40 | // Always use Query or QueryRow for SELECTS 41 | // Load distribution is round-robin only for now. 42 | var count int 43 | err = db.QueryRow("SELECT COUNT(*) FROM sometable").Scan(&count) 44 | if err != nil { 45 | log.Fatal(err) 46 | } 47 | 48 | // Write queries are directed to the master with Exec. 49 | // Always use Exec for INSERTS, UPDATES 50 | err = db.Exec("UPDATE sometable SET something = 1") 51 | if err != nil { 52 | log.Fatal(err) 53 | } 54 | 55 | // Prepared statements are aggregates. If any of the underlying 56 | // physical databases fails to prepare the statement, the call will 57 | // return an error. On success, if Exec is called, then the 58 | // master is used, if Query or QueryRow are called, then a slave 59 | // is used. 60 | stmt, err := db.Prepare("SELECT * FROM sometable WHERE something = ?") 61 | if err != nil { 62 | log.Fatal(err) 63 | } 64 | 65 | // Transactions always use the master 66 | tx, err := db.Begin() 67 | if err != nil { 68 | log.Fatal(err) 69 | } 70 | // Do something transactional ... 71 | if err = tx.Commit(); err != nil { 72 | log.Fatal(err) 73 | } 74 | 75 | // If needed, one can access the master or a slave explicitly. 76 | master, slave := db.Master(), db.Slave() 77 | } 78 | ``` 79 | 80 | ## Todo 81 | * Support other slave load balancing algorithms. 82 | 83 | ## License 84 | See [LICENSE](LICENSE) 85 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | Add support for smarter slave picking algorithms such as less delayed slave 2 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | package nap 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "database/sql/driver" 7 | "strings" 8 | "sync/atomic" 9 | "time" 10 | ) 11 | 12 | // DB is a logical database with multiple underlying physical databases 13 | // forming a single master multiple slaves topology. 14 | // Reads and writes are automatically directed to the correct physical db. 15 | type DB struct { 16 | pdbs []*sql.DB // Physical databases 17 | count uint64 // Monotonically incrementing counter on each query 18 | } 19 | 20 | // Open concurrently opens each underlying physical db. 21 | // dataSourceNames must be a semi-comma separated list of DSNs with the first 22 | // one being used as the master and the rest as slaves. 23 | func Open(driverName, dataSourceNames string) (*DB, error) { 24 | conns := strings.Split(dataSourceNames, ";") 25 | db := &DB{pdbs: make([]*sql.DB, len(conns))} 26 | 27 | err := scatter(len(db.pdbs), func(i int) (err error) { 28 | db.pdbs[i], err = sql.Open(driverName, conns[i]) 29 | return err 30 | }) 31 | 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | return db, nil 37 | } 38 | 39 | // Close closes all physical databases concurrently, releasing any open resources. 40 | func (db *DB) Close() error { 41 | return scatter(len(db.pdbs), func(i int) error { 42 | return db.pdbs[i].Close() 43 | }) 44 | } 45 | 46 | // Driver returns the physical database's underlying driver. 47 | func (db *DB) Driver() driver.Driver { 48 | return db.Master().Driver() 49 | } 50 | 51 | // Begin starts a transaction on the master. The isolation level is dependent on the driver. 52 | func (db *DB) Begin() (*sql.Tx, error) { 53 | return db.Master().Begin() 54 | } 55 | 56 | // BeginTx starts a transaction with the provided context on the master. 57 | // 58 | // The provided TxOptions is optional and may be nil if defaults should be used. 59 | // If a non-default isolation level is used that the driver doesn't support, 60 | // an error will be returned. 61 | func (db *DB) BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) { 62 | return db.Master().BeginTx(ctx, opts) 63 | } 64 | 65 | // Exec executes a query without returning any rows. 66 | // The args are for any placeholder parameters in the query. 67 | // Exec uses the master as the underlying physical db. 68 | func (db *DB) Exec(query string, args ...interface{}) (sql.Result, error) { 69 | return db.Master().Exec(query, args...) 70 | } 71 | 72 | // ExecContext executes a query without returning any rows. 73 | // The args are for any placeholder parameters in the query. 74 | // Exec uses the master as the underlying physical db. 75 | func (db *DB) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) { 76 | return db.Master().ExecContext(ctx, query, args...) 77 | } 78 | 79 | // Ping verifies if a connection to each physical database is still alive, 80 | // establishing a connection if necessary. 81 | func (db *DB) Ping() error { 82 | return scatter(len(db.pdbs), func(i int) error { 83 | return db.pdbs[i].Ping() 84 | }) 85 | } 86 | 87 | // PingContext verifies if a connection to each physical database is still 88 | // alive, establishing a connection if necessary. 89 | func (db *DB) PingContext(ctx context.Context) error { 90 | return scatter(len(db.pdbs), func(i int) error { 91 | return db.pdbs[i].PingContext(ctx) 92 | }) 93 | } 94 | 95 | // Prepare creates a prepared statement for later queries or executions 96 | // on each physical database, concurrently. 97 | func (db *DB) Prepare(query string) (Stmt, error) { 98 | stmts := make([]*sql.Stmt, len(db.pdbs)) 99 | 100 | err := scatter(len(db.pdbs), func(i int) (err error) { 101 | stmts[i], err = db.pdbs[i].Prepare(query) 102 | return err 103 | }) 104 | 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | return &stmt{db: db, stmts: stmts}, nil 110 | } 111 | 112 | // PrepareContext creates a prepared statement for later queries or executions 113 | // on each physical database, concurrently. 114 | // 115 | // The provided context is used for the preparation of the statement, not for 116 | // the execution of the statement. 117 | func (db *DB) PrepareContext(ctx context.Context, query string) (Stmt, error) { 118 | stmts := make([]*sql.Stmt, len(db.pdbs)) 119 | 120 | err := scatter(len(db.pdbs), func(i int) (err error) { 121 | stmts[i], err = db.pdbs[i].PrepareContext(ctx, query) 122 | return err 123 | }) 124 | 125 | if err != nil { 126 | return nil, err 127 | } 128 | return &stmt{db: db, stmts: stmts}, nil 129 | } 130 | 131 | // Query executes a query that returns rows, typically a SELECT. 132 | // The args are for any placeholder parameters in the query. 133 | // Query uses a slave as the physical db. 134 | func (db *DB) Query(query string, args ...interface{}) (*sql.Rows, error) { 135 | return db.Slave().Query(query, args...) 136 | } 137 | 138 | // QueryContext executes a query that returns rows, typically a SELECT. 139 | // The args are for any placeholder parameters in the query. 140 | // QueryContext uses a slave as the physical db. 141 | func (db *DB) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { 142 | return db.Slave().QueryContext(ctx, query, args...) 143 | } 144 | 145 | // QueryRow executes a query that is expected to return at most one row. 146 | // QueryRow always return a non-nil value. 147 | // Errors are deferred until Row's Scan method is called. 148 | // QueryRow uses a slave as the physical db. 149 | func (db *DB) QueryRow(query string, args ...interface{}) *sql.Row { 150 | return db.Slave().QueryRow(query, args...) 151 | } 152 | 153 | // QueryRowContext executes a query that is expected to return at most one row. 154 | // QueryRowContext always return a non-nil value. 155 | // Errors are deferred until Row's Scan method is called. 156 | // QueryRowContext uses a slave as the physical db. 157 | func (db *DB) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row { 158 | return db.Slave().QueryRowContext(ctx, query, args...) 159 | } 160 | 161 | // SetMaxIdleConns sets the maximum number of connections in the idle 162 | // connection pool for each underlying physical db. 163 | // If MaxOpenConns is greater than 0 but less than the new MaxIdleConns then the 164 | // new MaxIdleConns will be reduced to match the MaxOpenConns limit 165 | // If n <= 0, no idle connections are retained. 166 | func (db *DB) SetMaxIdleConns(n int) { 167 | for i := range db.pdbs { 168 | db.pdbs[i].SetMaxIdleConns(n) 169 | } 170 | } 171 | 172 | // SetMaxOpenConns sets the maximum number of open connections 173 | // to each physical database. 174 | // If MaxIdleConns is greater than 0 and the new MaxOpenConns 175 | // is less than MaxIdleConns, then MaxIdleConns will be reduced to match 176 | // the new MaxOpenConns limit. If n <= 0, then there is no limit on the number 177 | // of open connections. The default is 0 (unlimited). 178 | func (db *DB) SetMaxOpenConns(n int) { 179 | for i := range db.pdbs { 180 | db.pdbs[i].SetMaxOpenConns(n) 181 | } 182 | } 183 | 184 | // SetConnMaxLifetime sets the maximum amount of time a connection may be reused. 185 | // Expired connections may be closed lazily before reuse. 186 | // If d <= 0, connections are reused forever. 187 | func (db *DB) SetConnMaxLifetime(d time.Duration) { 188 | for i := range db.pdbs { 189 | db.pdbs[i].SetConnMaxLifetime(d) 190 | } 191 | } 192 | 193 | // Slave returns one of the physical databases which is a slave 194 | func (db *DB) Slave() *sql.DB { 195 | return db.pdbs[db.slave(len(db.pdbs))] 196 | } 197 | 198 | // Master returns the master physical database 199 | func (db *DB) Master() *sql.DB { 200 | return db.pdbs[0] 201 | } 202 | 203 | func (db *DB) slave(n int) int { 204 | if n <= 1 { 205 | return 0 206 | } 207 | return int(1 + (atomic.AddUint64(&db.count, 1) % uint64(n-1))) 208 | } 209 | -------------------------------------------------------------------------------- /db_test.go: -------------------------------------------------------------------------------- 1 | package nap 2 | 3 | import ( 4 | "testing" 5 | "testing/quick" 6 | 7 | _ "github.com/mattn/go-sqlite3" 8 | ) 9 | 10 | func TestOpen(t *testing.T) { 11 | // https://www.sqlite.org/inmemorydb.html 12 | db, err := Open("sqlite3", ":memory:;:memory:;:memory:") 13 | if err != nil { 14 | t.Fatal(err) 15 | } 16 | defer db.Close() 17 | 18 | if err = db.Ping(); err != nil { 19 | t.Error(err) 20 | } 21 | 22 | if want, got := 3, len(db.pdbs); want != got { 23 | t.Errorf("Unexpected number of physical dbs. Got: %d, Want: %d", got, want) 24 | } 25 | } 26 | 27 | func TestClose(t *testing.T) { 28 | db, err := Open("sqlite3", ":memory:;:memory:;:memory:") 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | 33 | if err = db.Close(); err != nil { 34 | t.Fatal(err) 35 | } 36 | 37 | if err = db.Ping(); err.Error() != "sql: database is closed" { 38 | t.Errorf("Physical dbs were not closed correctly. Got: %s", err) 39 | } 40 | } 41 | 42 | func TestSlave(t *testing.T) { 43 | db := &DB{} 44 | last := -1 45 | 46 | err := quick.Check(func(n int) bool { 47 | index := db.slave(n) 48 | if n <= 1 { 49 | return index == 0 50 | } 51 | 52 | result := index > 0 && index < n && index != last 53 | last = index 54 | 55 | return result 56 | }, nil) 57 | 58 | if err != nil { 59 | t.Error(err) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /scatter.go: -------------------------------------------------------------------------------- 1 | package nap 2 | 3 | func scatter(n int, fn func(i int) error) error { 4 | errors := make(chan error, n) 5 | 6 | var i int 7 | for i = 0; i < n; i++ { 8 | go func(i int) { errors <- fn(i) }(i) 9 | } 10 | 11 | var err, innerErr error 12 | for i = 0; i < cap(errors); i++ { 13 | if innerErr = <-errors; innerErr != nil { 14 | err = innerErr 15 | } 16 | } 17 | 18 | return err 19 | } 20 | -------------------------------------------------------------------------------- /scatter_test.go: -------------------------------------------------------------------------------- 1 | package nap 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "testing" 7 | ) 8 | 9 | func TestScatter(t *testing.T) { 10 | runtime.GOMAXPROCS(runtime.NumCPU()) 11 | 12 | seq := []int{1, 2, 3, 4, 5, 6, 7, 8} 13 | err := scatter(len(seq), func(i int) error { 14 | if seq[i]%2 == 0 { 15 | seq[i] *= seq[i] 16 | return nil 17 | } 18 | return fmt.Errorf("%d is an odd fellow", seq[i]) 19 | }) 20 | 21 | if err == nil { 22 | t.Fatal("Expected error, got nil") 23 | } 24 | 25 | want := []int{1, 4, 3, 16, 5, 36, 7, 64} 26 | for i := range want { 27 | if want[i] != seq[i] { 28 | t.Errorf("Wrong value at position %d. Want: %d, Got: %d", i, want[i], seq[i]) 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /stmt.go: -------------------------------------------------------------------------------- 1 | package nap 2 | 3 | import ( 4 | "database/sql" 5 | ) 6 | 7 | // Stmt is an aggregate prepared statement. 8 | // It holds a prepared statement for each underlying physical db. 9 | type Stmt interface { 10 | Close() error 11 | Exec(...interface{}) (sql.Result, error) 12 | Query(...interface{}) (*sql.Rows, error) 13 | QueryRow(...interface{}) *sql.Row 14 | } 15 | 16 | type stmt struct { 17 | db *DB 18 | stmts []*sql.Stmt 19 | } 20 | 21 | // Close closes the statement by concurrently closing all underlying 22 | // statements concurrently, returning the first non nil error. 23 | func (s *stmt) Close() error { 24 | return scatter(len(s.stmts), func(i int) error { 25 | return s.stmts[i].Close() 26 | }) 27 | } 28 | 29 | // Exec executes a prepared statement with the given arguments 30 | // and returns a Result summarizing the effect of the statement. 31 | // Exec uses the master as the underlying physical db. 32 | func (s *stmt) Exec(args ...interface{}) (sql.Result, error) { 33 | return s.stmts[0].Exec(args...) 34 | } 35 | 36 | // Query executes a prepared query statement with the given 37 | // arguments and returns the query results as a *sql.Rows. 38 | // Query uses a slave as the underlying physical db. 39 | func (s *stmt) Query(args ...interface{}) (*sql.Rows, error) { 40 | return s.stmts[s.db.slave(len(s.db.pdbs))].Query(args...) 41 | } 42 | 43 | // QueryRow executes a prepared query statement with the given arguments. 44 | // If an error occurs during the execution of the statement, that error 45 | // will be returned by a call to Scan on the returned *Row, which is always non-nil. 46 | // If the query selects no rows, the *Row's Scan will return ErrNoRows. 47 | // Otherwise, the *sql.Row's Scan scans the first selected row and discards the rest. 48 | // QueryRow uses a slave as the underlying physical db. 49 | func (s *stmt) QueryRow(args ...interface{}) *sql.Row { 50 | return s.stmts[s.db.slave(len(s.db.pdbs))].QueryRow(args...) 51 | } 52 | --------------------------------------------------------------------------------