├── .gitignore ├── README.md ├── cgo ├── go.mod ├── go.sum └── main.go └── nocgo ├── go.mod ├── go.sum └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.csv -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SQLite: CGO vs no CGO 2 | 3 | This repo benchmarks mattn/go-sqlite3 against modernc.org/sqlite which 4 | is a translation of SQLite3 from C to Go. This translation allows the 5 | latter package to avoid CGO since there is no C. 6 | 7 | My initial observations showed it being twice as slow as 8 | mattn/go-sqlite3 and this repo is to test that observation. 9 | 10 | See the [blog post for details](https://datastation.multiprocess.io/blog/2022-05-12-sqlite-in-go-with-and-without-cgo.html). 11 | -------------------------------------------------------------------------------- /cgo/go.mod: -------------------------------------------------------------------------------- 1 | module cgo 2 | 3 | go 1.18 4 | 5 | require github.com/mattn/go-sqlite3 v1.14.12 6 | -------------------------------------------------------------------------------- /cgo/go.sum: -------------------------------------------------------------------------------- 1 | github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0= 2 | github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 3 | -------------------------------------------------------------------------------- /cgo/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "math/rand" 7 | "os" 8 | "strings" 9 | "time" 10 | 11 | _ "github.com/mattn/go-sqlite3" 12 | ) 13 | 14 | var perfTune = ` 15 | pragma journal_mode = WAL; 16 | pragma synchronous = normal; 17 | pragma temp_store = memory; 18 | pragma mmap_size = 30000000000;` 19 | 20 | func main() { 21 | db, err := sql.Open("sqlite3", ":memory:") 22 | if err != nil { 23 | panic(err) 24 | } 25 | 26 | wordsFile, err := os.ReadFile("/usr/share/dict/words") 27 | if err != nil { 28 | panic(err) 29 | } 30 | words := strings.Split(string(wordsFile), "\n") 31 | 32 | rand.Seed(0) 33 | times := 10 34 | rowTests := []int{10_000, len(words), len(words) * 10} 35 | 36 | _, err = db.Exec(perfTune) 37 | if err != nil { 38 | panic(err) 39 | } 40 | 41 | fmt.Println("time,rows,category,version") 42 | 43 | for _, rows := range rowTests { 44 | for i := 0; i < times; i++ { 45 | _, err = db.Exec("DROP TABLE IF EXISTS people") 46 | if err != nil { 47 | panic(err) 48 | } 49 | _, err = db.Exec(` 50 | CREATE TABLE people ( 51 | name TEXT, 52 | country TEXT, 53 | region TEXT, 54 | occupation TEXT, 55 | age INT, 56 | company TEXT, 57 | favorite_team TEXT, 58 | favorite_sport TEXT 59 | )`) 60 | if err != nil { 61 | panic(err) 62 | } 63 | 64 | stmt, err := db.Prepare("INSERT INTO people VALUES (?, ?, ?, ?, ?, ?, ?, ?)") 65 | if err != nil { 66 | panic(err) 67 | } 68 | 69 | t1 := time.Now() 70 | for i := 0; i < rows; i++ { 71 | rnd_name := words[rand.Int()%len(words)] 72 | rnd_country := words[rand.Int()%len(words)] 73 | rnd_region := words[rand.Int()%len(words)] 74 | rnd_occupation := words[rand.Int()%len(words)] 75 | rnd_company := words[rand.Int()%len(words)] 76 | rnd_age := rand.Int() % 110 77 | rnd_fav_team := words[rand.Int()%len(words)] 78 | rnd_fav_sport := words[rand.Int()%len(words)] 79 | 80 | _, err := stmt.Exec(rnd_name, rnd_country, rnd_region, rnd_occupation, rnd_age, rnd_company, rnd_fav_team, rnd_fav_sport) 81 | if err != nil { 82 | panic(err) 83 | } 84 | } 85 | fmt.Printf("%f,%d,insert,cgo\n", float64(time.Now().Sub(t1)) / 1e9, rows) 86 | 87 | t1 = time.Now() 88 | _, err = db.Query("SELECT COUNT(1), age FROM people GROUP BY age ORDER BY COUNT(1) DESC") 89 | fmt.Printf("%f,%d,group_by,cgo\n", float64(time.Now().Sub(t1)) / 1e9, rows) 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /nocgo/go.mod: -------------------------------------------------------------------------------- 1 | module nocgo 2 | 3 | go 1.18 4 | 5 | require modernc.org/sqlite v1.17.2 6 | 7 | require ( 8 | github.com/google/uuid v1.3.0 // indirect 9 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect 10 | github.com/mattn/go-isatty v0.0.12 // indirect 11 | github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect 12 | golang.org/x/mod v0.3.0 // indirect 13 | golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect 14 | golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 // indirect 15 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 16 | lukechampine.com/uint128 v1.1.1 // indirect 17 | modernc.org/cc/v3 v3.36.0 // indirect 18 | modernc.org/ccgo/v3 v3.16.6 // indirect 19 | modernc.org/libc v1.16.7 // indirect 20 | modernc.org/mathutil v1.4.1 // indirect 21 | modernc.org/memory v1.1.1 // indirect 22 | modernc.org/opt v0.1.1 // indirect 23 | modernc.org/strutil v1.1.1 // indirect 24 | modernc.org/token v1.0.0 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /nocgo/go.sum: -------------------------------------------------------------------------------- 1 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= 2 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 3 | github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo= 4 | github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 5 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 6 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 7 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= 8 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= 9 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 10 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 11 | github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0= 12 | github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 13 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 14 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 15 | github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= 16 | github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= 17 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 18 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 19 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 20 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 21 | golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= 22 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 23 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 24 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 25 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 26 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 27 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 28 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 29 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 30 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 31 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 32 | golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= 33 | golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 34 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 35 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 36 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 37 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 38 | golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 h1:M8tBwCtWD/cZV9DZpFYRUgaymAYAr+aIUTWzDaM3uPs= 39 | golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 40 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 41 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 42 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 43 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 44 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 45 | lukechampine.com/uint128 v1.1.1 h1:pnxCASz787iMf+02ssImqk6OLt+Z5QHMoZyUXR4z6JU= 46 | lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= 47 | modernc.org/cc/v3 v3.36.0 h1:0kmRkTmqNidmu3c7BNDSdVHCxXCkWLmWmCIVX4LUboo= 48 | modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= 49 | modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= 50 | modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= 51 | modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= 52 | modernc.org/ccgo/v3 v3.16.6 h1:3l18poV+iUemQ98O3X5OMr97LOqlzis+ytivU4NqGhA= 53 | modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= 54 | modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= 55 | modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= 56 | modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= 57 | modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= 58 | modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= 59 | modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= 60 | modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= 61 | modernc.org/libc v1.16.7 h1:qzQtHhsZNpVPpeCu+aMIQldXeV1P0vRhSqCL0nOIJOA= 62 | modernc.org/libc v1.16.7/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= 63 | modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= 64 | modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8= 65 | modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= 66 | modernc.org/memory v1.1.1 h1:bDOL0DIDLQv7bWhP3gMvIrnoFw+Eo6F7a2QK9HPDiFU= 67 | modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= 68 | modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A= 69 | modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= 70 | modernc.org/sqlite v1.17.2 h1:TjmF36Wi5QcPYqRoAacV1cAyJ7xB/CD0ExpVUEMebnw= 71 | modernc.org/sqlite v1.17.2/go.mod h1:GOQmuiXd6pTTes1Fi2s9apiCcD/wbKQtBZ0Nw6/etjM= 72 | modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs= 73 | modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= 74 | modernc.org/tcl v1.13.1 h1:npxzTwFTZYM8ghWicVIX1cRWzj7Nd8i6AqqX2p+IYao= 75 | modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= 76 | modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk= 77 | modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= 78 | modernc.org/z v1.5.1 h1:RTNHdsrOpeoSeOF4FbzTo8gBYByaJ5xT7NgZ9ZqRiJM= 79 | modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= 80 | -------------------------------------------------------------------------------- /nocgo/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "math/rand" 7 | "os" 8 | "strings" 9 | "time" 10 | 11 | _ "modernc.org/sqlite" 12 | ) 13 | 14 | var perfTune = ` 15 | pragma journal_mode = WAL; 16 | pragma synchronous = normal; 17 | pragma temp_store = memory; 18 | pragma mmap_size = 30000000000;` 19 | 20 | func main() { 21 | db, err := sql.Open("sqlite", ":memory:") 22 | if err != nil { 23 | panic(err) 24 | } 25 | 26 | wordsFile, err := os.ReadFile("/usr/share/dict/words") 27 | if err != nil { 28 | panic(err) 29 | } 30 | words := strings.Split(string(wordsFile), "\n") 31 | 32 | rand.Seed(0) 33 | times := 10 34 | rowTests := []int{10_000, len(words), len(words)*10} 35 | 36 | _, err = db.Exec(perfTune) 37 | if err != nil { 38 | panic(err) 39 | } 40 | 41 | fmt.Println("time,rows,category,version") 42 | for _, rows := range rowTests { 43 | for i := 0; i < times; i++ { 44 | _, err := db.Exec("DROP TABLE IF EXISTS people") 45 | if err != nil { 46 | panic(err) 47 | } 48 | _, err = db.Exec(` 49 | CREATE TABLE people ( 50 | name TEXT, 51 | country TEXT, 52 | region TEXT, 53 | occupation TEXT, 54 | age INT, 55 | company TEXT, 56 | favorite_team TEXT, 57 | favorite_sport TEXT 58 | )`) 59 | if err != nil { 60 | panic(err) 61 | } 62 | 63 | stmt, err := db.Prepare("INSERT INTO people VALUES (?, ?, ?, ?, ?, ?, ?, ?)") 64 | if err != nil { 65 | panic(err) 66 | } 67 | 68 | t1 := time.Now() 69 | for i := 0; i < rows; i++ { 70 | rnd_name := words[rand.Int()%len(words)] 71 | rnd_country := words[rand.Int()%len(words)] 72 | rnd_region := words[rand.Int()%len(words)] 73 | rnd_occupation := words[rand.Int()%len(words)] 74 | rnd_company := words[rand.Int()%len(words)] 75 | rnd_age := rand.Int() % 110 76 | rnd_fav_team := words[rand.Int()%len(words)] 77 | rnd_fav_sport := words[rand.Int()%len(words)] 78 | 79 | _, err := stmt.Exec(rnd_name, rnd_country, rnd_region, rnd_occupation, rnd_age, rnd_company, rnd_fav_team, rnd_fav_sport) 80 | if err != nil { 81 | panic(err) 82 | } 83 | } 84 | 85 | fmt.Printf("%f,%d,insert,nocgo\n", float64(time.Now().Sub(t1)) / 1e9, rows) 86 | 87 | t1 = time.Now() 88 | _, err = db.Query("SELECT COUNT(1), age FROM people GROUP BY age ORDER BY COUNT(1) DESC") 89 | fmt.Printf("%f,%d,group_by,nocgo\n", float64(time.Now().Sub(t1)) / 1e9, rows) 90 | } 91 | } 92 | } 93 | --------------------------------------------------------------------------------