├── .gitignore ├── copy ├── bench-copy.py ├── bench-insert.py ├── datagen │ ├── go.mod │ ├── go.sum │ └── main.go ├── migrations │ └── 20230319014554_initial_tables_up.sql └── requirements.txt ├── index ├── bench.py ├── datagen │ ├── go.mod │ ├── go.sum │ └── main.go ├── insert.py ├── migrations │ └── 20230318023712_setup-users-table.sql └── requirements.txt ├── partition ├── bench-insert.py ├── datagen │ ├── go.mod │ ├── go.sum │ └── main.go ├── migrations │ └── 20230319014554_initial_tables_up.sql └── requirements.txt ├── prepare ├── .env ├── README.md ├── bench.py ├── migrations │ └── 20230316225351_initial_tables_up.sql ├── requirements.txt └── test.sql └── replicate ├── README.md └── docker-compose.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | *.csv 3 | .env 4 | -------------------------------------------------------------------------------- /copy/bench-copy.py: -------------------------------------------------------------------------------- 1 | import psycopg2 2 | from urllib.parse import urlparse 3 | import os 4 | from dotenv import load_dotenv 5 | import random 6 | import time 7 | import csv 8 | 9 | load_dotenv() 10 | 11 | conStr = os.getenv('DATABASE_URL', 'postgres://postgres:postgres@localhost:5432/postgres') 12 | p = urlparse(conStr) 13 | 14 | pg_connection_dict = { 15 | 'dbname': p.path[1:], 16 | 'user': p.username, 17 | 'password': p.password, 18 | 'port': p.port, 19 | 'host': p.hostname 20 | } 21 | 22 | conn = psycopg2.connect(**pg_connection_dict) 23 | 24 | now = time.time() 25 | with open('datagen/events_new.csv', newline='') as csvfile: 26 | with conn.cursor() as cur: 27 | cur.copy_from(csvfile, 'events_copy', sep=',', columns=('id', 'source', 'payload', 'event_timestamp')) 28 | conn.commit() 29 | 30 | end = time.time() 31 | 32 | conn.close() 33 | 34 | print(f"Total time for copy: {end - now:.2f} seconds") 35 | -------------------------------------------------------------------------------- /copy/bench-insert.py: -------------------------------------------------------------------------------- 1 | import psycopg2 2 | from urllib.parse import urlparse 3 | import os 4 | from dotenv import load_dotenv 5 | import random 6 | import time 7 | import csv 8 | 9 | load_dotenv() 10 | 11 | conStr = os.getenv('DATABASE_URL', 'postgres://postgres:postgres@localhost:5432/postgres') 12 | p = urlparse(conStr) 13 | 14 | pg_connection_dict = { 15 | 'dbname': p.path[1:], 16 | 'user': p.username, 17 | 'password': p.password, 18 | 'port': p.port, 19 | 'host': p.hostname 20 | } 21 | 22 | conn = psycopg2.connect(**pg_connection_dict) 23 | 24 | now = time.time() 25 | with open('datagen/events_new.csv', newline='') as csvfile: 26 | reader = csv.reader(csvfile, delimiter=',', quotechar='"') 27 | with conn.cursor() as cur: 28 | for row in reader: 29 | cur.execute("INSERT INTO events_insert (id, source, payload, event_timestamp) VALUES (%s, %s, %s, %s)", (row[0], row[1], row[2], row[3])) 30 | conn.commit() 31 | 32 | end = time.time() 33 | print(f"Total time for insert: {end - now:.2f} seconds") 34 | -------------------------------------------------------------------------------- /copy/datagen/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/dreamsofcode-io/postgres-brrr/partition/datagen 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/go-faker/faker/v4 v4.1.0 // indirect 7 | github.com/google/uuid v1.3.0 // indirect 8 | github.com/jaswdr/faker v1.16.0 // indirect 9 | golang.org/x/text v0.8.0 // indirect 10 | ) 11 | -------------------------------------------------------------------------------- /copy/datagen/go.sum: -------------------------------------------------------------------------------- 1 | github.com/go-faker/faker/v4 v4.1.0 h1:ffuWmpDrducIUOO0QSKSF5Q2dxAht+dhsT9FvVHhPEI= 2 | github.com/go-faker/faker/v4 v4.1.0/go.mod h1:uuNc0PSRxF8nMgjGrrrU4Nw5cF30Jc6Kd0/FUTTYbhg= 3 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 4 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 5 | github.com/jaswdr/faker v1.16.0 h1:5ZjusQbqIZwJnUymPirNKJI1yFCuozdSR9oeYPgD5Uk= 6 | github.com/jaswdr/faker v1.16.0/go.mod h1:x7ZlyB1AZqwqKZgyQlnqEG8FDptmHlncA5u2zY/yi6w= 7 | golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= 8 | golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 9 | -------------------------------------------------------------------------------- /copy/datagen/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/binary" 6 | "encoding/csv" 7 | "fmt" 8 | "math/rand" 9 | "net" 10 | "os" 11 | "reflect" 12 | "strings" 13 | "sync" 14 | "time" 15 | 16 | "github.com/go-faker/faker/v4" 17 | "github.com/google/uuid" 18 | ) 19 | 20 | type Event struct { 21 | ID uuid.UUID `faker:"uuid"` 22 | Source net.IP `faker:"ip"` 23 | Payload []byte `faker:"payload"` 24 | Timestamp string `faker:"timestamper"` 25 | } 26 | 27 | func setup() { 28 | rand.Seed(time.Now().UnixNano()) 29 | 30 | _ = faker.AddProvider("uuid", func(v reflect.Value) (interface{}, error) { 31 | return uuid.New(), nil 32 | }) 33 | 34 | _ = faker.AddProvider("ip", func(v reflect.Value) (interface{}, error) { 35 | return random_ip(), nil 36 | }) 37 | 38 | _ = faker.AddProvider("timestamper", func(v reflect.Value) (interface{}, error) { 39 | days := 1 //rand.Intn(3) + 1 40 | secs := rand.Intn(60*60*24*days - 1) 41 | dt := time.Date(2023, 3, 18, 23, 59, 59, 0, time.UTC).Add(time.Duration(secs) * time.Second * -1) 42 | return dt.Format(time.RFC3339), nil 43 | }) 44 | 45 | _ = faker.AddProvider("payload", func(v reflect.Value) (interface{}, error) { 46 | methods := strings.Split("GET,POST,PUT,DELETE,HEAD,OPTIONS,TRACE,CONNECT", ",") 47 | components := []string{ 48 | "admin", 49 | "api", 50 | "auth", 51 | "blog", 52 | "cdn", 53 | "cdn2", 54 | "cdn3", 55 | "cdn4", 56 | "web", 57 | "www", 58 | "files", 59 | "images", 60 | "img", 61 | "static", 62 | "static1", 63 | "static2", 64 | "ip", 65 | "users", 66 | "user", 67 | "login", 68 | "logout", 69 | "register", 70 | "signup", 71 | "signin", 72 | "signout", 73 | "account", 74 | "accounts", 75 | "profile", 76 | "profiles", 77 | "dashboard", 78 | "dashboards", 79 | "admin", 80 | "admins", 81 | "administrator", 82 | "administrators", 83 | "home", 84 | "index", 85 | "about", 86 | "contact", 87 | "contacts", 88 | "contact-us", 89 | "contact_us", 90 | } 91 | 92 | extensions := []string{ 93 | "html", 94 | "htm", 95 | "php", 96 | "asp", 97 | "aspx", 98 | "jsp", 99 | "js", 100 | "css", 101 | "png", 102 | "jpg", 103 | "jpeg", 104 | "gif", 105 | "svg", 106 | "ico", 107 | "json", 108 | "xml", 109 | "txt", 110 | "pdf", 111 | "doc", 112 | } 113 | 114 | isFile := rand.Intn(2) == 0 115 | 116 | numComponents := rand.Intn(5) 117 | 118 | pathComps := make([]string, numComponents) 119 | 120 | for i := 0; i < numComponents; i++ { 121 | pathComps[i] = components[rand.Intn(len(components))] 122 | } 123 | 124 | path := "/" + strings.Join(pathComps, "/") 125 | 126 | if isFile { 127 | path += "." + extensions[rand.Intn(len(extensions))] 128 | } 129 | 130 | method := methods[rand.Intn(len(methods))] 131 | 132 | return []byte(fmt.Sprintf("%s %s", method, path)), nil 133 | }) 134 | } 135 | 136 | func random_ip() net.IP { 137 | buf := make([]byte, 4) 138 | ip := rand.Uint32() 139 | 140 | binary.LittleEndian.PutUint32(buf, ip) 141 | 142 | return net.IP(buf) 143 | } 144 | 145 | func generateEvents() chan Event { 146 | threads := 40 147 | ch := make(chan Event) 148 | 149 | events := 100000 150 | 151 | wg := sync.WaitGroup{} 152 | for th := 0; th < threads; th++ { 153 | wg.Add(1) 154 | 155 | go func() { 156 | for i := 0; i < events/threads; i++ { 157 | var event Event 158 | _ = faker.FakeData(&event) 159 | ch <- event 160 | } 161 | 162 | wg.Done() 163 | }() 164 | } 165 | 166 | go func() { 167 | wg.Wait() 168 | fmt.Println("Closing chan") 169 | close(ch) 170 | }() 171 | 172 | return ch 173 | } 174 | 175 | func main() { 176 | setup() 177 | 178 | ch := generateEvents() 179 | 180 | f, err := os.Create("events_new.csv") 181 | if err != nil { 182 | panic(err) 183 | } 184 | 185 | w := csv.NewWriter(f) 186 | 187 | for ev := range ch { 188 | w.Write([]string{ 189 | ev.ID.String(), 190 | ev.Source.String(), 191 | base64.RawStdEncoding.EncodeToString(ev.Payload), 192 | ev.Timestamp, 193 | }) 194 | } 195 | 196 | w.Flush() 197 | 198 | f.Close() 199 | } 200 | -------------------------------------------------------------------------------- /copy/migrations/20230319014554_initial_tables_up.sql: -------------------------------------------------------------------------------- 1 | -- Add migration script here 2 | 3 | CREATE TABLE events_copy( 4 | id uuid primary key, 5 | source inet not null, 6 | payload bytea, 7 | event_timestamp timestamp not null default now() 8 | ); 9 | 10 | CREATE INDEX on events_copy (source); 11 | 12 | CREATE INDEX on events_copy (encode(payload, 'escape')); 13 | 14 | CREATE INDEX on events_copy (event_timestamp); 15 | 16 | CREATE TABLE events_insert( 17 | id uuid primary key, 18 | source inet not null, 19 | payload bytea, 20 | event_timestamp timestamp not null default now() 21 | ); 22 | 23 | CREATE INDEX on events_insert (source); 24 | 25 | CREATE INDEX on events_insert (encode(payload, 'escape')); 26 | 27 | CREATE INDEX on events_insert (event_timestamp); 28 | 29 | -------------------------------------------------------------------------------- /copy/requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg2-binary 2 | python-dotenv==1.0.0 3 | -------------------------------------------------------------------------------- /index/bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dreamsofcode-io/postgres-brrr/14a17979486ef4be60e3187644919f5efbaed498/index/bench.py -------------------------------------------------------------------------------- /index/datagen/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/dreamsofcode-io/postgres-brrr/index/datagen 2 | 3 | go 1.20 4 | 5 | require github.com/jaswdr/faker v1.16.0 // indirect 6 | -------------------------------------------------------------------------------- /index/datagen/go.sum: -------------------------------------------------------------------------------- 1 | github.com/jaswdr/faker v1.16.0 h1:5ZjusQbqIZwJnUymPirNKJI1yFCuozdSR9oeYPgD5Uk= 2 | github.com/jaswdr/faker v1.16.0/go.mod h1:x7ZlyB1AZqwqKZgyQlnqEG8FDptmHlncA5u2zY/yi6w= 3 | -------------------------------------------------------------------------------- /index/datagen/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "math/rand" 6 | "os" 7 | "strconv" 8 | "sync" 9 | "time" 10 | 11 | "github.com/jaswdr/faker" 12 | ) 13 | 14 | type Person struct { 15 | Index int 16 | Name string 17 | Age int 18 | CreatedAt time.Time 19 | } 20 | 21 | func generatePersons() <-chan Person { 22 | threads := 20 23 | 24 | ch := make(chan Person) 25 | 26 | persons := 20000000 27 | 28 | wg := sync.WaitGroup{} 29 | for i := 0; i < threads; i++ { 30 | wg.Add(1) 31 | 32 | go func(thread int) { 33 | fake := faker.New() 34 | 35 | for j := 0; j < persons/threads; j++ { 36 | ch <- Person{ 37 | Index: thread + (j * threads), 38 | Name: fake.Person().Name(), 39 | Age: rand.Intn(120), 40 | CreatedAt: time.Now(), 41 | } 42 | } 43 | 44 | wg.Done() 45 | }(i) 46 | } 47 | 48 | go func() { 49 | wg.Wait() 50 | close(ch) 51 | }() 52 | 53 | return ch 54 | } 55 | 56 | func main() { 57 | ch := generatePersons() 58 | 59 | f, err := os.Create("persons.csv") 60 | if err != nil { 61 | panic(err) 62 | } 63 | 64 | w := csv.NewWriter(f) 65 | 66 | for p := range ch { 67 | w.Write([]string{strconv.Itoa(p.Index), p.Name, strconv.Itoa(p.Age), p.CreatedAt.Format(time.RFC3339)}) 68 | } 69 | 70 | w.Flush() 71 | 72 | f.Close() 73 | } 74 | -------------------------------------------------------------------------------- /index/insert.py: -------------------------------------------------------------------------------- 1 | import psycopg 2 | from urllib.parse import urlparse 3 | import os 4 | from faker import Faker 5 | from dotenv import load_dotenv 6 | import random 7 | import time 8 | 9 | load_dotenv() 10 | 11 | conStr = os.getenv('DATABASE_URL', 'postgres://postgres:postgres@localhost:5432/postgres') 12 | p = urlparse(conStr) 13 | 14 | pg_connection_dict = { 15 | 'dbname': p.path[1:], 16 | 'user': p.username, 17 | 'password': p.password, 18 | 'port': p.port, 19 | 'host': p.hostname 20 | } 21 | 22 | conn = psycopg.connect(**pg_connection_dict) 23 | 24 | for i in range(2000000): 25 | cur = conn.cursor() 26 | cur.execute("INSERT INTO users (name, age) VALUES (%s, %s)", (Faker().name(), random.randint(0, 100))) 27 | conn.commit() 28 | cur.close() 29 | -------------------------------------------------------------------------------- /index/migrations/20230318023712_setup-users-table.sql: -------------------------------------------------------------------------------- 1 | -- Add migration script here 2 | CREATE TABLE users ( 3 | id SERIAL PRIMARY KEY, 4 | name VARCHAR(255) NOT NULL, 5 | age INTEGER NOT NULL, 6 | created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP 7 | ); 8 | -------------------------------------------------------------------------------- /index/requirements.txt: -------------------------------------------------------------------------------- 1 | Faker==17.6.0 2 | psycopg[binary] 3 | python-dotenv==1.0.0 4 | -------------------------------------------------------------------------------- /partition/bench-insert.py: -------------------------------------------------------------------------------- 1 | import psycopg2 2 | from urllib.parse import urlparse 3 | import os 4 | from dotenv import load_dotenv 5 | import random 6 | import time 7 | import csv 8 | 9 | load_dotenv() 10 | 11 | conStr = os.getenv('DATABASE_URL', 'postgres://postgres:postgres@localhost:5432/postgres') 12 | p = urlparse(conStr) 13 | 14 | pg_connection_dict = { 15 | 'dbname': p.path[1:], 16 | 'user': p.username, 17 | 'password': p.password, 18 | 'port': p.port, 19 | 'host': p.hostname 20 | } 21 | 22 | conn = psycopg2.connect(**pg_connection_dict) 23 | 24 | 25 | now = time.time() 26 | with open('datagen/events_new.csv', newline='') as csvfile: 27 | reader = csv.reader(csvfile, delimiter=',', quotechar='"') 28 | for row in reader: 29 | with conn.cursor() as cur: 30 | cur.execute("INSERT INTO events (id, source, payload, event_timestamp) VALUES (%s, %s, %s, %s)", (row[0], row[1], row[2], row[3])) 31 | conn.commit() 32 | 33 | end = time.time() 34 | print("Total time for partitioned table: ", end - now) 35 | -------------------------------------------------------------------------------- /partition/datagen/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/dreamsofcode-io/postgres-brrr/partition/datagen 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/go-faker/faker/v4 v4.1.0 // indirect 7 | github.com/google/uuid v1.3.0 // indirect 8 | github.com/jaswdr/faker v1.16.0 // indirect 9 | golang.org/x/text v0.8.0 // indirect 10 | ) 11 | -------------------------------------------------------------------------------- /partition/datagen/go.sum: -------------------------------------------------------------------------------- 1 | github.com/go-faker/faker/v4 v4.1.0 h1:ffuWmpDrducIUOO0QSKSF5Q2dxAht+dhsT9FvVHhPEI= 2 | github.com/go-faker/faker/v4 v4.1.0/go.mod h1:uuNc0PSRxF8nMgjGrrrU4Nw5cF30Jc6Kd0/FUTTYbhg= 3 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 4 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 5 | github.com/jaswdr/faker v1.16.0 h1:5ZjusQbqIZwJnUymPirNKJI1yFCuozdSR9oeYPgD5Uk= 6 | github.com/jaswdr/faker v1.16.0/go.mod h1:x7ZlyB1AZqwqKZgyQlnqEG8FDptmHlncA5u2zY/yi6w= 7 | golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= 8 | golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 9 | -------------------------------------------------------------------------------- /partition/datagen/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/binary" 6 | "encoding/csv" 7 | "fmt" 8 | "math/rand" 9 | "net" 10 | "os" 11 | "reflect" 12 | "strings" 13 | "sync" 14 | "time" 15 | 16 | "github.com/go-faker/faker/v4" 17 | "github.com/google/uuid" 18 | ) 19 | 20 | type Event struct { 21 | ID uuid.UUID `faker:"uuid"` 22 | Source net.IP `faker:"ip"` 23 | Payload []byte `faker:"payload"` 24 | Timestamp string `faker:"timestamper"` 25 | } 26 | 27 | func setup() { 28 | rand.Seed(time.Now().UnixNano()) 29 | 30 | _ = faker.AddProvider("uuid", func(v reflect.Value) (interface{}, error) { 31 | return uuid.New(), nil 32 | }) 33 | 34 | _ = faker.AddProvider("ip", func(v reflect.Value) (interface{}, error) { 35 | return random_ip(), nil 36 | }) 37 | 38 | _ = faker.AddProvider("timestamper", func(v reflect.Value) (interface{}, error) { 39 | days := 1 //rand.Intn(3) + 1 40 | secs := rand.Intn(60*60*24*days - 1) 41 | dt := time.Date(2023, 3, 18, 23, 59, 59, 0, time.UTC).Add(time.Duration(secs) * time.Second * -1) 42 | return dt.Format(time.RFC3339), nil 43 | }) 44 | 45 | _ = faker.AddProvider("payload", func(v reflect.Value) (interface{}, error) { 46 | methods := strings.Split("GET,POST,PUT,DELETE,HEAD,OPTIONS,TRACE,CONNECT", ",") 47 | components := []string{ 48 | "admin", 49 | "api", 50 | "auth", 51 | "blog", 52 | "cdn", 53 | "cdn2", 54 | "cdn3", 55 | "cdn4", 56 | "web", 57 | "www", 58 | "files", 59 | "images", 60 | "img", 61 | "static", 62 | "static1", 63 | "static2", 64 | "ip", 65 | "users", 66 | "user", 67 | "login", 68 | "logout", 69 | "register", 70 | "signup", 71 | "signin", 72 | "signout", 73 | "account", 74 | "accounts", 75 | "profile", 76 | "profiles", 77 | "dashboard", 78 | "dashboards", 79 | "admin", 80 | "admins", 81 | "administrator", 82 | "administrators", 83 | "home", 84 | "index", 85 | "about", 86 | "contact", 87 | "contacts", 88 | "contact-us", 89 | "contact_us", 90 | } 91 | 92 | extensions := []string{ 93 | "html", 94 | "htm", 95 | "php", 96 | "asp", 97 | "aspx", 98 | "jsp", 99 | "js", 100 | "css", 101 | "png", 102 | "jpg", 103 | "jpeg", 104 | "gif", 105 | "svg", 106 | "ico", 107 | "json", 108 | "xml", 109 | "txt", 110 | "pdf", 111 | "doc", 112 | } 113 | 114 | isFile := rand.Intn(2) == 0 115 | 116 | numComponents := rand.Intn(5) 117 | 118 | pathComps := make([]string, numComponents) 119 | 120 | for i := 0; i < numComponents; i++ { 121 | pathComps[i] = components[rand.Intn(len(components))] 122 | } 123 | 124 | path := "/" + strings.Join(pathComps, "/") 125 | 126 | if isFile { 127 | path += "." + extensions[rand.Intn(len(extensions))] 128 | } 129 | 130 | method := methods[rand.Intn(len(methods))] 131 | 132 | return []byte(fmt.Sprintf("%s %s", method, path)), nil 133 | }) 134 | } 135 | 136 | func random_ip() net.IP { 137 | buf := make([]byte, 4) 138 | ip := rand.Uint32() 139 | 140 | binary.LittleEndian.PutUint32(buf, ip) 141 | 142 | return net.IP(buf) 143 | } 144 | 145 | func generateEvents() chan Event { 146 | threads := 40 147 | ch := make(chan Event) 148 | 149 | events := 100100 150 | 151 | wg := sync.WaitGroup{} 152 | for i := 0; i < threads; i++ { 153 | wg.Add(1) 154 | 155 | go func() { 156 | for i := 0; i < events/threads; i++ { 157 | var event Event 158 | _ = faker.FakeData(&event) 159 | ch <- event 160 | } 161 | 162 | wg.Done() 163 | }() 164 | } 165 | 166 | go func() { 167 | wg.Wait() 168 | close(ch) 169 | }() 170 | 171 | return ch 172 | } 173 | 174 | func main() { 175 | setup() 176 | 177 | ch := generateEvents() 178 | 179 | f, err := os.Create("events_new.csv") 180 | if err != nil { 181 | panic(err) 182 | } 183 | 184 | w := csv.NewWriter(f) 185 | 186 | for ev := range ch { 187 | w.Write([]string{ 188 | ev.ID.String(), 189 | ev.Source.String(), 190 | base64.RawStdEncoding.EncodeToString(ev.Payload), 191 | ev.Timestamp, 192 | }) 193 | } 194 | 195 | w.Flush() 196 | 197 | f.Close() 198 | } 199 | -------------------------------------------------------------------------------- /partition/migrations/20230319014554_initial_tables_up.sql: -------------------------------------------------------------------------------- 1 | -- Add migration script here 2 | CREATE TABLE events ( 3 | id uuid, 4 | source inet not null, 5 | payload bytea, 6 | event_timestamp timestamp not null default now(), 7 | primary key (id, event_timestamp) 8 | ) PARTITION BY RANGE (event_timestamp); 9 | 10 | CREATE TABLE events_2023_03_16 PARTITION OF events 11 | FOR VALUES FROM ('2023-03-16') TO ('2023-03-17'); 12 | 13 | CREATE TABLE events_2023_03_17 PARTITION OF events 14 | FOR VALUES FROM ('2023-03-17') TO ('2023-03-18'); 15 | 16 | CREATE TABLE events_2023_03_18 PARTITION OF events 17 | FOR VALUES FROM ('2023-03-18') TO ('2023-03-19'); 18 | 19 | CREATE INDEX on events (source); 20 | 21 | CREATE INDEX on events (encode(payload, 'escape')); 22 | 23 | CREATE TABLE events_full( 24 | id uuid primary key, 25 | source inet not null, 26 | payload bytea, 27 | event_timestamp timestamp not null default now() 28 | ); 29 | 30 | CREATE INDEX on events_full (source); 31 | 32 | CREATE INDEX on events_full (encode(payload, 'escape')); 33 | 34 | CREATE INDEX on events_full (event_timestamp); 35 | -------------------------------------------------------------------------------- /partition/requirements.txt: -------------------------------------------------------------------------------- 1 | Faker==17.6.0 2 | psycopg2-binary 3 | python-dotenv==1.0.0 4 | -------------------------------------------------------------------------------- /prepare/.env: -------------------------------------------------------------------------------- 1 | DATABASE_URL=postgres://postgres:postgres@localhost:5432/arcania 2 | -------------------------------------------------------------------------------- /prepare/README.md: -------------------------------------------------------------------------------- 1 | # Prepare 2 | 3 | This directory can be used to test the performance of prepared statements. 4 | 5 | ## Setup 6 | 7 | - Python 3 8 | - pip 9 | - sqlx-cli 10 | 11 | 12 | To do so, first load up a virtual environment for python: 13 | 14 | ``` 15 | $ python -m venv .venv 16 | $ source .venv/bin/activate 17 | ``` 18 | 19 | then install the dependencies via pip 20 | 21 | ``` 22 | $ pip install -r requirements.txt 23 | ``` 24 | 25 | With python installed, you can set up the database using the `sqlx-cli` 26 | 27 | ``` 28 | $ sqlx migrate run 29 | ``` 30 | 31 | Finally, you can run the bench.py 32 | 33 | ``` 34 | $ python bench.py 35 | ``` 36 | -------------------------------------------------------------------------------- /prepare/bench.py: -------------------------------------------------------------------------------- 1 | import psycopg2 2 | from urllib.parse import urlparse 3 | import os 4 | from faker import Faker 5 | from dotenv import load_dotenv 6 | import random 7 | import time 8 | 9 | load_dotenv() 10 | 11 | conStr = os.getenv('DATABASE_URL', 'postgres://postgres:postgres@localhost:5432/postgres') 12 | p = urlparse(conStr) 13 | 14 | pg_connection_dict = { 15 | 'dbname': p.path[1:], 16 | 'user': p.username, 17 | 'password': p.password, 18 | 'port': p.port, 19 | 'host': p.hostname 20 | } 21 | 22 | conn = psycopg2.connect(**pg_connection_dict) 23 | 24 | 25 | MAX_LIMIT = 255 26 | MAX_NAME_LEN = 80 27 | 28 | fake = Faker() 29 | data = [] 30 | 31 | for i in range(100000): 32 | name = '' 33 | 34 | len = random.randint(0, MAX_NAME_LEN) 35 | for i in range(len): 36 | random_integer = random.randint(0, MAX_LIMIT) 37 | name += (chr(random_integer)) 38 | 39 | name = fake.job() 40 | date = fake.date_between(start_date='today', end_date='+30y') 41 | is_done = bool(random.getrandbits(1)) 42 | 43 | data.append((name, date, is_done)) 44 | 45 | cur = conn.cursor() 46 | cur.execute(""" 47 | PREPARE insert_task AS 48 | INSERT INTO task (name, due_date, is_done) 49 | VALUES ($1, $2, $3) 50 | """) 51 | cur.close() 52 | 53 | start = time.time() 54 | for row in data: 55 | cur = conn.cursor() 56 | cur.execute("EXECUTE insert_task (%s, %s, %s)", row, prepare=False) 57 | cur.close() 58 | 59 | end = time.time() 60 | print(f"using prepared statements: {end - start : .2f} seconds") 61 | 62 | start = time.time() 63 | for row in data: 64 | cur = conn.cursor() 65 | cur.execute("INSERT INTO task (name, due_date, is_done) VALUES (%s, %s, %s)", row, prepare=False) 66 | cur.close() 67 | 68 | end = time.time() 69 | print(f"direct queries: {end - start : .2f} seconds") 70 | 71 | cur = conn.cursor() 72 | cur.execute("DELETE FROM task") 73 | 74 | conn.commit() 75 | -------------------------------------------------------------------------------- /prepare/migrations/20230316225351_initial_tables_up.sql: -------------------------------------------------------------------------------- 1 | -- Add migration script here 2 | create table task ( 3 | id SERIAL PRIMARY KEY, 4 | name varchar not null, 5 | due_date date not null, 6 | is_done boolean not null default false, 7 | created_at timestamp NOT NULL DEFAULT now() 8 | ) 9 | -------------------------------------------------------------------------------- /prepare/requirements.txt: -------------------------------------------------------------------------------- 1 | Faker==17.6.0 2 | psycopg-binary==3.1.8 3 | python-dotenv==1.0.0 4 | -------------------------------------------------------------------------------- /prepare/test.sql: -------------------------------------------------------------------------------- 1 | PREPARE test_insert (text) AS 2 | INSERT INTO users (name) VALUES ($1); 3 | 4 | EXECUTE test_insert('User ' || :pgbench_tid); 5 | 6 | -------------------------------------------------------------------------------- /replicate/README.md: -------------------------------------------------------------------------------- 1 | # How to setup read repication with PostgreSQL. 2 | 3 | ## Setup Requirements 4 | 5 | - docker 6 | - docker-compose 7 | 8 | ## Usage 9 | 10 | With docker and docker composed installed run the following command: 11 | 12 | ``` 13 | $ docker-compose up 14 | ``` 15 | 16 | ### Adding a read user to the replica 17 | 18 | 1. Access the master container 19 | 20 | ``` 21 | $ docker-compose exec postgresql-master bash 22 | ``` 23 | 24 | 2. Access the psql client 25 | 26 | ``` 27 | $ psql -U postgres 28 | ``` 29 | 30 | 3. Run the SQL commands, creating the user `read_user` and giving them permisisons 31 | 32 | ``` 33 | CREATE USER read_user WITH PASSWORD 'reader_password'; 34 | GRANT CONNECT ON DATABASE my_database TO read_user; 35 | \connect my_database 36 | GRANT SELECT ON ALL TABLES IN SCHEMA public TO read_user; 37 | GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO read_user; 38 | GRANT USAGE ON SCHEMA public TO read_user; 39 | ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO read_user; 40 | ``` 41 | 42 | Restart all of the containers and possible connections. 43 | 44 | ## Acknowledgements 45 | 46 | Based off of the work by [JosimarCamargo](https://gist.github.com/JosimarCamargo/40f8636563c6e9ececf603e94c3affa7) 47 | 48 | -------------------------------------------------------------------------------- /replicate/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | postgresql-primary: 5 | image: 'docker.io/bitnami/postgresql:15' 6 | ports: 7 | - '5435:5432' 8 | volumes: 9 | - 'postgresql_primary_data:/bitnami/postgresql' 10 | environment: 11 | - POSTGRESQL_PGAUDIT_LOG=READ,WRITE 12 | - POSTGRESQL_LOG_HOSTNAME=true 13 | - POSTGRESQL_REPLICATION_MODE=master 14 | - POSTGRESQL_REPLICATION_USER=repl_user 15 | - POSTGRESQL_REPLICATION_PASSWORD=repl_password 16 | - POSTGRESQL_USERNAME=postgres 17 | - POSTGRESQL_DATABASE=my_database 18 | - ALLOW_EMPTY_PASSWORD=yes 19 | postgresql-replica: 20 | image: 'docker.io/bitnami/postgresql:15' 21 | ports: 22 | - '5433:5432' 23 | depends_on: 24 | - postgresql-primary 25 | environment: 26 | - POSTGRESQL_USERNAME=postgres 27 | - POSTGRESQL_PASSWORD=my_password 28 | - POSTGRESQL_MASTER_HOST=postgresql-primary 29 | - POSTGRESQL_PGAUDIT_LOG=READ,WRITE 30 | - POSTGRESQL_LOG_HOSTNAME=true 31 | - POSTGRESQL_REPLICATION_MODE=slave 32 | - POSTGRESQL_REPLICATION_USER=repl_user 33 | - POSTGRESQL_REPLICATION_PASSWORD=repl_password 34 | - POSTGRESQL_MASTER_PORT_NUMBER=5432 35 | 36 | volumes: 37 | postgresql_primary_data: 38 | driver: local 39 | --------------------------------------------------------------------------------