├── .gitignore ├── .vscode ├── c_cpp_properties.json ├── launch.json └── settings.json ├── Gopkg.lock ├── Gopkg.toml ├── README.md ├── boltexample ├── .gitignore ├── README.md └── main.go ├── chatserver ├── README.md ├── client │ ├── client.go │ └── tcp_client.go ├── protocol │ ├── command.go │ ├── reader.go │ ├── reader_test.go │ ├── writer.go │ └── writer_test.go ├── server │ ├── cmd │ │ └── main.go │ ├── server.go │ └── tcp_server.go └── tui │ ├── chatview.go │ ├── cmd │ └── main.go │ ├── loginview.go │ └── tui.go ├── dynonote ├── README.md ├── main.go ├── model │ └── model.go ├── restapi │ ├── item.go │ ├── server.go │ └── user.go └── service │ ├── notemanager.go │ ├── other.go │ └── star.go ├── echoserver ├── README.md ├── client │ └── client.go └── server │ └── server.go ├── externalmergesort ├── README.md ├── cmd │ ├── .gitignore │ └── main.go ├── quicksort.go ├── sort.go └── util.go ├── heap ├── README.md ├── heap.go └── heap_test.go ├── jni └── main.go ├── kms └── main.go ├── rocksdbexample ├── .gitignore ├── README.md ├── iterator.go ├── main.go ├── merge_operator.go ├── snapshot.go ├── transaction.go └── wrapper.go └── sys └── util.go /.gitignore: -------------------------------------------------------------------------------- 1 | vendor -------------------------------------------------------------------------------- /.vscode/c_cpp_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "Mac", 5 | "includePath": [ 6 | "${workspaceFolder}/**", 7 | "/Library/Java/JavaVirtualMachines/jdk1.8.0_121.jdk/Contents/Home/include/", 8 | "/Library/Java/JavaVirtualMachines/jdk1.8.0_121.jdk/Contents/Home/include/server" 9 | ], 10 | "defines": [], 11 | "macFrameworkPath": [ 12 | "/System/Library/Frameworks", 13 | "/Library/Frameworks" 14 | ], 15 | "compilerPath": "/usr/bin/clang", 16 | "cStandard": "c11", 17 | "cppStandard": "c++17", 18 | "intelliSenseMode": "clang-x64" 19 | } 20 | ], 21 | "version": 4 22 | } -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "auto", 12 | "remotePath": "", 13 | "port": 2345, 14 | "host": "127.0.0.1", 15 | "program": "${fileDirname}", 16 | "env": {}, 17 | "args": [], 18 | "showLog": false 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "files.associations": { 3 | "jni_wrapper.h": "c", 4 | "jni.h": "c" 5 | } 6 | } -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | digest = "1:3076a0751f5648f3ab1838abfba18bc5bf514cde7e3957500b62fba90633479a" 6 | name = "github.com/antlr/antlr4" 7 | packages = ["runtime/Go/antlr"] 8 | pruneopts = "UT" 9 | revision = "7a3f40bc341ddfb463d6e0aa1a6265064d020cb6" 10 | version = "4.8" 11 | 12 | [[projects]] 13 | branch = "master" 14 | digest = "1:d78b3e4f7ffa8d0bd2f4076d5fc2b68e7507ee407fcd3279b8c1f4f32938c1a0" 15 | name = "github.com/aws/aws-dax-go" 16 | packages = [ 17 | "dax", 18 | "dax/internal/cbor", 19 | "dax/internal/client", 20 | "dax/internal/lru", 21 | "dax/internal/parser", 22 | "dax/internal/parser/generated", 23 | ] 24 | pruneopts = "UT" 25 | revision = "2528715f78d8b40d5e7477e648069abf19cf3f8b" 26 | 27 | [[projects]] 28 | digest = "1:6f999996d0c3a89153df25da75b80cbd7a6784430a2971340aec246f82d921dc" 29 | name = "github.com/aws/aws-sdk-go" 30 | packages = [ 31 | "aws", 32 | "aws/awserr", 33 | "aws/awsutil", 34 | "aws/client", 35 | "aws/client/metadata", 36 | "aws/corehandlers", 37 | "aws/credentials", 38 | "aws/credentials/ec2rolecreds", 39 | "aws/credentials/endpointcreds", 40 | "aws/credentials/processcreds", 41 | "aws/credentials/stscreds", 42 | "aws/crr", 43 | "aws/csm", 44 | "aws/defaults", 45 | "aws/ec2metadata", 46 | "aws/endpoints", 47 | "aws/request", 48 | "aws/session", 49 | "aws/signer/v4", 50 | "internal/context", 51 | "internal/ini", 52 | "internal/sdkio", 53 | "internal/sdkmath", 54 | "internal/sdkrand", 55 | "internal/sdkuri", 56 | "internal/shareddefaults", 57 | "internal/strings", 58 | "internal/sync/singleflight", 59 | "private/protocol", 60 | "private/protocol/json/jsonutil", 61 | "private/protocol/jsonrpc", 62 | "private/protocol/query", 63 | "private/protocol/query/queryutil", 64 | "private/protocol/rest", 65 | "private/protocol/xml/xmlutil", 66 | "service/cognitoidentity", 67 | "service/dynamodb", 68 | "service/dynamodb/dynamodbattribute", 69 | "service/dynamodb/dynamodbiface", 70 | "service/dynamodb/expression", 71 | "service/kms", 72 | "service/sts", 73 | "service/sts/stsiface", 74 | ] 75 | pruneopts = "UT" 76 | revision = "1f57dbf66e23e62fa8b40039709c45afdd53e6d8" 77 | version = "v1.32.6" 78 | 79 | [[projects]] 80 | digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55" 81 | name = "github.com/dgrijalva/jwt-go" 82 | packages = ["."] 83 | pruneopts = "UT" 84 | revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e" 85 | version = "v3.2.0" 86 | 87 | [[projects]] 88 | branch = "master" 89 | digest = "1:40d0056c1b1f503c366ba441df92a82b5a2654d6f3747b1689a611eb5c9ce0a2" 90 | name = "github.com/gdamore/encoding" 91 | packages = ["."] 92 | pruneopts = "UT" 93 | revision = "b23993cbb6353f0e6aa98d0ee318a34728f628b9" 94 | 95 | [[projects]] 96 | digest = "1:00b9cce210566117aff926677c005aeaea6c85374e67bdcb72783af237c48f97" 97 | name = "github.com/gdamore/tcell" 98 | packages = [ 99 | ".", 100 | "terminfo", 101 | ] 102 | pruneopts = "UT" 103 | revision = "de7e78efa4a71b3f36c7154989c529dbdf9ae623" 104 | version = "v1.1.0" 105 | 106 | [[projects]] 107 | digest = "1:73770491417b7cb646b28610eec6481187aba7b5c3703da0d42f8d590d016f5f" 108 | name = "github.com/gofrs/uuid" 109 | packages = ["."] 110 | pruneopts = "UT" 111 | revision = "abfe1881e60ef34074c1b8d8c63b42565c356ed6" 112 | version = "v3.3.0" 113 | 114 | [[projects]] 115 | digest = "1:b117faf01abdbdadd7b8ceeae74cd1369c1b073fef41aee8b674c9d704a54c16" 116 | name = "github.com/gorilla/handlers" 117 | packages = ["."] 118 | pruneopts = "UT" 119 | revision = "8a3748addc242fc560bd6d4ff28b0374c010b1b4" 120 | version = "v1.4.2" 121 | 122 | [[projects]] 123 | digest = "1:25ebe6496abb289ef977c081b2d49f56dd97c32db4ca083d37f95923909ced02" 124 | name = "github.com/gorilla/mux" 125 | packages = ["."] 126 | pruneopts = "UT" 127 | revision = "75dcda0896e109a2a22c9315bca3bb21b87b2ba5" 128 | version = "v1.7.4" 129 | 130 | [[projects]] 131 | digest = "1:bb81097a5b62634f3e9fec1014657855610c82d19b9a40c17612e32651e35dca" 132 | name = "github.com/jmespath/go-jmespath" 133 | packages = ["."] 134 | pruneopts = "UT" 135 | revision = "c2b33e84" 136 | 137 | [[projects]] 138 | digest = "1:c65a16ac77d0b1aefc7009cabb6ac5ad05def02025f5be85f450c03f52cc6f86" 139 | name = "github.com/lucasb-eyer/go-colorful" 140 | packages = ["."] 141 | pruneopts = "UT" 142 | revision = "345fbb3dbcdb252d9985ee899a84963c0fa24c82" 143 | version = "v1.0" 144 | 145 | [[projects]] 146 | digest = "1:d8340a47046a0487cdc3f388eb5dad96998d3f6d44ef2c6dff3eb29010a16c28" 147 | name = "github.com/marcusolsson/tui-go" 148 | packages = [ 149 | ".", 150 | "wordwrap", 151 | ] 152 | pruneopts = "UT" 153 | revision = "cd94d703fac144c572eb929177adc5ae9a082e8e" 154 | 155 | [[projects]] 156 | digest = "1:cdb899c199f907ac9fb50495ec71212c95cb5b0e0a8ee0800da0238036091033" 157 | name = "github.com/mattn/go-runewidth" 158 | packages = ["."] 159 | pruneopts = "UT" 160 | revision = "ce7b0b5c7b45a81508558cd1dba6bb1e4ddb51bb" 161 | version = "v0.0.3" 162 | 163 | [[projects]] 164 | digest = "1:abf08734a6527df70ed361d7c369fb580e6840d8f7a6012e5f609fdfd93b4e48" 165 | name = "github.com/mitchellh/go-wordwrap" 166 | packages = ["."] 167 | pruneopts = "UT" 168 | revision = "9e67c67572bc5dd02aef930e2b0ae3c02a4b5a5c" 169 | version = "v1.0.0" 170 | 171 | [[projects]] 172 | branch = "master" 173 | digest = "1:e1c9609f03108cbbe7786b45858369809175c3d9d8bb319cabefe6e4f65bde8c" 174 | name = "github.com/nqbao/jnigo" 175 | packages = ["."] 176 | pruneopts = "UT" 177 | revision = "303e65215f3fb199eccbb781c8eac52b0732c707" 178 | 179 | [[projects]] 180 | digest = "1:860c25446f565f998eb698ddc033e9e43b2546eb950b20aca28f30ba73deb86d" 181 | name = "github.com/oklog/ulid" 182 | packages = ["."] 183 | pruneopts = "UT" 184 | revision = "be3bccf06dda9a40aa6800c4736ac77f2fef987f" 185 | version = "v2.0.2" 186 | 187 | [[projects]] 188 | branch = "master" 189 | digest = "1:1e38f591bdbaa29feb1ae8f05822307ff1f36c5a06f439477c424b1c749db5ed" 190 | name = "github.com/tecbot/gorocksdb" 191 | packages = ["."] 192 | pruneopts = "UT" 193 | revision = "8752a943348155073ae407010e2b75c40480271a" 194 | 195 | [[projects]] 196 | digest = "1:c28625428387b63dd7154eb857f51e700465cfbf7c06f619e71f2da33cefe47e" 197 | name = "go.etcd.io/bbolt" 198 | packages = ["."] 199 | pruneopts = "UT" 200 | revision = "583e8937c61f1af6513608ccc75c97b6abdf4ff9" 201 | version = "v1.3.0" 202 | 203 | [[projects]] 204 | branch = "master" 205 | digest = "1:008fb9f84ae9bf7994228b9c78810d44162071ec13caf28291677897dca819ee" 206 | name = "golang.org/x/sys" 207 | packages = ["unix"] 208 | pruneopts = "UT" 209 | revision = "d989b31c87461dc8ab2f1cac6792814e27fadea9" 210 | 211 | [[projects]] 212 | digest = "1:37672ad5821719e2df8509c2edd4ba5ae192463237c73c3a2d24ef8b2bc9e36f" 213 | name = "golang.org/x/text" 214 | packages = [ 215 | "encoding", 216 | "encoding/internal/identifier", 217 | "internal/gen", 218 | "transform", 219 | "unicode/cldr", 220 | ] 221 | pruneopts = "UT" 222 | revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" 223 | version = "v0.3.0" 224 | 225 | [solve-meta] 226 | analyzer-name = "dep" 227 | analyzer-version = 1 228 | input-imports = [ 229 | "github.com/aws/aws-dax-go/dax", 230 | "github.com/aws/aws-sdk-go/aws", 231 | "github.com/aws/aws-sdk-go/aws/credentials", 232 | "github.com/aws/aws-sdk-go/aws/session", 233 | "github.com/aws/aws-sdk-go/service/cognitoidentity", 234 | "github.com/aws/aws-sdk-go/service/dynamodb", 235 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute", 236 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface", 237 | "github.com/aws/aws-sdk-go/service/dynamodb/expression", 238 | "github.com/aws/aws-sdk-go/service/kms", 239 | "github.com/dgrijalva/jwt-go", 240 | "github.com/gorilla/handlers", 241 | "github.com/gorilla/mux", 242 | "github.com/marcusolsson/tui-go", 243 | "github.com/nqbao/jnigo", 244 | "github.com/oklog/ulid", 245 | "github.com/tecbot/gorocksdb", 246 | "go.etcd.io/bbolt", 247 | ] 248 | solver-name = "gps-cdcl" 249 | solver-version = 1 250 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | # Gopkg.toml example 2 | # 3 | # Refer to https://golang.github.io/dep/docs/Gopkg.toml.html 4 | # for detailed Gopkg.toml documentation. 5 | # 6 | # required = ["github.com/user/thing/cmd/thing"] 7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 8 | # 9 | # [[constraint]] 10 | # name = "github.com/user/project" 11 | # version = "1.0.0" 12 | # 13 | # [[constraint]] 14 | # name = "github.com/user/project2" 15 | # branch = "dev" 16 | # source = "github.com/myfork/project2" 17 | # 18 | # [[override]] 19 | # name = "github.com/x/y" 20 | # version = "2.4.0" 21 | # 22 | # [prune] 23 | # non-go = false 24 | # go-tests = true 25 | # unused-packages = true 26 | 27 | 28 | [prune] 29 | go-tests = true 30 | unused-packages = true 31 | 32 | [[constraint]] 33 | name = "github.com/marcusolsson/tui-go" 34 | revision = "cd94d703fac144c572eb929177adc5ae9a082e8e" 35 | 36 | [[constraint]] 37 | branch = "master" 38 | name = "github.com/tecbot/gorocksdb" 39 | 40 | [[constraint]] 41 | branch = "master" 42 | name = "github.com/aws/aws-dax-go" 43 | 44 | [[constraint]] 45 | name = "github.com/oklog/ulid" 46 | version = "2.0.2" 47 | 48 | [[constraint]] 49 | name = "github.com/dgrijalva/jwt-go" 50 | version = "3.2.0" 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Learning Go 2 | 3 | This is just a sandbox repository for me to put in random programming, mostly for me to learn go. The motivation is that I want to learn a new backend language to be able to write high-performance backend services, and there are many projects (k8s, docker, normad) are written in go. If i understand Go it will be easier for me to understand these projects. 4 | 5 | 6 | What i like about Go: 7 | 8 | * Clear and nice syntax. 9 | * Channel and goroutine is awesome, a lot more cleaner than in python where i have to use Condition for cordination. 10 | * Concurrency is easy to reason. Everything is blocking, unless you let it `go`. 11 | * Strong-typed 12 | * Fast compilation 13 | 14 | What I don't like about Go: 15 | 16 | * Single workspace is a confusing. 17 | * Error handling is just awful. It's too bad that Go does not have a try / catch block. 18 | * JSON deserialization is hard if you deal with generic interface 19 | * You tend to write more code due to the verbose syntax of Go 20 | * Lacking of Generic (coming soon) 21 | 22 | ## I/O 23 | 24 | * [Use bytes.Buffer for string writer](https://stackoverflow.com/questions/13765797/the-best-way-to-get-a-string-from-a-writer-in-go) 25 | * Use strings.Reader for string reader 26 | 27 | ## Channels 28 | 29 | It’s okie to leave channel open. GC will collect it 30 | 31 | * http://www.tapirgames.com/blog/golang-channel-closing 32 | 33 | ## Ser/der 34 | 35 | * [Gobs of data](https://blog.golang.org/gobs-of-data) 36 | * [Handling interace{} decoding with Gob](https://play.golang.org/p/xt4zNyPZ2W) 37 | * [Go codec series](http://ugorji.net/d/tag/go-codec/blog/) 38 | 39 | ## Slices 40 | 41 | * [Go slice is passed as reference](https://stackoverflow.com/questions/2439453/using-a-pointer-to-array). 42 | * [Use copy() to copy slice](https://stackoverflow.com/questions/30182538/why-can-not-i-duplicate-a-slice-with-copy-in-golang) 43 | * Use `[]byte(str)` to convert string to byte array. 44 | * Use `string(bytes)` to convert byte to string. 45 | * [Remove an item in slice](https://vbauerster.github.io/2017/04/removing-items-from-a-slice-while-iterating-in-go/) 46 | * [Use reflect.DeepEqual to compare two slices](https://yourbasic.org/golang/compare-slices/) 47 | 48 | ## Links 49 | 50 | * Package name convention 51 | * https://rakyll.org/style-packages 52 | * https://blog.golang.org/package-names 53 | * [Dependencies management with Dep](https://golang.github.io/dep) 54 | * [Should I commit vendor folder?](https://github.com/golang/dep/blob/master/docs/FAQ.md#should-i-commit-my-vendor-directory) 55 | * [appliedgo.net](https://appliedgo.net/tui/) 56 | * [Go by examples: Errors](https://gobyexample.com/errors) 57 | * [Traps, Gotchas, and Common Mistakes for New Golang Devs](http://devs.cloudimmunity.com/gotchas-and-common-mistakes-in-go-golang/) 58 | -------------------------------------------------------------------------------- /boltexample/.gitignore: -------------------------------------------------------------------------------- 1 | # test database file 2 | db 3 | -------------------------------------------------------------------------------- /boltexample/README.md: -------------------------------------------------------------------------------- 1 | # Bolt 2 | 3 | An example of trying [Bolt](https://github.com/etcd-io/bbolt) 4 | 5 | I prefer this because this is purely in Go, no need to compile external library like LevelDB or RocksDB. 6 | 7 | Remarks: 8 | 9 | * ACID support. 10 | * But do we really need ACID at this level? 11 | * Since this is a single process application. ACID can be achieved easily using a lock. 12 | * Use Cursor.Last() & Cursor.Prev() to navigate from the end of the key space. 13 | * Read() guarantees to work 14 | * Batch() blocks until all transactions finished. The transaction function may be called multiple time so the function must be idempotent 15 | 16 | References: 17 | * https://mycodesmells.com/post/first-steps-with-boltdb 18 | * [Badger vs Boltdb](https://blog.dgraph.io/post/badger-lmdb-boltdb/) 19 | * [The new InfluxDB storage engine: from LSM Tree to B+Tree and back again to create the Time Structured Merge Tree](https://docs.influxdata.com/influxdb/v1.6/concepts/storage_engine/) 20 | -------------------------------------------------------------------------------- /boltexample/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strconv" 7 | "time" 8 | 9 | bolt "go.etcd.io/bbolt" 10 | ) 11 | 12 | func main() { 13 | db, err := bolt.Open("./db", 0666, nil) 14 | 15 | if err != nil { 16 | panic(err) 17 | } 18 | 19 | defer db.Close() 20 | 21 | err = db.Update(func(tx *bolt.Tx) error { 22 | b, err := tx.CreateBucketIfNotExists([]byte("test")) 23 | 24 | if err != nil { 25 | return err 26 | } 27 | 28 | return b.Put([]byte("hello"), []byte("world")) 29 | }) 30 | 31 | if err != nil { 32 | panic(err) 33 | } 34 | 35 | err = db.Update(func(tx *bolt.Tx) error { 36 | b := tx.Bucket([]byte("test")) 37 | 38 | next, _ := b.NextSequence() 39 | 40 | return b.Put( 41 | []byte(time.Now().Format(time.RFC3339)), 42 | []byte(strconv.Itoa(int(next))), 43 | ) 44 | }) 45 | 46 | if err != nil { 47 | panic(err) 48 | } 49 | 50 | // try reading 51 | err = db.View(func(tx *bolt.Tx) error { 52 | c := tx.Bucket([]byte("test")).Cursor() 53 | 54 | // range read 55 | for k, v := c.Seek([]byte("2018-10")); k != nil && bytes.HasPrefix(k, []byte("2018-10")); k, v = c.Next() { 56 | fmt.Printf("key: %s, value: %s\n", k, v) 57 | } 58 | 59 | // reverse reading 60 | fmt.Println("reverse scanning") 61 | for k, _ := c.Last(); k != nil; k, _ = c.Prev() { 62 | fmt.Printf("key: %s\n", k) 63 | } 64 | 65 | return nil 66 | }) 67 | 68 | if err != nil { 69 | panic(err) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /chatserver/README.md: -------------------------------------------------------------------------------- 1 | # Simple Chat Server 2 | 3 | A simple chat server written in Golang, with very basic features: 4 | 5 | * There is only a single chat room for now 6 | * User can connect to the chat server 7 | * User can set their name 8 | * User can send message to the chat room 9 | 10 | ## Protocol 11 | 12 | For this excersie , I will use simple text-based message over TCP: 13 | 14 | * All messages are terminated with `\n` 15 | * To send a chat message, client will send: 16 | * `SEND chat message` 17 | * For now, chat message can not contain new line. 18 | * To set client name, client will send: 19 | * `NAME username` 20 | * For now, username can not contain space 21 | * Server will send the following command to all clients when there are new message: 22 | * `MESSAGE username the actual message` 23 | 24 | Later on I will try protobuf, or Gob to write protocol. 25 | 26 | ## References 27 | 28 | * https://gist.github.com/drewolson/3950226 29 | * https://scotch.io/bar-talk/build-a-realtime-chat-server-with-go-and-websockets 30 | * [tui-go](https://github.com/marcusolsson/tui-go) for chat client. tview may be a better option, but textare input is not available yet. 31 | -------------------------------------------------------------------------------- /chatserver/client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import "github.com/nqbao/learn-go/chatserver/protocol" 4 | 5 | type ChatClient interface { 6 | Dial(address string) error 7 | Start() 8 | Close() 9 | Send(command interface{}) error 10 | SetName(name string) error 11 | SendMessage(message string) error 12 | Error() chan error 13 | Incoming() chan protocol.MessageCommand 14 | } 15 | -------------------------------------------------------------------------------- /chatserver/client/tcp_client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "log" 5 | "net" 6 | 7 | "github.com/nqbao/learn-go/chatserver/protocol" 8 | ) 9 | 10 | type TcpChatClient struct { 11 | conn net.Conn 12 | cmdReader *protocol.CommandReader 13 | cmdWriter *protocol.CommandWriter 14 | name string 15 | error chan error 16 | incoming chan protocol.MessageCommand 17 | } 18 | 19 | func NewClient() *TcpChatClient { 20 | return &TcpChatClient{ 21 | incoming: make(chan protocol.MessageCommand), 22 | error: make(chan error), 23 | } 24 | } 25 | 26 | func (c *TcpChatClient) Dial(address string) error { 27 | conn, err := net.Dial("tcp", address) 28 | 29 | if err == nil { 30 | c.conn = conn 31 | c.cmdReader = protocol.NewCommandReader(conn) 32 | c.cmdWriter = protocol.NewCommandWriter(conn) 33 | } 34 | 35 | return err 36 | } 37 | 38 | func (c *TcpChatClient) Start() { 39 | for { 40 | cmd, err := c.cmdReader.Read() 41 | 42 | if err != nil { 43 | c.error <- err 44 | break // TODO: find a way to recover from this 45 | } 46 | 47 | if cmd != nil { 48 | switch v := cmd.(type) { 49 | case protocol.MessageCommand: 50 | c.incoming <- v 51 | default: 52 | log.Printf("Unknown command: %v", v) 53 | } 54 | } 55 | } 56 | } 57 | 58 | func (c *TcpChatClient) Close() { 59 | c.conn.Close() 60 | } 61 | 62 | func (c *TcpChatClient) Incoming() chan protocol.MessageCommand { 63 | return c.incoming 64 | } 65 | 66 | func (c *TcpChatClient) Error() chan error { 67 | return c.error 68 | } 69 | 70 | func (c *TcpChatClient) Send(command interface{}) error { 71 | return c.cmdWriter.Write(command) 72 | } 73 | 74 | func (c *TcpChatClient) SetName(name string) error { 75 | return c.Send(protocol.NameCommand{name}) 76 | } 77 | 78 | func (c *TcpChatClient) SendMessage(message string) error { 79 | return c.Send(protocol.SendCommand{ 80 | Message: message, 81 | }) 82 | } 83 | -------------------------------------------------------------------------------- /chatserver/protocol/command.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import "errors" 4 | 5 | var ( 6 | UnknownCommand = errors.New("Unknown command") 7 | ) 8 | 9 | // SendCommand is used for sending new message from client 10 | type SendCommand struct { 11 | Message string 12 | } 13 | 14 | // NameCommand is used for setting client display name 15 | type NameCommand struct { 16 | Name string 17 | } 18 | 19 | // MessageCommand is used for notifying new messages 20 | type MessageCommand struct { 21 | Name string 22 | Message string 23 | } 24 | -------------------------------------------------------------------------------- /chatserver/protocol/reader.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "log" 7 | ) 8 | 9 | type CommandReader struct { 10 | reader *bufio.Reader 11 | } 12 | 13 | func NewCommandReader(reader io.Reader) *CommandReader { 14 | return &CommandReader{ 15 | reader: bufio.NewReader(reader), 16 | } 17 | } 18 | 19 | func (r *CommandReader) Read() (interface{}, error) { 20 | // Read the first part 21 | commandName, err := r.reader.ReadString(' ') 22 | 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | switch commandName { 28 | case "MESSAGE ": 29 | user, err := r.reader.ReadString(' ') 30 | 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | message, err := r.reader.ReadString('\n') 36 | 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | return MessageCommand{ 42 | user[:len(user)-1], 43 | message[:len(message)-1], 44 | }, nil 45 | 46 | case "SEND ": 47 | message, err := r.reader.ReadString('\n') 48 | 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | return SendCommand{message[:len(message)-1]}, nil 54 | 55 | case "NAME ": 56 | name, err := r.reader.ReadString('\n') 57 | 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | return NameCommand{name[:len(name)-1]}, nil 63 | 64 | default: 65 | log.Printf("Unknown command: %v", commandName) 66 | } 67 | 68 | return nil, UnknownCommand 69 | } 70 | 71 | func (r *CommandReader) ReadAll() ([]interface{}, error) { 72 | commands := []interface{}{} 73 | 74 | for { 75 | command, err := r.Read() 76 | 77 | if command != nil { 78 | commands = append(commands, command) 79 | } 80 | 81 | if err == io.EOF { 82 | break 83 | } else if err != nil { 84 | return commands, err 85 | } 86 | } 87 | 88 | return commands, nil 89 | } 90 | -------------------------------------------------------------------------------- /chatserver/protocol/reader_test.go: -------------------------------------------------------------------------------- 1 | package protocol_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/nqbao/learn-go/chatserver/protocol" 9 | ) 10 | 11 | func TestCommandReader(t *testing.T) { 12 | tests := []struct { 13 | input string 14 | results []interface{} 15 | }{ 16 | { 17 | "SEND test\n", 18 | []interface{}{ 19 | protocol.SendCommand{"test"}, 20 | }, 21 | }, 22 | { 23 | "MESSAGE user1 hello\nMESSAGE user2 world\n", 24 | []interface{}{ 25 | protocol.MessageCommand{"user1", "hello"}, 26 | protocol.MessageCommand{"user2", "world"}, 27 | }, 28 | }, 29 | } 30 | 31 | for _, test := range tests { 32 | reader := protocol.NewCommandReader(strings.NewReader(test.input)) 33 | results, err := reader.ReadAll() 34 | 35 | t.Log(results) 36 | 37 | if err != nil { 38 | t.Errorf("Unable to read command, error %v", err) 39 | } else if !reflect.DeepEqual(results, test.results) { 40 | t.Errorf("Command output is not the same: %v %v", results, test.results) 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /chatserver/protocol/writer.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | ) 7 | 8 | type CommandWriter struct { 9 | writer io.Writer 10 | } 11 | 12 | func NewCommandWriter(writer io.Writer) *CommandWriter { 13 | return &CommandWriter{ 14 | writer: writer, 15 | } 16 | } 17 | 18 | func (w *CommandWriter) writeString(msg string) error { 19 | _, err := w.writer.Write([]byte(msg)) 20 | 21 | return err 22 | } 23 | 24 | func (w *CommandWriter) Write(command interface{}) error { 25 | // naive implementation ... 26 | var err error 27 | 28 | switch v := command.(type) { 29 | case SendCommand: 30 | err = w.writeString(fmt.Sprintf("SEND %v\n", v.Message)) 31 | case MessageCommand: 32 | err = w.writeString(fmt.Sprintf("MESSAGE %v %v\n", v.Name, v.Message)) 33 | case NameCommand: 34 | err = w.writeString(fmt.Sprintf("NAME %v\n", v.Name)) 35 | default: 36 | err = UnknownCommand 37 | } 38 | 39 | return err 40 | } 41 | -------------------------------------------------------------------------------- /chatserver/protocol/writer_test.go: -------------------------------------------------------------------------------- 1 | package protocol_test 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/nqbao/learn-go/chatserver/protocol" 8 | ) 9 | 10 | func TestWriteCommand(t *testing.T) { 11 | tests := []struct { 12 | commands []interface{} 13 | result string 14 | }{ 15 | { 16 | []interface{}{ 17 | protocol.SendCommand{"Hello"}, 18 | }, 19 | "SEND Hello\n", 20 | }, 21 | } 22 | 23 | buf := new(bytes.Buffer) 24 | for _, test := range tests { 25 | buf.Reset() 26 | cmdWriter := protocol.NewCommandWriter(buf) 27 | 28 | for _, cmd := range test.commands { 29 | if cmdWriter.Write(cmd) != nil { 30 | t.Errorf("Unable to write command %v", cmd) 31 | } 32 | } 33 | 34 | if buf.String() != test.result { 35 | t.Errorf("Command output is not the same: %v %v", buf.String(), test.result) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /chatserver/server/cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/nqbao/learn-go/chatserver/server" 5 | ) 6 | 7 | func main() { 8 | var s server.ChatServer 9 | s = server.NewServer() 10 | s.Listen(":3333") 11 | 12 | // start the server 13 | s.Start() 14 | } 15 | -------------------------------------------------------------------------------- /chatserver/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | type ChatServer interface { 4 | Listen(address string) error 5 | Broadcast(command interface{}) error 6 | Start() 7 | Close() 8 | } 9 | -------------------------------------------------------------------------------- /chatserver/server/tcp_server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "log" 7 | "net" 8 | "sync" 9 | 10 | "github.com/nqbao/learn-go/chatserver/protocol" 11 | ) 12 | 13 | type client struct { 14 | conn net.Conn 15 | name string 16 | writer *protocol.CommandWriter 17 | } 18 | 19 | type TcpChatServer struct { 20 | listener net.Listener 21 | clients []*client 22 | mutex *sync.Mutex 23 | } 24 | 25 | var ( 26 | UnknownClient = errors.New("Unknown client") 27 | ) 28 | 29 | func NewServer() *TcpChatServer { 30 | return &TcpChatServer{ 31 | mutex: &sync.Mutex{}, 32 | } 33 | } 34 | 35 | func (s *TcpChatServer) Listen(address string) error { 36 | l, err := net.Listen("tcp", address) 37 | 38 | if err == nil { 39 | s.listener = l 40 | } 41 | 42 | log.Printf("Listening on %v", address) 43 | 44 | return err 45 | } 46 | 47 | func (s *TcpChatServer) Close() { 48 | s.listener.Close() 49 | } 50 | 51 | func (s *TcpChatServer) Start() { 52 | for { 53 | // XXX: need a way to break the loop 54 | conn, err := s.listener.Accept() 55 | 56 | if err != nil { 57 | log.Print(err) 58 | } else { 59 | // handle connection 60 | client := s.accept(conn) 61 | go s.serve(client) 62 | } 63 | } 64 | } 65 | 66 | func (s *TcpChatServer) Broadcast(command interface{}) error { 67 | for _, client := range s.clients { 68 | // TODO: handle error here? 69 | client.writer.Write(command) 70 | } 71 | 72 | return nil 73 | } 74 | 75 | func (s *TcpChatServer) Send(name string, command interface{}) error { 76 | for _, client := range s.clients { 77 | if client.name == name { 78 | return client.writer.Write(command) 79 | } 80 | } 81 | 82 | return UnknownClient 83 | } 84 | 85 | func (s *TcpChatServer) accept(conn net.Conn) *client { 86 | log.Printf("Accepting connection from %v, total clients: %v", conn.RemoteAddr().String(), len(s.clients)+1) 87 | 88 | s.mutex.Lock() 89 | defer s.mutex.Unlock() 90 | 91 | client := &client{ 92 | conn: conn, 93 | writer: protocol.NewCommandWriter(conn), 94 | } 95 | 96 | s.clients = append(s.clients, client) 97 | 98 | return client 99 | } 100 | 101 | func (s *TcpChatServer) remove(client *client) { 102 | s.mutex.Lock() 103 | defer s.mutex.Unlock() 104 | 105 | // remove the connections from clients array 106 | for i, check := range s.clients { 107 | if check == client { 108 | s.clients = append(s.clients[:i], s.clients[i+1:]...) 109 | } 110 | } 111 | 112 | log.Printf("Closing connection from %v", client.conn.RemoteAddr().String()) 113 | client.conn.Close() 114 | } 115 | 116 | func (s *TcpChatServer) serve(client *client) { 117 | cmdReader := protocol.NewCommandReader(client.conn) 118 | 119 | defer s.remove(client) 120 | 121 | for { 122 | cmd, err := cmdReader.Read() 123 | 124 | if err != nil && err != io.EOF { 125 | log.Printf("Read error: %v", err) 126 | } 127 | 128 | if cmd != nil { 129 | switch v := cmd.(type) { 130 | case protocol.SendCommand: 131 | go s.Broadcast(protocol.MessageCommand{ 132 | Message: v.Message, 133 | Name: client.name, 134 | }) 135 | 136 | case protocol.NameCommand: 137 | client.name = v.Name 138 | } 139 | } 140 | 141 | if err == io.EOF { 142 | break 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /chatserver/tui/chatview.go: -------------------------------------------------------------------------------- 1 | package tui 2 | 3 | import ( 4 | tui "github.com/marcusolsson/tui-go" 5 | ) 6 | 7 | type SubmitMessageHandler func(string) 8 | 9 | type ChatView struct { 10 | tui.Box 11 | frame *tui.Box 12 | history *tui.Box 13 | onSubmit SubmitMessageHandler 14 | } 15 | 16 | func NewChatView() *ChatView { 17 | view := &ChatView{} 18 | 19 | // ref: https://github.com/marcusolsson/tui-go/blob/master/example/chat/main.go 20 | view.history = tui.NewVBox() 21 | 22 | historyScroll := tui.NewScrollArea(view.history) 23 | historyScroll.SetAutoscrollToBottom(true) 24 | 25 | historyBox := tui.NewVBox(historyScroll) 26 | historyBox.SetBorder(true) 27 | 28 | input := tui.NewEntry() 29 | input.SetFocused(true) 30 | input.SetSizePolicy(tui.Expanding, tui.Maximum) 31 | 32 | input.OnSubmit(func(e *tui.Entry) { 33 | if e.Text() != "" { 34 | if view.onSubmit != nil { 35 | view.onSubmit(e.Text()) 36 | } 37 | 38 | e.SetText("") 39 | } 40 | }) 41 | 42 | inputBox := tui.NewHBox(input) 43 | inputBox.SetBorder(true) 44 | inputBox.SetSizePolicy(tui.Expanding, tui.Maximum) 45 | 46 | view.frame = tui.NewVBox( 47 | historyBox, 48 | inputBox, 49 | ) 50 | 51 | view.frame.SetBorder(false) 52 | view.Append(view.frame) 53 | 54 | return view 55 | } 56 | 57 | func (c *ChatView) OnSubmit(handler SubmitMessageHandler) { 58 | c.onSubmit = handler 59 | } 60 | 61 | func (c *ChatView) AddMessage(msg string) { 62 | c.history.Append( 63 | tui.NewHBox( 64 | tui.NewLabel(msg), 65 | ), 66 | ) 67 | } 68 | -------------------------------------------------------------------------------- /chatserver/tui/cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | 7 | "github.com/nqbao/learn-go/chatserver/client" 8 | "github.com/nqbao/learn-go/chatserver/tui" 9 | ) 10 | 11 | func main() { 12 | address := flag.String("server", "localhost:3333", "Which server to connect to") 13 | 14 | flag.Parse() 15 | 16 | client := client.NewClient() 17 | err := client.Dial(*address) 18 | 19 | if err != nil { 20 | log.Fatal(err) 21 | } 22 | 23 | defer client.Close() 24 | 25 | // start the client to listen for incoming message 26 | go client.Start() 27 | 28 | tui.StartUi(client) 29 | } 30 | -------------------------------------------------------------------------------- /chatserver/tui/loginview.go: -------------------------------------------------------------------------------- 1 | package tui 2 | 3 | import tui "github.com/marcusolsson/tui-go" 4 | 5 | type LoginHandler func(string) 6 | 7 | type LoginView struct { 8 | tui.Box 9 | frame *tui.Box 10 | loginHandler LoginHandler 11 | } 12 | 13 | func NewLoginView() *LoginView { 14 | // https://github.com/marcusolsson/tui-go/blob/master/example/login/main.go 15 | user := tui.NewEntry() 16 | user.SetFocused(true) 17 | user.SetSizePolicy(tui.Maximum, tui.Maximum) 18 | 19 | label := tui.NewLabel("Enter your name: ") 20 | user.SetSizePolicy(tui.Expanding, tui.Maximum) 21 | 22 | userBox := tui.NewHBox( 23 | label, 24 | user, 25 | ) 26 | userBox.SetBorder(true) 27 | userBox.SetSizePolicy(tui.Expanding, tui.Maximum) 28 | 29 | view := &LoginView{} 30 | view.frame = tui.NewVBox( 31 | tui.NewSpacer(), 32 | tui.NewPadder(-4, 0, tui.NewPadder(4, 0, userBox)), 33 | tui.NewSpacer(), 34 | ) 35 | view.Append(view.frame) 36 | 37 | user.OnSubmit(func(e *tui.Entry) { 38 | if e.Text() != "" { 39 | if view.loginHandler != nil { 40 | view.loginHandler(e.Text()) 41 | } 42 | 43 | e.SetText("") 44 | } 45 | }) 46 | 47 | return view 48 | } 49 | 50 | func (v *LoginView) OnLogin(handler LoginHandler) { 51 | v.loginHandler = handler 52 | } 53 | -------------------------------------------------------------------------------- /chatserver/tui/tui.go: -------------------------------------------------------------------------------- 1 | package tui 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/marcusolsson/tui-go" 8 | "github.com/nqbao/learn-go/chatserver/client" 9 | ) 10 | 11 | func StartUi(c client.ChatClient) { 12 | loginView := NewLoginView() 13 | chatView := NewChatView() 14 | 15 | ui, err := tui.New(loginView) 16 | if err != nil { 17 | panic(err) 18 | } 19 | 20 | quit := func() { ui.Quit() } 21 | 22 | ui.SetKeybinding("Esc", quit) 23 | ui.SetKeybinding("Ctrl+c", quit) 24 | 25 | loginView.OnLogin(func(username string) { 26 | c.SetName(username) 27 | ui.SetWidget(chatView) 28 | }) 29 | 30 | chatView.OnSubmit(func(msg string) { 31 | c.SendMessage(msg) 32 | }) 33 | 34 | go func() { 35 | for { 36 | select { 37 | case err := <-c.Error(): 38 | 39 | if err == io.EOF { 40 | ui.Update(func() { 41 | chatView.AddMessage("Connection closed connection from server.") 42 | }) 43 | } else { 44 | panic(err) 45 | } 46 | case msg := <-c.Incoming(): 47 | // we need to make the change via ui update to make sure the ui is repaint correctly 48 | ui.Update(func() { 49 | chatView.AddMessage(fmt.Sprintf("%v: %v", msg.Name, msg.Message)) 50 | }) 51 | } 52 | } 53 | }() 54 | 55 | if err := ui.Run(); err != nil { 56 | panic(err) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /dynonote/README.md: -------------------------------------------------------------------------------- 1 | # Sample application for using DynamoDB in Go 2 | 3 | Adapted from [Udemy dynamodb course](https://udemy.com/course/dynamodb) 4 | 5 | Sample query: 6 | 7 | * Login / logout: aws cognito ✅ 8 | * Read/Write new note per user: UID,ULID ✅ 9 | * Get all notes: Scan ✅ (Not in UI) 10 | * Query notes of a single user: UID ✅ 11 | * Add note to category: Update using UID, ULID (not in UI) 12 | * Query note per category: GSI (not in UI) 13 | * Star/unstar notes: GSI ✅ (not in UI) 14 | -------------------------------------------------------------------------------- /dynonote/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/nqbao/learn-go/dynonote/model" 9 | "github.com/nqbao/learn-go/dynonote/restapi" 10 | "github.com/nqbao/learn-go/dynonote/service" 11 | ) 12 | 13 | func testHeavyWrite(user string) { 14 | // test heavy writing 15 | 16 | wg := sync.WaitGroup{} 17 | 18 | count := 100 19 | workers := 2 20 | 21 | nm := service.NewNoteManager(nil) 22 | 23 | for z := 0; z < workers; z++ { 24 | go func(starter int) { 25 | t := time.Now() 26 | 27 | for i := 0; i < count; i++ { 28 | note := &model.Note{} 29 | note.UserKey = user 30 | note.Title = fmt.Sprintf("my note %v", z*count+i) 31 | note.Content = "world" 32 | nm.CreateNote(note) 33 | 34 | if i > 0 && i%100 == 0 { 35 | fmt.Printf("Write %v items in %v seconds\n", i, (time.Now().Unix() - t.Unix())) 36 | 37 | t = time.Now() 38 | } 39 | } 40 | 41 | wg.Done() 42 | }(z) 43 | 44 | wg.Add(1) 45 | } 46 | 47 | wg.Wait() 48 | } 49 | 50 | func testHeavyRead() { 51 | t := time.Now() 52 | for i := 0; i < 100; i++ { 53 | l, err := service.GetAllNotes() 54 | 55 | if err != nil { 56 | fmt.Printf("%v\n", err) 57 | } else { 58 | fmt.Printf("Read %v in %v seconds\n", len(l), (time.Now().Unix() - t.Unix())) 59 | } 60 | 61 | t = time.Now() 62 | } 63 | } 64 | 65 | func main() { 66 | 67 | // note := service.GetNote("test", "01EB5GRBK31TZ37GA3KKC29SQG") 68 | // if note != nil { 69 | // fmt.Printf(note.Content) 70 | // note.Content = "again two three four" 71 | 72 | // service.UpdateNote(note) 73 | // } 74 | 75 | // service.DeleteNote("khanh", "01EB9C4K6PSR2JYHEJ5D39DEAQ") 76 | // service.StarNote("khanh", "01EB9C4K45AZ0PMDNFEC41YMWF", 0) 77 | // n := service.GetNote("khanh", "01EB9C4K45AZ0PMDNFEC41YMWF") 78 | // if n != nil { 79 | // fmt.Printf("%v", n) 80 | //} 81 | 82 | // notes := service.GetStarNotes("khanh") 83 | // for _, n := range notes { 84 | // fmt.Printf("%v %v %v\n", n.ULID, n.UserKey, n.Title) 85 | // } 86 | 87 | // testHeavyRead() 88 | 89 | restapi.StartServer() 90 | } 91 | -------------------------------------------------------------------------------- /dynonote/model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type Note struct { 4 | UserKey string `dynamodbav:"user_id" json:"user_id"` // hashkey 5 | Timestamp int64 `dynamodbav:"timestamp" json:"timestamp"` // sortkey 6 | UserName string `dynamodbav:"username" json:"username"` 7 | ULID string `dynamodbav:"note_id" json:"note_id"` 8 | Title string `dynamodbav:"title" json:"title"` 9 | Content string `dynamodbav:"content" json:"content"` 10 | CategoryKey string `dynamodbav:"cat,omitempty" json:"cat,omitempty"` 11 | Star int `dynamodb:"star,omitempty" json:"star,omitempty"` 12 | } 13 | -------------------------------------------------------------------------------- /dynonote/restapi/item.go: -------------------------------------------------------------------------------- 1 | package restapi 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "strconv" 8 | 9 | "github.com/gorilla/mux" 10 | "github.com/nqbao/learn-go/dynonote/model" 11 | "github.com/nqbao/learn-go/dynonote/service" 12 | ) 13 | 14 | type PostItemRequest struct { 15 | Item *model.Note 16 | } 17 | 18 | func postNote(w http.ResponseWriter, r *http.Request) { 19 | nm := service.NewNoteManager(getAwsCredentials(r)) 20 | 21 | req := &PostItemRequest{} 22 | writer := json.NewEncoder(w) 23 | err := json.NewDecoder(r.Body).Decode(&req) 24 | 25 | if err != nil { 26 | w.WriteHeader(http.StatusInternalServerError) 27 | writer.Encode(map[string]string{ 28 | "status": "error", 29 | "message": fmt.Sprintf("%v", err), 30 | }) 31 | } else { 32 | note := req.Item 33 | note.UserKey = getUser(r) 34 | note.UserName = getUserName(r) 35 | 36 | err := nm.CreateNote(note) 37 | 38 | if err != nil { 39 | w.WriteHeader(http.StatusInternalServerError) 40 | writer.Encode(map[string]string{ 41 | "status": "error", 42 | "message": fmt.Sprintf("%v", err), 43 | }) 44 | } else { 45 | writer.Encode(note) 46 | } 47 | } 48 | } 49 | 50 | func updateNote(w http.ResponseWriter, r *http.Request) { 51 | nm := service.NewNoteManager(getAwsCredentials(r)) 52 | 53 | req := &PostItemRequest{} 54 | writer := json.NewEncoder(w) 55 | err := json.NewDecoder(r.Body).Decode(&req) 56 | 57 | if err != nil { 58 | w.WriteHeader(http.StatusInternalServerError) 59 | writer.Encode(map[string]string{ 60 | "status": "error", 61 | "message": fmt.Sprintf("%v", err), 62 | }) 63 | } else { 64 | note := req.Item 65 | note.UserKey = getUser(r) 66 | note.UserName = getUserName(r) 67 | 68 | err := nm.UpdateNote(note) 69 | 70 | if err != nil { 71 | w.WriteHeader(http.StatusInternalServerError) 72 | writer.Encode(map[string]string{ 73 | "status": "error", 74 | "message": fmt.Sprintf("%v", err), 75 | }) 76 | } else { 77 | writer.Encode(note) 78 | } 79 | } 80 | } 81 | 82 | func deleteNote(w http.ResponseWriter, r *http.Request) { 83 | nm := service.NewNoteManager(getAwsCredentials(r)) 84 | writer := json.NewEncoder(w) 85 | 86 | vars := mux.Vars(r) 87 | ts, _ := strconv.Atoi(vars["timestamp"]) 88 | err := nm.DeleteNote(getUser(r), ts) 89 | 90 | if err != nil { 91 | w.WriteHeader(http.StatusInternalServerError) 92 | writer.Encode(map[string]string{ 93 | "status": "error", 94 | "message": fmt.Sprintf("%v", err), 95 | }) 96 | } else { 97 | w.Write([]byte("{\"success\": true}")) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /dynonote/restapi/server.go: -------------------------------------------------------------------------------- 1 | package restapi 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "os" 9 | "strconv" 10 | 11 | "github.com/gorilla/handlers" 12 | "github.com/gorilla/mux" 13 | "github.com/nqbao/learn-go/dynonote/model" 14 | "github.com/nqbao/learn-go/dynonote/service" 15 | ) 16 | 17 | const address = ":5000" 18 | 19 | type ListNoteResponses struct { 20 | Items []*model.Note 21 | LastEvaluatedKey map[string]string `json:",omitempty"` 22 | } 23 | 24 | func listUserNote(w http.ResponseWriter, r *http.Request) { 25 | vars := mux.Vars(r) 26 | 27 | limitArg := r.URL.Query()["limit"] 28 | limit := 5 29 | 30 | if len(limitArg) >= 1 { 31 | limit, _ = strconv.Atoi(limitArg[0]) 32 | } 33 | 34 | nm := service.NewNoteManager(nil) 35 | 36 | notes, startKey, err := nm.GetUserNote(getUser(r), limit, vars["start"]) 37 | 38 | writer := json.NewEncoder(w) 39 | 40 | if err != nil { 41 | w.WriteHeader(http.StatusInternalServerError) 42 | writer.Encode(map[string]string{ 43 | "status": "error", 44 | "message": fmt.Sprintf("%v", err), 45 | }) 46 | } else { 47 | repr := ListNoteResponses{ 48 | Items: notes, 49 | } 50 | 51 | if startKey != "" { 52 | repr.LastEvaluatedKey = map[string]string{ 53 | "timestamp": startKey, 54 | } 55 | } 56 | 57 | writer.Encode(repr) 58 | } 59 | } 60 | 61 | func StartServer() { 62 | router := mux.NewRouter().StrictSlash(true) 63 | 64 | router.Path("/api/note"). 65 | Methods("POST"). 66 | HandlerFunc(postNote) 67 | 68 | router.Path("/api/note"). 69 | Methods("PATCH"). 70 | HandlerFunc(updateNote) 71 | 72 | router.Path("/api/note/{timestamp}"). 73 | Methods("DELETE"). 74 | HandlerFunc(deleteNote) 75 | 76 | router.Path("/api/notes"). 77 | Queries("start", "{start}"). 78 | HandlerFunc(listUserNote) 79 | 80 | router.HandleFunc("/api/notes", listUserNote) 81 | 82 | router.Path("/api/tokensignin").Methods("POST").HandlerFunc(tokenSignIn) 83 | 84 | log.Printf("Listening on address %s", address) 85 | 86 | corsRoute := handlers.CORS( 87 | handlers.AllowedOrigins([]string{"*"}), 88 | handlers.AllowedHeaders([]string{"content-type", "authorization"}), 89 | handlers.AllowedMethods([]string{"GET", "POST", "HEAD", "PATCH", "DELETE"}), 90 | )(authHandler(router)) 91 | loggingRouter := handlers.LoggingHandler(os.Stdout, corsRoute) 92 | 93 | log.Fatal(http.ListenAndServe(address, loggingRouter)) 94 | } 95 | -------------------------------------------------------------------------------- /dynonote/restapi/user.go: -------------------------------------------------------------------------------- 1 | package restapi 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/credentials" 13 | "github.com/aws/aws-sdk-go/service/cognitoidentity" 14 | "github.com/dgrijalva/jwt-go" 15 | "github.com/nqbao/learn-go/dynonote/service" 16 | ) 17 | 18 | type tokenRequest struct { 19 | IDToken string `json:"id_token"` 20 | } 21 | 22 | type _authHandler struct { 23 | handler http.Handler 24 | } 25 | 26 | func (h _authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 27 | // TODO: don't restrict access to all endpoint 28 | if r.Method != "OPTIONS" && r.URL.Path != "/api/tokensignin" { 29 | authHeader := r.Header["Authorization"] 30 | if len(authHeader) > 0 { 31 | token, err := jwt.Parse(authHeader[0], func(token *jwt.Token) (interface{}, error) { 32 | return nil, nil 33 | }) 34 | 35 | // if err != nil { 36 | // w.WriteHeader(http.StatusForbidden) 37 | // json.NewEncoder(w).Encode(map[string]string{ 38 | // "status": "error", 39 | // "message": fmt.Sprintf("%v", err), 40 | // }) 41 | // return 42 | // } 43 | 44 | svc := cognitoidentity.New(service.NewSession()) 45 | 46 | logins := map[string]*string{ 47 | "accounts.google.com": aws.String(authHeader[0]), 48 | } 49 | 50 | ids, err := svc.GetId(&cognitoidentity.GetIdInput{ 51 | IdentityPoolId: aws.String("us-east-1:660f9f63-aae4-4954-96a6-12589e06e40c"), 52 | Logins: logins, 53 | }) 54 | 55 | if err != nil { 56 | w.WriteHeader(http.StatusForbidden) 57 | json.NewEncoder(w).Encode(map[string]string{ 58 | "status": "error", 59 | "message": fmt.Sprintf("%v", err), 60 | }) 61 | return 62 | } 63 | 64 | // now get credentials 65 | creds, err := svc.GetCredentialsForIdentity(&cognitoidentity.GetCredentialsForIdentityInput{ 66 | IdentityId: ids.IdentityId, 67 | Logins: logins, 68 | }) 69 | 70 | if err != nil { 71 | w.WriteHeader(http.StatusForbidden) 72 | json.NewEncoder(w).Encode(map[string]string{ 73 | "status": "error", 74 | "message": fmt.Sprintf("%v", err), 75 | }) 76 | return 77 | } 78 | 79 | ctx := context.WithValue(r.Context(), "aws_credentials", creds) 80 | ctx = context.WithValue(ctx, "token", token) 81 | 82 | r = r.WithContext(ctx) 83 | } 84 | } 85 | 86 | h.handler.ServeHTTP(w, r) 87 | } 88 | 89 | func getUser(r *http.Request) string { 90 | creds, ok := r.Context().Value("aws_credentials").(*cognitoidentity.GetCredentialsForIdentityOutput) 91 | 92 | if ok { 93 | return *creds.IdentityId 94 | } 95 | 96 | return "" 97 | } 98 | 99 | func getUserName(r *http.Request) string { 100 | token, ok := r.Context().Value("token").(*jwt.Token) 101 | 102 | if ok { 103 | return token.Claims.(jwt.MapClaims)["email"].(string) 104 | } 105 | 106 | return "" 107 | } 108 | 109 | func getAwsCredentials(r *http.Request) *credentials.Credentials { 110 | creds, ok := r.Context().Value("aws_credentials").(*cognitoidentity.GetCredentialsForIdentityOutput) 111 | 112 | if ok { 113 | return credentials.NewStaticCredentials( 114 | *creds.Credentials.AccessKeyId, 115 | *creds.Credentials.SecretKey, 116 | *creds.Credentials.SessionToken, 117 | ) 118 | } 119 | 120 | return nil 121 | } 122 | 123 | func authHandler(h http.Handler) http.Handler { 124 | return _authHandler{handler: h} 125 | } 126 | 127 | func tokenSignIn(w http.ResponseWriter, r *http.Request) { 128 | req := &tokenRequest{} 129 | err := json.NewDecoder(r.Body).Decode(req) 130 | writer := json.NewEncoder(w) 131 | 132 | if err != nil { 133 | w.WriteHeader(http.StatusBadRequest) 134 | writer.Encode(map[string]string{ 135 | "status": "error", 136 | "message": fmt.Sprintf("%v", err), 137 | }) 138 | } else { 139 | endpoint := fmt.Sprintf("https://www.googleapis.com/oauth2/v3/tokeninfo?id_token=%v", req.IDToken) 140 | 141 | client := http.Client{ 142 | Timeout: 5 * time.Second, 143 | } 144 | 145 | repr, err := client.Get(endpoint) 146 | 147 | if err != nil { 148 | w.WriteHeader(http.StatusBadRequest) 149 | writer.Encode(map[string]string{ 150 | "status": "error", 151 | "message": fmt.Sprintf("%v", err), 152 | }) 153 | } else { 154 | w.WriteHeader(repr.StatusCode) 155 | io.Copy(w, repr.Body) 156 | } 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /dynonote/service/notemanager.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/aws/aws-dax-go/dax" 10 | "github.com/aws/aws-sdk-go/aws" 11 | "github.com/aws/aws-sdk-go/aws/credentials" 12 | "github.com/aws/aws-sdk-go/aws/session" 13 | "github.com/aws/aws-sdk-go/service/dynamodb" 14 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" 15 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" 16 | "github.com/aws/aws-sdk-go/service/dynamodb/expression" 17 | "github.com/nqbao/learn-go/dynonote/model" 18 | "github.com/oklog/ulid" 19 | ) 20 | 21 | var ( 22 | region = "us-east-1" 23 | tableName = "test.notes" 24 | daxEndpoint = "test2.8mmam2.clustercfg.dax.use1.cache.amazonaws.com:8111" 25 | ) 26 | 27 | type NoteManager struct { 28 | session *session.Session 29 | } 30 | 31 | func NewNoteManager(creds *credentials.Credentials) *NoteManager { 32 | nm := &NoteManager{} 33 | 34 | if creds != nil { 35 | nm.session, _ = session.NewSession(&aws.Config{ 36 | Region: ®ion, 37 | Credentials: creds, 38 | }) 39 | } else { 40 | nm.session = newSession() 41 | } 42 | 43 | return nm 44 | } 45 | 46 | func newID() string { 47 | entropy := ulid.Monotonic(rand.New(rand.NewSource(time.Now().UnixNano())), 0) 48 | 49 | id := ulid.MustNew(ulid.Now(), entropy) 50 | return id.String() 51 | } 52 | 53 | func newSession() *session.Session { 54 | sess, _ := session.NewSession(&aws.Config{ 55 | Region: aws.String(region), 56 | }) 57 | 58 | return sess 59 | } 60 | 61 | func NewSession() *session.Session { 62 | return newSession() 63 | } 64 | 65 | func newDaxClient() dynamodbiface.DynamoDBAPI { 66 | cfg := dax.DefaultConfig() 67 | cfg.HostPorts = []string{daxEndpoint} 68 | cfg.Region = region 69 | cli, err := dax.New(cfg) 70 | 71 | if err != nil { 72 | panic(err) 73 | } 74 | 75 | return cli 76 | } 77 | 78 | func (nm *NoteManager) CreateNote(n *model.Note) error { 79 | n.Timestamp = time.Now().Unix() 80 | n.ULID = newID() 81 | return nm.putNote(n, false) 82 | } 83 | 84 | func (nm *NoteManager) UpdateNote(n *model.Note) error { 85 | return nm.putNote(n, true) 86 | } 87 | 88 | // put item will replace item with same key 89 | func (nm *NoteManager) putNote(n *model.Note, check bool) error { 90 | client := dynamodb.New(nm.session) 91 | 92 | av, err := dynamodbattribute.MarshalMap(n) 93 | 94 | input := &dynamodb.PutItemInput{ 95 | TableName: aws.String(tableName), 96 | Item: av, 97 | ReturnConsumedCapacity: aws.String("TOTAL"), 98 | } 99 | 100 | if check { 101 | expr, err := expression.NewBuilder().WithCondition( 102 | expression.And( 103 | expression.Name("user_id").Equal(expression.Value(n.UserKey)), 104 | expression.Name("timestamp").Equal(expression.Value(n.Timestamp)), 105 | ), 106 | ).Build() 107 | 108 | if err != nil { 109 | return err 110 | } 111 | 112 | input.SetConditionExpression(*expr.Condition()) 113 | input.SetExpressionAttributeNames(expr.Names()) 114 | input.SetExpressionAttributeValues(expr.Values()) 115 | } 116 | 117 | _, err = client.PutItem(input) 118 | 119 | return err 120 | // fmt.Printf("consumed WCU: %v\n", output.ConsumedCapacity) 121 | } 122 | 123 | func GetNote(user string, id string) (note *model.Note) { 124 | client := dynamodb.New(newSession()) 125 | 126 | output, err := client.GetItem(&dynamodb.GetItemInput{ 127 | TableName: aws.String(tableName), 128 | Key: map[string]*dynamodb.AttributeValue{ 129 | "uid": &dynamodb.AttributeValue{ 130 | S: aws.String(user), 131 | }, 132 | "nid": &dynamodb.AttributeValue{ 133 | S: aws.String(id), 134 | }, 135 | }, 136 | }) 137 | 138 | if err != nil { 139 | panic(err) 140 | } 141 | 142 | if output.Item != nil { 143 | note = &model.Note{} 144 | dynamodbattribute.UnmarshalMap(output.Item, note) 145 | } 146 | 147 | return 148 | } 149 | 150 | func (nm *NoteManager) DeleteNote(user string, id int) error { 151 | client := dynamodb.New(nm.session) 152 | // client := newDaxClient() 153 | 154 | output, err := client.DeleteItem(&dynamodb.DeleteItemInput{ 155 | TableName: aws.String(tableName), 156 | Key: map[string]*dynamodb.AttributeValue{ 157 | "user_id": &dynamodb.AttributeValue{ 158 | S: aws.String(user), 159 | }, 160 | "timestamp": &dynamodb.AttributeValue{ 161 | N: aws.String(strconv.Itoa(id)), 162 | }, 163 | }, 164 | ReturnConsumedCapacity: aws.String("TOTAL"), 165 | }) 166 | 167 | fmt.Printf("%v\n", output.ConsumedCapacity) 168 | return err 169 | } 170 | 171 | func (nm *NoteManager) GetUserNote(user string, limit int, startKey string) (result []*model.Note, newStartKey string, err error) { 172 | result = nil 173 | 174 | input := &dynamodb.QueryInput{ 175 | TableName: aws.String(tableName), 176 | ReturnConsumedCapacity: aws.String("TOTAL"), 177 | ScanIndexForward: aws.Bool(false), 178 | } 179 | 180 | expr, err := expression.NewBuilder().WithKeyCondition( 181 | expression.Key("user_id").Equal(expression.Value(user)), 182 | ).Build() 183 | 184 | if err != nil { 185 | return 186 | } 187 | 188 | input.SetKeyConditionExpression(*expr.KeyCondition()) 189 | input.SetExpressionAttributeNames((expr.Names())) 190 | input.SetExpressionAttributeValues(expr.Values()) 191 | 192 | if limit > 0 { 193 | input.SetLimit(int64(limit)) 194 | } 195 | 196 | if startKey != "" { 197 | input.SetExclusiveStartKey(map[string]*dynamodb.AttributeValue{ 198 | "user_id": &dynamodb.AttributeValue{S: &user}, 199 | "timestamp": &dynamodb.AttributeValue{N: &startKey}, 200 | }) 201 | } 202 | 203 | result, newStartKey, err = queryNotes(input) 204 | 205 | return 206 | } 207 | 208 | func queryNotes(input *dynamodb.QueryInput) (result []*model.Note, startKey string, err error) { 209 | client := dynamodb.New(newSession()) 210 | // client := newDaxClient() 211 | 212 | if *input.Limit > 0 { 213 | output, err := client.Query(input) 214 | 215 | if err != nil { 216 | return nil, "", err 217 | } 218 | 219 | for _, item := range output.Items { 220 | note := &model.Note{} 221 | dynamodbattribute.UnmarshalMap(item, ¬e) 222 | 223 | result = append(result, note) 224 | } 225 | 226 | // fmt.Printf("%v\n", *input.Limit) 227 | // fmt.Printf("%v\n", output.LastEvaluatedKey) 228 | 229 | if len(output.LastEvaluatedKey) > 0 { 230 | startKey = *(output.LastEvaluatedKey["timestamp"].N) 231 | } 232 | } else { 233 | err = client.QueryPages(input, func(output *dynamodb.QueryOutput, lastPage bool) bool { 234 | for _, item := range output.Items { 235 | note := &model.Note{} 236 | dynamodbattribute.UnmarshalMap(item, ¬e) 237 | 238 | result = append(result, note) 239 | } 240 | 241 | return true 242 | }) 243 | } 244 | 245 | return 246 | } 247 | 248 | func GetAllNotes() (result []*model.Note, err error) { 249 | client := dynamodb.New(newSession()) 250 | err = client.ScanPages(&dynamodb.ScanInput{ 251 | TableName: aws.String(tableName), 252 | ReturnConsumedCapacity: aws.String("TOTAL"), 253 | }, func(page *dynamodb.ScanOutput, lastPage bool) bool { 254 | for _, item := range page.Items { 255 | note := &model.Note{} 256 | dynamodbattribute.UnmarshalMap(item, note) 257 | 258 | result = append(result, note) 259 | } 260 | 261 | fmt.Printf("%v\n", page.ConsumedCapacity) 262 | 263 | return true 264 | }) 265 | 266 | return 267 | } 268 | 269 | func IncrNoteViews(user string, id string, counter int) { 270 | client := dynamodb.New(newSession()) 271 | 272 | expr, err := expression.NewBuilder().WithUpdate( 273 | expression.Add(expression.Name("views"), expression.Value(counter)), 274 | ).Build() 275 | 276 | if err != nil { 277 | panic(err) 278 | } 279 | 280 | out, err := client.UpdateItem(&dynamodb.UpdateItemInput{ 281 | TableName: aws.String(tableName), 282 | Key: map[string]*dynamodb.AttributeValue{ 283 | "uid": &dynamodb.AttributeValue{ 284 | S: aws.String(user), 285 | }, 286 | "nid": &dynamodb.AttributeValue{ 287 | S: aws.String(id), 288 | }, 289 | }, 290 | ReturnConsumedCapacity: aws.String("TOTAL"), 291 | UpdateExpression: expr.Update(), 292 | ExpressionAttributeNames: expr.Names(), 293 | ExpressionAttributeValues: expr.Values(), 294 | }) 295 | 296 | if err != nil { 297 | panic(err) 298 | } 299 | 300 | fmt.Printf("%v\n", out.ConsumedCapacity) 301 | } 302 | -------------------------------------------------------------------------------- /dynonote/service/other.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/service/dynamodb" 9 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" 10 | "github.com/nqbao/learn-go/dynonote/model" 11 | ) 12 | 13 | func ResetDatabase() error { 14 | client := dynamodb.New(newSession()) 15 | writeErr := 0 16 | 17 | for { 18 | output, err := client.Scan(&dynamodb.ScanInput{ 19 | TableName: &tableName, 20 | Limit: aws.Int64(25), 21 | }) 22 | 23 | if err != nil { 24 | return err 25 | } 26 | 27 | if len(output.Items) == 0 { 28 | break 29 | } 30 | 31 | // batch write max is 25 32 | writeRequests := []*dynamodb.WriteRequest{} 33 | for _, item := range output.Items { 34 | note := &model.Note{} 35 | dynamodbattribute.UnmarshalMap(item, note) 36 | 37 | writeRequest := &dynamodb.WriteRequest{ 38 | DeleteRequest: &dynamodb.DeleteRequest{ 39 | Key: map[string]*dynamodb.AttributeValue{ 40 | "uid": &dynamodb.AttributeValue{S: ¬e.UserKey}, 41 | "nid": &dynamodb.AttributeValue{S: ¬e.ULID}, 42 | }, 43 | }, 44 | } 45 | 46 | writeRequests = append(writeRequests, writeRequest) 47 | } 48 | 49 | fmt.Printf("Deleting %v items\n", len(writeRequests)) 50 | 51 | batchOutput, err := client.BatchWriteItem(&dynamodb.BatchWriteItemInput{ 52 | RequestItems: map[string][]*dynamodb.WriteRequest{ 53 | tableName: writeRequests, 54 | }, 55 | }) 56 | 57 | if err != nil { 58 | writeErr++ 59 | log.Printf("error: %v", err) 60 | 61 | if writeErr > 5 { 62 | return err 63 | } 64 | } 65 | 66 | fmt.Printf("Unprocessed items %v\n", len(batchOutput.UnprocessedItems)) 67 | } 68 | 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /dynonote/service/star.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/service/dynamodb" 8 | "github.com/aws/aws-sdk-go/service/dynamodb/expression" 9 | "github.com/nqbao/learn-go/dynonote/model" 10 | ) 11 | 12 | func StarNote(user string, id string, star int) { 13 | client := dynamodb.New(newSession()) 14 | 15 | var expr expression.Expression 16 | var err error 17 | 18 | if star != 0 { 19 | expr, err = expression.NewBuilder().WithUpdate( 20 | expression.Set(expression.Name("star"), expression.Value(star)), 21 | ).Build() 22 | 23 | if err != nil { 24 | panic(err) 25 | } 26 | } else { 27 | expr, err = expression.NewBuilder().WithUpdate( 28 | expression.Remove(expression.Name("star")), 29 | ).Build() 30 | } 31 | 32 | out, err := client.UpdateItem(&dynamodb.UpdateItemInput{ 33 | TableName: aws.String(tableName), 34 | Key: map[string]*dynamodb.AttributeValue{ 35 | "uid": &dynamodb.AttributeValue{ 36 | S: aws.String(user), 37 | }, 38 | "nid": &dynamodb.AttributeValue{ 39 | S: aws.String(id), 40 | }, 41 | }, 42 | ReturnConsumedCapacity: aws.String("TOTAL"), 43 | UpdateExpression: expr.Update(), 44 | ExpressionAttributeNames: expr.Names(), 45 | ExpressionAttributeValues: expr.Values(), 46 | }) 47 | 48 | if err != nil { 49 | panic(err) 50 | } 51 | 52 | fmt.Printf("%v\n", out.ConsumedCapacity) 53 | } 54 | 55 | func GetStarNotes(user string) (result []*model.Note) { 56 | result = nil 57 | 58 | input := &dynamodb.QueryInput{ 59 | TableName: aws.String(tableName), 60 | IndexName: aws.String("uid-star-index"), 61 | ReturnConsumedCapacity: aws.String("TOTAL"), 62 | } 63 | 64 | expr, err := expression.NewBuilder().WithKeyCondition( 65 | expression.Key("uid").Equal(expression.Value(user)), 66 | ).Build() 67 | 68 | if err != nil { 69 | panic(err) 70 | } 71 | 72 | input.SetKeyConditionExpression(*expr.KeyCondition()) 73 | input.SetExpressionAttributeNames((expr.Names())) 74 | input.SetExpressionAttributeValues(expr.Values()) 75 | 76 | result, _, err = queryNotes(input) 77 | 78 | if err != nil { 79 | panic(err) 80 | } 81 | 82 | return 83 | } 84 | -------------------------------------------------------------------------------- /echoserver/README.md: -------------------------------------------------------------------------------- 1 | # TCP Echo Server 2 | 3 | A simple TCP server just echos back what you type. 4 | 5 | * https://coderwall.com/p/wohavg/creating-a-simple-tcp-server-in-go 6 | * https://jameshfisher.com/2017/04/18/golang-tcp-server.html 7 | -------------------------------------------------------------------------------- /echoserver/client/client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net" 7 | "time" 8 | ) 9 | 10 | func main() { 11 | c, err := net.Dial("tcp", "localhost:3333") 12 | 13 | if err != nil { 14 | log.Fatal(err) 15 | } 16 | 17 | defer c.Close() 18 | 19 | // make some noise 20 | go func() { 21 | for { 22 | c.Write([]byte("hello\n")) 23 | time.Sleep(5 * time.Second) 24 | } 25 | }() 26 | 27 | for { 28 | // c.SetReadDeadline(time.Now().Add(2 * time.Second)) 29 | l := 10 30 | b := make([]byte, l) 31 | n, err := c.Read(b) 32 | 33 | if err != nil { 34 | log.Printf("%v", err) 35 | } 36 | 37 | fmt.Printf("%s %v\n", b, n) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /echoserver/server/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net" 9 | "strings" 10 | ) 11 | 12 | func handleRequest(conn net.Conn) { 13 | log.Printf("Accepting new connection %v", conn.RemoteAddr()) 14 | 15 | close := func() { 16 | log.Print("Closing connection") 17 | conn.Close() 18 | } 19 | 20 | defer close() 21 | 22 | reader := bufio.NewReader(conn) 23 | writer := bufio.NewWriter(conn) 24 | 25 | for { 26 | str, err := reader.ReadString('\n') 27 | 28 | if err == io.EOF { 29 | log.Print("Client close connection") 30 | break 31 | } else if err != nil { 32 | log.Panic(err) 33 | } 34 | 35 | str = strings.TrimSpace(str) 36 | 37 | if str == "STOP" { 38 | log.Printf("Receive stop signal") 39 | close() 40 | break 41 | } else { 42 | writer.WriteString(fmt.Sprintf("> %s\n", str)) 43 | writer.Flush() 44 | } 45 | } 46 | } 47 | 48 | func main() { 49 | // 1st: create a listener 50 | l, err := net.Listen("tcp", ":3333") 51 | 52 | if err != nil { 53 | log.Fatal(err) 54 | } 55 | 56 | log.Print("Listen on port 3333") 57 | 58 | // close the socket when server is ended 59 | defer l.Close() 60 | 61 | for { 62 | conn, err := l.Accept() 63 | 64 | if err != nil { 65 | log.Fatal(err) 66 | } 67 | 68 | go handleRequest(conn) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /externalmergesort/README.md: -------------------------------------------------------------------------------- 1 | # External Merge Sort 2 | 3 | 0. Build the binary from the [cmd](./cmd) folder. 4 | 5 | 1. Generate a test file 6 | 7 | ``` 8 | ./cmd -command generate -input 1000M.txt -count 1000000000 9 | ``` 10 | 11 | 2. Sort the file using external merge sort with chunk size of 100M. The result will be written to `chunk/final` 12 | 13 | ``` 14 | ./cmd -command sort -input 1000M.txt -chunk 100 15 | ``` 16 | 17 | 3. Validate the file 18 | 19 | ``` 20 | ./cmd -command validate -input chunk/final 21 | ``` 22 | -------------------------------------------------------------------------------- /externalmergesort/cmd/.gitignore: -------------------------------------------------------------------------------- 1 | *.txt 2 | chunk/* -------------------------------------------------------------------------------- /externalmergesort/cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | 7 | sort "github.com/nqbao/learn-go/externalmergesort" 8 | ) 9 | 10 | func main() { 11 | var command = flag.String("command", "", "Which command to run") 12 | var input = flag.String("input", "", "Input file") 13 | var count = flag.Int("count", 0, "How many number to generate") 14 | var chunkSize = flag.Float64("chunk", 1.0, "Chunk size") 15 | flag.Parse() 16 | 17 | if *command == "" || *input == "" { 18 | fmt.Printf("Usage: sort -command [command] -input [input_file]\n") 19 | } 20 | 21 | // generate the test data to a file 22 | if *command == "generate" { 23 | sort.GenerateData(input, *count) 24 | } else if *command == "validate" { 25 | sort.ValidateData(input) 26 | } else if *command == "sort" { 27 | sort.ExternalMergeSort(input, *chunkSize) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /externalmergesort/quicksort.go: -------------------------------------------------------------------------------- 1 | package externalmergesort 2 | 3 | // perform inline quicksort 4 | // I choose quicksort because it does not require extra memory and it is generally faster than 5 | // heapsort because it does not need to swap the element around. 6 | // see https://medium.com/@k2u4yt/quicksort-vs-heapsort-3b6dc5395083 7 | func quicksort(input []int32, size int) { 8 | quicksortFrom(input, 0, size-1) 9 | } 10 | 11 | func quicksortFrom(input []int32, left int, right int) { 12 | if left < right { 13 | p := quicksortParition(input, left, right) 14 | quicksortFrom(input, left, p-1) 15 | quicksortFrom(input, p+1, right) 16 | } 17 | } 18 | 19 | func quicksortParition(input []int32, left int, right int) int { 20 | start := left 21 | value := input[left] 22 | left = left + 1 23 | 24 | for { 25 | // find the left mark 26 | for { 27 | if left <= right && input[left] <= value { 28 | left = left + 1 29 | } else { 30 | break 31 | } 32 | } 33 | 34 | // find the right mark 35 | for { 36 | if right >= left && input[right] >= value { 37 | right = right - 1 38 | } else { 39 | break 40 | } 41 | } 42 | 43 | // swap 44 | if right < left { 45 | break 46 | } else { 47 | input[left], input[right] = input[right], input[left] 48 | } 49 | } 50 | 51 | // swap the right mark 52 | input[start], input[right] = input[right], input[start] 53 | 54 | return right 55 | } 56 | -------------------------------------------------------------------------------- /externalmergesort/sort.go: -------------------------------------------------------------------------------- 1 | package externalmergesort 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "log" 7 | "math" 8 | "os" 9 | "runtime" 10 | "strconv" 11 | 12 | sys "github.com/nqbao/learn-go/sys" 13 | ) 14 | 15 | func ExternalMergeSort(input *string, chunkSize float64) { 16 | f, err := os.OpenFile(*input, os.O_RDONLY, 0666) 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | defer f.Close() 21 | 22 | sliceSize := int(chunkSize*1024*1024) / 4 23 | chunk := make([]int32, sliceSize) 24 | 25 | log.Printf("Slice size %v", sliceSize) 26 | 27 | scanner := bufio.NewScanner(f) 28 | 29 | chunkIndex := 0 30 | count := 0 31 | for scanner.Scan() { 32 | parsed, _ := strconv.Atoi(scanner.Text()) 33 | chunk[count] = int32(parsed) 34 | count = count + 1 35 | 36 | // finish 1 chunk, write it down 37 | if count >= sliceSize { 38 | log.Printf("Sorting and writing chunk %v", chunkIndex) 39 | quicksort(chunk, count) 40 | runtime.GC() 41 | writeChunk(chunk, count, chunkIndex) 42 | runtime.GC() 43 | sys.PrintMemUsage() 44 | 45 | // reset counter and increase chunk index 46 | count = 0 47 | chunkIndex = chunkIndex + 1 48 | } 49 | } 50 | 51 | // don't forget the final chunk 52 | if count > 0 { 53 | log.Printf("Sorting and writing chunk %v", chunkIndex) 54 | quicksort(chunk, count) 55 | runtime.GC() 56 | writeChunk(chunk, count, chunkIndex) 57 | runtime.GC() 58 | sys.PrintMemUsage() 59 | } 60 | 61 | log.Printf("Merging chunks ...") 62 | mergeChunks(chunkIndex + 1) 63 | } 64 | 65 | func writeChunk(input []int32, count int, chunkIndex int) { 66 | os.MkdirAll("chunk", 0766) 67 | 68 | chunkName := fmt.Sprintf("chunk/%v", chunkIndex) 69 | f, err := os.OpenFile(chunkName, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0666) 70 | 71 | if err != nil { 72 | log.Fatal(err) 73 | } 74 | 75 | defer f.Close() 76 | 77 | writer := bufio.NewWriter(f) 78 | 79 | for i := 0; i < count; i++ { 80 | writer.WriteString(fmt.Sprintf("%v\n", input[i])) 81 | } 82 | 83 | err = writer.Flush() 84 | if err != nil { 85 | log.Fatal(err) 86 | } 87 | } 88 | 89 | func mergeChunks(chunks int) { 90 | f, err := os.OpenFile("chunk/final", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0666) 91 | 92 | if err != nil { 93 | log.Fatal(err) 94 | } 95 | 96 | defer f.Close() 97 | writer := bufio.NewWriter(f) 98 | 99 | readers := make([]*bufio.Scanner, chunks) 100 | for i := 0; i < chunks; i++ { 101 | f2, err := os.Open(fmt.Sprintf("chunk/%v", i)) 102 | 103 | if err != nil { 104 | log.Fatal(err) 105 | } 106 | 107 | defer f2.Close() 108 | readers[i] = bufio.NewScanner(f2) 109 | } 110 | 111 | top := make([]*int, chunks) 112 | for i := 0; i < chunks; i++ { 113 | if top[i] == nil && readers[i].Scan() { 114 | tmp, _ := strconv.Atoi(readers[i].Text()) 115 | top[i] = &tmp 116 | } 117 | } 118 | 119 | for { 120 | // find current min 121 | cur := int(math.MaxInt32) 122 | j := -1 123 | for i := 0; i < chunks; i++ { 124 | if top[i] != nil && cur >= *top[i] { 125 | // fmt.Printf("Selecting %v: %v %v\n", i, cur, *top[i]) 126 | cur = *top[i] 127 | j = i 128 | } 129 | } 130 | 131 | // done 132 | if j == -1 { 133 | break 134 | } 135 | 136 | // fmt.Printf("Reading from chunk %v: %v\n", j, cur) 137 | writer.WriteString(fmt.Sprintf("%v\n", cur)) 138 | 139 | if readers[j].Scan() { 140 | tmp, _ := strconv.Atoi(readers[j].Text()) 141 | top[j] = &tmp 142 | } else { 143 | top[j] = nil 144 | } 145 | } 146 | 147 | writer.Flush() 148 | } 149 | -------------------------------------------------------------------------------- /externalmergesort/util.go: -------------------------------------------------------------------------------- 1 | package externalmergesort 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "log" 7 | "math" 8 | "math/rand" 9 | "os" 10 | "strconv" 11 | ) 12 | 13 | func GenerateData(input *string, count int) { 14 | fmt.Printf("Generating %v random numbers\n", count) 15 | f, err := os.OpenFile(*input, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) 16 | 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | 21 | defer f.Close() 22 | 23 | writer := bufio.NewWriter(f) 24 | for i := 0; i < count; i++ { 25 | writer.WriteString(fmt.Sprintf("%v\n", rand.Int31())) 26 | } 27 | 28 | writer.Flush() 29 | } 30 | 31 | func ValidateData(input *string) { 32 | f, err := os.OpenFile(*input, os.O_RDONLY, 0666) 33 | if err != nil { 34 | log.Fatal(err) 35 | } 36 | 37 | defer f.Close() 38 | 39 | scanner := bufio.NewScanner(f) 40 | 41 | prev := math.MinInt32 42 | line := 1 43 | for scanner.Scan() { 44 | cur, _ := strconv.Atoi(scanner.Text()) 45 | 46 | if cur < prev { 47 | fmt.Printf("Error at line %v: %v > %v\n", line, cur, prev) 48 | break 49 | } 50 | 51 | prev = cur 52 | line++ 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /heap/README.md: -------------------------------------------------------------------------------- 1 | ## Links 2 | 3 | - https://www.hackerearth.com/practice/notes/heaps-and-priority-queues/ 4 | - https://www.geeksforgeeks.org/heap-sort/ 5 | -------------------------------------------------------------------------------- /heap/heap.go: -------------------------------------------------------------------------------- 1 | package heap 2 | 3 | // VerifyHeap verify if an array of integer satisfies 4 | // the heap property, which parent node is larger or equal to its child 5 | func VerifyHeap(heap []int) bool { 6 | return verifyHeapNode(heap, 0) 7 | } 8 | 9 | // Sort sorts an array using HeapSort algorithm 10 | func Sort(heap []int) { 11 | for size := len(heap); size > 1; size-- { 12 | heapify(heap, size) 13 | 14 | // move the first to the last 15 | heap[size-1], heap[0] = heap[0], heap[size-1] 16 | } 17 | } 18 | 19 | func verifyHeapNode(heap []int, node int) bool { 20 | leftChild := node * 2 21 | rightChild := node*2 + 1 22 | size := len(heap) 23 | 24 | if leftChild < size && heap[node] < heap[leftChild] { 25 | return false 26 | } 27 | 28 | if rightChild < size && heap[node] < heap[rightChild] { 29 | return false 30 | } 31 | 32 | return true 33 | } 34 | 35 | // Heapify converts an array to ensure it satisfies the heap property 36 | // this function performs inplace edit of the given array 37 | func Heapify(heap []int) { 38 | heapify(heap, len(heap)) 39 | } 40 | 41 | func heapify(heap []int, size int) { 42 | // convert all subtrees of the heaps 43 | for i := len(heap) / 2; i >= 0; i-- { 44 | heapifySubtree(heap, i, size) 45 | } 46 | } 47 | 48 | func heapifySubtree(heap []int, node int, size int) { 49 | left := 2*node + 1 50 | right := 2*node + 2 51 | 52 | largest := node 53 | 54 | if left < size && heap[left] > heap[largest] { 55 | largest = left 56 | } 57 | 58 | if right < size && heap[right] > heap[largest] { 59 | largest = right 60 | } 61 | 62 | // move down 63 | if largest != node { 64 | heap[largest], heap[node] = heap[node], heap[largest] 65 | heapifySubtree(heap, largest, size) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /heap/heap_test.go: -------------------------------------------------------------------------------- 1 | package heap_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/nqbao/learn-go/heap" 7 | ) 8 | 9 | func TestVerifyHeap(t *testing.T) { 10 | tables := []struct { 11 | input []int 12 | result bool 13 | }{ 14 | {[]int{}, true}, 15 | {[]int{}, true}, 16 | {[]int{5}, true}, 17 | {[]int{10, 5}, true}, 18 | {[]int{5, 10}, false}, 19 | {[]int{3, 2, 1}, true}, 20 | {[]int{3, 2, 1, 0}, true}, 21 | {[]int{6, 4, 5, 3, 2, 1}, true}, 22 | {[]int{8, 10, 5, 3, 2, 1}, false}, 23 | } 24 | 25 | for _, test := range tables { 26 | if heap.VerifyHeap(test.input) != test.result { 27 | t.Errorf("Expect heap %v to be %v", test.input, test.result) 28 | } 29 | } 30 | } 31 | 32 | func TestHeapify(t *testing.T) { 33 | tables := [][]int{ 34 | []int{}, 35 | []int{5}, 36 | []int{10, 20}, 37 | []int{10, 20, 15}, 38 | []int{4, 7, 8, 3, 2, 6, 5}, 39 | []int{1, 4, 3, 7, 8, 9, 10}, 40 | } 41 | 42 | for _, test := range tables { 43 | original := make([]int, len(test)) 44 | copy(original, test) 45 | heap.Heapify(test) 46 | 47 | t.Logf("Result %v", test) 48 | 49 | if !heap.VerifyHeap(test) { 50 | t.Errorf("Unable to heapify %v", original) 51 | } 52 | } 53 | } 54 | 55 | func TestHeapSort(t *testing.T) { 56 | tables := [][]int{ 57 | []int{}, 58 | []int{5}, 59 | []int{10, 20}, 60 | []int{10, 20, 15}, 61 | []int{4, 7, 8, 3, 2, 6, 5}, 62 | []int{12, 11, 13, 5, 6, 7}, 63 | []int{1, 4, 3, 7, 8, 9, 10}, 64 | []int{9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 100}, 65 | } 66 | 67 | for _, test := range tables { 68 | original := make([]int, len(test)) 69 | copy(original, test) 70 | heap.Sort(test) 71 | 72 | t.Logf("Result %v", test) 73 | 74 | // verify if this is ascending 75 | for i := 0; i < len(test)-1; i++ { 76 | if test[i] > test[i+1] { 77 | t.Errorf("Unable to sort %v, result: %v", original, test) 78 | } 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /jni/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | 7 | "github.com/nqbao/jnigo" 8 | ) 9 | 10 | func getJavaVersion(vm *jnigo.JVM) string { 11 | args := []jnigo.JObject{} 12 | 13 | if str, e := vm.NewString("java.version"); e == nil { 14 | args = append(args, str) 15 | } 16 | 17 | v, err := vm.CallStaticFunction("java/lang/System", "getProperty", "(Ljava/lang/String;)Ljava/lang/String;", args) 18 | 19 | if err != nil { 20 | log.Fatal(err) 21 | } 22 | 23 | return fmt.Sprintf("%v", v.GoValue()) 24 | } 25 | 26 | func testHashcode(vm *jnigo.JVM) int32 { 27 | // simulate the example from https://github.com/timob/jnigi#example 28 | obj, err := vm.NewJClass("java/lang/Object", []jnigo.JObject{}) 29 | 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | 34 | v, err := obj.CallFunction("hashCode", "()I", nil) 35 | 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | 40 | return v.GoValue().(int32) 41 | } 42 | 43 | func main() { 44 | vm := jnigo.CreateJVM() 45 | 46 | fmt.Printf("Java version: %v\n", getJavaVersion(vm)) 47 | fmt.Printf("Object hash code: %v\n", testHashcode(vm)) 48 | } 49 | -------------------------------------------------------------------------------- /kms/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/session" 8 | "github.com/aws/aws-sdk-go/service/kms" 9 | ) 10 | 11 | func main() { 12 | sess := session.Must(session.NewSession( 13 | &aws.Config{Region: aws.String("us-east-1")}, 14 | )) 15 | 16 | c := kms.New(sess) 17 | 18 | aliases, err := c.ListAliases(&kms.ListAliasesInput{}) 19 | 20 | if err != nil { 21 | panic(err) 22 | } 23 | 24 | fmt.Printf("%v\n", aliases) 25 | 26 | encReq := &kms.EncryptInput{ 27 | KeyId: aws.String("alias/vault"), 28 | Plaintext: []byte("hello world"), 29 | } 30 | 31 | enc, err := c.Encrypt(encReq) 32 | 33 | if err != nil { 34 | panic(err) 35 | } 36 | 37 | fmt.Printf("%s\n", enc.CiphertextBlob) 38 | 39 | decReq := &kms.DecryptInput{ 40 | CiphertextBlob: enc.CiphertextBlob, 41 | } 42 | 43 | dec, err := c.Decrypt(decReq) 44 | 45 | if err != nil { 46 | panic(err) 47 | } 48 | 49 | fmt.Printf("%v\n", string(dec.Plaintext)) 50 | } 51 | -------------------------------------------------------------------------------- /rocksdbexample/.gitignore: -------------------------------------------------------------------------------- 1 | # Test database folder 2 | db 3 | -------------------------------------------------------------------------------- /rocksdbexample/README.md: -------------------------------------------------------------------------------- 1 | # RocksDB example 2 | 3 | First I need to clone and make rockdbs: 4 | 5 | * Clone rocksdb 6 | * `make shared_lib` to compile RocksDB 7 | 8 | Make sure to set the following environments: 9 | 10 | ``` 11 | export ROCKSDB_PATH=~/Projects/sandbox/rocksdb 12 | export CGO_CFLAGS="-I$ROCKSDB_PATH/include" 13 | export CGO_LDFLAGS="-L$ROCKSDB_PATH -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4" 14 | export LD_LIBRARY_PATH="$ROCKSDB_PATH:$LD_LIBRARY_PATH" 15 | export DYLD_LIBRARY_PATH="$ROCKSDB_PATH:$DYLD_LIBRARY_PATH" 16 | ``` 17 | 18 | Remarks: 19 | 20 | * Interface is quite verbose, not as clean as BoltDB 21 | * You need to create Read/Write options. Make sure to free each of them. 22 | * There is no Go document, so you need to read Go code or [the document for C++](https://github.com/facebook/rocksdb/wiki) 23 | * Iterator is similar to Cursor in BoltDB 24 | * Tailing Iterator allows you to see new data as it come in. 25 | * Merge: Provide a way to perform read-modify-write operation, such as Counter 26 | * You need to define a merge operator 27 | * Merge happpens lazily, until a Put/Delete/Get 28 | * Full merge is to merge existing value and an operand 29 | * Partial merge to merge two operands. It can be used to speed up performance (reduce number of full merge) 30 | * Column Familiy are like Buckets in BoltDB 31 | * You need to make sure to specify column families when you open the DB. Or you need to open ALL column families. 32 | * It is quite ugly when you want to combine with `SetCreateIfMissing` 33 | * Compaction Filter: a background garbage collector to remove unused keys 34 | * Snapshot: provide a view of the DB at the time creation 35 | * Transactions: provide transaction support, but with some caveat 36 | * You have to explicitly create a `TransactionDB`, and there seems to be a problem if you try to open the db twice. 37 | * Merge is not available, at least in the Golang binding 38 | * Use `GetForUpdate` instead of `Get` 39 | * Since this is single process, ACID can be easily achieved using a lock 40 | * Checkpont: create a backup of the database to another directory. 41 | -------------------------------------------------------------------------------- /rocksdbexample/iterator.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | rocksdb "github.com/tecbot/gorocksdb" 8 | ) 9 | 10 | func IteratorWithPrefixExample(db *rocksdb.DB) error { 11 | readOpts := rocksdb.NewDefaultReadOptions() 12 | defer readOpts.Destroy() 13 | 14 | it2 := db.NewIterator(readOpts) 15 | defer it2.Close() 16 | it2.Seek([]byte("2018-")) 17 | 18 | fmt.Printf("prefix scan\n") 19 | for ; it2.ValidForPrefix([]byte("2018-")); it2.Next() { 20 | fmt.Printf("Key: %s\n", it2.Key().Data()) 21 | } 22 | 23 | return nil 24 | } 25 | 26 | func IteratorExample(db *rocksdb.DB) error { 27 | readOpts := rocksdb.NewDefaultReadOptions() 28 | defer readOpts.Destroy() 29 | 30 | // loop through all keys 31 | it := db.NewIterator(readOpts) 32 | defer it.Close() 33 | it.SeekToFirst() 34 | 35 | for ; it.Valid(); it.Next() { 36 | fmt.Printf("Key: %s\n", it.Key().Data()) 37 | } 38 | return nil 39 | } 40 | 41 | // simple example of using tailing iterator to read newly inserted data 42 | func WatchChangeExample(db *rocksdb.DB, stop chan bool) { 43 | readOpts := rocksdb.NewDefaultReadOptions() 44 | readOpts.SetTailing(true) 45 | defer readOpts.Destroy() 46 | 47 | it := db.NewIterator(readOpts) 48 | defer it.Close() 49 | 50 | prefix := []byte("2018-") 51 | it.Seek(prefix) 52 | 53 | shouldStop := false 54 | for { 55 | select { 56 | case <-stop: 57 | shouldStop = true 58 | 59 | default: 60 | // copy the key to new byte buffer 61 | curkey := make([]byte, len(it.Key().Data())) 62 | copy(curkey, it.Key().Data()) 63 | 64 | fmt.Printf("got key: %s %v\n", curkey, it.Valid()) 65 | it.Next() 66 | 67 | if !it.ValidForPrefix(prefix) { 68 | // means there is no more available key right now, seek to previous key 69 | // fmt.Printf("seek %s\n", curkey) 70 | it.Seek(curkey) 71 | time.Sleep(10 * time.Microsecond) 72 | } 73 | } 74 | 75 | if shouldStop { 76 | break 77 | } 78 | } 79 | 80 | fmt.Printf("stopping the loop") 81 | } 82 | -------------------------------------------------------------------------------- /rocksdbexample/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "time" 7 | 8 | rocksdb "github.com/tecbot/gorocksdb" 9 | ) 10 | 11 | func main() { 12 | stop := make(chan bool) 13 | 14 | db, cfs, err := OpenDbWithAllCFs("./db") 15 | 16 | if err != nil { 17 | panic(err) 18 | } 19 | 20 | defer func() { 21 | fmt.Printf("shutting down\n") 22 | 23 | time.Sleep(1 * time.Second) 24 | close(stop) 25 | 26 | for _, cf := range cfs { 27 | cf.Destroy() 28 | } 29 | db.Close() 30 | }() 31 | 32 | // go WatchChangeExample(db, stop) 33 | 34 | // write a key 35 | writeOpts := rocksdb.NewDefaultWriteOptions() 36 | writeOpts.SetSync(true) 37 | defer writeOpts.Destroy() 38 | err = db.Put(writeOpts, []byte("hello"), []byte("world")) 39 | 40 | if err != nil { 41 | panic(err) 42 | } 43 | 44 | // write multiple keys at a time 45 | batch := rocksdb.NewWriteBatch() 46 | // batch.Delete([]byte("hello")) 47 | batch.Put([]byte("test"), []byte("value")) 48 | batch.Put([]byte("test2"), []byte("value2")) 49 | 50 | newkey := time.Now().Format(time.RFC3339) 51 | 52 | batch.Put( 53 | []byte(newkey), 54 | []byte(strconv.Itoa(int(time.Now().Unix()))), 55 | ) 56 | 57 | err = db.Write(writeOpts, batch) 58 | 59 | if err != nil { 60 | panic(err) 61 | } 62 | 63 | fmt.Printf("write done: %v\n", newkey) 64 | 65 | // read a key 66 | readOpts := rocksdb.NewDefaultReadOptions() 67 | defer readOpts.Destroy() 68 | 69 | result, err := db.Get(readOpts, []byte("hello")) 70 | defer result.Free() 71 | 72 | if err != nil { 73 | panic(err) 74 | } 75 | 76 | fmt.Printf("%s\n", result.Data()) 77 | 78 | IteratorWithPrefixExample(db) 79 | 80 | // err = TransactionExample() 81 | // if err != nil { 82 | // panic(err) 83 | // } 84 | 85 | /*err = SnapshotExample(db) 86 | if err != nil { 87 | panic(err) 88 | }*/ 89 | 90 | /*err = MergeExample(db) 91 | if err != nil { 92 | panic(err) 93 | }*/ 94 | } 95 | -------------------------------------------------------------------------------- /rocksdbexample/merge_operator.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | 7 | rocksdb "github.com/tecbot/gorocksdb" 8 | ) 9 | 10 | type NumberMergeOperator struct { 11 | rocksdb.MergeOperator 12 | } 13 | 14 | func (mo NumberMergeOperator) Name() string { 15 | return "NumberMergeOperator" 16 | } 17 | 18 | func (mo NumberMergeOperator) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { 19 | var current uint16 = 0 20 | var result = make([]byte, 4) 21 | 22 | if len(existingValue) > 0 { 23 | current = binary.BigEndian.Uint16(existingValue) 24 | } 25 | 26 | for _, v := range operands { 27 | op := binary.BigEndian.Uint16(v) 28 | current = current + op 29 | } 30 | 31 | binary.BigEndian.PutUint16(result, current) 32 | return result, true 33 | } 34 | 35 | func (mo NumberMergeOperator) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { 36 | return nil, false 37 | } 38 | 39 | func MergeExample(db *rocksdb.DB) error { 40 | opts := rocksdb.NewDefaultWriteOptions() 41 | defer opts.Destroy() 42 | 43 | bs := make([]byte, 4) 44 | binary.BigEndian.PutUint16(bs, 1) 45 | 46 | err := db.Merge(opts, []byte("add_me"), bs) 47 | 48 | if err != nil { 49 | return err 50 | } 51 | 52 | readOpts := rocksdb.NewDefaultReadOptions() 53 | defer readOpts.Destroy() 54 | 55 | // this will materialize the merge 56 | slice, err := db.Get(readOpts, []byte("add_me")) 57 | 58 | if err != nil { 59 | return err 60 | } 61 | 62 | defer slice.Free() 63 | fmt.Printf("After merge: %v\n", binary.BigEndian.Uint16(slice.Data())) 64 | 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /rocksdbexample/snapshot.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "reflect" 6 | 7 | rocksdb "github.com/tecbot/gorocksdb" 8 | ) 9 | 10 | func SnapshotExample(db *rocksdb.DB) error { 11 | testKey := []byte("this is new") 12 | 13 | writeOpts := rocksdb.NewDefaultWriteOptions() 14 | defer writeOpts.Destroy() 15 | 16 | // make sure key is not there before the snapshot 17 | db.Delete(writeOpts, testKey) 18 | 19 | snapshot := db.NewSnapshot() 20 | defer db.ReleaseSnapshot(snapshot) 21 | 22 | // add a new key to the db 23 | err := db.Put(writeOpts, testKey, []byte("123")) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | // verify if the key is not in the snapshot 29 | readOpt := rocksdb.NewDefaultReadOptions() 30 | defer readOpt.Destroy() 31 | readOpt.SetSnapshot(snapshot) 32 | it := db.NewIterator(readOpt) 33 | it.SeekToFirst() 34 | 35 | foundKey := false 36 | for ; it.Valid(); it.Next() { 37 | if reflect.DeepEqual(it.Key().Data(), testKey) { 38 | foundKey = true 39 | } 40 | } 41 | 42 | if foundKey { 43 | return errors.New("key should not be found as it is in snapshot") 44 | } 45 | 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /rocksdbexample/transaction.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import rocksdb "github.com/tecbot/gorocksdb" 4 | 5 | func TransactionExample() error { 6 | opts := rocksdb.NewDefaultOptions() 7 | opts.SetCompression(rocksdb.ZLibCompression) 8 | defer opts.Destroy() 9 | 10 | txdbOpts := rocksdb.NewDefaultTransactionDBOptions() 11 | defer txdbOpts.Destroy() 12 | 13 | _, err := rocksdb.OpenTransactionDb(opts, txdbOpts, "./db") 14 | 15 | if err != nil { 16 | return err 17 | } 18 | 19 | // txdb.Name 20 | 21 | // defer txdb.Close() 22 | 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /rocksdbexample/wrapper.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | 8 | rocksdb "github.com/tecbot/gorocksdb" 9 | ) 10 | 11 | func OpenDbWithAllCFs(name string) (*rocksdb.DB, map[string]*rocksdb.ColumnFamilyHandle, error) { 12 | mo := NumberMergeOperator{} 13 | 14 | opts := rocksdb.NewDefaultOptions() 15 | opts.SetMergeOperator(mo) 16 | 17 | defer opts.Destroy() 18 | opts.SetCreateIfMissing(true) 19 | 20 | // how many LOG.old file to keep 21 | opts.SetKeepLogFileNum(10) 22 | opts.SetCompression(rocksdb.ZLibCompression) 23 | 24 | // ugly hack when db is not available 25 | if _, err := os.Stat(path.Join(name, "CURRENT")); err != nil { 26 | db, err := rocksdb.OpenDb(opts, name) 27 | 28 | return db, nil, err 29 | } else { 30 | // XXX: this won't work if the database does not exists 31 | cfNames, err := rocksdb.ListColumnFamilies(opts, name) 32 | if err != nil { 33 | return nil, nil, err 34 | } 35 | fmt.Printf("column families: %v\n", cfNames) 36 | 37 | cfOpts := make([]*rocksdb.Options, len(cfNames)) 38 | for i := range cfNames { 39 | cfOpts[i] = opts 40 | } 41 | 42 | db, cfs, err := rocksdb.OpenDbColumnFamilies(opts, name, cfNames, cfOpts) 43 | 44 | if err != nil { 45 | return nil, nil, err 46 | } 47 | 48 | cfMap := make(map[string]*rocksdb.ColumnFamilyHandle) 49 | 50 | for i := range cfNames { 51 | cfMap[cfNames[i]] = cfs[i] 52 | } 53 | 54 | return db, cfMap, nil 55 | } 56 | } 57 | 58 | func CreateCFExample(db *rocksdb.DB) error { 59 | cfOpts := rocksdb.NewDefaultOptions() 60 | defer cfOpts.Destroy() 61 | cf, err := db.CreateColumnFamily(cfOpts, "cf1") 62 | if err != nil { 63 | return err 64 | } 65 | defer cf.Destroy() 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /sys/util.go: -------------------------------------------------------------------------------- 1 | package sys 2 | 3 | import ( 4 | "log" 5 | "runtime" 6 | ) 7 | 8 | // PrintMemUsage print current memory consumption of Go 9 | func PrintMemUsage() { 10 | var m runtime.MemStats 11 | runtime.ReadMemStats(&m) 12 | // For info on each, see: https://golang.org/pkg/runtime/#MemStats 13 | log.Printf( 14 | "Alloc = %v MiB, TotalAlloc = %v MiB, Sys = %v MiB, NumGC = %v", 15 | (m.Alloc / 1024 / 1024), 16 | (m.TotalAlloc / 1024 / 1024), 17 | m.Sys, 18 | m.NumGC, 19 | ) 20 | } 21 | --------------------------------------------------------------------------------