├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── all_test.go ├── cmd └── gitdb │ ├── gitdb.go │ ├── gitdb_test.go │ └── template.go ├── config.go ├── db.go ├── db_mock.go ├── db_mock_test.go ├── db_test.go ├── driver.go ├── driver_git.go ├── driver_gitbinary.go ├── driver_local.go ├── errors.go ├── event.go ├── example ├── booking │ ├── collection.go │ ├── enums.go │ └── model.go └── main.go ├── go.mod ├── go.sum ├── index.go ├── init.go ├── init_test.go ├── internal ├── crypto │ └── security.go ├── db │ ├── block.go │ ├── dataset.go │ └── record.go ├── digital │ └── size.go └── errors │ └── errors.go ├── lock.go ├── log.go ├── mail.go ├── mail_test.go ├── model.go ├── paths.go ├── read.go ├── read_test.go ├── schema.go ├── schema_test.go ├── ssh.go ├── static ├── css │ └── app.css ├── errors.html ├── index.html ├── js │ └── app.js ├── list.html ├── sidebar.html └── view.html ├── sync.go ├── sync_test.go ├── testdata ├── v1 │ └── data │ │ └── data │ │ ├── Message │ │ └── b0.json │ │ └── MessageV2 │ │ └── 202003.json └── v2 │ └── data │ └── data │ ├── Message │ └── b0.json │ └── MessageV2 │ └── 202003.json ├── tools ├── fix_corrupt_head.sh └── fix_corrupt_missing_objects.sh ├── transaction.go ├── transaction_test.go ├── ui.go ├── ui_static.go ├── ui_test.go ├── ui_view.go ├── ui_viewmodels.go ├── upload.go ├── upload_test.go ├── user.go ├── user_test.go ├── write.go └── write_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | example/data 2 | example/example 3 | example/ui_gitdb.go 4 | .DS_Store 5 | .vscode 6 | .idea 7 | vendor 8 | *.test 9 | .gitdb 10 | cover.out 11 | prof.cpu 12 | prof.mem 13 | bin -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go_import_path: github.com/gogitdb/github 3 | 4 | sudo: false 5 | go: 6 | - 1.14 7 | branches: 8 | only: 9 | - master 10 | script: 11 | - make test 12 | after_success: 13 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020 Okechukwu Ugwu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test testdel example install release 2 | testdel: 3 | go test ./... -coverprofile=cover.out -v 4 | go tool cover -html=cover.out 5 | rm -f cover.out 6 | test: 7 | go test ./... -coverprofile=cover.out 8 | go tool cover -func=cover.out 9 | example: 10 | cd example && rm -Rf data && go run *.go && cd - 11 | install: 12 | go install github.com/gogitdb/gitdb/v2/cmd/gitdb 13 | release: 14 | go install github.com/gogitdb/gitdb/v2/cmd/gitdb 15 | go generate 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | GitDB 2 | ===== 3 | 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/gogitdb/gitdb?style=flat-square)](https://goreportcard.com/report/github.com/gogitdb/gitdb) 5 | [![Coverage](https://codecov.io/gh/gogitdb/gitdb/branch/develop/graph/badge.svg)](https://codecov.io/gh/gogitdb/gitdb) 6 | [![Build Status Travis](https://img.shields.io/travis/gogitdb/gitdb.svg?style=flat-square&&branch=master)](https://travis-ci.com/gogitdb/gitdb) 7 | [![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/gogitdb/gitdb) 8 | [![Releases](https://img.shields.io/github/release/gogitdb/gitdb/all.svg?style=flat-square)](https://github.com/gogitdb/gitdb/releases) 9 | [![LICENSE](https://img.shields.io/github/license/gogitdb/gitdb.svg?style=flat-square)](https://github.com/gogitdb/gitdb/blob/master/LICENSE) 10 | 11 | ## What is GitDB? 12 | 13 | > GitDB is not a binary. It’s a library! 14 | 15 | GitDB is a decentralized document database written in Go that uses [Git](https://git-scm.com/) under the hood to provide database-like functionalities via strictly defined interfaces. 16 | 17 | GitDB allows developers to create Models of objects in their application which implement a Model Interface that can access it's persistence features. This allows GitDB to work with these objects in database operations. 18 | 19 | ## Why GitDB - motivation behind project? 20 | 21 | - A need for a database that was quick and simple to set up 22 | - A need for a database that was decentralized and each participating client in a system can store their data independent of other clients. 23 | 24 | 25 | ## Features 26 | 27 | - Decentralized 28 | - Document store 29 | - Embedded into your go application 30 | - Encryption (encrypt on write, decrypt on read) 31 | - Record locking. 32 | - Simple Indexing System 33 | - Transactions 34 | - Web UI 35 | 36 | 37 | ## Project versioning 38 | 39 | GitDB uses [semantic versioning](http://semver.org). 40 | API should not change between patch and minor releases. 41 | New minor versions may add additional features to the API. 42 | 43 | ## Table of Contents 44 | 45 | - [Getting Started](#getting-started) 46 | - [Installing](#installing) 47 | - [Configuration](#configuration) 48 | - [Opening a database](#opening-a-database) 49 | - [Inserting/Updating a record](#insertingupdating-a-record) 50 | - [Fetching a single record](#fetching-a-single-record) 51 | - [Fetching all records in a dataset](#fetching-all-records-in-a-dataset) 52 | - [Deleting a record](#deleting-a-record) 53 | - [Search for records](#search-for-records) 54 | - [Transactions](#transactions) 55 | - [Encryption](#encryption) 56 | - [Resources](#resources) 57 | - [Caveats & Limitations](#caveats--limitations) 58 | - [Reading the Source](#reading-the-source) 59 | 60 | 61 | ## Getting Started 62 | 63 | ### Installing 64 | 65 | To start using GitDB, install Go and run `go get`: 66 | 67 | ```sh 68 | $ go get github.com/gogitdb/gitdb/v2 69 | ``` 70 | 71 | 72 | ### Configuration 73 | 74 | Below are configuration options provided by GitDB 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 |
NameDescriptionTypeRequiredDefault
DbPathPath on your machine where you want GitDB to create/clone your databasestringYN/A
ConnectionNameUnique name for gitdb connection. Use this when opening multiple GitDB connectionsstringN"default"
OnlineRemoteURL for remote git server you want GitDB to sync with e.g git@github.com:user/db.git or https://github.com/user/db.git. 100 |

Note: The first time GitDB runs, it will automatically generate ssh keys and will automatically attempt to use this key to sync with the OnlineRemote, 101 | therefore ensure that the generated keys are added to this git server. The ssh keys can be found at Config.DbPath/.gitdb/ssh

102 | 		stringN""
SyncIntervalThis controls how often you want GitDB to sync with the online remotetime.Duration.N5s
EncryptionKey16,24 or 32 byte string used to provide AES encryption for Models that implement ShouldEncryptstringN""
UserThis specifies the user connected to the Gitdb and will be used to commit all changes to the databasegitdb.UserNghost <ghost@gitdb.local>
EnableUIUse this option to enable GitDB web user interfaceboolNfalse
UIPortUse this option to change the default port which GitDB uses to serve it's web user interfaceintN4120
FactoryFor backward compatibity with v1. In v1 GitDB needed a factory method to be able construct concrete Model for certain database operations. 145 | This has now been dropped in v2 146 | func(dataset string) gitdb.ModelNnil
MockFlag used for testing apps. If true, will return a mock GitDB connectionboolNfalse
159 | 160 | You can configure GitDB either using the constructor or constructing it yourself 161 | 162 | ```go 163 | cfg := gitdb.NewConfig(path) 164 | //or 165 | cfg := gitdb.Config{ 166 | DbPath: path 167 | } 168 | ``` 169 | 170 | 171 | 173 | 174 | ### Importing GitDB 175 | 176 | To use GitDB as an embedded document store, import as: 177 | 178 | ```go 179 | import "github.com/gogitdb/gitdb/v2" 180 | 181 | cfg := gitdb.NewConfig(path) 182 | db, err := gitdb.Open(cfg) 183 | if err != nil { 184 | log.Fatal(err) 185 | } 186 | defer db.Close() 187 | ``` 188 | 189 | 190 | ### Opening a database 191 | ```go 192 | package main 193 | 194 | import ( 195 | "log" 196 | "github.com/gogitdb/gitdb/v2" 197 | ) 198 | 199 | func main() { 200 | 201 | cfg := gitdb.NewConfig("/tmp/data") 202 | // Open will create or clone down a git repo 203 | // in configured path if it does not exist. 204 | db, err := gitdb.Open(cfg) 205 | if err != nil { 206 | log.Fatal(err) 207 | } 208 | defer db.Close() 209 | 210 | ... 211 | } 212 | ``` 213 | 214 | ### Models 215 | 216 | A Model is a struct that represents a record in GitDB. GitDB only works with models that implement the gidb.Model interface 217 | 218 | gitdb.TimeStampedModel is a simple struct that allows you to easily add CreatedAt and UpdatedAt to all the Models in your application and will automatically time stamp them before persisting to GitDB. You can write your own base Models to embed common fields across your application Models 219 | 220 | ```go 221 | type BankAccount struct { 222 | //TimeStampedModel will add CreatedAt and UpdatedAt fields this Model 223 | gitdb.TimeStampedModel 224 | AccountType string 225 | AccountNo string 226 | Currency string 227 | Name string 228 | } 229 | 230 | func (b *BankAccount) GetSchema() *gitdb.Schema { 231 | //Dataset Name 232 | name := "Accounts" 233 | //Block ID 234 | block := b.CreatedAt.Format("200601") 235 | //Record ID 236 | record := b.AccountNo 237 | 238 | //Indexes speed up searching 239 | indexes := make(map[string]interface{}) 240 | indexes["AccountType"] = b.AccountType 241 | 242 | return gitdb.NewSchema(name, block, record, indexes) 243 | } 244 | 245 | func (b *BankAccount) Validate() error { return nil } 246 | func (b *BankAccount) IsLockable() bool { return false } 247 | func (b *BankAccount) ShouldEncrypt() bool { return false } 248 | func (b *BankAccount) GetLockFileNames() []string { return []string{} } 249 | 250 | ... 251 | 252 | ``` 253 | 254 | ### Inserting/Updating a record 255 | ```go 256 | package main 257 | 258 | import ( 259 | "log" 260 | "github.com/gogitdb/gitdb/v2" 261 | ) 262 | 263 | func main(){ 264 | cfg := gitdb.NewConfig("/tmp/data") 265 | db, err := gitdb.Open(cfg) 266 | if err != nil { 267 | log.Fatal(err) 268 | } 269 | defer db.Close() 270 | 271 | //populate model 272 | account := &BankAccount{} 273 | account.AccountNo = "0123456789" 274 | account.AccountType = "Savings" 275 | account.Currency = "GBP" 276 | account.Name = "Foo Bar" 277 | 278 | err = db.Insert(account) 279 | if err != nil { 280 | log.Println(err) 281 | } 282 | 283 | //get account id 284 | log.Println(gitdb.Id(account)) 285 | 286 | //update account name 287 | account.Name = "Bar Foo" 288 | err = db.Insert(account) 289 | if err != nil { 290 | log.Println(err) 291 | } 292 | } 293 | ``` 294 | 295 | ### Fetching a single record 296 | ```go 297 | package main 298 | import ( 299 | "log" 300 | "github.com/gogitdb/gitdb/v2" 301 | ) 302 | 303 | func main(){ 304 | cfg := gitdb.NewConfig("/tmp/data") 305 | db, err := gitdb.Open(cfg) 306 | if err != nil { 307 | log.Fatal(err) 308 | } 309 | defer db.Close() 310 | 311 | //model to passed to Get to store result 312 | var account BankAccount 313 | if err := db.Get("Accounts/202003/0123456789", &account); err != nil { 314 | log.Println(err) 315 | } 316 | } 317 | ``` 318 | ### Fetching all records in a dataset 319 | ```go 320 | package main 321 | 322 | import ( 323 | "fmt" 324 | "log" 325 | "github.com/gogitdb/gitdb/v2" 326 | ) 327 | 328 | func main(){ 329 | cfg := gitdb.NewConfig("/tmp/data") 330 | db, err := gitdb.Open(cfg) 331 | if err != nil { 332 | log.Fatal(err) 333 | } 334 | defer db.Close() 335 | 336 | records, err := db.Fetch("Accounts") 337 | if err != nil { 338 | log.Print(err) 339 | return 340 | } 341 | 342 | accounts := []*BankAccount{} 343 | for _, r := range records { 344 | b := &BankAccount{} 345 | r.Hydrate(b) 346 | accounts = append(accounts, b) 347 | log.Print(fmt.Sprintf("%s-%s", gitdb.ID(b), b.AccountNo)) 348 | } 349 | } 350 | 351 | ``` 352 | 353 | ### Fetching all records from specific block in a dataset 354 | ```go 355 | package main 356 | 357 | import ( 358 | "fmt" 359 | "log" 360 | "github.com/gogitdb/gitdb/v2" 361 | ) 362 | 363 | func main(){ 364 | cfg := gitdb.NewConfig("/tmp/data") 365 | db, err := gitdb.Open(cfg) 366 | if err != nil { 367 | log.Fatal(err) 368 | } 369 | defer db.Close() 370 | 371 | records, err := db.Fetch("Accounts", "b0", "b1") 372 | if err != nil { 373 | log.Print(err) 374 | return 375 | } 376 | 377 | var accounts []*BankAccount 378 | for _, r := range records { 379 | b := &BankAccount{} 380 | r.Hydrate(b) 381 | accounts = append(accounts, b) 382 | log.Print(fmt.Sprintf("%s-%s", gitdb.ID(b), b.AccountNo)) 383 | } 384 | } 385 | 386 | ``` 387 | 388 | ### Deleting a record 389 | ```go 390 | package main 391 | 392 | import ( 393 | "log" 394 | "github.com/gogitdb/gitdb/v2" 395 | ) 396 | 397 | func main(){ 398 | cfg := gitdb.NewConfig("/tmp/data") 399 | db, err := gitdb.Open(cfg) 400 | if err != nil { 401 | log.Fatal(err) 402 | } 403 | defer db.Close() 404 | 405 | if err := db.Delete("Accounts/202003/0123456789"); err != nil { 406 | log.Print(err) 407 | } 408 | } 409 | ``` 410 | 411 | ### Search for records 412 | ```go 413 | package main 414 | 415 | import ( 416 | "fmt" 417 | "log" 418 | "github.com/gogitdb/gitdb/v2" 419 | ) 420 | 421 | func main(){ 422 | cfg := gitdb.NewConfig("/tmp/data") 423 | db, err := gitdb.Open(cfg) 424 | if err != nil { 425 | log.Fatal(err) 426 | } 427 | defer db.Close() 428 | 429 | //Find all records that have savings account type 430 | searchParam := &db.SearchParam{Index: "AccountType", Value: "Savings"} 431 | records, err := dbconn.Search("Accounts", []*db.SearchParam{searchParam}, gitdb.SearchEquals) 432 | if err != nil { 433 | log.Println(err.Error()) 434 | return 435 | } 436 | 437 | accounts := []*BankAccount{} 438 | for _, r := range records { 439 | b := &BankAccount{} 440 | r.Hydrate(b) 441 | accounts = append(accounts, b) 442 | log.Print(fmt.Sprintf("%s-%s", b.ID, b.CreatedAt)) 443 | } 444 | } 445 | ``` 446 | 447 | ### Transactions 448 | ```go 449 | package main 450 | 451 | import ( 452 | "log" 453 | "github.com/gogitdb/gitdb/v2" 454 | ) 455 | 456 | func main() { 457 | cfg := gitdb.NewConfig("/tmp/data") 458 | db, err := gitdb.Open(cfg) 459 | if err != nil { 460 | log.Fatal(err) 461 | } 462 | defer db.Close() 463 | 464 | func accountUpgradeFuncOne() error { println("accountUpgradeFuncOne..."); return nil } 465 | func accountUpgradeFuncTwo() error { println("accountUpgradeFuncTwo..."); return errors.New("accountUpgradeFuncTwo failed") } 466 | func accountUpgradeFuncThree() error { println("accountUpgradeFuncThree"); return nil } 467 | 468 | tx := db.StartTransaction("AccountUpgrade") 469 | tx.AddOperation(accountUpgradeFuncOne) 470 | tx.AddOperation(accountUpgradeFuncTwo) 471 | tx.AddOperation(accountUpgradeFuncThree) 472 | terr := tx.Commit() 473 | if terr != nil { 474 | log.Print(terr) 475 | } 476 | } 477 | ``` 478 | 479 | ### Encryption 480 | 481 | GitDB suppports AES encryption and is done on a Model level, which means you can have a database with different Models where some are encrypted and others are not. To encrypt your data, your Model must implement `ShouldEncrypt()` to return true and you must set `gitdb.Config.EncryptionKey`. For maximum security set this key to a 32 byte string to select AES-256 482 | 483 | ```go 484 | package main 485 | 486 | import ( 487 | "log" 488 | "github.com/gogitdb/gitdb/v2" 489 | ) 490 | 491 | func main(){ 492 | cfg := gitdb.NewConfig("/tmp/data") 493 | cfg.EncryptionKey = "a_32_bytes_string_for_AES-256" 494 | db, err := gitdb.Open(cfg) 495 | if err != nil { 496 | log.Fatal(err) 497 | } 498 | defer db.Close() 499 | 500 | //populate model 501 | account := &BankAccount{} 502 | account.AccountNo = "0123456789" 503 | account.AccountType = "Savings" 504 | account.Currency = "GBP" 505 | account.Name = "Foo Bar" 506 | 507 | //Insert will encrypt the account 508 | err = db.Insert(account) 509 | if err != nil { 510 | log.Println(err) 511 | } 512 | 513 | //Get will automatically decrypt account 514 | var account BankAccount 515 | err = db.Get("Accounts/202003/0123456789", &account) 516 | if err != nil { 517 | log.Println(err) 518 | } 519 | } 520 | ``` 521 | 522 | ## Resources 523 | 524 | For more information on getting started with Gitdb, check out the following articles: 525 | * [Gtidb - an embedded distributed document database for Go](https://docs.google.com/document/d/1OPalq-7J_Vo_uks35up_4a_V0BsRrpwL1J4JG6ZFoEs/edit?usp=sharing) by Oke Ugwu 526 | 527 | 528 | ## Caveats & Limitations 529 | 530 | It's important to pick the right tool for the job and GitDB is no exception. 531 | Here are a few things to note when evaluating and using GitDB: 532 | 533 | * GitDB is good for systems where data producers are indpendent. 534 | * GitDB currently depends on the git binary to work 535 | 536 | ## Reading the Source 537 | 538 | GitDB is a relatively small code base (<5KLOC) for an embedded, distributed, 539 | document database so it can be a good starting point for people 540 | interested in how databases work. 541 | 542 | The best places to start are the main entry points into GitDB: 543 | 544 | - [`Open()`](https://github.com/gogitdb/gitdb/blob/a8088f138b072edd64021591e34adf878f2d0bd5/init.go#L17) - Initializes the reference to the database. It's responsible for 545 | creating the database if it doesn't exist and pulling down existing database 546 | if an online remote is specified. 547 | 548 | If you have additional notes that could be helpful for others, please submit 549 | them via pull request. 550 | 551 | 552 | 560 | -------------------------------------------------------------------------------- /all_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "io/ioutil" 9 | "os" 10 | "os/exec" 11 | "path/filepath" 12 | "strconv" 13 | "strings" 14 | "testing" 15 | 16 | "github.com/bouggo/log" 17 | "github.com/gogitdb/gitdb/v2" 18 | ) 19 | 20 | var testDb gitdb.GitDb 21 | var messageId int 22 | 23 | const testData = "/tmp/gitdb-test" 24 | const dbPath = testData + "/data" 25 | const fakeRemote = testData + "/online" 26 | 27 | //Test flags for more interactivity 28 | var flagLogLevel int 29 | var flagFakeRemote bool 30 | 31 | func TestMain(m *testing.M) { 32 | flag.IntVar(&flagLogLevel, "loglevel", int(log.LevelTest), "control verbosity of test logs") 33 | flag.BoolVar(&flagFakeRemote, "fakerepo", true, "create fake remote repo for tests") 34 | flag.Parse() 35 | 36 | //fail test if git is not installed 37 | if _, err := exec.LookPath("git"); err != nil { 38 | log.Test("git is required to run tests") 39 | return 40 | } 41 | 42 | gitdb.SetLogLevel(gitdb.LogLevel(flagLogLevel)) 43 | m.Run() 44 | } 45 | 46 | func setup(t testing.TB, cfg *gitdb.Config) func(t testing.TB) { 47 | 48 | if cfg == nil { 49 | cfg = getConfig() 50 | } 51 | 52 | if flagFakeRemote && !strings.HasPrefix(cfg.OnlineRemote, "git@") { 53 | fakeOnlineRepo(t) 54 | } 55 | 56 | //if DbPath is pointing to existing test data, create a fake .git folder 57 | //so that it passes the git repo check on db.boot 58 | if strings.HasPrefix(cfg.DBPath, "./testdata") { 59 | if err := os.MkdirAll(filepath.Join(cfg.DBPath, "data", ".git"), 0755); err != nil { 60 | t.Errorf("fake .git failed: %s", err.Error()) 61 | } 62 | } 63 | 64 | testDb = getDbConn(t, cfg) 65 | messageId = 0 66 | 67 | testDb.RegisterModel("Message", &Message{}) 68 | testDb.RegisterModel("MessageV2", &MessageV2{}) 69 | 70 | return teardown 71 | } 72 | 73 | func teardown(t testing.TB) { 74 | testDb.Close() 75 | 76 | log.Test("truncating test data") 77 | if err := os.RemoveAll(testData); err != nil { 78 | t.Errorf("cleanup failed - %s", err.Error()) 79 | } 80 | } 81 | 82 | func fakeOnlineRepo(t testing.TB) { 83 | log.Test("creating fake online repo") 84 | if err := os.MkdirAll(fakeRemote, 0755); err != nil { 85 | t.Errorf("fake repo failed: %s", err.Error()) 86 | return 87 | } 88 | 89 | cmd := exec.Command("git", "-C", fakeRemote, "init", "--bare") 90 | if _, err := cmd.CombinedOutput(); err != nil { 91 | t.Errorf("fake repo failed: %s", err.Error()) 92 | return 93 | } 94 | } 95 | 96 | func getDbConn(t testing.TB, cfg *gitdb.Config) gitdb.GitDb { 97 | conn, err := gitdb.Open(cfg) 98 | if err != nil { 99 | t.Errorf("getDbConn failed: %s", err) 100 | return nil 101 | } 102 | log.Test("test db connection opened") 103 | conn.SetUser(gitdb.NewUser("Tester", "tester@io")) 104 | return conn 105 | } 106 | 107 | func getConfig() *gitdb.Config { 108 | config := gitdb.NewConfig(dbPath) 109 | config.EncryptionKey = "b61ba8270ccc3c1d42b4417e7bd60b71" 110 | if flagFakeRemote { 111 | config.OnlineRemote = fakeRemote 112 | } 113 | 114 | return config 115 | } 116 | 117 | func getMockConfig() *gitdb.Config { 118 | config := gitdb.NewConfig("/mock") 119 | config.Mock = true 120 | return config 121 | } 122 | 123 | func getReadTestConfig(version string) *gitdb.Config { 124 | cfg := gitdb.NewConfig("./testdata/" + version + "/data") 125 | if version == "v1" { 126 | //add a factory method 127 | cfg.Factory = func(dataset string) gitdb.Model { 128 | switch dataset { 129 | case "Message": 130 | return &Message{} 131 | case "MessageV2": 132 | return &MessageV2{} 133 | } 134 | return &Message{} 135 | } 136 | } 137 | return cfg 138 | } 139 | 140 | func getTestMessage() *Message { 141 | m := getTestMessageWithId(messageId) 142 | messageId++ 143 | 144 | return m 145 | } 146 | 147 | func getTestMessageWithId(messageId int) *Message { 148 | m := &Message{ 149 | MessageId: messageId, 150 | From: "alice@example.com", 151 | To: "bob@example.com", 152 | Body: "Hello", 153 | } 154 | 155 | return m 156 | } 157 | 158 | type Message struct { 159 | gitdb.TimeStampedModel 160 | MessageId int 161 | From string 162 | To string 163 | Body string 164 | } 165 | 166 | func (m *Message) GetSchema() *gitdb.Schema { 167 | 168 | name := "Message" 169 | block := "b0" 170 | // block := gitdb.AutoBlock(dbPath, m, gitdb.BlockByCount, 2) 171 | record := fmt.Sprintf("%d", m.MessageId) 172 | 173 | //Indexes speed up searching 174 | indexes := make(map[string]interface{}) 175 | indexes["From"] = m.From 176 | 177 | return gitdb.NewSchema(name, block, record, indexes) 178 | } 179 | 180 | func (m *Message) Validate() error { return nil } 181 | func (m *Message) IsLockable() bool { return true } 182 | func (m *Message) ShouldEncrypt() bool { return true } 183 | func (m *Message) GetLockFileNames() []string { 184 | return []string{ 185 | fmt.Sprintf("%d-%s", m.MessageId, m.From), 186 | } 187 | } 188 | 189 | // func (m *Message) BeforeInsert() error { return nil } 190 | 191 | type MessageV2 struct { 192 | gitdb.TimeStampedModel 193 | MessageId int 194 | From string 195 | To string 196 | Body string 197 | } 198 | 199 | func (m *MessageV2) GetSchema() *gitdb.Schema { 200 | 201 | name := "MessageV2" 202 | block := m.CreatedAt.Format("200601") 203 | record := fmt.Sprintf("%d", m.MessageId) 204 | 205 | //Indexes speed up searching 206 | indexes := make(map[string]interface{}) 207 | indexes["From"] = m.From 208 | 209 | return gitdb.NewSchema(name, block, record, indexes) 210 | } 211 | 212 | func (m *MessageV2) Validate() error { return nil } 213 | func (m *MessageV2) IsLockable() bool { return false } 214 | func (m *MessageV2) ShouldEncrypt() bool { return false } 215 | func (m *MessageV2) GetLockFileNames() []string { return []string{} } 216 | 217 | // func (m *MessageV2) BeforeInsert() error { return nil } 218 | 219 | //count the number of records in fetched block 220 | func countRecords(dataset string) int { 221 | 222 | datasetPath := getConfig().DBPath + "/data/" + dataset + "/" 223 | 224 | cmd := exec.Command("/bin/bash", "-c", "grep "+dataset+" "+datasetPath+"*.json | wc -l | awk '{print $1}'") 225 | b, err := cmd.CombinedOutput() 226 | if err != nil { 227 | println(err.Error()) 228 | } 229 | 230 | v := strings.TrimSpace(string(b)) 231 | want, err := strconv.Atoi(v) 232 | if err != nil { 233 | println(v) 234 | println(err.Error()) 235 | want = 0 236 | } 237 | 238 | return want 239 | } 240 | 241 | func generateInserts(t testing.TB, count int) { 242 | log.Test(fmt.Sprintf("inserting %d records\n", count)) 243 | for i := 0; i < count; i++ { 244 | if err := testDb.Insert(getTestMessage()); err != nil { 245 | t.Errorf("generateInserts failed: %s", err) 246 | } 247 | 248 | } 249 | log.Test("done inserting") 250 | } 251 | 252 | func insert(m gitdb.Model, benchmark bool) error { 253 | if err := testDb.Insert(m); err != nil { 254 | return err 255 | } 256 | 257 | if !benchmark && !m.ShouldEncrypt() { 258 | //check that block file exist 259 | recordID := gitdb.ID(m) 260 | dataset, block, _, err := gitdb.ParseID(recordID) 261 | if err != nil { 262 | return err 263 | } 264 | cfg := getConfig() 265 | blockFile := filepath.Join(cfg.DBPath, "data", dataset, block+".json") 266 | if _, err := os.Stat(blockFile); err != nil { 267 | //return err 268 | return errors.New("insert test stat failed - " + blockFile) 269 | } else { 270 | b, err := ioutil.ReadFile(blockFile) 271 | if err != nil { 272 | return err 273 | } 274 | 275 | rep := strings.NewReplacer("\n", "", "\\", "", "\t", "", "\"{", "{", "}\"", "}", " ", "") 276 | got := rep.Replace(string(b)) 277 | 278 | w := map[string]interface{}{ 279 | recordID: struct { 280 | Version string 281 | Indexes map[string]interface{} 282 | Data gitdb.Model 283 | }{ 284 | gitdb.RecVersion, 285 | gitdb.Indexes(m), 286 | m, 287 | }, 288 | } 289 | 290 | x, _ := json.Marshal(w) 291 | want := string(x) 292 | 293 | want = want[1 : len(want)-1] 294 | 295 | if !strings.Contains(got, want) { 296 | return fmt.Errorf("Want: %s, Got: %s", want, got) 297 | } 298 | } 299 | } 300 | 301 | return nil 302 | } 303 | -------------------------------------------------------------------------------- /cmd/gitdb/gitdb.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "flag" 7 | "fmt" 8 | "io/ioutil" 9 | "os" 10 | "path" 11 | "path/filepath" 12 | "runtime" 13 | "strings" 14 | "time" 15 | ) 16 | 17 | var ( 18 | packageRoot = "../../" 19 | 20 | embedCommand = flag.NewFlagSet("embed", flag.ExitOnError) 21 | output = embedCommand.String("o", "./ui_static.go", "output file name; default ./ui_static.go") 22 | 23 | // dbpath = flag.String("p", "", "path do gitdb") 24 | ) 25 | 26 | func main() { 27 | 28 | command := os.Args[1] 29 | switch command { 30 | case "embed-ui": 31 | embedCommand.Parse(os.Args[2:]) 32 | err := embedUI() 33 | if err != nil { 34 | fmt.Println(err.Error()) 35 | } 36 | default: 37 | fmt.Println("invalid command; try gitdb embed-ui") 38 | //future commands 39 | //clean-db i.e git gc 40 | //repair 41 | //dataset 42 | //dataset blocks 43 | //dataset records 44 | return 45 | } 46 | } 47 | 48 | type staticFile struct { 49 | Name string 50 | Content string 51 | } 52 | 53 | func embedUI() error { 54 | _, filename, _, ok := runtime.Caller(0) 55 | if ok { 56 | packageRoot = path.Dir(path.Dir(path.Dir(filename))) + "/" 57 | } 58 | 59 | var files []staticFile 60 | if err := readAllStaticFiles(filepath.Join(packageRoot, "static"), &files); err != nil { 61 | return err 62 | } 63 | 64 | _, err := os.Stat(path.Dir(*output)) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | w, err := os.Create(*output) 70 | if err != nil { 71 | fmt.Println(err) 72 | return err 73 | } 74 | 75 | return packageTmpl.Execute(w, struct { 76 | Files []staticFile 77 | Date string 78 | }{ 79 | Files: files, 80 | Date: time.Now().Format(time.RFC1123), 81 | }) 82 | } 83 | 84 | func readAllStaticFiles(path string, files *[]staticFile) error { 85 | 86 | dirs, err := ioutil.ReadDir(path) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | for _, dir := range dirs { 92 | fileName := filepath.Join(path, dir.Name()) 93 | fmt.Println(fileName) 94 | if !dir.IsDir() { 95 | b, err := ioutil.ReadFile(fileName) 96 | if err != nil { 97 | return err 98 | } 99 | 100 | b = bytes.Replace(b, []byte(" "), []byte(""), -1) 101 | b = bytes.Replace(b, []byte("\n"), []byte(""), -1) 102 | 103 | content := base64.StdEncoding.EncodeToString(b) 104 | 105 | fileKey := strings.Replace(fileName, packageRoot, "", 1) 106 | *files = append(*files, staticFile{fileKey, content}) 107 | } 108 | 109 | if !strings.HasPrefix(dir.Name(), ".") && dir.IsDir() { 110 | readAllStaticFiles(fileName, files) 111 | } 112 | } 113 | 114 | return nil 115 | } 116 | -------------------------------------------------------------------------------- /cmd/gitdb/gitdb_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func Test_embedUI(t *testing.T) { 9 | err := embedUI() 10 | if err != nil { 11 | t.Errorf("embedUI() failed: %s", err) 12 | } 13 | os.Remove("./ui_static.go") 14 | } 15 | -------------------------------------------------------------------------------- /cmd/gitdb/template.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "text/template" 4 | 5 | var packageTmpl = template.Must(template.New("package").Parse(`package gitdb 6 | // Code generated by gitdb embed-ui on {{.Date}}; DO NOT EDIT. 7 | 8 | func init() { 9 | //Embed Files 10 | {{range .Files}} 11 | getFs().embed("{{.Name}}", "{{.Content}}") 12 | {{end}} 13 | } 14 | `)) 15 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | ) 7 | 8 | // Config represents configuration options for GitDB 9 | type Config struct { 10 | ConnectionName string 11 | DBPath string 12 | OnlineRemote string 13 | EncryptionKey string 14 | SyncInterval time.Duration 15 | User *User 16 | Factory func(string) Model 17 | EnableUI bool 18 | UIPort int 19 | // Mock is a hook for testing apps. If true will return a Mock DB connection 20 | Mock bool 21 | Driver dbDriver 22 | } 23 | 24 | const defaultConnectionName = "default" 25 | const defaultSyncInterval = time.Second * 5 26 | const defaultUserName = "ghost" 27 | const defaultUserEmail = "ghost@gitdb.local" 28 | const defaultUIPort = 4120 29 | 30 | // NewConfig constructs a *Config 31 | func NewConfig(dbPath string) *Config { 32 | return &Config{ 33 | DBPath: dbPath, 34 | SyncInterval: defaultSyncInterval, 35 | User: NewUser(defaultUserName, defaultUserEmail), 36 | ConnectionName: defaultConnectionName, 37 | UIPort: defaultUIPort, 38 | Driver: &gitDriver{driver: &gitBinaryDriver{}}, 39 | } 40 | } 41 | 42 | // NewConfigWithLocalDriver constructs a *Config 43 | func NewConfigWithLocalDriver(dbPath string) *Config { 44 | return &Config{ 45 | DBPath: dbPath, 46 | SyncInterval: defaultSyncInterval, 47 | User: NewUser(defaultUserName, defaultUserEmail), 48 | ConnectionName: defaultConnectionName, 49 | UIPort: defaultUIPort, 50 | Driver: &localDriver{}, 51 | } 52 | } 53 | 54 | // Validate returns an error is *Config.DBPath is not set 55 | func (c *Config) Validate() error { 56 | if len(c.DBPath) == 0 { 57 | return errors.New("Config.DbPath must be set") 58 | } 59 | 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "path/filepath" 7 | "sync" 8 | "time" 9 | 10 | "github.com/bouggo/log" 11 | "github.com/gogitdb/gitdb/v2/internal/db" 12 | ) 13 | 14 | // RecVersion of gitdb 15 | const RecVersion = "v2" 16 | 17 | // SearchMode defines how gitdb should search with SearchParam 18 | type SearchMode int 19 | 20 | const ( 21 | // SearchEquals will search index for records whose values equal SearchParam.Value 22 | SearchEquals SearchMode = 1 23 | // SearchContains will search index for records whose values contain SearchParam.Value 24 | SearchContains SearchMode = 2 25 | // SearchStartsWith will search index for records whose values start with SearchParam.Value 26 | SearchStartsWith SearchMode = 3 27 | // SearchEndsWith will search index for records whose values ends with SearchParam.Value 28 | SearchEndsWith SearchMode = 4 29 | ) 30 | 31 | // SearchParam represents search parameters against GitDB index 32 | type SearchParam struct { 33 | Index string 34 | Value string 35 | } 36 | 37 | // GitDb interface defines all exported funcs an implementation must have 38 | type GitDb interface { 39 | Close() error 40 | Insert(m Model) error 41 | InsertMany(m []Model) error 42 | Get(id string, m Model) error 43 | Exists(id string) error 44 | Fetch(dataset string, block ...string) ([]*db.Record, error) 45 | Search(dataDir string, searchParams []*SearchParam, searchMode SearchMode) ([]*db.Record, error) 46 | Delete(id string) error 47 | DeleteOrFail(id string) error 48 | Lock(m Model) error 49 | Unlock(m Model) error 50 | Upload() *Upload 51 | Migrate(from Model, to Model) error 52 | GetMails() []*mail 53 | StartTransaction(name string) Transaction 54 | GetLastCommitTime() (time.Time, error) 55 | SetUser(user *User) error 56 | Config() Config 57 | Sync() error 58 | RegisterModel(dataset string, m Model) bool 59 | } 60 | 61 | type gitdb struct { 62 | mu sync.Mutex 63 | indexMu sync.Mutex 64 | writeMu sync.Mutex 65 | syncMu sync.Mutex 66 | commit sync.WaitGroup 67 | locked chan bool 68 | shutdown chan bool 69 | events chan *dbEvent 70 | 71 | config Config 72 | driver dbDriver 73 | 74 | autoCommit bool 75 | indexUpdated bool 76 | loopStarted bool 77 | closed bool 78 | 79 | indexCache gdbSimpleIndexCache 80 | loadedBlocks map[string]*db.Block 81 | 82 | mails []*mail 83 | registry map[string]Model 84 | } 85 | 86 | func newConnection() *gitdb { 87 | // autocommit defaults to true 88 | db := &gitdb{autoCommit: true, indexCache: make(gdbSimpleIndexCache)} 89 | // initialize channels 90 | db.events = make(chan *dbEvent, 1) 91 | db.locked = make(chan bool, 1) 92 | // initialize shutdown channel with capacity 3 93 | // to represent the event loop, sync clock, UI server 94 | // goroutines 95 | db.shutdown = make(chan bool, 3) 96 | 97 | return db 98 | } 99 | 100 | func (g *gitdb) Config() Config { 101 | return g.config 102 | } 103 | 104 | func (g *gitdb) Close() error { 105 | if g == nil { 106 | return errors.New("gitdb is nil") 107 | } 108 | 109 | g.mu.Lock() 110 | defer g.mu.Unlock() 111 | log.Test("shutting down gitdb") 112 | if g.closed { 113 | log.Test("connection already closed") 114 | return nil 115 | } 116 | 117 | // flush index to disk 118 | if err := g.flushIndex(); err != nil { 119 | return err 120 | } 121 | 122 | // send shutdown event to event loop and sync clock 123 | g.shutdown <- true 124 | g.waitForCommit() 125 | 126 | // remove cached connection 127 | delete(conns, g.config.ConnectionName) 128 | g.closed = true 129 | log.Info("closed gitdb conn") 130 | return nil 131 | } 132 | 133 | func (g *gitdb) configure(cfg Config) { 134 | if len(cfg.ConnectionName) == 0 { 135 | cfg.ConnectionName = defaultConnectionName 136 | } 137 | 138 | if int64(cfg.SyncInterval) == 0 { 139 | cfg.SyncInterval = defaultSyncInterval 140 | } 141 | 142 | if cfg.UIPort == 0 { 143 | cfg.UIPort = defaultUIPort 144 | } 145 | 146 | g.driver = cfg.Driver 147 | if cfg.Driver == nil { 148 | g.driver = &gitDriver{driver: &gitBinaryDriver{}} 149 | } 150 | 151 | g.config = cfg 152 | } 153 | 154 | // Migrate model from one schema to another 155 | func (g *gitdb) Migrate(from Model, to Model) error { 156 | 157 | // TODO add test case for this 158 | // schema has not changed 159 | /*if from.GetSchema().recordID() == to.GetSchema().recordID() { 160 | return errors.New("Invalid migration - no change found in schema") 161 | }*/ 162 | 163 | block := db.NewEmptyBlock(g.config.EncryptionKey) 164 | if err := g.doFetch(from.GetSchema().name(), block); err != nil { 165 | return err 166 | } 167 | 168 | oldBlocks := map[string]string{} 169 | var migrate []Model 170 | for _, record := range block.Records() { 171 | dataset, blockID, _, _ := ParseID(record.ID()) 172 | if _, ok := oldBlocks[blockID]; !ok { 173 | blockFilePath := filepath.Join(g.dbDir(), dataset, blockID+".json") 174 | oldBlocks[blockID] = blockFilePath 175 | } 176 | 177 | if err := record.Hydrate(to); err != nil { 178 | return err 179 | } 180 | 181 | migrate = append(migrate, to) 182 | } 183 | 184 | // InsertMany will rollback if any insert fails 185 | if err := g.InsertMany(migrate); err != nil { 186 | return err 187 | } 188 | 189 | // remove all old block files 190 | for _, blockFilePath := range oldBlocks { 191 | log.Info("Removing old block: " + blockFilePath) 192 | err := os.Remove(blockFilePath) 193 | if err != nil { 194 | return err 195 | } 196 | } 197 | 198 | return nil 199 | } 200 | 201 | func (g *gitdb) GetLastCommitTime() (time.Time, error) { 202 | return g.driver.lastCommitTime() 203 | } 204 | -------------------------------------------------------------------------------- /db_mock.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "reflect" 8 | "strings" 9 | "time" 10 | 11 | "github.com/bouggo/log" 12 | "github.com/gogitdb/gitdb/v2/internal/db" 13 | ) 14 | 15 | type mockdb struct { 16 | config Config 17 | data map[string]Model 18 | index map[string]map[string]interface{} 19 | locks map[string]bool 20 | } 21 | 22 | type mocktransaction struct { 23 | name string 24 | operations []operation 25 | db *mockdb 26 | } 27 | 28 | func (t *mocktransaction) Commit() error { 29 | for _, o := range t.operations { 30 | if err := o(); err != nil { 31 | log.Info("Reverting transaction: " + err.Error()) 32 | return err 33 | } 34 | } 35 | return nil 36 | } 37 | 38 | func (t *mocktransaction) AddOperation(o operation) { 39 | t.operations = append(t.operations, o) 40 | } 41 | 42 | func newMockConnection() *mockdb { 43 | db := &mockdb{ 44 | data: make(map[string]Model), 45 | index: make(map[string]map[string]interface{}), 46 | locks: make(map[string]bool), 47 | } 48 | return db 49 | } 50 | 51 | func (g *mockdb) Close() error { 52 | g.data = nil 53 | return nil 54 | } 55 | 56 | func (g *mockdb) Insert(m Model) error { 57 | g.data[ID(m)] = m 58 | 59 | for name, value := range m.GetSchema().indexes { 60 | key := m.GetSchema().dataset + "." + name 61 | if _, ok := g.index[key]; !ok { 62 | g.index[key] = make(map[string]interface{}) 63 | } 64 | g.index[key][ID(m)] = value 65 | } 66 | 67 | return nil 68 | } 69 | 70 | func (g *mockdb) InsertMany(m []Model) error { 71 | for _, model := range m { 72 | g.Insert(model) 73 | } 74 | return nil 75 | } 76 | 77 | func (g *mockdb) Get(id string, result Model) error { 78 | 79 | if reflect.ValueOf(result).Kind() != reflect.Ptr || reflect.ValueOf(result).IsNil() { 80 | return errors.New("Second argument to Get must be a non-nil pointer") 81 | } 82 | 83 | model, exists := g.data[id] 84 | if exists { 85 | 86 | resultType := reflect.ValueOf(result).Type() 87 | modelType := reflect.ValueOf(model).Type() 88 | if resultType != modelType { 89 | return fmt.Errorf("Second argument to Get must be of type %v", modelType) 90 | } 91 | 92 | b, err := json.Marshal(model) 93 | if err != nil { 94 | return err 95 | } 96 | return json.Unmarshal(b, result) 97 | } 98 | 99 | dataset, _, _, _ := ParseID(id) 100 | return fmt.Errorf("Record %s not found in %s", id, dataset) 101 | } 102 | 103 | func (g *mockdb) Exists(id string) error { 104 | _, exists := g.data[id] 105 | if !exists { 106 | dataset, _, _, _ := ParseID(id) 107 | return fmt.Errorf("Record %s not found in %s", id, dataset) 108 | } 109 | 110 | return nil 111 | } 112 | 113 | func (g *mockdb) Fetch(dataset string, blocks ...string) ([]*db.Record, error) { 114 | var result []*db.Record 115 | blockStream := "|" + strings.Join(blocks, "|") + "|" 116 | for id, model := range g.data { 117 | ds, b, _, err := ParseID(id) 118 | if err != nil { 119 | log.Error(err.Error()) 120 | continue 121 | } 122 | 123 | if len(blocks) > 0 { 124 | if ds == dataset && strings.Contains(blockStream, "|"+b+"|") { 125 | result = append(result, db.ConvertModel(ID(model), model)) 126 | } 127 | } else { 128 | if ds == dataset { 129 | result = append(result, db.ConvertModel(ID(model), model)) 130 | } 131 | } 132 | } 133 | 134 | return result, nil 135 | } 136 | 137 | func (g *mockdb) Search(dataset string, searchParams []*SearchParam, searchMode SearchMode) ([]*db.Record, error) { 138 | result := []*db.Record{} 139 | for _, searchParam := range searchParams { 140 | key := dataset + "." + searchParam.Index 141 | 142 | queryValue := strings.ToLower(searchParam.Value) 143 | for recordID, value := range g.index[key] { 144 | addResult := false 145 | dbValue := strings.ToLower(value.(string)) 146 | switch searchMode { 147 | case SearchEquals: 148 | addResult = dbValue == queryValue 149 | case SearchContains: 150 | addResult = strings.Contains(dbValue, queryValue) 151 | case SearchStartsWith: 152 | addResult = strings.HasPrefix(dbValue, queryValue) 153 | case SearchEndsWith: 154 | addResult = strings.HasSuffix(dbValue, queryValue) 155 | } 156 | 157 | if addResult { 158 | result = append(result, db.ConvertModel(recordID, g.data[recordID])) 159 | } 160 | } 161 | } 162 | 163 | return result, nil 164 | } 165 | 166 | func (g *mockdb) Delete(id string) error { 167 | delete(g.data, id) 168 | return nil 169 | } 170 | 171 | func (g *mockdb) DeleteOrFail(id string) error { 172 | _, exists := g.data[id] 173 | if !exists { 174 | return fmt.Errorf("record %s does not exist", id) 175 | } 176 | 177 | delete(g.data, id) 178 | return nil 179 | } 180 | 181 | func (g *mockdb) Lock(m Model) error { 182 | 183 | if _, ok := m.(LockableModel); !ok { 184 | return errors.New("Model is not lockable") 185 | } 186 | 187 | for _, l := range m.(LockableModel).GetLockFileNames() { 188 | key := m.GetSchema().dataset + "." + l 189 | if _, ok := g.locks[l]; !ok { 190 | g.locks[key] = true 191 | } else { 192 | return errors.New("Lock file already exist: " + l) 193 | } 194 | } 195 | return nil 196 | } 197 | 198 | func (g *mockdb) Unlock(m Model) error { 199 | if _, ok := m.(LockableModel); !ok { 200 | return errors.New("Model is not lockable") 201 | } 202 | 203 | for _, l := range m.(LockableModel).GetLockFileNames() { 204 | key := m.GetSchema().dataset + "." + l 205 | delete(g.locks, key) 206 | } 207 | return nil 208 | } 209 | 210 | func (g *mockdb) Upload() *Upload { 211 | //todo 212 | return nil 213 | } 214 | 215 | func (g *mockdb) GetMails() []*mail { 216 | return []*mail{} 217 | } 218 | 219 | func (g *mockdb) StartTransaction(name string) Transaction { 220 | //todo return mock transaction 221 | return &mocktransaction{name: name, db: g} 222 | } 223 | 224 | func (g *mockdb) GetLastCommitTime() (time.Time, error) { 225 | return time.Now(), nil 226 | } 227 | 228 | func (g *mockdb) SetUser(user *User) error { 229 | g.config.User = user 230 | return nil 231 | } 232 | 233 | func (g *mockdb) Migrate(from Model, to Model) error { 234 | 235 | migrate := []Model{} 236 | records, err := g.Fetch(from.GetSchema().dataset) 237 | if err != nil { 238 | return err 239 | } 240 | 241 | for _, record := range records { 242 | if err := record.Hydrate(to); err != nil { 243 | return err 244 | } 245 | 246 | migrate = append(migrate, to) 247 | } 248 | 249 | if err := g.InsertMany(migrate); err != nil { 250 | return err 251 | } 252 | 253 | return nil 254 | } 255 | 256 | func (g *mockdb) Config() Config { 257 | return g.config 258 | } 259 | 260 | func (g *mockdb) configure(cfg Config) { 261 | if len(cfg.ConnectionName) == 0 { 262 | cfg.ConnectionName = defaultConnectionName 263 | } 264 | g.config = cfg 265 | } 266 | 267 | func (g *mockdb) Sync() error { 268 | return nil 269 | } 270 | 271 | func (g *mockdb) RegisterModel(dataset string, m Model) bool { 272 | return true 273 | } 274 | -------------------------------------------------------------------------------- /db_mock_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import ( 4 | "errors" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/gogitdb/gitdb/v2" 9 | ) 10 | 11 | func setupMock(t *testing.T) gitdb.GitDb { 12 | cfg := getMockConfig() 13 | db, err := gitdb.Open(cfg) 14 | if err != nil { 15 | t.Error("mock db setup failed") 16 | } 17 | 18 | //insert some test models for testing read operations 19 | i := 101 20 | for i < 111 { 21 | m := getTestMessageWithId(i) 22 | db.Insert(m) 23 | i++ 24 | } 25 | 26 | return db 27 | } 28 | 29 | func TestMockClose(t *testing.T) { 30 | db := setupMock(t) 31 | if err := db.Close(); err != nil { 32 | t.Errorf("db.Close() failed: %s", err) 33 | } 34 | } 35 | 36 | func TestMockInsert(t *testing.T) { 37 | db := setupMock(t) 38 | m := getTestMessage() 39 | 40 | if err := db.Insert(m); err != nil { 41 | t.Errorf("db.Insert failed: %s", err) 42 | } 43 | } 44 | 45 | func TestMockInsertMany(t *testing.T) { 46 | 47 | db := setupMock(t) 48 | msgs := []gitdb.Model{} 49 | for i := 0; i < 10; i++ { 50 | m := getTestMessage() 51 | msgs = append(msgs, m) 52 | } 53 | 54 | err := db.InsertMany(msgs) 55 | if err != nil { 56 | t.Errorf("db.InsertMany failed: %s", err) 57 | } 58 | } 59 | 60 | func TestMockGet(t *testing.T) { 61 | db := setupMock(t) 62 | 63 | m := &Message{} 64 | 65 | id := "Message/b0/1" 66 | if err := db.Get(id, m); err == nil { 67 | t.Errorf("db.Get(%s) failed: %s", id, err) 68 | } 69 | 70 | id = "Message/b0/110" 71 | if err := db.Get(id, m); err != nil { 72 | t.Errorf("db.Get(%s) failed: %s", id, err) 73 | } 74 | 75 | if mId := gitdb.ID(m); mId != id { 76 | t.Errorf("db.Get(%s) want: %s, got: %s", id, id, mId) 77 | } 78 | } 79 | 80 | func TestMockExists(t *testing.T) { 81 | db := setupMock(t) 82 | 83 | id := "Message/b0/1" 84 | if err := db.Exists(id); err == nil { 85 | t.Errorf("db.Get(%s) failed: %s", id, err) 86 | } 87 | 88 | id = "Message/b0/110" 89 | if err := db.Exists(id); err != nil { 90 | t.Errorf("db.Get(%s) failed: %s", id, err) 91 | } 92 | } 93 | 94 | func TestMockFetch(t *testing.T) { 95 | db := setupMock(t) 96 | 97 | dataset := "Message" 98 | records, err := db.Fetch(dataset) 99 | if err != nil { 100 | t.Errorf("db.Fetch(%s) failed: %s", dataset, err) 101 | } 102 | 103 | want := 10 104 | if got := len(records); got != want { 105 | t.Errorf("db.Fetch(%s) failed: want %d, got %d", dataset, want, got) 106 | } 107 | } 108 | 109 | func TestMockFetchBlock(t *testing.T) { 110 | db := setupMock(t) 111 | 112 | dataset := "Message" 113 | records, err := db.Fetch(dataset, "b0") 114 | if err != nil { 115 | t.Errorf("db.Fetch(%s) failed: %s", dataset, err) 116 | } 117 | 118 | want := 10 119 | if got := len(records); got != want { 120 | t.Errorf("db.Fetch(%s) failed: want %d, got %d", dataset, want, got) 121 | } 122 | } 123 | 124 | func TestMockSearch(t *testing.T) { 125 | db := setupMock(t) 126 | 127 | count := 10 128 | sp := &gitdb.SearchParam{ 129 | Index: "From", 130 | Value: "alice@example.com", 131 | } 132 | 133 | results, err := db.Search("Message", []*gitdb.SearchParam{sp}, gitdb.SearchEquals) 134 | if err != nil { 135 | t.Errorf("search failed with error - %s", err) 136 | } 137 | 138 | if len(results) != count { 139 | t.Errorf("search result count wrong. want: %d, got: %d", count, len(results)) 140 | } 141 | } 142 | 143 | func TestMockDelete(t *testing.T) { 144 | db := setupMock(t) 145 | 146 | //delete non-existent record should pass 147 | id := "Message/b0/1" 148 | if err := db.Delete(id); err != nil { 149 | t.Errorf("db.Delete(%s) failed: %s", id, err) 150 | } 151 | 152 | //delete existent record should pass 153 | id = "Message/b0/110" 154 | if err := db.Delete(id); err != nil { 155 | t.Errorf("db.Delete(%s) failed: %s", id, err) 156 | } 157 | } 158 | func TestMockDeleteOrFail(t *testing.T) { 159 | db := setupMock(t) 160 | 161 | //delete non-existent record should fail 162 | id := "Message/b0/1" 163 | if err := db.DeleteOrFail(id); err == nil { 164 | t.Errorf("db.DeleteOrFail(%s) failed: %s", id, err) 165 | } 166 | 167 | id = "Message/b0/110" 168 | if err := db.DeleteOrFail(id); err != nil { 169 | t.Errorf("db.DeleteOrFail(%s) failed: %s", id, err) 170 | } 171 | } 172 | func TestMockLock(t *testing.T) { 173 | db := setupMock(t) 174 | 175 | m := getTestMessage() 176 | if err := db.Lock(m); err != nil { 177 | t.Errorf("db.Lock(m) failed: %s", err) 178 | } 179 | } 180 | 181 | func TestMockUnlock(t *testing.T) { 182 | db := setupMock(t) 183 | 184 | m := getTestMessage() 185 | if err := db.Unlock(m); err != nil { 186 | t.Errorf("db.Lock(m) failed: %s", err) 187 | } 188 | } 189 | 190 | func TestMockMigrate(t *testing.T) { 191 | db := setupMock(t) 192 | 193 | m := &Message{} 194 | m2 := &MessageV2{} 195 | 196 | if err := db.Migrate(m, m2); err != nil { 197 | t.Errorf("db.Migrate(m, m2) returned error - %s", err) 198 | } 199 | } 200 | 201 | func TestMockGetMails(t *testing.T) { 202 | db := setupMock(t) 203 | mails := db.GetMails() 204 | if len(mails) > 0 { 205 | t.Errorf("db.GetMails() should be 0") 206 | } 207 | } 208 | 209 | func TestMockStartTransaction(t *testing.T) { 210 | db := setupMock(t) 211 | tx := db.StartTransaction("test") 212 | if tx == nil { 213 | t.Errorf("db.StartTransaction() returned: %v", nil) 214 | } 215 | 216 | tx.AddOperation(func() error { return nil }) 217 | tx.AddOperation(func() error { return nil }) 218 | tx.AddOperation(func() error { return errors.New("test error") }) 219 | tx.AddOperation(func() error { return nil }) 220 | if err := tx.Commit(); err == nil { 221 | t.Error("transaction should fail on 3rd operation") 222 | } 223 | } 224 | 225 | func TestMockGetLastCommitTime(t *testing.T) { 226 | db := setupMock(t) 227 | if _, err := db.GetLastCommitTime(); err != nil { 228 | t.Errorf("db.GetLastCommitTime() returned error - %s", err) 229 | } 230 | } 231 | 232 | func TestMockSetUser(t *testing.T) { 233 | db := setupMock(t) 234 | 235 | user := gitdb.NewUser("test", "tester@gitdb.io") 236 | if err := db.SetUser(user); err != nil { 237 | t.Errorf("db.SetUser failed: %s", err) 238 | } 239 | } 240 | 241 | func TestMockConfig(t *testing.T) { 242 | db := setupMock(t) 243 | // cfg := getMockConfig() 244 | if reflect.DeepEqual(db.Config(), getMockConfig()) { 245 | t.Errorf("db.Config != getMockConfig()") 246 | } 247 | } 248 | -------------------------------------------------------------------------------- /db_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/gogitdb/gitdb/v2" 8 | ) 9 | 10 | func TestMigrate(t *testing.T) { 11 | teardown := setup(t, nil) 12 | defer teardown(t) 13 | 14 | m := getTestMessageWithId(0) 15 | if err := insert(m, false); err != nil { 16 | t.Errorf("insert failed: %s", err) 17 | } 18 | 19 | m2 := &MessageV2{} 20 | 21 | if err := testDb.Migrate(m, m2); err != nil { 22 | t.Errorf("testDb.Migrate() returned error - %s", err) 23 | } 24 | } 25 | 26 | func TestNewConfig(t *testing.T) { 27 | cfg := gitdb.NewConfig(dbPath) 28 | db, err := gitdb.Open(cfg) 29 | if err != nil { 30 | t.Errorf("gitdb.Open failed: %s", err) 31 | } 32 | 33 | if reflect.DeepEqual(db.Config(), cfg) { 34 | t.Errorf("Config does not match. want: %v, got: %v", cfg, db.Config()) 35 | } 36 | } 37 | 38 | func TestNewConfigWithLocalDriver(t *testing.T) { 39 | cfg := gitdb.NewConfigWithLocalDriver(dbPath) 40 | db, err := gitdb.Open(cfg) 41 | if err != nil { 42 | t.Errorf("gitdb.Open failed: %s", err) 43 | } 44 | 45 | if reflect.DeepEqual(db.Config(), cfg) { 46 | t.Errorf("Config does not match. want: %v, got: %v", cfg, db.Config()) 47 | } 48 | } 49 | 50 | func TestConfigValidate(t *testing.T) { 51 | cfg := &gitdb.Config{} 52 | if err := cfg.Validate(); err == nil { 53 | t.Errorf("cfg.Validate should fail if DbPath is %s", cfg.DBPath) 54 | } 55 | } 56 | 57 | func TestGetLastCommitTime(t *testing.T) { 58 | teardown := setup(t, nil) 59 | defer teardown(t) 60 | 61 | _, err := testDb.GetLastCommitTime() 62 | if err == nil { 63 | t.Errorf("dbConn.GetLastCommitTime() returned error - %s", err) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /driver.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import "time" 4 | 5 | type dbDriver interface { 6 | name() string 7 | setup(db *gitdb) error 8 | sync() error 9 | commit(filePath string, msg string, user *User) error 10 | undo() error 11 | changedFiles() []string 12 | lastCommitTime() (time.Time, error) 13 | } 14 | 15 | type gitDBDriver interface { 16 | dbDriver 17 | init() error 18 | clone() error 19 | addRemote() error 20 | pull() error 21 | push() error 22 | } 23 | -------------------------------------------------------------------------------- /driver_git.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "time" 10 | 11 | "github.com/bouggo/log" 12 | ) 13 | 14 | type gitDriver struct { 15 | driver gitDBDriver 16 | absDBPath string 17 | } 18 | 19 | func (d *gitDriver) name() string { 20 | return d.driver.name() 21 | } 22 | 23 | // if .db directory does not exist, create it and attempt 24 | // to do a git clone from remote 25 | func (d *gitDriver) setup(db *gitdb) error { 26 | d.absDBPath = db.dbDir() 27 | if err := d.driver.setup(db); err != nil { 28 | return err 29 | } 30 | 31 | // create .ssh dir 32 | if err := db.generateSSHKeyPair(); err != nil { 33 | return err 34 | } 35 | 36 | // force git to only use generated ssh key and not fallback to ssh_config or ssh-agent 37 | sshCmd := fmt.Sprintf("ssh -F none -i '%s' -o IdentitiesOnly=yes -o StrictHostKeyChecking=no", db.privateKeyFilePath()) 38 | if err := os.Setenv("GIT_SSH_COMMAND", sshCmd); err != nil { 39 | return err 40 | } 41 | 42 | dataDir := db.dbDir() 43 | dotGitDir := filepath.Join(dataDir, ".git") 44 | if _, err := os.Stat(dataDir); err != nil { 45 | log.Info("database not initialized") 46 | 47 | // create db directory 48 | if err := os.MkdirAll(dataDir, 0755); err != nil { 49 | return err 50 | } 51 | 52 | if len(db.config.OnlineRemote) > 0 { 53 | if err := d.clone(); err != nil { 54 | return err 55 | } 56 | 57 | if err := d.addRemote(); err != nil { 58 | return err 59 | } 60 | } else if err := d.init(); err != nil { 61 | return err 62 | } 63 | } else if _, err := os.Stat(dotGitDir); err != nil { 64 | log.Info(err.Error()) 65 | return errors.New(db.config.DBPath + " is not a git repository") 66 | } else if len(db.config.OnlineRemote) > 0 { // TODO Review this properly 67 | // if remote is configured i.e stat .git/refs/remotes/online 68 | // if remote dir does not exist add remotes 69 | remotesPath := filepath.Join(dataDir, ".git", "refs", "remotes", "online") 70 | if _, err := os.Stat(remotesPath); err != nil { 71 | if err := d.addRemote(); err != nil { 72 | return err 73 | } 74 | } 75 | } 76 | 77 | return nil 78 | } 79 | 80 | // this function is only called once. I.e when a initializing the database for the 81 | // very first time. In this case we must clone the online repo 82 | func (d *gitDriver) init() error { 83 | // we take this very seriously 84 | if err := d.driver.init(); err != nil { 85 | if err := os.RemoveAll(d.absDBPath); err != nil { 86 | return err 87 | } 88 | } 89 | 90 | return nil 91 | } 92 | 93 | func (d *gitDriver) clone() error { 94 | // we take this very seriously 95 | log.Info("cloning down database...") 96 | if err := d.driver.clone(); err != nil { 97 | // TODO if err is authentication related generate key pair 98 | if err := os.RemoveAll(d.absDBPath); err != nil { 99 | return err 100 | } 101 | 102 | if strings.Contains(err.Error(), "denied") { 103 | return ErrAccessDenied 104 | } 105 | return err 106 | } 107 | 108 | return nil 109 | } 110 | 111 | func (d *gitDriver) addRemote() error { 112 | // we take this very seriously 113 | if err := d.driver.addRemote(); err != nil { 114 | if !strings.Contains(err.Error(), "already exists") { 115 | if err := os.RemoveAll(d.absDBPath); err != nil { // TODO is this necessary? 116 | return err 117 | } 118 | return err 119 | } 120 | } 121 | 122 | return nil 123 | } 124 | 125 | func (d *gitDriver) sync() error { 126 | return d.driver.sync() 127 | } 128 | 129 | func (d *gitDriver) commit(filePath string, msg string, user *User) error { 130 | mu.Lock() 131 | defer mu.Unlock() 132 | if err := d.driver.commit(filePath, msg, user); err != nil { 133 | // todo: update to return this error but for now at least log it 134 | log.Error(err.Error()) 135 | } 136 | 137 | return nil 138 | } 139 | 140 | func (d *gitDriver) undo() error { 141 | return d.driver.undo() 142 | } 143 | 144 | func (d *gitDriver) changedFiles() []string { 145 | return d.driver.changedFiles() 146 | } 147 | 148 | func (d *gitDriver) lastCommitTime() (time.Time, error) { 149 | return d.driver.lastCommitTime() 150 | } 151 | -------------------------------------------------------------------------------- /driver_gitbinary.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "errors" 5 | "os/exec" 6 | "strings" 7 | "time" 8 | 9 | "github.com/bouggo/log" 10 | ) 11 | 12 | type gitBinaryDriver struct { 13 | config Config 14 | absDBPath string 15 | } 16 | 17 | func (d *gitBinaryDriver) name() string { 18 | return "gitBinary" 19 | } 20 | 21 | func (d *gitBinaryDriver) setup(db *gitdb) error { 22 | d.config = db.config 23 | d.absDBPath = db.dbDir() 24 | return nil 25 | } 26 | 27 | func (d *gitBinaryDriver) init() error { 28 | cmd := exec.Command("git", "-C", d.absDBPath, "init") 29 | // log(utils.CmdToString(cmd)) 30 | if out, err := cmd.CombinedOutput(); err != nil { 31 | log.Info(string(out)) 32 | return err 33 | } 34 | 35 | return nil 36 | } 37 | 38 | func (d *gitBinaryDriver) clone() error { 39 | 40 | cmd := exec.Command("git", "clone", "--depth", "10", d.config.OnlineRemote, d.absDBPath) 41 | if out, err := cmd.CombinedOutput(); err != nil { 42 | return errors.New(string(out)) 43 | } 44 | 45 | return nil 46 | } 47 | 48 | func (d *gitBinaryDriver) addRemote() error { 49 | // check to see if we have origin / online remotes 50 | cmd := exec.Command("git", "-C", d.absDBPath, "remote") 51 | out, err := cmd.CombinedOutput() 52 | 53 | if err != nil { 54 | // log(string(out)) 55 | return err 56 | } 57 | 58 | remoteStr := string(out) 59 | hasOriginRemote := strings.Contains(remoteStr, "origin") 60 | hasOnlineRemote := strings.Contains(remoteStr, "online") 61 | 62 | if hasOriginRemote { 63 | cmd := exec.Command("git", "-C", d.absDBPath, "remote", "rm", "origin") 64 | if out, err := cmd.CombinedOutput(); err != nil { 65 | log.Info(string(out)) 66 | } 67 | } 68 | 69 | if !hasOnlineRemote { 70 | cmd = exec.Command("git", "-C", d.absDBPath, "remote", "add", "online", d.config.OnlineRemote) 71 | // log(utils.CmdToString(cmd)) 72 | if out, err := cmd.CombinedOutput(); err != nil { 73 | return errors.New(string(out)) 74 | } 75 | } 76 | 77 | return nil 78 | } 79 | 80 | func (d *gitBinaryDriver) sync() error { 81 | if err := d.pull(); err != nil { 82 | return err 83 | } 84 | if err := d.push(); err != nil { 85 | return err 86 | } 87 | 88 | return nil 89 | } 90 | 91 | func (d *gitBinaryDriver) pull() error { 92 | cmd := exec.Command("git", "-C", d.absDBPath, "pull", "online", "master") 93 | // log(utils.CmdToString(cmd)) 94 | if out, err := cmd.CombinedOutput(); err != nil { 95 | log.Error(string(out)) 96 | 97 | return errors.New("failed to pull data from online remote") 98 | } 99 | 100 | return nil 101 | } 102 | 103 | func (d *gitBinaryDriver) push() error { 104 | cmd := exec.Command("git", "-C", d.absDBPath, "push", "online", "master") 105 | // log(utils.CmdToString(cmd)) 106 | if out, err := cmd.CombinedOutput(); err != nil { 107 | log.Error(string(out)) 108 | return errors.New("failed to push data to online remotes") 109 | } 110 | 111 | return nil 112 | } 113 | 114 | func (d *gitBinaryDriver) commit(filePath string, msg string, user *User) error { 115 | cmd := exec.Command("git", "-C", d.absDBPath, "config", "user.email", user.Email) 116 | // log(utils.CmdToString(cmd)) 117 | if out, err := cmd.CombinedOutput(); err != nil { 118 | log.Error(string(out)) 119 | return err 120 | } 121 | 122 | cmd = exec.Command("git", "-C", d.absDBPath, "config", "user.name", user.Name) 123 | // log(utils.CmdToString(cmd)) 124 | if out, err := cmd.CombinedOutput(); err != nil { 125 | log.Error(string(out)) 126 | return err 127 | } 128 | 129 | cmd = exec.Command("git", "-C", d.absDBPath, "add", filePath) 130 | // log(utils.CmdToString(cmd)) 131 | if out, err := cmd.CombinedOutput(); err != nil { 132 | log.Error(string(out)) 133 | return err 134 | } 135 | 136 | cmd = exec.Command("git", "-C", d.absDBPath, "commit", "-am", msg) 137 | // log(utils.CmdToString(cmd)) 138 | if out, err := cmd.CombinedOutput(); err != nil { 139 | log.Error(string(out)) 140 | return err 141 | } 142 | 143 | log.Info("new changes committed") 144 | return nil 145 | } 146 | 147 | func (d *gitBinaryDriver) undo() error { 148 | cmd := exec.Command("git", "-C", d.absDBPath, "checkout", ".") 149 | // log(utils.CmdToString(cmd)) 150 | if out, err := cmd.CombinedOutput(); err != nil { 151 | log.Error(string(out)) 152 | return err 153 | } 154 | 155 | cmd = exec.Command("git", "-C", d.absDBPath, "clean", "-fd") 156 | // log(utils.CmdToString(cmd)) 157 | if out, err := cmd.CombinedOutput(); err != nil { 158 | log.Error(string(out)) 159 | return err 160 | } 161 | 162 | log.Info("changes reverted") 163 | return nil 164 | } 165 | 166 | func (d *gitBinaryDriver) changedFiles() []string { 167 | var files []string 168 | if len(d.config.OnlineRemote) > 0 { 169 | log.Test("getting list of changed files...") 170 | // git fetch 171 | cmd := exec.Command("git", "-C", d.absDBPath, "fetch", "online", "master") 172 | if out, err := cmd.CombinedOutput(); err != nil { 173 | log.Error(string(out)) 174 | return files 175 | } 176 | 177 | // git diff --name-only ..online/master 178 | cmd = exec.Command("git", "-C", d.absDBPath, "diff", "--name-only", "..online/master") 179 | out, err := cmd.CombinedOutput() 180 | if err != nil { 181 | log.Error(string(out)) 182 | return files 183 | } 184 | 185 | output := string(out) 186 | if len(output) > 0 { 187 | // strip out lock files 188 | for _, file := range strings.Split(output, "\n") { 189 | if strings.HasSuffix(file, ".json") { 190 | files = append(files, file) 191 | } 192 | } 193 | 194 | return files 195 | } 196 | } 197 | 198 | return files 199 | } 200 | 201 | func (d *gitBinaryDriver) lastCommitTime() (time.Time, error) { 202 | var t time.Time 203 | cmd := exec.Command("git", "-C", d.absDBPath, "log", "-1", "--remotes=online", "--format=%cd", "--date=iso") 204 | // log.PutInfo(utils.CmdToString(cmd)) 205 | out, err := cmd.CombinedOutput() 206 | if err != nil { 207 | // log.PutError("gitLastCommit Failed") 208 | return t, err 209 | } 210 | 211 | timeString := string(out) 212 | if len(timeString) >= 25 { 213 | return time.Parse("2006-01-02 15:04:05 -0700", timeString[:25]) 214 | } 215 | 216 | return t, errors.New("no commit history in repo") 217 | } 218 | -------------------------------------------------------------------------------- /driver_local.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "time" 7 | 8 | "github.com/bouggo/log" 9 | ) 10 | 11 | type localDriver struct { 12 | config Config 13 | absDBPath string 14 | } 15 | 16 | func (d *localDriver) name() string { 17 | return "local" 18 | } 19 | 20 | func (d *localDriver) setup(db *gitdb) error { 21 | d.config = db.config 22 | d.absDBPath = db.dbDir() 23 | // create db directory 24 | if err := os.MkdirAll(db.dbDir(), 0755); err != nil { 25 | return err 26 | } 27 | return nil 28 | } 29 | 30 | func (d *localDriver) sync() error { 31 | return nil 32 | } 33 | 34 | func (d *localDriver) commit(filePath string, msg string, user *User) error { 35 | log.Info("new changes committed") 36 | return nil 37 | } 38 | 39 | func (d *localDriver) undo() error { 40 | log.Info("changes reverted") 41 | return nil 42 | } 43 | 44 | func (d *localDriver) changedFiles() []string { 45 | var files []string 46 | return files 47 | } 48 | 49 | func (d *localDriver) lastCommitTime() (time.Time, error) { 50 | return time.Now(), errors.New("no commit history in repo") 51 | } 52 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import "github.com/gogitdb/gitdb/v2/internal/errors" 4 | 5 | var ( 6 | ErrNoRecords = errors.ErrNoRecords 7 | ErrRecordNotFound = errors.ErrRecordNotFound 8 | ErrInvalidRecordID = errors.ErrInvalidRecordID 9 | ErrDBSyncFailed = errors.ErrDBSyncFailed 10 | ErrLowBattery = errors.ErrLowBattery 11 | ErrNoOnlineRemote = errors.ErrNoOnlineRemote 12 | ErrAccessDenied = errors.ErrAccessDenied 13 | ErrInvalidDataset = errors.ErrInvalidDataset 14 | ) 15 | 16 | type ResolvableError interface { 17 | Resolution() string 18 | } 19 | 20 | type Error struct { 21 | err error 22 | resolution string 23 | } 24 | 25 | func (e Error) Error() string { 26 | return e.err.Error() 27 | } 28 | 29 | func (e Error) Resolution() string { 30 | return e.resolution 31 | } 32 | 33 | func ErrorWithResolution(e error, resolution string) Error { 34 | return Error{err: e, resolution: resolution} 35 | } 36 | -------------------------------------------------------------------------------- /event.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "fmt" 5 | "github.com/bouggo/log" 6 | "github.com/distatus/battery" 7 | ) 8 | 9 | type eventType string 10 | 11 | var ( 12 | w eventType = "write" //write 13 | wBefore eventType = "writeBefore" //writeBefore 14 | d eventType = "delete" //delete 15 | r eventType = "read" //read 16 | ) 17 | 18 | type dbEvent struct { 19 | Type eventType 20 | Dataset string 21 | Description string 22 | Commit bool 23 | } 24 | 25 | func newWriteEvent(description string, dataset string, commit bool) *dbEvent { 26 | return &dbEvent{Type: w, Description: description, Dataset: dataset, Commit: commit} 27 | } 28 | 29 | func newWriteBeforeEvent(description string, dataset string) *dbEvent { 30 | return &dbEvent{Type: wBefore, Description: description, Dataset: dataset} 31 | } 32 | 33 | func newReadEvent(description string, dataset string) *dbEvent { 34 | return &dbEvent{Type: r, Description: description, Dataset: dataset} 35 | } 36 | 37 | func newDeleteEvent(description string, dataset string, commit bool) *dbEvent { 38 | return &dbEvent{Type: w, Description: description, Dataset: dataset, Commit: commit} 39 | } 40 | 41 | func (g *gitdb) startEventLoop() { 42 | go func(g *gitdb) { 43 | log.Test("starting event loop") 44 | 45 | for { 46 | select { 47 | case <-g.shutdown: 48 | log.Info("event shutdown") 49 | log.Test("shutting down event loop") 50 | return 51 | case e := <-g.events: 52 | switch e.Type { 53 | case w, d: 54 | if e.Commit { 55 | if err := g.driver.commit(e.Dataset, e.Description, g.config.User); err != nil { 56 | log.Error(err.Error()) 57 | } 58 | log.Test("handled write event for " + e.Description) 59 | } 60 | g.commit.Done() 61 | default: 62 | log.Test("No handler found for " + string(e.Type) + " event") 63 | } 64 | } 65 | } 66 | }(g) 67 | 68 | } 69 | 70 | func hasSufficientBatteryPower(threshold float64) bool { 71 | batt, err := battery.Get(0) 72 | if err != nil || batt.State == battery.Charging { 73 | //device is probably running on direct power 74 | return true 75 | } 76 | 77 | percentageCharge := batt.Current / batt.Full * 100 78 | 79 | log.Info(fmt.Sprintf("Battery Level: %6.2f%%", percentageCharge)) 80 | 81 | //return true if battery life is above threshold 82 | return percentageCharge >= threshold 83 | } 84 | -------------------------------------------------------------------------------- /example/booking/collection.go: -------------------------------------------------------------------------------- 1 | package booking 2 | 3 | //implement sort interface to allow us sort an array of files i.e []os.FileInfo 4 | // type collection []*Model 5 | 6 | // func (c collection) Len() int { 7 | // return len(c) 8 | // } 9 | // func (c collection) Less(i, j int) bool { 10 | // return c[i].CreatedAt.Before(c[j].CreatedAt) 11 | // } 12 | // func (c collection) Swap(i, j int) { 13 | // c[i], c[j] = c[j], c[i] 14 | // } 15 | 16 | // type Collection struct { 17 | // SortFunc func(i int, j int) bool 18 | // Data []*db.Model 19 | // } 20 | 21 | // func NewCollection(data []*db.Model) *Collection { 22 | // return &Collection{Data: data} 23 | // } 24 | 25 | // func (c *Collection) Len() int { 26 | // return len(c.Data) 27 | // } 28 | // func (c *Collection) Less(i, j int) bool { 29 | // return c.SortFunc(i, j) 30 | // } 31 | // func (c *Collection) Swap(i, j int) { 32 | // c.Data[i], c.Data[j] = c.Data[j], c.Data[i] 33 | // } 34 | 35 | // func (c *Collection) SortByDate(i, j int) bool { 36 | // return c.Data[i].CreatedAt.Before(c.Data[j].CreatedAt) 37 | // } 38 | 39 | // func (c *Collection) SortByName(i, j int) bool { 40 | // return c.Data[i].Name.Before(c.Data[j].CreatedAt) 41 | // } 42 | 43 | // c := NewCollection(a) 44 | // c.SortFunc = c.SortByDate 45 | 46 | // sort(c) 47 | -------------------------------------------------------------------------------- /example/booking/enums.go: -------------------------------------------------------------------------------- 1 | package booking 2 | 3 | type Status string 4 | 5 | const ( 6 | Cancelled Status = "cancelled" 7 | CheckedOut Status = "checkedout" 8 | CheckedIn Status = "checkedin" 9 | Booked Status = "booked" 10 | ) 11 | 12 | type PaymentMode string 13 | 14 | const ( 15 | Daily PaymentMode = "daily" 16 | Hourly PaymentMode = "hourly" 17 | ) 18 | 19 | 20 | type RoomType string 21 | 22 | const ( 23 | Room RoomType = "room" 24 | Hall RoomType = "hall" 25 | ) -------------------------------------------------------------------------------- /example/booking/model.go: -------------------------------------------------------------------------------- 1 | package booking 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/gogitdb/gitdb/v2" 7 | ) 8 | 9 | type BookingModel struct { 10 | //extends.. 11 | gitdb.TimeStampedModel 12 | Type RoomType 13 | CheckInDate time.Time 14 | CheckOutDate time.Time 15 | CheckedOutAt time.Time 16 | Guests int 17 | CardsIssued int 18 | Status Status 19 | PaymentMode PaymentMode 20 | RoomPrice float64 21 | RoomId string 22 | CustomerId string 23 | UserId string 24 | NextOfKin string 25 | Purpose string 26 | } 27 | 28 | func NewBookingModel() *BookingModel { 29 | return &BookingModel{} 30 | } 31 | 32 | func (b *BookingModel) NumberOfHours() int { 33 | return int(b.CheckOutDate.Sub(b.CheckInDate).Hours()) 34 | } 35 | 36 | func (b *BookingModel) NumberOfNights() int { 37 | n := int(b.CheckOutDate.Sub(b.CheckInDate).Hours() / 24) 38 | if n <= 0 { 39 | n = 1 40 | } 41 | 42 | return n 43 | } 44 | 45 | //GetSchema example 46 | func (b *BookingModel) GetSchema() *gitdb.Schema { 47 | 48 | //Name of schema 49 | name := "Booking" 50 | //Block of schema 51 | block := b.CreatedAt.Format("200601") 52 | //Record of schema 53 | record := string(b.Type) + "_" + b.CreatedAt.Format("20060102150405") 54 | 55 | //Indexes speed up searching 56 | indexes := make(map[string]interface{}) 57 | indexes["RoomId"] = b.RoomId 58 | indexes["Guests"] = b.Guests 59 | indexes["CustomerId"] = b.CustomerId 60 | indexes["CreationDate"] = b.CreatedAt.Format("2006-01-02") 61 | 62 | return gitdb.NewSchema(name, block, record, indexes) 63 | } 64 | 65 | //GetLockFileNames example 66 | //Todo add comment and date expansion function 67 | func (b *BookingModel) GetLockFileNames() []string { 68 | var names []string 69 | names = append(names, "lock_"+b.CheckInDate.Format("2006-01-02")+"_"+b.RoomId) 70 | return names 71 | } 72 | 73 | //Validate example 74 | func (b *BookingModel) Validate() error { 75 | //write validation logic here 76 | return nil 77 | } 78 | 79 | //IsLockable example 80 | func (b *BookingModel) IsLockable() bool { 81 | return false 82 | } 83 | 84 | //ShouldEncrypt example 85 | func (b *BookingModel) ShouldEncrypt() bool { 86 | return false 87 | } 88 | -------------------------------------------------------------------------------- /example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "errors" 8 | "log" 9 | "os" 10 | 11 | db "github.com/gogitdb/gitdb/v2" 12 | "github.com/gogitdb/gitdb/v2/example/booking" 13 | ) 14 | 15 | var dbconn db.GitDb 16 | var logToFile bool 17 | 18 | func init() { 19 | cfg := &db.Config{ 20 | DBPath: "./data", 21 | OnlineRemote: os.Getenv("GITDB_REPO"), 22 | SyncInterval: time.Second * 5, 23 | EncryptionKey: "XVlBzgbaiCMRAjWwhTHctcuAxhxKQFDa", //this has to be 32 bytes to select AES-256 24 | User: db.NewUser("dev", "dev@gitdb.io"), 25 | EnableUI: true, 26 | } 27 | 28 | if logToFile { 29 | runLogFile, err := os.OpenFile("./db.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 30 | if err == nil { 31 | logger := log.New(runLogFile, "GITDB: ", log.Ldate|log.Ltime|log.Lshortfile) 32 | db.SetLogger(logger) 33 | } 34 | } 35 | 36 | var err error 37 | dbconn, err = db.Open(cfg) 38 | if err != nil { 39 | fmt.Println(err) 40 | os.Exit(1) 41 | } 42 | } 43 | 44 | func main() { 45 | db.SetLogLevel(db.LogLevelInfo) 46 | defer dbconn.Close() 47 | write() 48 | // fetch() 49 | testUI() 50 | } 51 | 52 | func testUI() { 53 | //simulate a block so UI can run 54 | c := make(chan int) 55 | c <- 1 56 | } 57 | 58 | func testTransaction() { 59 | t := dbconn.StartTransaction("booking") 60 | t.AddOperation(updateRoom) 61 | t.AddOperation(lockRoom) 62 | t.AddOperation(saveBooking) 63 | t.Commit() 64 | } 65 | 66 | func updateRoom() error { println("updating room..."); return nil } 67 | func lockRoom() error { println("locking room"); return errors.New("cannot lock room") } 68 | func saveBooking() error { println("saving booking"); return nil } 69 | 70 | func testWrite() { 71 | ticker := time.NewTicker(time.Second * 4) 72 | for { 73 | select { 74 | case <-ticker.C: 75 | write() 76 | } 77 | } 78 | } 79 | 80 | func write() { 81 | //populate model 82 | bm := booking.NewBookingModel() 83 | bm.Type = booking.Room 84 | bm.CheckInDate = time.Now() 85 | bm.CheckOutDate = time.Now() 86 | bm.CustomerId = "customer_1" 87 | bm.Guests = 2 88 | bm.CardsIssued = 1 89 | bm.NextOfKin = "Kin 1" 90 | bm.Status = booking.CheckedIn 91 | bm.UserId = "user_1" 92 | bm.RoomId = "room_1" 93 | 94 | err := dbconn.Insert(bm) 95 | if err != nil { 96 | fmt.Println(err) 97 | } 98 | } 99 | 100 | func read() { 101 | b := &booking.BookingModel{} 102 | err := dbconn.Get("Booking/201802/room_201802070030", b) 103 | if err != nil { 104 | fmt.Println(err.Error()) 105 | } else { 106 | fmt.Println(b.NextOfKin) 107 | } 108 | } 109 | 110 | func delete() { 111 | err := dbconn.Delete("Booking/201801/room_201801111823") 112 | if err != nil { 113 | fmt.Println(err.Error()) 114 | } else { 115 | fmt.Println("Deleted") 116 | } 117 | } 118 | 119 | func search() { 120 | searchParam := &db.SearchParam{Index: "CustomerId", Value: "customer_2"} 121 | rows, err := dbconn.Search("Booking", []*db.SearchParam{searchParam}, db.SearchEquals) 122 | if err != nil { 123 | fmt.Println(err.Error()) 124 | } else { 125 | for _, r := range rows { 126 | fmt.Println(r) 127 | } 128 | } 129 | } 130 | 131 | func fetch() { 132 | rows, err := dbconn.Fetch("Booking") 133 | if err != nil { 134 | fmt.Println(err.Error()) 135 | } else { 136 | bookings := []*booking.BookingModel{} 137 | for _, r := range rows { 138 | b := &booking.BookingModel{} 139 | r.Hydrate(b) 140 | bookings = append(bookings, b) 141 | 142 | fmt.Println(fmt.Sprintf("%s-%s", db.ID(b), b.CreatedAt)) 143 | } 144 | } 145 | } 146 | 147 | func mail() { 148 | mails := dbconn.GetMails() 149 | for _, m := range mails { 150 | fmt.Println(m.Body) 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gogitdb/gitdb/v2 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/bouggo/log v0.0.1 7 | github.com/distatus/battery v0.10.0 8 | github.com/gorilla/mux v1.7.4 9 | github.com/valyala/fastjson v1.5.1 10 | golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 11 | golang.org/x/sys v0.0.0-20220721230656-c6bc011c0c49 // indirect 12 | howett.net/plist v0.0.0-20200419221736-3b63eb3a43b5 // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/bouggo/log v0.0.1 h1:ki+t3NRgbcLtO3UpnzRwtWHsmB9Q/nYGlY+aM7I5AM8= 2 | github.com/bouggo/log v0.0.1/go.mod h1:3gQbYNgxubDvcQHMWOMcoCIbhw0x/Q3dCNZcLTy/bPI= 3 | github.com/distatus/battery v0.10.0 h1:YbizvmV33mqqC1fPCAEaQGV3bBhfYOfM+2XmL+mvt5o= 4 | github.com/distatus/battery v0.10.0/go.mod h1:STnSvFLX//eEpkaN7qWRxCWxrWOcssTDgnG4yqq9BRE= 5 | github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= 6 | github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 7 | github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 8 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 9 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 10 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 11 | github.com/valyala/fastjson v1.5.1 h1:SXaQZVSwLjZOVhDEhjiCcDtnX0Feu7Z7A1+C5atpoHM= 12 | github.com/valyala/fastjson v1.5.1/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= 13 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 14 | golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88= 15 | golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 16 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 17 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 18 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 19 | golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 20 | golang.org/x/sys v0.0.0-20220721230656-c6bc011c0c49 h1:TMjZDarEwf621XDryfitp/8awEhiZNiwgphKlTMGRIg= 21 | golang.org/x/sys v0.0.0-20220721230656-c6bc011c0c49/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 22 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 23 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 24 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 25 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 26 | howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= 27 | howett.net/plist v0.0.0-20200419221736-3b63eb3a43b5 h1:AQkaJpH+/FmqRjmXZPELom5zIERYZfwTjnHpfoVMQEc= 28 | howett.net/plist v0.0.0-20200419221736-3b63eb3a43b5/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= 29 | -------------------------------------------------------------------------------- /index.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/json" 7 | "fmt" 8 | "io/ioutil" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | 13 | "github.com/bouggo/log" 14 | "github.com/gogitdb/gitdb/v2/internal/db" 15 | ) 16 | 17 | type gdbIndex map[string]gdbIndexValue 18 | type gdbIndexCache map[string]gdbIndex 19 | type gdbSimpleIndex map[string]interface{} //recordID => value 20 | type gdbSimpleIndexCache map[string]gdbSimpleIndex //index => (recordID => value) 21 | type gdbIndexValue struct { 22 | Offset int `json:"o"` 23 | Len int `json:"l"` 24 | Value interface{} `json:"v"` 25 | } 26 | 27 | //TODO handle deletes 28 | func (g *gitdb) updateIndexes(dataBlock *db.Block) { 29 | g.indexMu.Lock() 30 | defer g.indexMu.Unlock() 31 | 32 | g.indexUpdated = true 33 | dataset := dataBlock.Dataset().Name() 34 | indexPath := g.indexPath(dataset) 35 | 36 | log.Info("updating in-memory index: " + dataset) 37 | //get line position of each record in the block 38 | //p := extractPositions(dataBlock) 39 | 40 | model := g.registry[dataset] 41 | if model == nil { 42 | if g.config.Factory != nil { 43 | model = g.config.Factory(dataset) 44 | } 45 | } 46 | 47 | if model == nil { 48 | log.Error(fmt.Sprintf("model not found in registry or factory: %s", dataset)) 49 | return 50 | } 51 | 52 | var indexes map[string]interface{} 53 | for _, record := range dataBlock.Records() { 54 | if err := record.Hydrate(model); err != nil { 55 | log.Error(fmt.Sprintf("record.Hydrate failed: %s %s", record.ID(), err)) 56 | } 57 | indexes = model.GetSchema().indexes 58 | 59 | //append index for id 60 | recordID := record.ID() 61 | indexes["id"] = recordID 62 | 63 | for name, value := range indexes { 64 | indexFile := filepath.Join(indexPath, name+".json") 65 | if _, ok := g.indexCache[indexFile]; !ok { 66 | g.indexCache[indexFile] = g.readIndex(indexFile) 67 | } 68 | //g.indexCache[indexFile][recordID] = gdbIndexValue{ 69 | // Offset: p[recordID][0], 70 | // Len: p[recordID][1], 71 | // Value: value, 72 | //} 73 | g.indexCache[indexFile][recordID] = value 74 | } 75 | } 76 | } 77 | 78 | func (g *gitdb) flushIndex() error { 79 | g.indexMu.Lock() 80 | defer g.indexMu.Unlock() 81 | 82 | if g.indexUpdated { 83 | log.Test("flushing index") 84 | for indexFile, data := range g.indexCache { 85 | 86 | indexPath := filepath.Dir(indexFile) 87 | if _, err := os.Stat(indexPath); err != nil { 88 | if err := os.MkdirAll(indexPath, 0755); err != nil { 89 | log.Error("Failed to write to index: " + indexFile) 90 | return err 91 | } 92 | } 93 | 94 | // indexBytes, err := json.MarshalIndent(data, "", "\t") 95 | indexBytes, err := json.Marshal(data) 96 | if err != nil { 97 | log.Error("Failed to write to index [" + indexFile + "]: " + err.Error()) 98 | return err 99 | } 100 | 101 | if err := ioutil.WriteFile(indexFile, indexBytes, 0744); err != nil { 102 | log.Error("Failed to write to index: " + indexFile) 103 | return err 104 | } 105 | } 106 | g.indexUpdated = false 107 | } 108 | 109 | return nil 110 | } 111 | 112 | func (g *gitdb) readIndex(indexFile string) gdbSimpleIndex { 113 | rMap := make(gdbSimpleIndex) 114 | if _, err := os.Stat(indexFile); err == nil { 115 | data, err := ioutil.ReadFile(indexFile) 116 | if err == nil { 117 | err = json.Unmarshal(data, &rMap) 118 | } 119 | 120 | if err != nil { 121 | log.Error(err.Error()) 122 | } 123 | } 124 | return rMap 125 | } 126 | 127 | func (g *gitdb) buildIndexSmart(changedFiles []string) { 128 | for _, blockFile := range changedFiles { 129 | log.Info("Building index for block: " + blockFile) 130 | block := db.LoadBlock(filepath.Join(g.dbDir(), blockFile), g.config.EncryptionKey) 131 | g.updateIndexes(block) 132 | } 133 | log.Info("Building index complete") 134 | } 135 | 136 | func (g *gitdb) buildIndexTargeted(target string) { 137 | ds := db.LoadDataset(filepath.Join(g.dbDir(), target), g.config.EncryptionKey) 138 | for _, block := range ds.Blocks() { 139 | g.updateIndexes(block) 140 | } 141 | } 142 | 143 | func (g *gitdb) buildIndexFull() { 144 | datasets := db.LoadDatasets(g.dbDir(), g.config.EncryptionKey) 145 | for _, ds := range datasets { 146 | g.buildIndexTargeted(ds.Name()) 147 | } 148 | if err := g.flushIndex(); err != nil { 149 | log.Error("gitDB: flushIndex failed: " + err.Error()) 150 | } 151 | } 152 | 153 | //extractPositions returns the position of all records in a block 154 | //as they would appear in the physical block file 155 | //a block can contain records from multiple physical block files 156 | //especially when *gitdb.doFetch is called so proceed with caution 157 | func extractPositions(b *db.Block) map[string][]int { 158 | var positions = map[string][]int{} 159 | 160 | data, err := json.MarshalIndent(b, "", "\t") 161 | if err != nil { 162 | log.Error(err.Error()) 163 | } 164 | 165 | offset := 3 166 | scanner := bufio.NewScanner(bytes.NewReader(data)) 167 | for scanner.Scan() { 168 | line := scanner.Text() 169 | if strings.TrimSpace(line) != "{" && strings.TrimSpace(line) != "}" { 170 | length := len(line) 171 | 172 | record := strings.SplitN(line, ":", 2) 173 | if len(record) != 2 { 174 | continue 175 | } 176 | 177 | recordID := strings.TrimSpace(record[0]) 178 | recordID = recordID[1 : len(recordID)-1] 179 | positions[recordID] = []int{offset, length} 180 | 181 | //account for newline 182 | offset += length + 1 183 | } 184 | } 185 | 186 | if len(positions) < 1 { 187 | panic("no positions extracted from: " + b.Path()) 188 | } 189 | 190 | return positions 191 | } 192 | -------------------------------------------------------------------------------- /init.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "sync" 9 | 10 | "github.com/bouggo/log" 11 | ) 12 | 13 | var mu sync.Mutex 14 | var conns map[string]GitDb 15 | 16 | // Open opens a connection to GitDB 17 | func Open(config *Config) (GitDb, error) { 18 | cfg := *config 19 | 20 | if err := cfg.Validate(); err != nil { 21 | return nil, err 22 | } 23 | 24 | if conns == nil { 25 | conns = make(map[string]GitDb) 26 | } 27 | 28 | if cfg.Mock { 29 | conn := newMockConnection() 30 | conn.configure(cfg) 31 | conns[cfg.ConnectionName] = conn 32 | return conn, nil 33 | } 34 | 35 | conn := newConnection() 36 | conn.configure(cfg) 37 | 38 | err := conn.boot() 39 | logMsg := "Db booted fine" 40 | if err != nil { 41 | logMsg = fmt.Sprintf("Db booted with errors - %s", err) 42 | } 43 | 44 | log.Info(logMsg) 45 | 46 | if err != nil { 47 | if errors.Is(err, ErrAccessDenied) { 48 | fb, readErr := ioutil.ReadFile(conn.publicKeyFilePath()) 49 | if readErr != nil { 50 | return nil, readErr 51 | } 52 | 53 | // inform users to ask admin to add their public key to repo 54 | resolution := "Contact your database admin to add your public key to git server\n" 55 | resolution += "Public key: " + fmt.Sprintf("%s", fb) 56 | 57 | return nil, ErrorWithResolution(err, resolution) 58 | } 59 | 60 | return nil, err 61 | } 62 | 63 | // if boot() returned an error do not start event loop 64 | if !conn.loopStarted { 65 | conn.startEventLoop() 66 | if cfg.SyncInterval > 0 { 67 | conn.startSyncClock() 68 | } 69 | if cfg.EnableUI { 70 | conn.startUI() 71 | } 72 | conn.loopStarted = true 73 | } 74 | 75 | conns[cfg.ConnectionName] = conn 76 | return conn, nil 77 | } 78 | 79 | // Conn returns the last connection started by Open(*Config) 80 | // if you opened more than one connection use GetConn(name) instead 81 | func Conn() GitDb { 82 | if len(conns) > 1 { 83 | panic("Multiple gitdb connections found. Use GetConn function instead") 84 | } 85 | 86 | if len(conns) == 0 { 87 | panic("No open gitdb connections found") 88 | } 89 | 90 | var connName string 91 | for k := range conns { 92 | connName = k 93 | break 94 | } 95 | 96 | if _, ok := conns[connName]; !ok { 97 | panic("No gitdb connection found - " + connName) 98 | } 99 | 100 | return conns[connName] 101 | } 102 | 103 | // GetConn returns a specific gitdb connection by name 104 | func GetConn(name string) GitDb { 105 | if _, ok := conns[name]; !ok { 106 | panic("No gitdb connection found") 107 | } 108 | 109 | return conns[name] 110 | } 111 | 112 | func (g *gitdb) boot() error { 113 | log.Info("Booting up db using " + g.driver.name() + " driver") 114 | 115 | if err := g.driver.setup(g); err != nil { 116 | return err 117 | } 118 | 119 | // rebuild index if we have to 120 | if _, err := os.Stat(g.indexDir()); err != nil { 121 | // no index directory found so we need to re-index the whole db 122 | go g.buildIndexFull() 123 | } 124 | 125 | return nil 126 | } 127 | -------------------------------------------------------------------------------- /init_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gogitdb/gitdb/v2" 7 | ) 8 | 9 | func TestConn(t *testing.T) { 10 | setup(t, nil) 11 | defer testDb.Close() 12 | got := gitdb.Conn() 13 | if got != testDb { 14 | t.Errorf("connection don't match") 15 | } 16 | } 17 | 18 | func TestGetConn(t *testing.T) { 19 | setup(t, nil) 20 | defer testDb.Close() 21 | got := gitdb.GetConn("default") 22 | if got != testDb { 23 | t.Errorf("connection don't match") 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /internal/crypto/security.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "crypto/aes" 5 | "crypto/cipher" 6 | "crypto/rand" 7 | "encoding/base64" 8 | "io" 9 | ) 10 | 11 | //Encrypt message with key 12 | func Encrypt(key string, message string) string { 13 | plainText := []byte(message) 14 | 15 | block, err := aes.NewCipher([]byte(key)) 16 | if err != nil { 17 | println(err.Error()) 18 | return "" 19 | } 20 | 21 | //IV needs to be unique, but doesn't have to be secure. 22 | //It's common to put it at the beginning of the ciphertext. 23 | cipherText := make([]byte, aes.BlockSize+len(plainText)) 24 | iv := cipherText[:aes.BlockSize] 25 | if _, err = io.ReadFull(rand.Reader, iv); err != nil { 26 | println(err.Error()) 27 | return "" 28 | } 29 | 30 | stream := cipher.NewCFBEncrypter(block, iv) 31 | stream.XORKeyStream(cipherText[aes.BlockSize:], plainText) 32 | 33 | //returns to base64 encoded string 34 | encmess := base64.URLEncoding.EncodeToString(cipherText) 35 | return encmess 36 | } 37 | 38 | //Decrypt message with key 39 | func Decrypt(key string, secureMessage string) string { 40 | cipherText, err := base64.URLEncoding.DecodeString(secureMessage) 41 | if err != nil { 42 | return "" 43 | } 44 | 45 | block, err := aes.NewCipher([]byte(key)) 46 | if err != nil { 47 | return "" 48 | } 49 | 50 | if len(cipherText) < aes.BlockSize { 51 | return "" 52 | } 53 | 54 | iv := cipherText[:aes.BlockSize] 55 | cipherText = cipherText[aes.BlockSize:] 56 | 57 | stream := cipher.NewCFBDecrypter(block, iv) 58 | // XORKeyStream can work in-place if the two arguments are the same. 59 | stream.XORKeyStream(cipherText, cipherText) 60 | 61 | decodedmess := string(cipherText) 62 | return decodedmess 63 | } 64 | -------------------------------------------------------------------------------- /internal/db/block.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "github.com/gogitdb/gitdb/v2/internal/errors" 7 | "io/ioutil" 8 | "os" 9 | "path/filepath" 10 | "sort" 11 | 12 | "github.com/bouggo/log" 13 | "github.com/gogitdb/gitdb/v2/internal/digital" 14 | ) 15 | 16 | //Block represents a block file 17 | type Block struct { 18 | dataset *Dataset 19 | path string 20 | key string 21 | size int64 22 | badRecords []string 23 | records map[string]*Record 24 | } 25 | 26 | //EmptyBlock is used for hydration 27 | type EmptyBlock struct { 28 | Block 29 | } 30 | 31 | //HydrateByPositions should be called on EmptyBlock 32 | //pos must be []int{offset, position} 33 | func (b *EmptyBlock) HydrateByPositions(blockFilePath string, positions ...[]int) error { 34 | fd, err := os.Open(blockFilePath) 35 | if err != nil { 36 | return err 37 | } 38 | defer fd.Close() 39 | 40 | blockJSON := []byte("{") 41 | for i, pos := range positions { 42 | 43 | fd.Seek(int64(pos[0]), 0) 44 | line := make([]byte, pos[1]) 45 | fd.Read(line) 46 | 47 | line = bytes.TrimSpace(line) 48 | ln := len(line) - 1 49 | //are we at the end of seek 50 | if i < len(positions)-1 { 51 | //ensure line ends with a comma 52 | if line[ln] != ',' { 53 | line = append(line, ',') 54 | 55 | } 56 | } else { 57 | //make sure last line has no comma 58 | if line[ln] == ',' { 59 | line = line[0:ln] 60 | } 61 | } 62 | line = append(line, "\n"...) 63 | blockJSON = append(blockJSON, line...) 64 | } 65 | blockJSON = append(blockJSON, '}') 66 | return json.Unmarshal(blockJSON, b) 67 | } 68 | 69 | //Hydrate should be called on EmptyBlock 70 | func (b *EmptyBlock) Hydrate(blockFilePath string) error { 71 | data, err := ioutil.ReadFile(blockFilePath) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | if err := json.Unmarshal(data, b); err != nil { 77 | return err //errBadBlock 78 | } 79 | 80 | return err 81 | } 82 | 83 | //Dataset returns the dataset *Block belongs to 84 | func (b *Block) Dataset() *Dataset { 85 | return b.dataset 86 | } 87 | 88 | //HumanSize returns human readable size of a block 89 | func (b *Block) HumanSize() string { 90 | return digital.FormatBytes(uint64(b.size)) 91 | } 92 | 93 | //RecordCount returns the number of records in a block 94 | func (b *Block) RecordCount() int { 95 | return len(b.records) 96 | } 97 | 98 | //BadRecords returns all bad records found in a block 99 | func (b *Block) BadRecords() []string { 100 | return b.badRecords 101 | } 102 | 103 | //Path returns path to block file 104 | func (b *Block) Path() string { 105 | return b.path 106 | } 107 | 108 | //NewEmptyBlock should be used to store records from multiple blocks 109 | func NewEmptyBlock(key string) *EmptyBlock { 110 | return &EmptyBlock{Block{ 111 | key: key, 112 | records: map[string]*Record{}, 113 | badRecords: []string{}, 114 | }} 115 | } 116 | 117 | //LoadBlock loads a block at a particular path 118 | func LoadBlock(blockFilePath, key string) *Block { 119 | block := &Block{ 120 | path: blockFilePath, 121 | key: key, 122 | records: map[string]*Record{}, 123 | badRecords: []string{}, 124 | //TODO figure out a neat way to inject key 125 | dataset: &Dataset{path: filepath.Dir(blockFilePath), key: key}, 126 | } 127 | 128 | if err := block.load(); err != nil { 129 | log.Error(err.Error()) 130 | block.dataset.badBlocks = append(block.dataset.badBlocks, blockFilePath) 131 | } 132 | 133 | return block 134 | } 135 | 136 | //MarshalJSON implements json.MarshalJSON 137 | func (b *Block) MarshalJSON() ([]byte, error) { 138 | raw := map[string]string{} 139 | for k, v := range b.records { 140 | raw[k] = v.data 141 | } 142 | 143 | return json.Marshal(raw) 144 | } 145 | 146 | //UnmarshalJSON implements json.UnmarshalJSON 147 | func (b *Block) UnmarshalJSON(data []byte) error { 148 | var raw map[string]string 149 | if err := json.Unmarshal(data, &raw); err != nil { 150 | return err 151 | } 152 | 153 | //populate recs 154 | for k, v := range raw { 155 | r := newRecord(k, v) 156 | b.records[k] = r 157 | } 158 | 159 | return nil 160 | } 161 | 162 | func (b *Block) load() error { 163 | blockFile := filepath.Join(b.path) 164 | log.Info("Reading block: " + blockFile) 165 | data, err := ioutil.ReadFile(blockFile) 166 | if err != nil { 167 | return err 168 | } 169 | 170 | b.size = int64(len(data)) 171 | return json.Unmarshal(data, b) 172 | } 173 | 174 | //Record returns record in specifed index i 175 | func (b *Block) Record(i int) *Record { 176 | records := b.Records() 177 | record := records[i] 178 | return record 179 | } 180 | 181 | //Add a record to Block 182 | func (b *Block) Add(recordID, value string) { 183 | b.records[recordID] = newRecord(recordID, value) 184 | } 185 | 186 | //Get a record by key from a Block 187 | func (b *Block) Get(key string) (*Record, error) { 188 | if _, ok := b.records[key]; ok { 189 | b.records[key].key = b.key 190 | return b.records[key], nil 191 | } 192 | 193 | return nil, errors.ErrRecordNotFound 194 | } 195 | 196 | //Delete a record by key from a Block 197 | func (b *Block) Delete(key string) error { 198 | if _, ok := b.records[key]; ok { 199 | delete(b.records, key) 200 | return nil 201 | } 202 | 203 | return errors.ErrRecordNotFound 204 | } 205 | 206 | //Len returns length of block 207 | func (b *Block) Len() int { 208 | return len(b.records) 209 | } 210 | 211 | //Records returns decrypted slice of all Records in a Block 212 | //sorted in asc order of id 213 | func (b *Block) Records() []*Record { 214 | var records []*Record 215 | for _, v := range b.records { 216 | v.decrypt(b.key) 217 | records = append(records, v) 218 | } 219 | 220 | sort.Sort(collection(records)) 221 | 222 | return records 223 | } 224 | 225 | func (b *Block) Filter(recordIDs map[string]string) { 226 | records := make(map[string]*Record, len(recordIDs)) 227 | for recordID, value := range b.records { 228 | if _, ok := recordIDs[recordID]; ok { 229 | records[recordID] = value 230 | } 231 | 232 | } 233 | b.records = records 234 | } 235 | -------------------------------------------------------------------------------- /internal/db/dataset.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "io/ioutil" 5 | "path" 6 | "path/filepath" 7 | "strings" 8 | "time" 9 | 10 | "github.com/bouggo/log" 11 | "github.com/gogitdb/gitdb/v2/internal/digital" 12 | ) 13 | 14 | //Dataset represent a collection of blocks 15 | type Dataset struct { 16 | path string 17 | blocks []*Block 18 | badBlocks []string 19 | badRecords []string 20 | lastModified time.Time 21 | 22 | key string 23 | } 24 | 25 | //LoadDataset loads the dataset at path 26 | func LoadDataset(datasetPath, key string) *Dataset { 27 | ds := &Dataset{ 28 | path: datasetPath, 29 | key: key, 30 | } 31 | ds.loadBlocks() 32 | 33 | return ds 34 | } 35 | 36 | //LoadDatasets loads all datasets in given gitdb path 37 | func LoadDatasets(dbPath, key string) []*Dataset { 38 | var datasets []*Dataset 39 | 40 | dirs, err := ioutil.ReadDir(dbPath) 41 | if err != nil { 42 | log.Error(err.Error()) 43 | return datasets 44 | } 45 | 46 | for _, dir := range dirs { 47 | if !strings.HasPrefix(dir.Name(), ".") && dir.IsDir() { 48 | ds := &Dataset{ 49 | path: filepath.Join(dbPath, dir.Name()), 50 | lastModified: dir.ModTime(), 51 | key: key, 52 | } 53 | 54 | datasets = append(datasets, ds) 55 | } 56 | } 57 | return datasets 58 | } 59 | 60 | //Name returns name of dataset 61 | func (d *Dataset) Name() string { 62 | return filepath.Base(d.path) 63 | } 64 | 65 | //Path returns path to dataset 66 | func (d *Dataset) Path() string { 67 | return d.path 68 | } 69 | 70 | //Size returns the total size of all blocks in a DataSet 71 | func (d *Dataset) Size() int64 { 72 | size := int64(0) 73 | for _, block := range d.blocks { 74 | size += block.size 75 | } 76 | 77 | return size 78 | } 79 | 80 | //HumanSize returns human friendly size of a DataSet 81 | func (d *Dataset) HumanSize() string { 82 | return digital.FormatBytes(uint64(d.Size())) 83 | } 84 | 85 | //BlockCount returns the number of blocks in a DataSet 86 | func (d *Dataset) BlockCount() int { 87 | if len(d.blocks) == 0 { 88 | d.loadBlocks() 89 | } 90 | return len(d.blocks) 91 | } 92 | 93 | //RecordCount returns the number of records in a DataSet 94 | func (d *Dataset) RecordCount() int { 95 | count := 0 96 | if d.BlockCount() > 0 { 97 | for _, block := range d.blocks { 98 | count += block.RecordCount() 99 | } 100 | } 101 | 102 | return count 103 | } 104 | 105 | //BadBlocksCount returns the number of bad blocks in a DataSet 106 | func (d *Dataset) BadBlocksCount() int { 107 | if len(d.badBlocks) == 0 { 108 | d.RecordCount() //hack to get records loaded so errors can be populated in dataset 109 | } 110 | 111 | return len(d.badBlocks) 112 | } 113 | 114 | //BadRecordsCount returns the number of bad records in a DataSet 115 | func (d *Dataset) BadRecordsCount() int { 116 | if len(d.badRecords) == 0 { 117 | d.RecordCount() //hack to get records loaded so errors can be populated in dataset 118 | } 119 | 120 | return len(d.badRecords) 121 | } 122 | 123 | //Block returns block at index i of a DataSet 124 | func (d *Dataset) Block(i int) *Block { 125 | if len(d.blocks) == 0 { 126 | //load blocks so errors can be populated in dataset 127 | d.loadBlocks() 128 | } 129 | 130 | if i <= len(d.blocks)-1 { 131 | return d.blocks[i] 132 | } 133 | 134 | return nil 135 | } 136 | 137 | //Blocks returns all the Blocks in a dataset 138 | func (d *Dataset) Blocks() []*Block { 139 | return d.blocks 140 | } 141 | 142 | //BadBlocks returns all the bad blocks in a dataset 143 | func (d *Dataset) BadBlocks() []string { 144 | return d.badBlocks 145 | } 146 | 147 | //BadRecords returns all the bad records in a dataset 148 | func (d *Dataset) BadRecords() []string { 149 | return d.badRecords 150 | } 151 | 152 | //LastModifiedDate returns the last modification time of a DataSet 153 | func (d *Dataset) LastModifiedDate() string { 154 | return d.lastModified.Format("02 Jan 2006 15:04:05") 155 | } 156 | 157 | //loadBlocks reads all blocks in a Dataset into memory 158 | func (d *Dataset) loadBlocks() { 159 | blks, err := ioutil.ReadDir(d.path) 160 | if err != nil { 161 | log.Error(err.Error()) 162 | } 163 | 164 | for _, blk := range blks { 165 | if !blk.IsDir() && strings.HasSuffix(blk.Name(), ".json") { 166 | b := LoadBlock(filepath.Join(d.path, blk.Name()), d.key) 167 | d.blocks = append(d.blocks, b) 168 | } 169 | } 170 | } 171 | 172 | //Indexes returns the indexes set on a DataSet 173 | func (d *Dataset) Indexes() []string { 174 | //grab indexes 175 | var indexes []string 176 | 177 | indexFiles, err := ioutil.ReadDir(filepath.Join(path.Dir(d.path), ".gitdb/index/", d.Name())) 178 | if err != nil { 179 | return indexes 180 | } 181 | 182 | for _, indexFile := range indexFiles { 183 | indexes = append(indexes, strings.TrimSuffix(indexFile.Name(), ".json")) 184 | } 185 | 186 | return indexes 187 | } 188 | -------------------------------------------------------------------------------- /internal/db/record.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/bouggo/log" 9 | "github.com/gogitdb/gitdb/v2/internal/crypto" 10 | "github.com/valyala/fastjson" 11 | ) 12 | 13 | //Record represents a model stored in gitdb 14 | type Record struct { 15 | id string 16 | data string 17 | key string 18 | 19 | p fastjson.Parser 20 | decrypted bool 21 | } 22 | 23 | //newRecord constructs a Record 24 | func newRecord(id, data string) *Record { 25 | return &Record{id: id, data: data} 26 | } 27 | 28 | //ID returns record id 29 | func (r *Record) ID() string { 30 | return r.id 31 | } 32 | 33 | //Data returns record data unmodified 34 | func (r *Record) Data() string { 35 | return r.data 36 | } 37 | 38 | //Hydrate populates given interfacce with underlying record data 39 | func (r *Record) Hydrate(model interface{}) error { 40 | r.decrypt(r.key) 41 | version := r.Version() 42 | switch version { 43 | case "v1": 44 | if err := json.Unmarshal([]byte(r.data), model); err != nil { 45 | return err 46 | } 47 | return nil 48 | case "v2": //TODO Optimize Unmarshall-Marshall technique 49 | v, err := r.p.Parse(r.data) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | obj := v.GetObject("Data") 55 | buf := make([]byte, obj.Len()) 56 | buf = obj.MarshalTo(buf) 57 | buf = bytes.Trim(buf, "\x00") 58 | 59 | // fmt.Printf("%s\n", oh) 60 | return json.Unmarshal(buf, model) 61 | default: 62 | return fmt.Errorf("Unable to hydrate version : %s", version) 63 | } 64 | } 65 | 66 | func (r *Record) decrypt(key string) { 67 | if len(key) > 0 && !r.decrypted { 68 | log.Test("decrypting with: " + key) 69 | dec := crypto.Decrypt(key, r.data) 70 | if len(dec) > 0 { 71 | r.data = dec 72 | } 73 | r.decrypted = true 74 | } 75 | } 76 | 77 | //JSON returns data decrypted and indented 78 | func (r *Record) JSON() string { 79 | var buf bytes.Buffer 80 | r.decrypt(r.key) 81 | if err := json.Indent(&buf, []byte(r.data), "", "\t"); err != nil { 82 | log.Error(err.Error()) 83 | } 84 | 85 | return buf.String() 86 | } 87 | 88 | //Version returns the version of the record 89 | func (r *Record) Version() string { 90 | v, err := r.p.Parse(r.data) 91 | if err != nil { 92 | return "v1" 93 | } 94 | 95 | versionBytes := v.GetStringBytes("Version") 96 | version := string(versionBytes) 97 | if len(version) == 0 { 98 | version = "v1" 99 | } 100 | 101 | return version 102 | } 103 | 104 | //ConvertModel converts a Model to a record 105 | func ConvertModel(id string, m interface{}) *Record { 106 | b, _ := json.Marshal(m) 107 | return newRecord(id, string(b)) 108 | } 109 | 110 | //collection represents a sortable slice of Records 111 | type collection []*Record 112 | 113 | func (c collection) Len() int { 114 | return len(c) 115 | } 116 | func (c collection) Less(i, j int) bool { 117 | return c[i].id < c[j].id 118 | } 119 | func (c collection) Swap(i, j int) { 120 | c[i], c[j] = c[j], c[i] 121 | } 122 | -------------------------------------------------------------------------------- /internal/digital/size.go: -------------------------------------------------------------------------------- 1 | package digital 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | const ( 9 | sizeByte = 1.0 << (10 * iota) 10 | sizeKb 11 | sizeMb 12 | sizeGb 13 | sizeTb 14 | ) 15 | 16 | func FormatBytes(bytes uint64) string { 17 | unit := "" 18 | value := float32(bytes) 19 | 20 | switch { 21 | case bytes >= sizeTb: 22 | unit = "TB" 23 | value = value / sizeTb 24 | case bytes >= sizeGb: 25 | unit = "GB" 26 | value = value / sizeGb 27 | case bytes >= sizeMb: 28 | unit = "MB" 29 | value = value / sizeMb 30 | case bytes >= sizeKb: 31 | unit = "KB" 32 | value = value / sizeKb 33 | case bytes >= sizeByte: 34 | unit = "B" 35 | case bytes == 0: 36 | return "0" 37 | } 38 | 39 | stringValue := fmt.Sprintf("%.1f", value) 40 | stringValue = strings.TrimSuffix(stringValue, ".0") 41 | return fmt.Sprintf("%s%s", stringValue, unit) 42 | } 43 | -------------------------------------------------------------------------------- /internal/errors/errors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import "errors" 4 | 5 | var ( 6 | //internal errors 7 | errDB = errors.New("gitDB: database error") 8 | errBadBlock = errors.New("gitDB: Bad block error - invalid json") 9 | errBadRecord = errors.New("gitDB: Bad record error") 10 | errConnectionClosed = errors.New("gitDB: connection is closed") 11 | errConnectionInvalid = errors.New("gitDB: connection is not valid. use gitdb.Start to construct a valid connection") 12 | 13 | //external errors 14 | ErrNoRecords = errors.New("gitDB: no records found") 15 | ErrRecordNotFound = errors.New("gitDB: record not found") 16 | ErrInvalidRecordID = errors.New("gitDB: invalid record id") 17 | ErrDBSyncFailed = errors.New("gitDB: Database sync failed") 18 | ErrLowBattery = errors.New("gitDB: Insufficient battery power. Syncing disabled") 19 | ErrNoOnlineRemote = errors.New("gitDB: Online remote is not set. Syncing disabled") 20 | ErrAccessDenied = errors.New("gitDB: Access was denied to online repository") 21 | ErrInvalidDataset = errors.New("gitDB: invalid dataset. Dataset not in registry") 22 | ) 23 | -------------------------------------------------------------------------------- /lock.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "errors" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | 10 | "github.com/bouggo/log" 11 | ) 12 | 13 | func (g *gitdb) Lock(mo Model) error { 14 | 15 | m := wrap(mo) 16 | if _, ok := mo.(LockableModel); !ok { 17 | return errors.New("Model is not lockable") 18 | } 19 | 20 | var lockFilesWritten []string 21 | 22 | fullPath := g.lockDir(m) 23 | if _, err := os.Stat(fullPath); err != nil { 24 | err := os.MkdirAll(fullPath, 0755) 25 | if err != nil { 26 | return err 27 | } 28 | } 29 | 30 | lockFiles := mo.(LockableModel).GetLockFileNames() 31 | for _, file := range lockFiles { 32 | lockFile := filepath.Join(fullPath, file+".lock") 33 | g.events <- newWriteBeforeEvent("...", lockFile) 34 | 35 | //when locking a model, lockfile should not exist 36 | if _, err := os.Stat(lockFile); err == nil { 37 | if derr := g.deleteLockFiles(lockFilesWritten); derr != nil { 38 | log.Error(derr.Error()) 39 | } 40 | return errors.New("Lock file already exist: " + lockFile) 41 | } 42 | 43 | err := ioutil.WriteFile(lockFile, []byte(""), 0644) 44 | if err != nil { 45 | if derr := g.deleteLockFiles(lockFilesWritten); derr != nil { 46 | log.Error(derr.Error()) 47 | } 48 | return errors.New("Failed to write lock " + lockFile + ": " + err.Error()) 49 | } 50 | 51 | lockFilesWritten = append(lockFilesWritten, lockFile) 52 | } 53 | 54 | g.commit.Add(1) 55 | commitMsg := "Created Lock Files for: " + ID(m) 56 | g.events <- newWriteEvent(commitMsg, fullPath, g.autoCommit) 57 | 58 | //block here until write has been committed 59 | g.waitForCommit() 60 | return nil 61 | } 62 | 63 | func (g *gitdb) Unlock(mo Model) error { 64 | 65 | m := wrap(mo) 66 | if _, ok := mo.(LockableModel); !ok { 67 | return errors.New("Model is not lockable") 68 | } 69 | 70 | fullPath := g.lockDir(m) 71 | 72 | lockFiles := mo.(LockableModel).GetLockFileNames() 73 | for _, file := range lockFiles { 74 | lockFile := filepath.Join(fullPath, file+".lock") 75 | 76 | if _, err := os.Stat(lockFile); err == nil { 77 | //log.PutInfo("Removing " + lockFile) 78 | err := os.Remove(lockFile) 79 | if err != nil { 80 | return errors.New("Could not delete lock file: " + lockFile) 81 | } 82 | } 83 | } 84 | 85 | g.commit.Add(1) 86 | commitMsg := "Removing Lock Files for: " + ID(m) 87 | g.events <- newWriteEvent(commitMsg, fullPath, g.autoCommit) 88 | 89 | //block here until write has been committed 90 | g.waitForCommit() 91 | return nil 92 | } 93 | 94 | func (g *gitdb) deleteLockFiles(files []string) error { 95 | var err error 96 | var failedDeletes []string 97 | if len(files) > 0 { 98 | for _, file := range files { 99 | err = os.Remove(file) 100 | if err != nil { 101 | failedDeletes = append(failedDeletes, file) 102 | } 103 | } 104 | } 105 | 106 | if len(failedDeletes) > 0 { 107 | return errors.New("Could not delete lock files: " + strings.Join(failedDeletes, ",")) 108 | } 109 | 110 | return err 111 | } 112 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | golog "log" 5 | 6 | "github.com/bouggo/log" 7 | ) 8 | 9 | //LogLevel is used to set verbosity of GitDB 10 | type LogLevel int 11 | 12 | const ( 13 | //LogLevelNone - log nothing 14 | LogLevelNone LogLevel = iota 15 | //LogLevelError - logs only errors 16 | LogLevelError 17 | //LogLevelWarning - logs warning and errors 18 | LogLevelWarning 19 | //LogLevelTest - logs only debug messages 20 | LogLevelTest 21 | //LogLevelInfo - logs info, warining and errors 22 | LogLevelInfo 23 | ) 24 | 25 | //SetLogLevel sets log level 26 | func SetLogLevel(l LogLevel) { 27 | log.SetLogLevel(log.Level(l)) 28 | } 29 | 30 | //SetLogger sets Logger 31 | func SetLogger(l *golog.Logger) { 32 | log.SetLogger(l) 33 | } 34 | -------------------------------------------------------------------------------- /mail.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type mail struct { 8 | Subject string 9 | Body string 10 | Date time.Time 11 | } 12 | 13 | func newMail(subject string, body string) *mail { 14 | return &mail{Subject: subject, Body: body, Date: time.Now()} 15 | } 16 | 17 | func (g *gitdb) GetMails() []*mail { 18 | mails := g.mails 19 | g.mails = []*mail{} 20 | return mails 21 | } 22 | 23 | func (g *gitdb) sendMail(m *mail) { 24 | g.mails = append(g.mails, m) 25 | 26 | } 27 | -------------------------------------------------------------------------------- /mail_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestMailGetMails(t *testing.T) { 8 | teardown := setup(t, nil) 9 | defer teardown(t) 10 | 11 | mails := testDb.GetMails() 12 | if len(mails) > 0 { 13 | t.Errorf("testDb.GetMails() should be 0") 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /model.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | //Model interface describes methods GitDB supports 8 | type Model interface { 9 | GetSchema() *Schema 10 | //Validate validates a Model 11 | Validate() error 12 | //ShouldEncrypt informs GitDb if a Model support encryption 13 | ShouldEncrypt() bool 14 | //BeforeInsert is called by gitdb before insert 15 | BeforeInsert() error 16 | } 17 | 18 | type LockableModel interface { 19 | //GetLockFileNames informs GitDb of files a Models using for locking 20 | GetLockFileNames() []string 21 | } 22 | 23 | //TimeStampedModel provides time stamp fields 24 | type TimeStampedModel struct { 25 | CreatedAt time.Time 26 | UpdatedAt time.Time 27 | } 28 | 29 | //BeforeInsert implements Model.BeforeInsert 30 | func (m *TimeStampedModel) BeforeInsert() error { 31 | stampTime := time.Now() 32 | if m.CreatedAt.IsZero() { 33 | m.CreatedAt = stampTime 34 | } 35 | m.UpdatedAt = stampTime 36 | 37 | return nil 38 | } 39 | 40 | type model struct { 41 | Version string 42 | Data Model 43 | } 44 | 45 | func wrap(m Model) *model { 46 | return &model{ 47 | Version: RecVersion, 48 | Data: m, 49 | } 50 | } 51 | 52 | func (m *model) GetSchema() *Schema { 53 | return m.Data.GetSchema() 54 | } 55 | 56 | func (m *model) Validate() error { 57 | return m.Data.Validate() 58 | } 59 | 60 | func (m *model) ShouldEncrypt() bool { 61 | return m.Data.ShouldEncrypt() 62 | } 63 | 64 | func (m *model) BeforeInsert() error { 65 | err := m.Data.BeforeInsert() 66 | return err 67 | } 68 | 69 | func (g *gitdb) RegisterModel(dataset string, m Model) bool { 70 | if g.registry == nil { 71 | g.registry = make(map[string]Model) 72 | } 73 | g.registry[dataset] = m 74 | return true 75 | } 76 | 77 | func (g *gitdb) isRegistered(dataset string) bool { 78 | if _, ok := g.registry[dataset]; ok { 79 | return true 80 | } 81 | 82 | if g.config.Factory != nil && g.config.Factory(dataset) != nil { 83 | return true 84 | } 85 | 86 | return false 87 | } 88 | -------------------------------------------------------------------------------- /paths.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "path/filepath" 5 | ) 6 | 7 | func (g *gitdb) absDbPath() string { 8 | absDbPath, err := filepath.Abs(g.config.DBPath) 9 | if err != nil { 10 | panic(err) 11 | } 12 | 13 | return absDbPath 14 | } 15 | 16 | func (g *gitdb) dbDir() string { 17 | return filepath.Join(g.absDbPath(), "data") 18 | } 19 | 20 | func (g *gitdb) fullPath(m Model) string { 21 | return filepath.Join(g.dbDir(), m.GetSchema().name()) 22 | } 23 | 24 | func (g *gitdb) blockFilePath(dataset, block string) string { 25 | return filepath.Join(g.dbDir(), dataset, block+".json") 26 | } 27 | 28 | func (g *gitdb) lockDir(m Model) string { 29 | return filepath.Join(g.fullPath(m), "Lock") 30 | } 31 | 32 | //index path 33 | func (g *gitdb) indexDir() string { 34 | return filepath.Join(g.absDbPath(), g.internalDirName(), "index") 35 | } 36 | 37 | func (g *gitdb) indexPath(dataset string) string { 38 | return filepath.Join(g.indexDir(), dataset) 39 | } 40 | 41 | //ssh paths 42 | func (g *gitdb) sshDir() string { 43 | return filepath.Join(g.absDbPath(), g.internalDirName(), "ssh") 44 | } 45 | 46 | //ssh paths 47 | func (g *gitdb) publicKeyFilePath() string { 48 | return filepath.Join(g.sshDir(), "gitdb.pub") 49 | } 50 | 51 | func (g *gitdb) privateKeyFilePath() string { 52 | return filepath.Join(g.sshDir(), "gitdb") 53 | } 54 | 55 | func (g *gitdb) internalDirName() string { 56 | return ".gitdb" //todo rename 57 | } 58 | -------------------------------------------------------------------------------- /read.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | 10 | "github.com/bouggo/log" 11 | "github.com/gogitdb/gitdb/v2/internal/db" 12 | ) 13 | 14 | func (g *gitdb) loadBlock(blockFile string) (*db.Block, error) { 15 | 16 | if g.loadedBlocks == nil { 17 | g.loadedBlocks = map[string]*db.Block{} 18 | } 19 | 20 | //if block file is not cached, load into cache 21 | if _, ok := g.loadedBlocks[blockFile]; !ok { 22 | g.loadedBlocks[blockFile] = db.LoadBlock(blockFile, g.config.EncryptionKey) 23 | } 24 | 25 | return g.loadedBlocks[blockFile], nil 26 | } 27 | 28 | func (g *gitdb) doGet(id string) (*db.Record, error) { 29 | 30 | dataset, block, _, err := ParseID(id) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | if !g.isRegistered(dataset) { 36 | return nil, ErrInvalidDataset 37 | } 38 | 39 | blockFilePath := filepath.Join(g.dbDir(), dataset, block+".json") 40 | if _, err := os.Stat(blockFilePath); err != nil { 41 | return nil, ErrNoRecords 42 | } 43 | 44 | //we used to to a doGetByIndex here but it doesn't work properly 45 | //TODO revisit doGetByIndex 46 | dataBlock := db.NewEmptyBlock(g.config.EncryptionKey) 47 | if err := dataBlock.Hydrate(blockFilePath); err != nil { 48 | return nil, err 49 | } 50 | 51 | return dataBlock.Get(id) 52 | } 53 | 54 | //func (g *gitdb) doGetByIndex(id, dataset string) (*db.Record, error) { 55 | // //read id index 56 | // indexFile := filepath.Join(g.indexDir(), dataset, "id.json") 57 | // if _, ok := g.indexCache[indexFile]; !ok { 58 | // g.buildIndexTargeted(dataset) 59 | // } 60 | // 61 | // iv, ok := g.indexCache[indexFile][id] 62 | // if ok { 63 | // dataBlock := db.NewEmptyBlock(g.config.EncryptionKey) 64 | // err = dataBlock.HydrateByPositions(blockFilePath, []int{iv.Offset, iv.Len}) 65 | // if err != nil { 66 | // log.Error(err.Error()) 67 | // return nil, fmt.Errorf("Record %s not found in %s", id, dataset) 68 | // } 69 | // 70 | // record, err := dataBlock.Get(id) 71 | // if err != nil { 72 | // log.Error(err.Error()) 73 | // return nil, fmt.Errorf("Record %s not found in %s", id, dataset) 74 | // } 75 | // 76 | // return record, nil 77 | // } 78 | // 79 | // return nil, fmt.Errorf("Record %s not found in %s", id, dataset) 80 | //} 81 | 82 | //Get hydrates a model with specified id into result Model 83 | func (g *gitdb) Get(id string, result Model) error { 84 | record, err := g.doGet(id) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | g.events <- newReadEvent("...", id) 90 | 91 | return record.Hydrate(result) 92 | } 93 | 94 | func (g *gitdb) Exists(id string) error { 95 | _, err := g.doGet(id) 96 | if err == nil { 97 | g.events <- newReadEvent("...", id) 98 | } 99 | 100 | return err 101 | } 102 | 103 | func (g *gitdb) Fetch(dataset string, blocks ...string) ([]*db.Record, error) { 104 | if !g.isRegistered(dataset) { 105 | return nil, ErrInvalidDataset 106 | } 107 | 108 | dataBlock := db.NewEmptyBlock(g.config.EncryptionKey) 109 | 110 | if len(blocks) > 0 { 111 | fullPath := filepath.Join(g.dbDir(), dataset) 112 | for _, block := range blocks { 113 | blockFile := filepath.Join(fullPath, block+".json") 114 | log.Test("Fetching BLOCK records from - " + blockFile) 115 | if err := dataBlock.Hydrate(blockFile); err != nil { 116 | return nil, err 117 | } 118 | } 119 | 120 | return dataBlock.Records(), nil 121 | } 122 | 123 | if err := g.doFetch(dataset, dataBlock); err != nil { 124 | return nil, err 125 | } 126 | 127 | log.Info(fmt.Sprintf("%d records found in %s", dataBlock.Len(), dataset)) 128 | return dataBlock.Records(), nil 129 | } 130 | 131 | func (g *gitdb) doFetch(dataset string, dataBlock *db.EmptyBlock) error { 132 | 133 | fullPath := filepath.Join(g.dbDir(), dataset) 134 | //events <- newReadEvent("...", fullPath) 135 | log.Info("Fetching records from - " + fullPath) 136 | files, err := ioutil.ReadDir(fullPath) 137 | if err != nil && !os.IsNotExist(err) { 138 | return err 139 | } 140 | 141 | if os.IsNotExist(err) { 142 | return ErrNoRecords 143 | } 144 | 145 | var fileName string 146 | for _, file := range files { 147 | fileName = filepath.Join(fullPath, file.Name()) 148 | if filepath.Ext(fileName) == ".json" { 149 | if err := dataBlock.Hydrate(fileName); err != nil { 150 | return err 151 | } 152 | } 153 | } 154 | 155 | return nil 156 | } 157 | 158 | func (g *gitdb) Search(dataset string, searchParams []*SearchParam, searchMode SearchMode) ([]*db.Record, error) { 159 | if !g.isRegistered(dataset) { 160 | return nil, ErrInvalidDataset 161 | } 162 | 163 | //searchBlocks return the position of the record in the block 164 | //searchBlocks := map[string][][]int{} //index based 165 | searchBlocks := map[string]bool{} 166 | matchingRecords := map[string]string{} 167 | 168 | for _, searchParam := range searchParams { 169 | indexFile := filepath.Join(g.indexDir(), dataset, searchParam.Index+".json") 170 | if _, ok := g.indexCache[indexFile]; !ok { 171 | g.buildIndexTargeted(dataset) 172 | } 173 | 174 | g.events <- newReadEvent("...", indexFile) 175 | 176 | queryValue := strings.ToLower(searchParam.Value) 177 | for recordID, iv := range g.indexCache[indexFile] { 178 | addResult := false 179 | dbValue := strings.ToLower(iv.(string)) 180 | switch searchMode { 181 | case SearchEquals: 182 | addResult = dbValue == queryValue 183 | case SearchContains: 184 | addResult = strings.Contains(dbValue, queryValue) 185 | case SearchStartsWith: 186 | addResult = strings.HasPrefix(dbValue, queryValue) 187 | case SearchEndsWith: 188 | addResult = strings.HasSuffix(dbValue, queryValue) 189 | } 190 | 191 | if addResult { 192 | dataset, block, _, err := ParseID(recordID) 193 | if err != nil { 194 | return nil, err 195 | } 196 | 197 | matchingRecords[recordID] = recordID 198 | searchBlocks[g.blockFilePath(dataset, block)] = true 199 | } 200 | } 201 | } 202 | 203 | resultBlock := db.NewEmptyBlock(g.config.EncryptionKey) 204 | //TODO revisit index based search 205 | //for block, pos := range searchBlocks { 206 | // blockFile := filepath.Join(g.dbDir(), dataset, block+".json") 207 | // err := resultBlock.HydrateByPositions(blockFile, pos...) 208 | // if err != nil { 209 | // return nil, err 210 | // } 211 | //} 212 | 213 | for block := range searchBlocks { 214 | if err := resultBlock.Hydrate(block); err != nil { 215 | log.Error(err.Error()) 216 | continue 217 | } 218 | } 219 | 220 | resultBlock.Filter(matchingRecords) 221 | return resultBlock.Records(), nil 222 | } 223 | -------------------------------------------------------------------------------- /read_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/bouggo/log" 7 | "github.com/gogitdb/gitdb/v2" 8 | ) 9 | 10 | //TODO write negative tests (e.g record not found) 11 | func TestGet(t *testing.T) { 12 | teardown := setup(t, getReadTestConfig(gitdb.RecVersion)) 13 | defer teardown(t) 14 | 15 | m := getTestMessage() 16 | 17 | recId := gitdb.ID(m) 18 | result := &Message{} 19 | err := testDb.Get(recId, result) 20 | if err != nil { 21 | t.Error(err.Error()) 22 | } 23 | 24 | if err == nil && gitdb.ID(result) != recId { 25 | t.Errorf("Want: %v, Got: %v", recId, gitdb.ID(result)) 26 | } 27 | } 28 | 29 | func TestGetV1(t *testing.T) { 30 | teardown := setup(t, getReadTestConfig("v1")) 31 | defer teardown(t) 32 | 33 | m := getTestMessage() 34 | 35 | recId := gitdb.ID(m) 36 | result := &Message{} 37 | err := testDb.Get(recId, result) 38 | if err != nil { 39 | t.Error(err.Error()) 40 | } 41 | 42 | if err == nil && gitdb.ID(result) != recId { 43 | t.Errorf("Want: %v, Got: %v", recId, gitdb.ID(result)) 44 | } 45 | } 46 | 47 | func TestExists(t *testing.T) { 48 | teardown := setup(t, getReadTestConfig(gitdb.RecVersion)) 49 | defer teardown(t) 50 | 51 | m := getTestMessage() 52 | 53 | recId := gitdb.ID(m) 54 | err := testDb.Exists(recId) 55 | if err != nil { 56 | t.Error(err.Error()) 57 | } 58 | } 59 | 60 | func TestFetch(t *testing.T) { 61 | teardown := setup(t, getReadTestConfig(gitdb.RecVersion)) 62 | defer teardown(t) 63 | 64 | dataset := "Message" 65 | messages, err := testDb.Fetch(dataset) 66 | if err != nil { 67 | t.Error(err.Error()) 68 | } 69 | 70 | want := 10 71 | got := len(messages) 72 | if got != want { 73 | t.Errorf("Want: %d, Got: %d", want, got) 74 | } 75 | } 76 | 77 | func TestFetchBlock(t *testing.T) { 78 | teardown := setup(t, getReadTestConfig(gitdb.RecVersion)) 79 | defer teardown(t) 80 | 81 | dataset := "Message" 82 | messages, err := testDb.Fetch(dataset, "b0") 83 | if err != nil { 84 | t.Error(err.Error()) 85 | } 86 | 87 | want := 10 88 | got := len(messages) 89 | if got != want { 90 | t.Errorf("Want: %d, Got: %d", want, got) 91 | } 92 | } 93 | 94 | //TODO test correctness of search results 95 | func TestSearch(t *testing.T) { 96 | teardown := setup(t, getReadTestConfig(gitdb.RecVersion)) 97 | defer teardown(t) 98 | 99 | count := 10 100 | sp := &gitdb.SearchParam{ 101 | Index: "From", 102 | Value: "alice@example.com", 103 | } 104 | 105 | results, err := testDb.Search("Message", []*gitdb.SearchParam{sp}, gitdb.SearchEquals) 106 | if err != nil { 107 | t.Errorf("search failed with error - %s", err) 108 | return 109 | } 110 | 111 | if len(results) != count { 112 | t.Errorf("search result count wrong. want: %d, got: %d", count, len(results)) 113 | } 114 | } 115 | 116 | func BenchmarkFetch(b *testing.B) { 117 | teardown := setup(b, getReadTestConfig(gitdb.RecVersion)) 118 | defer teardown(b) 119 | 120 | b.ReportAllocs() 121 | for i := 0; i <= b.N; i++ { 122 | testDb.Fetch("Message") 123 | } 124 | } 125 | 126 | func BenchmarkGet(b *testing.B) { 127 | teardown := setup(b, getReadTestConfig(gitdb.RecVersion)) 128 | defer teardown(b) 129 | 130 | b.ReportAllocs() 131 | m := &Message{} 132 | for i := 0; i <= b.N; i++ { 133 | testDb.Get("Message/b0/1", m) 134 | log.Test(m.Body) 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /schema.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | 12 | "github.com/bouggo/log" 13 | ) 14 | 15 | //Schema holds functions for generating a model id 16 | type Schema struct { 17 | dataset string 18 | block string 19 | record string 20 | indexes map[string]interface{} 21 | 22 | internal bool 23 | } 24 | 25 | //NewSchema constructs a *Schema 26 | func NewSchema(name, block, record string, indexes map[string]interface{}) *Schema { 27 | return &Schema{dataset: name, block: block, record: record, indexes: indexes} 28 | } 29 | 30 | func newSchema(name, block, record string, indexes map[string]interface{}) *Schema { 31 | return &Schema{dataset: name, block: block, record: record, indexes: indexes, internal: true} 32 | } 33 | 34 | //name returns name of schema 35 | func (a *Schema) name() string { 36 | return a.dataset 37 | } 38 | 39 | //blockID retuns block id of schema 40 | func (a *Schema) blockID() string { 41 | return a.dataset + "/" + a.block 42 | } 43 | 44 | //recordID returns record id of schema 45 | func (a *Schema) recordID() string { 46 | return a.blockID() + "/" + a.record 47 | } 48 | 49 | //Validate ensures *Schema is valid 50 | func (a *Schema) Validate() error { 51 | if len(a.dataset) == 0 { 52 | return errors.New("Invalid Schema Name") 53 | } 54 | 55 | if !a.internal && !a.validDatasetName(a.dataset) { 56 | return fmt.Errorf("%s is a reserved Schema Name", a.dataset) 57 | } 58 | 59 | if !a.validName(a.block) { 60 | return errors.New("Invalid Schema Block ID") 61 | } 62 | 63 | if !a.validName(a.record) { 64 | return errors.New("Invalid Schema Record ID") 65 | } 66 | 67 | if _, ok := a.indexes["id"]; ok && !a.internal { 68 | return fmt.Errorf("%s is a reserved index name", "id") 69 | } 70 | 71 | return nil 72 | } 73 | 74 | func (a *Schema) validDatasetName(name string) bool { 75 | reservedName := []string{"gitdb", "bucket", "upload"} 76 | lcname := strings.ToLower(name) 77 | for _, rname := range reservedName { 78 | if lcname == rname { 79 | return false 80 | } 81 | } 82 | 83 | return a.validName(name) 84 | } 85 | 86 | func (a *Schema) validName(name string) bool { 87 | if len(name) < 1 { 88 | return false 89 | } 90 | 91 | allowedChars := `abcdefghijklmnopqrstuvwxyz0123456789_-.` 92 | return strings.ContainsAny(strings.ToLower(name), allowedChars) 93 | } 94 | 95 | //Indexes returns the index map of a given Model 96 | func Indexes(m Model) map[string]interface{} { 97 | return m.GetSchema().indexes 98 | } 99 | 100 | //ID returns the id of a given Model 101 | func ID(m Model) string { 102 | return m.GetSchema().recordID() 103 | } 104 | 105 | //ParseID parses a record id and returns it's metadata 106 | func ParseID(id string) (dataset string, block string, record string, err error) { 107 | recordMeta := strings.Split(id, "/") 108 | if len(recordMeta) != 3 { 109 | err = ErrInvalidRecordID 110 | } else { 111 | dataset = recordMeta[0] 112 | block = recordMeta[1] 113 | record = recordMeta[2] 114 | } 115 | 116 | return dataset, block, record, err 117 | } 118 | 119 | //BlockMethod type of method to use with AutoBlock 120 | type BlockMethod string 121 | 122 | var ( 123 | //BlockBySize generates a new block when current block has reached a specified size 124 | BlockBySize BlockMethod = "size" 125 | //BlockByCount generates a new block when the number of records has reached a specified count 126 | BlockByCount BlockMethod = "count" 127 | ) 128 | 129 | //AutoBlock automatically generates block id for a given Model depending on a BlockMethod 130 | func AutoBlock(dbPath string, m Model, method BlockMethod, n int64) string { 131 | 132 | var currentBlock int 133 | var currentBlockFile os.FileInfo 134 | var currentBlockrecords map[string]interface{} 135 | 136 | //being sensible 137 | if n <= 0 { 138 | n = 1000 139 | } 140 | 141 | dataset := m.GetSchema().name() 142 | fullPath := filepath.Join(dbPath, "data", dataset) 143 | 144 | if _, err := os.Stat(fullPath); err != nil { 145 | return fmt.Sprintf("b%d", currentBlock) 146 | } 147 | 148 | files, err := ioutil.ReadDir(fullPath) 149 | if err != nil { 150 | log.Error(err.Error()) 151 | log.Test("AutoBlock: " + err.Error()) 152 | return "" 153 | } 154 | 155 | if len(files) == 0 { 156 | log.Test("AutoBlock: no blocks found at " + fullPath) 157 | return fmt.Sprintf("b%d", currentBlock) 158 | } 159 | 160 | currentBlock = -1 161 | for _, currentBlockFile = range files { 162 | currentBlockFileName := filepath.Join(fullPath, currentBlockFile.Name()) 163 | if filepath.Ext(currentBlockFileName) != ".json" { 164 | continue 165 | } 166 | 167 | currentBlock++ 168 | //TODO OPTIMIZE read file 169 | b, err := ioutil.ReadFile(currentBlockFileName) 170 | if err != nil { 171 | log.Test("AutoBlock: " + err.Error()) 172 | log.Error(err.Error()) 173 | continue 174 | } 175 | 176 | currentBlockrecords = make(map[string]interface{}) 177 | if err := json.Unmarshal(b, ¤tBlockrecords); err != nil { 178 | log.Error(err.Error()) 179 | continue 180 | } 181 | 182 | block := strings.Replace(filepath.Base(currentBlockFileName), filepath.Ext(currentBlockFileName), "", 1) 183 | id := fmt.Sprintf("%s/%s/%s", dataset, block, m.GetSchema().record) 184 | 185 | log.Test("AutoBlock: searching for - " + id) 186 | //model already exists return its block 187 | if _, ok := currentBlockrecords[id]; ok { 188 | log.Test("AutoBlock: found - " + id) 189 | return block 190 | } 191 | } 192 | 193 | //is current block at it's size limit? 194 | if method == BlockBySize && currentBlockFile.Size() >= n { 195 | currentBlock++ 196 | return fmt.Sprintf("b%d", currentBlock) 197 | } 198 | 199 | //record size check 200 | log.Test(fmt.Sprintf("AutoBlock: current block count - %d", len(currentBlockrecords))) 201 | if method == BlockByCount && len(currentBlockrecords) >= int(n) { 202 | currentBlock++ 203 | } 204 | 205 | return fmt.Sprintf("b%d", currentBlock) 206 | } 207 | -------------------------------------------------------------------------------- /schema_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/gogitdb/gitdb/v2" 8 | ) 9 | 10 | func TestAutoBlock(t *testing.T) { 11 | cfg := getReadTestConfig(gitdb.RecVersion) 12 | teardown := setup(t, cfg) 13 | defer teardown(t) 14 | 15 | m := getTestMessage() 16 | 17 | want := "b0" 18 | got := gitdb.AutoBlock(cfg.DBPath, m, gitdb.BlockByCount, 10) 19 | if got != want { 20 | t.Errorf("want: %s, got: %s", want, got) 21 | } 22 | 23 | m.MessageId = 11 24 | want = "b1" 25 | got = gitdb.AutoBlock(cfg.DBPath, m, gitdb.BlockByCount, 10) 26 | if got != want { 27 | t.Errorf("want: %s, got: %s", want, got) 28 | } 29 | 30 | m.MessageId = 11 31 | want = "b0" 32 | got = gitdb.AutoBlock("/non/existent/path", m, gitdb.BlockByCount, 10) 33 | if got != want { 34 | t.Errorf("want: %s, got: %s", want, got) 35 | } 36 | } 37 | 38 | func TestHydrate(t *testing.T) { 39 | teardown := setup(t, getReadTestConfig(gitdb.RecVersion)) 40 | defer teardown(t) 41 | 42 | result := &Message{} 43 | records, err := testDb.Fetch("Message") 44 | if err != nil { 45 | t.Errorf("testDb.Fetch failed: %s", err) 46 | } 47 | 48 | err = records[0].Hydrate(result) 49 | if err != nil { 50 | t.Errorf("record.Hydrate failed: %s", err) 51 | } 52 | } 53 | 54 | func TestParseId(t *testing.T) { 55 | testId := "DatasetName/Block/RecordId" 56 | ds, block, recordId, err := gitdb.ParseID(testId) 57 | 58 | passed := ds == "DatasetName" && block == "Block" && recordId == "RecordId" && err == nil 59 | if !passed { 60 | t.Errorf("want: DatasetName|Block|RecordId, Got:%s|%s|%s", ds, block, recordId) 61 | } 62 | } 63 | 64 | func TestValidate(t *testing.T) { 65 | cases := []struct { 66 | dataset string 67 | block string 68 | record string 69 | indexes map[string]interface{} 70 | pass bool 71 | }{ 72 | {"d1", "b0", "r0", nil, true}, 73 | {"", "b0", "r0", nil, false}, 74 | {"d1", "", "r0", nil, false}, 75 | {"d1", "b0", "", nil, false}, 76 | {"", "", "", nil, false}, 77 | } 78 | 79 | for _, tc := range cases { 80 | t.Run(fmt.Sprintf("%s/%s/%s", tc.dataset, tc.block, tc.record), func(t *testing.T) { 81 | err := gitdb.NewSchema(tc.dataset, tc.block, tc.record, tc.indexes).Validate() 82 | if (err == nil) != tc.pass { 83 | t.Errorf("test failed: %s", err) 84 | } 85 | }) 86 | } 87 | } 88 | 89 | func BenchmarkParseId(b *testing.B) { 90 | b.ReportAllocs() 91 | for i := 0; i <= b.N; i++ { 92 | gitdb.ParseID("DatasetName/Block/RecordId") 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /ssh.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "crypto/rand" 5 | "crypto/rsa" 6 | "crypto/x509" 7 | "encoding/pem" 8 | "io/ioutil" 9 | "os" 10 | 11 | "github.com/bouggo/log" 12 | "golang.org/x/crypto/ssh" 13 | ) 14 | 15 | // generateSSHKeyPair make a pair of public and private keys for SSH access. 16 | // Public key is encoded in the format for inclusion in an OpenSSH authorized_keys file. 17 | // Private Key generated is PEM encoded 18 | func (g *gitdb) generateSSHKeyPair() error { 19 | 20 | if _, err := os.Stat(g.privateKeyFilePath()); err == nil { 21 | 22 | if _, err := os.Stat(g.publicKeyFilePath()); err == nil { 23 | return nil 24 | } 25 | 26 | log.Info("Re-generating public key") 27 | //public key is missing - recreate public key 28 | pkPem, err := ioutil.ReadFile(g.privateKeyFilePath()) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | b, _ := pem.Decode(pkPem) 34 | privateKey, err := x509.ParsePKCS1PrivateKey(b.Bytes) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | return g.generatePublicKey(privateKey) 40 | } 41 | 42 | if err := os.MkdirAll(g.sshDir(), os.ModePerm); err != nil { 43 | return err 44 | } 45 | 46 | log.Info("Generating ssh key pairs") 47 | privateKey, err := rsa.GenerateKey(rand.Reader, 2048) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | err = g.generatePrivateKey(privateKey) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | return g.generatePublicKey(privateKey) 58 | } 59 | 60 | func (g *gitdb) generatePrivateKey(pk *rsa.PrivateKey) error { 61 | // generate and write private key as PEM 62 | privateKeyFile, err := os.OpenFile(g.privateKeyFilePath(), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0400) 63 | defer privateKeyFile.Close() 64 | if err != nil { 65 | return err 66 | } 67 | privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(pk)} 68 | if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil { 69 | return err 70 | } 71 | 72 | return nil 73 | } 74 | 75 | func (g *gitdb) generatePublicKey(pk *rsa.PrivateKey) error { 76 | // generate and write public key 77 | pub, err := ssh.NewPublicKey(&pk.PublicKey) 78 | if err != nil { 79 | return err 80 | } 81 | return ioutil.WriteFile(g.publicKeyFilePath(), ssh.MarshalAuthorizedKey(pub), 0655) 82 | } 83 | -------------------------------------------------------------------------------- /static/css/app.css: -------------------------------------------------------------------------------- 1 | body { 2 | padding: 0; 3 | margin: 0; 4 | font-family: Arial, Helvetica, sans-serif; 5 | } 6 | 7 | div { 8 | box-sizing: border-box; 9 | } 10 | 11 | h1 { 12 | padding: 0; 13 | margin: 0; 14 | margin-bottom: 30px; 15 | } 16 | 17 | h1 a { 18 | text-decoration: none; 19 | color: darkseagreen; 20 | } 21 | 22 | .sidebar { 23 | float: left; 24 | width: 20%; 25 | height: 800px; 26 | background-color: #eee; 27 | border-right: 1px solid #ddd; 28 | padding: 10px; 29 | } 30 | 31 | .content { 32 | padding: 30px; 33 | padding-top: 10px; 34 | float: left; 35 | width: 80%; 36 | height: 800px; 37 | } 38 | 39 | .nav { 40 | list-style: none; 41 | margin: 0; 42 | padding: 0 43 | } 44 | 45 | .nav li { 46 | color: #000; 47 | } 48 | 49 | .nav a { 50 | color: #000; 51 | text-decoration: none; 52 | display: block; 53 | padding-top: 10px; 54 | padding-bottom: 5px; 55 | padding-left: 5px; 56 | border-bottom: 1px solid #ddd; 57 | } 58 | 59 | .nav a:hover { 60 | background-color: #ddd; 61 | } 62 | 63 | table tr:hover td { 64 | cursor: pointer; 65 | background-color: #ccc; 66 | } 67 | 68 | table th { 69 | background-color: darkseagreen; 70 | color: #fff; 71 | text-align: left; 72 | } 73 | 74 | table { 75 | width: 100%; 76 | /* border: 1px solid #000; */ 77 | border-spacing: 0px; 78 | } 79 | 80 | table td, 81 | table th { 82 | padding: 10px; 83 | border-bottom: 1px solid #ddd; 84 | } 85 | 86 | pre { 87 | background-color: #222; 88 | color: #fff; 89 | padding: 10px; 90 | font-size: 14px; 91 | width: 800px; 92 | overflow: hidden; 93 | } 94 | 95 | textarea { 96 | display: block; 97 | } 98 | 99 | .listWindow { 100 | width: 100%; 101 | overflow-x: scroll; 102 | } -------------------------------------------------------------------------------- /static/errors.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | {{template "sidebar" $}} 8 |
9 |

{{.Title}}

10 | 11 | {{if .DataSet.BadBlocks}} 12 |

Bad Blocks

13 |
    14 | {{range $key, $value := .DataSet.BadBlocks}} 15 |
  • {{ $value }}
    • 16 | {{end}} 17 |
18 | {{end}} {{if .DataSet.BadRecords}} 19 |

Bad Records

20 |
    21 | {{range $key, $value := .DataSet.BadRecords}} 22 |
  • {{ $value }}
    • 23 | {{end}} 24 |
25 | {{end}} 26 |
27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | {{template "sidebar" $}} 9 |
10 |

{{.Title}}

11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | {{range $key, $value := .DataSets}} 23 | 24 | 25 | 26 | 27 | 28 | 29 | 36 | 37 | 38 | {{end}} 39 |
DatasetNo. of blocksNo. of recordsSizeErrorsIndexesLast Modified
{{ $value.Name }}{{ $value.BlockCount }}{{ $value.RecordCount }}{{ $value.HumanSize }}{{ $value.BadBlocksCount }} block(s) / {{ $value.BadRecordsCount }} record(s) 30 |
    31 | {{range $indexName := $value.Indexes}} 32 |
  • {{ $indexName }}
    • 33 | {{end}} 34 |
35 | 		{{ $value.LastModifiedDate }}
40 |
41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /static/js/app.js: -------------------------------------------------------------------------------- 1 | window.addEventListener('load', (event) => { 2 | makeDatasetRowsClickable(); 3 | makeRecordRowsClickable(); 4 | }); 5 | 6 | function makeDatasetRowsClickable() { 7 | document.querySelectorAll('.datasetRow').forEach(row => { 8 | row.addEventListener('click', event => { 9 | window.location = row.dataset.view 10 | }); 11 | }) 12 | } 13 | 14 | function makeRecordRowsClickable() { 15 | document.querySelectorAll('.recordRow').forEach(row => { 16 | row.addEventListener('click', event => { 17 | window.location = row.dataset.view 18 | }); 19 | }) 20 | } -------------------------------------------------------------------------------- /static/list.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | {{template "sidebar" $}} 9 |
10 |

{{.DataSet.Name}}

11 |
{{.DataSet.BlockCount}} blocks {{.DataSet.HumanSize}}
12 | 13 |
14 | 15 | 16 | {{range $key, $value := .Table.Headers}} 17 | 18 | {{end}} 19 | 20 | {{range $key, $value := .Table.Rows}} 21 | 22 | {{range $k, $v := $value}} {{if eq $k 0}} 23 | 24 | {{else}} 25 | 26 | {{end}} {{end}} 27 | 28 | {{end}} 29 |
{{ $value }}
{{ $v }}{{ $v }}
30 |
31 | 32 |
33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /static/sidebar.html: -------------------------------------------------------------------------------- 1 | {{define "sidebar"}} 2 |
3 |

GitDB

4 | Data Sets 5 | 10 |
11 | {{end}} -------------------------------------------------------------------------------- /static/view.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | {{template "sidebar" $}} 9 |
10 |

{{.DataSet.Name}}

11 |
{{.DataSet.BlockCount}} blocks {{.Block.HumanSize}}/{{.DataSet.HumanSize}}
12 | 13 | Prev Block | Next Block 14 | 
15 |   {{.Content}}
16 |
17 | Prev Record | Next Record 18 |
19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /sync.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "fmt" 5 | "github.com/bouggo/log" 6 | "time" 7 | ) 8 | 9 | func (g *gitdb) Sync() error { 10 | g.syncMu.Lock() 11 | defer g.syncMu.Unlock() 12 | 13 | if len(g.config.OnlineRemote) == 0 { 14 | return ErrNoOnlineRemote 15 | } 16 | 17 | // if client PC has at least 20% battery life 18 | if !hasSufficientBatteryPower(20) { 19 | return ErrLowBattery 20 | } 21 | 22 | log.Info("Syncing database...") 23 | changedFiles := g.driver.changedFiles() 24 | if err := g.driver.sync(); err != nil { 25 | log.Error(err.Error()) 26 | return ErrDBSyncFailed 27 | } 28 | 29 | // reset loaded blocks 30 | g.loadedBlocks = nil 31 | 32 | g.buildIndexSmart(changedFiles) 33 | return nil 34 | } 35 | 36 | func (g *gitdb) startSyncClock() { 37 | go func(g *gitdb) { 38 | log.Test(fmt.Sprintf("starting sync clock @ interval %s", g.config.SyncInterval)) 39 | ticker := time.NewTicker(g.config.SyncInterval) 40 | for { 41 | select { 42 | case <-g.shutdown: 43 | log.Test("shutting down sync clock") 44 | return 45 | case <-ticker.C: 46 | g.writeMu.Lock() 47 | if err := g.Sync(); err != nil { 48 | log.Error(err.Error()) 49 | } 50 | g.writeMu.Unlock() 51 | } 52 | } 53 | }(g) 54 | } 55 | -------------------------------------------------------------------------------- /sync_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "testing" 7 | ) 8 | 9 | func TestSync(t *testing.T) { 10 | t.Skip() 11 | cfg := getConfig() 12 | cfg.DBPath = "/tmp/voguedb" 13 | cfg.OnlineRemote = "git@bitbucket.org:voguehotel/db-dev.git" 14 | cfg.EncryptionKey = "" 15 | cfg.SyncInterval = 0 //disables sync clock 16 | teardown := setup(t, cfg) 17 | defer teardown(t) 18 | 19 | wg := sync.WaitGroup{} 20 | n := 10 21 | for { 22 | if n <= 0 { 23 | break 24 | } 25 | wg.Add(1) 26 | fmt.Println("dispatching ", n) 27 | go func() { 28 | if err := testDb.Sync(); err != nil { 29 | t.Errorf("db.Sync failed: %s", err) 30 | } 31 | wg.Done() 32 | }() 33 | n-- 34 | } 35 | 36 | wg.Wait() 37 | } 38 | -------------------------------------------------------------------------------- /testdata/v1/data/data/Message/b0.json: -------------------------------------------------------------------------------- 1 | { 2 | "Message/b0/0": "{\"CreatedAt\":\"2020-03-29T19:02:36.576218+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.576218+01:00\",\"MessageId\":0,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}", 3 | "Message/b0/1": "{\"CreatedAt\":\"2020-03-29T19:02:36.577797+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.577797+01:00\",\"MessageId\":1,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}", 4 | "Message/b0/2": "{\"CreatedAt\":\"2020-03-29T19:02:36.578336+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.578336+01:00\",\"MessageId\":2,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}", 5 | "Message/b0/3": "{\"CreatedAt\":\"2020-03-29T19:02:36.578835+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.578835+01:00\",\"MessageId\":3,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}", 6 | "Message/b0/4": "{\"CreatedAt\":\"2020-03-29T19:02:36.579284+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.579284+01:00\",\"MessageId\":4,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}", 7 | "Message/b0/5": "{\"CreatedAt\":\"2020-03-29T19:02:36.57987+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.57987+01:00\",\"MessageId\":5,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}", 8 | "Message/b0/6": "{\"CreatedAt\":\"2020-03-29T19:02:36.580396+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.580396+01:00\",\"MessageId\":6,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}", 9 | "Message/b0/7": "{\"CreatedAt\":\"2020-03-29T19:02:36.581039+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.581039+01:00\",\"MessageId\":7,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}", 10 | "Message/b0/8": "{\"CreatedAt\":\"2020-03-29T19:02:36.58172+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.58172+01:00\",\"MessageId\":8,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}", 11 | "Message/b0/9": "{\"CreatedAt\":\"2020-03-29T19:02:36.582319+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.582319+01:00\",\"MessageId\":9,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}" 12 | } 13 | -------------------------------------------------------------------------------- /testdata/v1/data/data/MessageV2/202003.json: -------------------------------------------------------------------------------- 1 | { 2 | "MessageV2/202003/0": "{\"CreatedAt\":\"2020-03-29T19:00:32.994814+01:00\",\"UpdatedAt\":\"2020-03-29T19:00:33.06084+01:00\",\"MessageId\":0,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}" 3 | } 4 | -------------------------------------------------------------------------------- /testdata/v2/data/data/Message/b0.json: -------------------------------------------------------------------------------- 1 | { 2 | "Message/b0/0": "{\"Version\":\"v2\",\"Indexes\":{\"From\":\"alice@example.com\"},\"Data\":{\"CreatedAt\":\"2020-03-29T19:02:36.576218+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.576218+01:00\",\"MessageId\":0,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}}", 3 | "Message/b0/1": "{\"Version\":\"v2\",\"Indexes\":{\"From\":\"alice@example.com\"},\"Data\":{\"CreatedAt\":\"2020-03-29T19:02:36.577797+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.577797+01:00\",\"MessageId\":1,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}}", 4 | "Message/b0/2": "{\"Version\":\"v2\",\"Indexes\":{\"From\":\"alice@example.com\"},\"Data\":{\"CreatedAt\":\"2020-03-29T19:02:36.578336+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.578336+01:00\",\"MessageId\":2,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}}", 5 | "Message/b0/3": "{\"Version\":\"v2\",\"Indexes\":{\"From\":\"alice@example.com\"},\"Data\":{\"CreatedAt\":\"2020-03-29T19:02:36.578835+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.578835+01:00\",\"MessageId\":3,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}}", 6 | "Message/b0/4": "{\"Version\":\"v2\",\"Indexes\":{\"From\":\"alice@example.com\"},\"Data\":{\"CreatedAt\":\"2020-03-29T19:02:36.579284+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.579284+01:00\",\"MessageId\":4,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}}", 7 | "Message/b0/5": "{\"Version\":\"v2\",\"Indexes\":{\"From\":\"alice@example.com\"},\"Data\":{\"CreatedAt\":\"2020-03-29T19:02:36.57987+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.57987+01:00\",\"MessageId\":5,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}}", 8 | "Message/b0/6": "{\"Version\":\"v2\",\"Indexes\":{\"From\":\"alice@example.com\"},\"Data\":{\"CreatedAt\":\"2020-03-29T19:02:36.580396+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.580396+01:00\",\"MessageId\":6,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}}", 9 | "Message/b0/7": "{\"Version\":\"v2\",\"Indexes\":{\"From\":\"alice@example.com\"},\"Data\":{\"CreatedAt\":\"2020-03-29T19:02:36.581039+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.581039+01:00\",\"MessageId\":7,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}}", 10 | "Message/b0/8": "{\"Version\":\"v2\",\"Indexes\":{\"From\":\"alice@example.com\"},\"Data\":{\"CreatedAt\":\"2020-03-29T19:02:36.58172+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.58172+01:00\",\"MessageId\":8,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}}", 11 | "Message/b0/9": "{\"Version\":\"v2\",\"Indexes\":{\"From\":\"alice@example.com\"},\"Data\":{\"CreatedAt\":\"2020-03-29T19:02:36.582319+01:00\",\"UpdatedAt\":\"2020-03-29T19:02:36.582319+01:00\",\"MessageId\":9,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}}" 12 | } 13 | -------------------------------------------------------------------------------- /testdata/v2/data/data/MessageV2/202003.json: -------------------------------------------------------------------------------- 1 | { 2 | "MessageV2/202003/0": "{\"Version\":\"v2\",\"Indexes\":{\"From\":\"alice@example.com\"},\"Data\":{\"CreatedAt\":\"2020-03-29T19:00:32.994814+01:00\",\"UpdatedAt\":\"2020-03-29T19:00:33.06084+01:00\",\"MessageId\":0,\"From\":\"alice@example.com\",\"To\":\"bob@example.com\",\"Body\":\"Hello\"}}" 3 | } 4 | -------------------------------------------------------------------------------- /tools/fix_corrupt_head.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash -------------------------------------------------------------------------------- /tools/fix_corrupt_missing_objects.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | #https://stackoverflow.com/questions/33012869/broken-branch-in-git-fatal-your-current-branch-appears-to-be-broken 4 | 5 | git fsck --full >& fsck_report.txt 6 | cat fsck_report.txt | grep "missing" | awk '{print $7}' > fsck_report.txt 7 | 8 | 9 | Did it report a corrupted file? 10 | If so delete the file, go back to step #1. 11 | 12 | for item in `cat fsck_report.txt` 13 | do 14 | echo "deleting $item" 15 | rm -f $item 16 | done 17 | 18 | Do del .git/index 19 | Do git reset 20 | 21 | rm -f fsck_report 22 | 23 | 24 | #remove invalid reflog references 25 | git reflog expire --stale-fix --all 26 | 27 | clone down repo to a separate dir 28 | and run 29 | cat ../fresh/.git/objects/pack/pack-*.pack | git unpack-objects 30 | within the broken repo -------------------------------------------------------------------------------- /transaction.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/bouggo/log" 7 | ) 8 | 9 | type operation func() error 10 | 11 | // Transaction represents a db transaction 12 | type Transaction interface { 13 | Commit() error 14 | AddOperation(o operation) 15 | } 16 | 17 | type transaction struct { 18 | name string 19 | operations []operation 20 | db *gitdb 21 | } 22 | 23 | func (t *transaction) Commit() error { 24 | t.db.autoCommit = false 25 | for _, o := range t.operations { 26 | if err := o(); err != nil { 27 | log.Info("Reverting transaction: " + err.Error()) 28 | err2 := t.db.driver.undo() 29 | t.db.autoCommit = true 30 | if err2 != nil { 31 | err = fmt.Errorf("%s - %s", err.Error(), err2.Error()) 32 | } 33 | 34 | return err 35 | } 36 | } 37 | 38 | t.db.autoCommit = true 39 | commitMsg := "Committing transaction: " + t.name 40 | t.db.commit.Add(1) 41 | t.db.events <- newWriteEvent(commitMsg, ".", t.db.autoCommit) 42 | t.db.waitForCommit() 43 | return nil 44 | } 45 | 46 | func (t *transaction) AddOperation(o operation) { 47 | t.operations = append(t.operations, o) 48 | } 49 | 50 | func (g *gitdb) StartTransaction(name string) Transaction { 51 | return &transaction{name: name, db: g} 52 | } 53 | -------------------------------------------------------------------------------- /transaction_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/gogitdb/gitdb/v2" 8 | ) 9 | 10 | func TestTransaction(t *testing.T) { 11 | //intentionally crafted to increase converage on *gitdb.configure 12 | cfg := &gitdb.Config{ 13 | DBPath: dbPath, 14 | } 15 | teardown := setup(t, cfg) 16 | defer teardown(t) 17 | 18 | tx := testDb.StartTransaction("test") 19 | tx.AddOperation(func() error { return nil }) 20 | tx.AddOperation(func() error { return nil }) 21 | tx.AddOperation(func() error { return errors.New("test error") }) 22 | tx.AddOperation(func() error { return nil }) 23 | if err := tx.Commit(); err == nil { 24 | t.Error("transaction should fail on 3rd operation") 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /ui.go: -------------------------------------------------------------------------------- 1 | //go:generate gitdb embed-ui -o ./ui_static.go 2 | package gitdb 3 | 4 | import ( 5 | "context" 6 | "encoding/base64" 7 | "fmt" 8 | "html/template" 9 | "io/ioutil" 10 | "net/http" 11 | "path/filepath" 12 | "time" 13 | 14 | "github.com/bouggo/log" 15 | "github.com/gogitdb/gitdb/v2/internal/db" 16 | "github.com/gorilla/mux" 17 | ) 18 | 19 | var fs *fileSystem 20 | 21 | func getFs() *fileSystem { 22 | if fs == nil { 23 | fs = &fileSystem{} 24 | } 25 | return fs 26 | } 27 | 28 | func (g *gitdb) startUI() { 29 | 30 | server := &http.Server{ 31 | Addr: fmt.Sprintf("localhost:%d", g.config.UIPort), 32 | Handler: new(router).configure(g.config), 33 | } 34 | 35 | log.Info("GitDB GUI will run at http://" + server.Addr) 36 | go func() { 37 | if err := server.ListenAndServe(); err != nil { 38 | log.Error(err.Error()) 39 | } 40 | }() 41 | 42 | //listen for shutdown event 43 | go func() { 44 | <-g.shutdown 45 | if server != nil { 46 | log.Test("shutting down UI server") 47 | server.Shutdown(context.TODO()) 48 | } 49 | return 50 | }() 51 | } 52 | 53 | type fileSystem struct { 54 | files map[string]string 55 | } 56 | 57 | func (e *fileSystem) embed(name, src string) { 58 | if e.files == nil { 59 | e.files = make(map[string]string) 60 | } 61 | e.files[name] = src 62 | } 63 | 64 | func (e *fileSystem) has(name string) bool { 65 | _, ok := e.files[name] 66 | return ok 67 | } 68 | 69 | func (e *fileSystem) get(name string) []byte { 70 | content := e.files[name] 71 | decoded, err := base64.StdEncoding.DecodeString(content) 72 | if err != nil { 73 | log.Error(err.Error()) 74 | return []byte("") 75 | } 76 | 77 | return decoded 78 | } 79 | 80 | //router provides all the http handlers for the UI 81 | type router struct { 82 | datasets []*db.Dataset 83 | refreshAt time.Time 84 | } 85 | 86 | func (u *router) configure(cfg Config) *mux.Router { 87 | router := mux.NewRouter() 88 | for path, handler := range u.getEndpoints() { 89 | router.HandleFunc(path, handler) 90 | } 91 | 92 | //refresh dataset after 1 minute 93 | router.Use(func(h http.Handler) http.Handler { 94 | if u.refreshAt.IsZero() || u.refreshAt.Before(time.Now()) { 95 | u.datasets = db.LoadDatasets(filepath.Join(cfg.DBPath, "data"), cfg.EncryptionKey) 96 | u.refreshAt = time.Now().Add(time.Second * 10) 97 | } 98 | 99 | return h 100 | }) 101 | 102 | return router 103 | } 104 | 105 | //getEndpoints maps a path to a http handler 106 | func (u *router) getEndpoints() map[string]http.HandlerFunc { 107 | return map[string]http.HandlerFunc{ 108 | "/css/app.css": u.appCSS, 109 | "/js/app.js": u.appJS, 110 | "/": u.overview, 111 | "/errors/{dataset}": u.viewErrors, 112 | "/list/{dataset}": u.list, 113 | "/view/{dataset}": u.view, 114 | "/view/{dataset}/b{b}/r{r}": u.view, 115 | } 116 | } 117 | 118 | func (u *router) appCSS(w http.ResponseWriter, r *http.Request) { 119 | src := readView("static/css/app.css") 120 | w.Header().Set("Content-Type", "text/css") 121 | w.Write(src) 122 | } 123 | 124 | func (u *router) appJS(w http.ResponseWriter, r *http.Request) { 125 | src := readView("static/js/app.js") 126 | w.Header().Set("Content-Type", "text/javascript") 127 | w.Write(src) 128 | } 129 | 130 | func (u *router) overview(w http.ResponseWriter, r *http.Request) { 131 | viewModel := &overviewViewModel{} 132 | viewModel.Title = "Overview" 133 | viewModel.DataSets = u.datasets 134 | 135 | render(w, viewModel, "static/index.html", "static/sidebar.html") 136 | } 137 | 138 | func (u *router) list(w http.ResponseWriter, r *http.Request) { 139 | vars := mux.Vars(r) 140 | viewDs := vars["dataset"] 141 | 142 | dataset := u.findDataset(viewDs) 143 | if dataset == nil { 144 | w.Write([]byte("Dataset (" + viewDs + ") does not exist")) 145 | return 146 | } 147 | 148 | block := dataset.Block(0) 149 | table := tablulate(block) 150 | viewModel := &listDataSetViewModel{DataSet: dataset, Table: table} 151 | viewModel.DataSets = u.datasets 152 | 153 | render(w, viewModel, "static/list.html", "static/sidebar.html") 154 | } 155 | 156 | func (u *router) view(w http.ResponseWriter, r *http.Request) { 157 | vars := mux.Vars(r) 158 | viewDs := vars["dataset"] 159 | 160 | dataset := u.findDataset(viewDs) 161 | if dataset == nil { 162 | w.Write([]byte("Dataset (" + viewDs + ") does not exist")) 163 | return 164 | } 165 | 166 | viewModel := &viewDataSetViewModel{ 167 | DataSet: dataset, 168 | Content: "No record found", 169 | Pager: &pager{totalBlocks: dataset.BlockCount()}, 170 | } 171 | viewModel.DataSets = u.datasets 172 | if vars["b"] != "" && vars["r"] != "" { 173 | viewModel.Pager.set(vars["b"], vars["r"]) 174 | } 175 | block := dataset.Block(viewModel.Pager.blockPage) 176 | viewModel.Block = block 177 | viewModel.Pager.totalRecords = block.RecordCount() 178 | if viewModel.Pager.totalRecords > viewModel.Pager.recordPage { 179 | viewModel.Content = block.Record(viewModel.Pager.recordPage).JSON() 180 | } 181 | 182 | render(w, viewModel, "static/view.html", "static/sidebar.html") 183 | } 184 | 185 | func (u *router) viewErrors(w http.ResponseWriter, r *http.Request) { 186 | vars := mux.Vars(r) 187 | viewDs := vars["dataset"] 188 | 189 | dataset := u.findDataset(viewDs) 190 | if dataset == nil { 191 | w.Write([]byte("Dataset (" + viewDs + ") does not exist")) 192 | return 193 | } 194 | viewModel := &errorsViewModel{DataSet: dataset} 195 | viewModel.Title = "Errors" 196 | viewModel.DataSets = u.datasets 197 | 198 | render(w, viewModel, "static/errors.html", "static/sidebar.html") 199 | } 200 | 201 | func (u *router) findDataset(name string) *db.Dataset { 202 | for _, ds := range u.datasets { 203 | if ds.Name() == name { 204 | return ds 205 | } 206 | } 207 | return nil 208 | } 209 | 210 | func render(w http.ResponseWriter, data interface{}, templates ...string) { 211 | 212 | parseFiles := false 213 | for _, template := range templates { 214 | if !getFs().has(template) { 215 | parseFiles = true 216 | } 217 | } 218 | 219 | var t *template.Template 220 | var err error 221 | if parseFiles { 222 | t, err = template.ParseFiles(templates...) 223 | if err != nil { 224 | log.Error(err.Error()) 225 | } 226 | } else { 227 | t = template.New("overview") 228 | for _, template := range templates { 229 | log.Test("Reading EMBEDDED file - " + template) 230 | t, err = t.Parse(string(getFs().get(template))) 231 | if err != nil { 232 | log.Error(err.Error()) 233 | } 234 | } 235 | } 236 | 237 | t.Execute(w, data) 238 | } 239 | 240 | func readView(fileName string) []byte { 241 | if getFs().has(fileName) { 242 | return getFs().get(fileName) 243 | } 244 | 245 | data, err := ioutil.ReadFile(fileName) 246 | if err != nil { 247 | log.Error(err.Error()) 248 | return []byte("") 249 | } 250 | 251 | return data 252 | } 253 | -------------------------------------------------------------------------------- /ui_static.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | // Code generated by gitdb embed-ui on Tue, 07 Apr 2020 14:46:09 BST; DO NOT EDIT. 3 | 4 | func init() { 5 | //Embed Files 6 | 7 | getFs().embed("static/css/app.css", "Ym9keSB7cGFkZGluZzogMDttYXJnaW46IDA7Zm9udC1mYW1pbHk6IEFyaWFsLCBIZWx2ZXRpY2EsIHNhbnMtc2VyaWY7fWRpdiB7Ym94LXNpemluZzogYm9yZGVyLWJveDt9aDEge3BhZGRpbmc6IDA7bWFyZ2luOiAwO21hcmdpbi1ib3R0b206IDMwcHg7fWgxIGEge3RleHQtZGVjb3JhdGlvbjogbm9uZTtjb2xvcjogZGFya3NlYWdyZWVuO30uc2lkZWJhciB7ZmxvYXQ6IGxlZnQ7d2lkdGg6IDIwJTtoZWlnaHQ6IDgwMHB4O2JhY2tncm91bmQtY29sb3I6ICNlZWU7Ym9yZGVyLXJpZ2h0OiAxcHggc29saWQgI2RkZDtwYWRkaW5nOiAxMHB4O30uY29udGVudCB7cGFkZGluZzogMzBweDtwYWRkaW5nLXRvcDogMTBweDtmbG9hdDogbGVmdDt3aWR0aDogODAlO2hlaWdodDogODAwcHg7fS5uYXYge2xpc3Qtc3R5bGU6IG5vbmU7bWFyZ2luOiAwO3BhZGRpbmc6IDB9Lm5hdiBsaSB7Y29sb3I6ICMwMDA7fS5uYXYgYSB7Y29sb3I6ICMwMDA7dGV4dC1kZWNvcmF0aW9uOiBub25lO2Rpc3BsYXk6IGJsb2NrO3BhZGRpbmctdG9wOiAxMHB4O3BhZGRpbmctYm90dG9tOiA1cHg7cGFkZGluZy1sZWZ0OiA1cHg7Ym9yZGVyLWJvdHRvbTogMXB4IHNvbGlkICNkZGQ7fS5uYXYgYTpob3ZlciB7YmFja2dyb3VuZC1jb2xvcjogI2RkZDt9dGFibGUgdHI6aG92ZXIgdGQge2N1cnNvcjogcG9pbnRlcjtiYWNrZ3JvdW5kLWNvbG9yOiAjY2NjO310YWJsZSB0aCB7YmFja2dyb3VuZC1jb2xvcjogZGFya3NlYWdyZWVuO2NvbG9yOiAjZmZmO3RleHQtYWxpZ246IGxlZnQ7fXRhYmxlIHt3aWR0aDogMTAwJTsvKiBib3JkZXI6IDFweCBzb2xpZCAjMDAwOyAqL2JvcmRlci1zcGFjaW5nOiAwcHg7fXRhYmxlIHRkLHRhYmxlIHRoIHtwYWRkaW5nOiAxMHB4O2JvcmRlci1ib3R0b206IDFweCBzb2xpZCAjZGRkO31wcmUge2JhY2tncm91bmQtY29sb3I6ICMyMjI7Y29sb3I6ICNmZmY7cGFkZGluZzogMTBweDtmb250LXNpemU6IDE0cHg7d2lkdGg6IDgwMHB4O292ZXJmbG93OiBoaWRkZW47fXRleHRhcmVhIHtkaXNwbGF5OiBibG9jazt9Lmxpc3RXaW5kb3cge3dpZHRoOiAxMDAlO292ZXJmbG93LXg6IHNjcm9sbDt9") 8 | 9 | getFs().embed("static/errors.html", "PGh0bWw+PGhlYWQ+PC9oZWFkPjxsaW5rIHJlbD0ic3R5bGVzaGVldCIgaHJlZj0iL2Nzcy9hcHAuY3NzIj48Ym9keT57e3RlbXBsYXRlICJzaWRlYmFyIiAkfX08ZGl2IGNsYXNzPSJjb250ZW50Ij48aDE+e3suVGl0bGV9fTwvaDE+e3tpZiAuRGF0YVNldC5CYWRCbG9ja3N9fTxoMj5CYWQgQmxvY2tzPC9oMj48dWw+e3tyYW5nZSAka2V5LCAkdmFsdWUgOj0gLkRhdGFTZXQuQmFkQmxvY2tzfX08bGk+PGEgaHJlZj0iL2VkaXQve3sgJHZhbHVlIH19Ij57eyAkdmFsdWUgfX08L2E+PC9saT57e2VuZH19PC91bD57e2VuZH19IHt7aWYgLkRhdGFTZXQuQmFkUmVjb3Jkc319PGgyPkJhZCBSZWNvcmRzPC9oMj48dWw+e3tyYW5nZSAka2V5LCAkdmFsdWUgOj0gLkRhdGFTZXQuQmFkUmVjb3Jkc319PGxpPjxhIGhyZWY9IiMiPnt7ICR2YWx1ZSB9fTwvYT48L2xpPnt7ZW5kfX08L3VsPnt7ZW5kfX08L2Rpdj48L2JvZHk+PC9odG1sPg==") 10 | 11 | getFs().embed("static/index.html", "PGh0bWw+PGhlYWQ+PC9oZWFkPjxsaW5rIHJlbD0ic3R5bGVzaGVldCIgaHJlZj0iL2Nzcy9hcHAuY3NzIj48c2NyaXB0IHNyYz0iL2pzL2FwcC5qcyI+PC9zY3JpcHQ+PGJvZHk+e3t0ZW1wbGF0ZSAic2lkZWJhciIgJH19PGRpdiBjbGFzcz0iY29udGVudCI+PGgxPnt7LlRpdGxlfX08L2gxPjx0YWJsZT48dHI+PHRoPkRhdGFzZXQ8L3RoPjx0aD5Oby4gb2YgYmxvY2tzPC90aD48dGg+Tm8uIG9mIHJlY29yZHM8L3RoPjx0aD5TaXplPC90aD48dGg+RXJyb3JzPC90aD48dGg+SW5kZXhlczwvdGg+PHRoPkxhc3QgTW9kaWZpZWQ8L3RoPjwvdHI+e3tyYW5nZSAka2V5LCAkdmFsdWUgOj0gLkRhdGFTZXRzfX08dHIgY2xhc3M9ImRhdGFzZXRSb3ciIGRhdGEtdmlldz0iL2xpc3Qve3sgJHZhbHVlLk5hbWUgfX0iPjx0ZD57eyAkdmFsdWUuTmFtZSB9fTwvdGQ+PHRkPnt7ICR2YWx1ZS5CbG9ja0NvdW50IH19PC90ZD48dGQ+e3sgJHZhbHVlLlJlY29yZENvdW50IH19PC90ZD48dGQ+e3sgJHZhbHVlLkh1bWFuU2l6ZSB9fTwvdGQ+PHRkPjxhIGhyZWY9Ii9lcnJvcnMve3sgJHZhbHVlLk5hbWUgfX0iPnt7ICR2YWx1ZS5CYWRCbG9ja3NDb3VudCB9fSBibG9jayhzKSAvIHt7ICR2YWx1ZS5CYWRSZWNvcmRzQ291bnQgfX0gcmVjb3JkKHMpPC9hPjwvdGQ+PHRkPjx1bD57e3JhbmdlICRpbmRleE5hbWUgOj0gJHZhbHVlLkluZGV4ZXN9fTxsaT57eyAkaW5kZXhOYW1lIH19PC9saT57e2VuZH19PC91bD48L3RkPjx0ZD57eyAkdmFsdWUuTGFzdE1vZGlmaWVkRGF0ZSB9fTwvdGQ+PC90cj57e2VuZH19PC90YWJsZT48L2Rpdj48L2JvZHk+PC9odG1sPg==") 12 | 13 | getFs().embed("static/js/app.js", "d2luZG93LmFkZEV2ZW50TGlzdGVuZXIoJ2xvYWQnLCAoZXZlbnQpID0+IHttYWtlRGF0YXNldFJvd3NDbGlja2FibGUoKTttYWtlUmVjb3JkUm93c0NsaWNrYWJsZSgpO30pO2Z1bmN0aW9uIG1ha2VEYXRhc2V0Um93c0NsaWNrYWJsZSgpIHtkb2N1bWVudC5xdWVyeVNlbGVjdG9yQWxsKCcuZGF0YXNldFJvdycpLmZvckVhY2gocm93ID0+IHtyb3cuYWRkRXZlbnRMaXN0ZW5lcignY2xpY2snLCBldmVudCA9PiB7d2luZG93LmxvY2F0aW9uID0gcm93LmRhdGFzZXQudmlld30pO30pfWZ1bmN0aW9uIG1ha2VSZWNvcmRSb3dzQ2xpY2thYmxlKCkge2RvY3VtZW50LnF1ZXJ5U2VsZWN0b3JBbGwoJy5yZWNvcmRSb3cnKS5mb3JFYWNoKHJvdyA9PiB7cm93LmFkZEV2ZW50TGlzdGVuZXIoJ2NsaWNrJywgZXZlbnQgPT4ge3dpbmRvdy5sb2NhdGlvbiA9IHJvdy5kYXRhc2V0LnZpZXd9KTt9KX0=") 14 | 15 | getFs().embed("static/list.html", "PGh0bWw+PGhlYWQ+PC9oZWFkPjxsaW5rIHJlbD0ic3R5bGVzaGVldCIgaHJlZj0iL2Nzcy9hcHAuY3NzIj48c2NyaXB0IHNyYz0iL2pzL2FwcC5qcyI+PC9zY3JpcHQ+PGJvZHk+e3t0ZW1wbGF0ZSAic2lkZWJhciIgJH19PGRpdiBjbGFzcz0iY29udGVudCI+PGgxPnt7LkRhdGFTZXQuTmFtZX19PC9oMT48ZGl2PjxzcGFuPnt7LkRhdGFTZXQuQmxvY2tDb3VudH19IGJsb2Nrczwvc3Bhbj4gPHNwYW4+e3suRGF0YVNldC5IdW1hblNpemV9fTwvc3Bhbj48L2Rpdj48ZGl2IGNsYXNzPSJsaXN0V2luZG93Ij48dGFibGU+PHRyPnt7cmFuZ2UgJGtleSwgJHZhbHVlIDo9IC5UYWJsZS5IZWFkZXJzfX08dGg+e3sgJHZhbHVlIH19PC90aD57e2VuZH19PC90cj57e3JhbmdlICRrZXksICR2YWx1ZSA6PSAuVGFibGUuUm93c319PHRyIGNsYXNzPSJyZWNvcmRSb3ciIGRhdGEtdmlldz0iL3ZpZXcve3skLkRhdGFTZXQuTmFtZX19L2IwL3J7eyAka2V5IH19Ij57e3JhbmdlICRrLCAkdiA6PSAkdmFsdWV9fSB7e2lmIGVxICRrIDB9fTx0ZD57eyAkdiB9fTwvdGQ+e3tlbHNlfX08dGQ+e3sgJHYgfX08L3RkPnt7ZW5kfX0ge3tlbmR9fTx0cj57e2VuZH19PC90YWJsZT48L2Rpdj48L2Rpdj48L2JvZHk+PC9odG1sPg==") 16 | 17 | getFs().embed("static/sidebar.html", "e3tkZWZpbmUgInNpZGViYXIifX08ZGl2IGNsYXNzPSJzaWRlYmFyIj48aDE+PGEgaHJlZj0iLyI+R2l0REI8L2E+PC9oMT48c3Ryb25nPkRhdGEgU2V0czwvc3Ryb25nPjx1bCBjbGFzcz0ibmF2Ij57e3JhbmdlICRrZXksICR2YWx1ZSA6PSAuRGF0YVNldHN9fTxsaT48YSBocmVmPSIvbGlzdC97eyAkdmFsdWUuTmFtZSB9fSI+e3sgJHZhbHVlLk5hbWUgfX08L2E+PC9saT57e2VuZH19PC91bD48L2Rpdj57e2VuZH19") 18 | 19 | getFs().embed("static/view.html", "PGh0bWw+PGhlYWQ+PC9oZWFkPjxsaW5rIHJlbD0ic3R5bGVzaGVldCIgaHJlZj0iL2Nzcy9hcHAuY3NzIj48Ym9keT57e3RlbXBsYXRlICJzaWRlYmFyIiAkfX08ZGl2IGNsYXNzPSJjb250ZW50Ij48aDE+e3suRGF0YVNldC5OYW1lfX08L2gxPjxkaXY+PHNwYW4+e3suRGF0YVNldC5CbG9ja0NvdW50fX0gYmxvY2tzPC9zcGFuPiA8c3Bhbj57ey5CbG9jay5IdW1hblNpemV9fS97ey5EYXRhU2V0Lkh1bWFuU2l6ZX19PC9zcGFuPjwvZGl2PjxhIGhyZWY9Ii92aWV3L3t7LkRhdGFTZXQuTmFtZX19L3t7LlBhZ2VyLlByZXZCbG9ja1VSSX19Ij5QcmV2IEJsb2NrPC9hPiB8IDxhIGhyZWY9Ii92aWV3L3t7LkRhdGFTZXQuTmFtZX19L3t7LlBhZ2VyLk5leHRCbG9ja1VSSX19Ij5OZXh0IEJsb2NrPC9hPjxwcmU+e3suQ29udGVudH19PC9wcmU+PGEgaHJlZj0iL3ZpZXcve3suRGF0YVNldC5OYW1lfX0ve3suUGFnZXIuUHJldlJlY29yZFVSSX19Ij5QcmV2IFJlY29yZDwvYT4gfCA8YSBocmVmPSIvdmlldy97ey5EYXRhU2V0Lk5hbWV9fS97ey5QYWdlci5OZXh0UmVjb3JkVVJJfX0iPk5leHQgUmVjb3JkPC9hPjwvZGl2PjwvYm9keT48L2h0bWw+") 20 | 21 | } 22 | -------------------------------------------------------------------------------- /ui_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | ) 7 | 8 | func TestServer(t *testing.T) { 9 | cfg := getConfig() 10 | cfg.EnableUI = true 11 | teardown := setup(t, cfg) 12 | 13 | insert(getTestMessageWithId(1), false) 14 | 15 | //fire off some requests 16 | client := http.DefaultClient 17 | requests := []*http.Request{ 18 | request(http.MethodGet, "http://localhost:4120/css/app.css"), 19 | request(http.MethodGet, "http://localhost:4120/js/app.js"), 20 | request(http.MethodGet, "http://localhost:4120/"), 21 | request(http.MethodGet, "http://localhost:4120/errors/Message"), 22 | request(http.MethodGet, "http://localhost:4120/list/Message"), 23 | request(http.MethodGet, "http://localhost:4120/view/Message"), 24 | request(http.MethodGet, "http://localhost:4120/view/Message/b0/r0"), 25 | } 26 | 27 | for _, req := range requests { 28 | t.Logf("Testing %s", req.URL.String()) 29 | resp, err := client.Do(req) 30 | if err != nil { 31 | t.Errorf("GitDB UI Server request failed: %s", err) 32 | } 33 | 34 | //todo use golden files to check response 35 | // b, _ := ioutil.ReadAll(resp.Body) 36 | resp.Body.Close() 37 | // t.Log(string(b)) 38 | } 39 | 40 | teardown(t) 41 | } 42 | 43 | func request(method, url string) *http.Request { 44 | req, _ := http.NewRequest(method, url, nil) 45 | return req 46 | } 47 | -------------------------------------------------------------------------------- /ui_view.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strconv" 7 | 8 | "github.com/bouggo/log" 9 | "github.com/gogitdb/gitdb/v2/internal/db" 10 | ) 11 | 12 | //table represents a tabular view 13 | type table struct { 14 | Headers []string 15 | Rows [][]string 16 | } 17 | 18 | //tablulate returns a tabular representation of a Block 19 | func tablulate(b *db.Block) *table { 20 | t := &table{} 21 | var jsonMap map[string]interface{} 22 | 23 | for i, record := range b.Records() { 24 | if err := record.Hydrate(&jsonMap); err != nil { 25 | log.Error(err.Error()) 26 | continue 27 | } 28 | 29 | var row []string 30 | if i == 0 { 31 | t.Headers = sortHeaderFields(jsonMap) 32 | } 33 | for _, key := range t.Headers { 34 | val := fmt.Sprintf("%v", jsonMap[key]) 35 | if len(val) > 40 { 36 | val = val[0:40] 37 | } 38 | row = append(row, val) 39 | } 40 | 41 | t.Rows = append(t.Rows, row) 42 | } 43 | 44 | return t 45 | } 46 | 47 | func sortHeaderFields(recMap map[string]interface{}) []string { 48 | // To store the keys in slice in sorted order 49 | var keys []string 50 | for k := range recMap { 51 | keys = append(keys, k) 52 | } 53 | sort.Strings(keys) 54 | return keys 55 | } 56 | 57 | //pager is used to paginate records for the UI 58 | type pager struct { 59 | blockPage int 60 | recordPage int 61 | totalBlocks int 62 | totalRecords int 63 | } 64 | 65 | //set page of Pager 66 | func (p *pager) set(blockFlag string, recordFlag string) { 67 | log.Test("Setting pager: " + blockFlag + "," + recordFlag) 68 | p.blockPage, _ = strconv.Atoi(blockFlag) 69 | p.recordPage, _ = strconv.Atoi(recordFlag) 70 | } 71 | 72 | //NextRecordURI returns the uri for the next record 73 | func (p *pager) NextRecordURI() string { 74 | recordPage := p.recordPage 75 | if p.recordPage < p.totalRecords-1 { 76 | recordPage = p.recordPage + 1 77 | } 78 | 79 | return fmt.Sprintf("b%d/r%d", p.blockPage, recordPage) 80 | } 81 | 82 | //PrevRecordURI returns uri for the previous record 83 | func (p *pager) PrevRecordURI() string { 84 | recordPage := p.recordPage 85 | if p.recordPage > 0 { 86 | recordPage = p.recordPage - 1 87 | } 88 | 89 | return fmt.Sprintf("b%d/r%d", p.blockPage, recordPage) 90 | } 91 | 92 | //NextBlockURI returns uri for the next block 93 | func (p *pager) NextBlockURI() string { 94 | blockPage := p.blockPage 95 | if p.blockPage < p.totalBlocks-1 { 96 | blockPage = p.blockPage + 1 97 | } 98 | 99 | return fmt.Sprintf("b%d/r%d", blockPage, 0) 100 | } 101 | 102 | //PrevBlockURI returns uri for the previous block 103 | func (p *pager) PrevBlockURI() string { 104 | blockPage := p.blockPage 105 | if p.blockPage > 0 { 106 | blockPage = p.blockPage - 1 107 | } 108 | 109 | return fmt.Sprintf("b%d/r%d", blockPage, 0) 110 | } 111 | -------------------------------------------------------------------------------- /ui_viewmodels.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import "github.com/gogitdb/gitdb/v2/internal/db" 4 | 5 | type baseViewModel struct { 6 | Title string 7 | DataSets []*db.Dataset 8 | } 9 | 10 | type overviewViewModel struct { 11 | baseViewModel 12 | } 13 | 14 | type viewDataSetViewModel struct { 15 | baseViewModel 16 | DataSet *db.Dataset 17 | Block *db.Block 18 | Pager *pager 19 | Content string 20 | } 21 | 22 | type listDataSetViewModel struct { 23 | baseViewModel 24 | DataSet *db.Dataset 25 | Table *table 26 | } 27 | 28 | type errorsViewModel struct { 29 | baseViewModel 30 | DataSet *db.Dataset 31 | } 32 | -------------------------------------------------------------------------------- /upload.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path" 9 | "path/filepath" 10 | "strings" 11 | ) 12 | 13 | const uploadDataset = "Bucket" 14 | 15 | var validExtensions = map[string]bool{ 16 | ".html": true, 17 | ".pdf": true, 18 | ".doc": true, 19 | ".xlsx": true, 20 | ".csv": true, 21 | ".jpg": true, 22 | ".gif": true, 23 | ".png": true, 24 | ".svg": true, 25 | ".json": true, 26 | ".yaml": true, 27 | ".yml": true, 28 | ".md": true, 29 | } 30 | 31 | //UploadModel represents a file upload 32 | type UploadModel struct { 33 | Bucket string 34 | File string 35 | Path string 36 | UploadedBy string 37 | TimeStampedModel 38 | } 39 | 40 | //GetSchema implements Model.GetSchema 41 | func (u *UploadModel) GetSchema() *Schema { 42 | return newSchema( 43 | uploadDataset, 44 | // AutoBlock(u.db.dbDir(), name, BlockByCount, 1000), 45 | u.Bucket, 46 | u.File, 47 | make(map[string]interface{}), 48 | ) 49 | } 50 | 51 | //Validate implements Model.Validate 52 | func (u *UploadModel) Validate() error { 53 | //todo 54 | return nil 55 | } 56 | 57 | func (U *UploadModel) ShouldEncrypt() bool { return false } 58 | 59 | //Upload provides API for managing file uploads 60 | type Upload struct { 61 | db *gitdb 62 | model *UploadModel 63 | } 64 | 65 | //Get returns an upload by id 66 | func (u *Upload) Get(id string, result *UploadModel) error { 67 | return u.db.Get(id, result) 68 | } 69 | 70 | //Delete an upload by id 71 | func (u *Upload) Delete(id string) error { 72 | var data UploadModel 73 | if err := u.Get(id, &data); err != nil { 74 | return err 75 | } 76 | 77 | err := u.db.Delete(id) 78 | if err == nil { 79 | err = os.Remove(data.Path) 80 | } 81 | 82 | return err 83 | } 84 | 85 | //New uploads specified file into bucket 86 | func (u *Upload) New(bucket, file string) error { 87 | err := u.db.Exists(u.id(bucket, file)) 88 | if err == nil { 89 | return errors.New("file already exists") 90 | } 91 | 92 | return u.upload(bucket, file) 93 | } 94 | 95 | //Replace overrides a specified file in bucket 96 | func (u *Upload) Replace(bucket, file string) error { 97 | err := u.db.Exists(u.id(bucket, file)) 98 | if err != nil { 99 | return err 100 | } 101 | 102 | return u.upload(bucket, file) 103 | } 104 | 105 | func (u *Upload) id(bucket, file string) string { 106 | return uploadDataset + "/" + bucket + "/" + u.cleanFileName(file) 107 | } 108 | 109 | func (u *Upload) cleanFileName(filename string) string { 110 | //todo a better func to clean up filenames 111 | filename = strings.ReplaceAll(path.Clean(filename), "/", "-") 112 | filename = strings.ReplaceAll(filename, " ", "-") 113 | return filename 114 | } 115 | 116 | func (u *Upload) upload(bucket, file string) error { 117 | var src *os.File 118 | var dst *os.File 119 | var err error 120 | 121 | if src, err = os.Open(file); err != nil { 122 | return err 123 | } 124 | 125 | ext := filepath.Ext(file) 126 | if _, ok := validExtensions[ext]; !ok { 127 | return fmt.Errorf("%s files are not allowed", ext) 128 | } 129 | 130 | filename := u.cleanFileName(file) 131 | uploadPath := filepath.Join(u.db.dbDir(), uploadDataset, bucket, filename) 132 | fmt.Println(uploadPath) 133 | if err = os.MkdirAll(path.Dir(uploadPath), os.ModePerm); err != nil { 134 | return err 135 | } 136 | 137 | if dst, err = os.Create(uploadPath); err != nil { 138 | return err 139 | } 140 | if _, err = io.Copy(dst, src); err != nil { 141 | return err 142 | } 143 | 144 | //neutralise file 145 | if err = dst.Chmod(0640); err != nil { 146 | return err 147 | } 148 | 149 | //Uploads/b0/bucket-file.jpg 150 | m := &UploadModel{ 151 | Bucket: bucket, 152 | File: filename, 153 | Path: uploadPath, 154 | UploadedBy: u.db.config.User.String(), 155 | } 156 | return u.db.Insert(m) 157 | } 158 | 159 | func (g *gitdb) Upload() *Upload { 160 | g.RegisterModel(uploadDataset, &UploadModel{}) 161 | return &Upload{db: g} 162 | } 163 | -------------------------------------------------------------------------------- /upload_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import "testing" 4 | 5 | func TestUploadNew(t *testing.T) { 6 | teardown := setup(t, getConfig()) 7 | defer teardown(t) 8 | 9 | err := testDb.Upload().New("creds", "./README.md") 10 | if err != nil { 11 | t.Errorf("Upload.New() failed: %s", err) 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /user.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | //User represents the user currently connected to the database 4 | //and will be used to identify who made changes to it 5 | type User struct { 6 | Name string 7 | Email string 8 | } 9 | 10 | //AuthorName return commit author git style 11 | func (u *User) AuthorName() string { 12 | return u.Name + " <" + u.Email + ">" 13 | } 14 | 15 | //String is an alias for AuthorName 16 | func (u *User) String() string { 17 | return u.AuthorName() 18 | } 19 | 20 | //NewUser constructs a *DbUser 21 | func NewUser(name string, email string) *User { 22 | return &User{Name: name, Email: email} 23 | } 24 | 25 | //SetUser sets the user connection 26 | func (g *gitdb) SetUser(user *User) error { 27 | g.config.User = user 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /user_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gogitdb/gitdb/v2" 7 | ) 8 | 9 | func TestNewUser(t *testing.T) { 10 | user := gitdb.NewUser("test", "tester@gitdb.io") 11 | want := "test " 12 | got := user.AuthorName() 13 | if want != got { 14 | t.Errorf("want: %s, got: %s", want, got) 15 | } 16 | 17 | got = user.String() 18 | if want != got { 19 | t.Errorf("want: %s, got: %s", want, got) 20 | } 21 | } 22 | 23 | func TestSetUser(t *testing.T) { 24 | teardown := setup(t, nil) 25 | defer teardown(t) 26 | 27 | user := gitdb.NewUser("test", "tester@gitdb.io") 28 | if err := testDb.SetUser(user); err != nil { 29 | t.Errorf("testDb.SetUser failed: %s", err) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /write.go: -------------------------------------------------------------------------------- 1 | package gitdb 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | 10 | "github.com/bouggo/log" 11 | "github.com/gogitdb/gitdb/v2/internal/crypto" 12 | "github.com/gogitdb/gitdb/v2/internal/db" 13 | ) 14 | 15 | func (g *gitdb) Insert(mo Model) error { 16 | m := wrap(mo) 17 | 18 | if err := m.Validate(); err != nil { 19 | return err 20 | } 21 | 22 | if err := m.BeforeInsert(); err != nil { 23 | return fmt.Errorf("Model.BeforeInsert failed: %s", err) 24 | } 25 | 26 | if err := m.GetSchema().Validate(); err != nil { 27 | return err 28 | } 29 | 30 | return g.insert(m) 31 | } 32 | 33 | func (g *gitdb) InsertMany(models []Model) error { 34 | tx := g.StartTransaction("InsertMany") 35 | for _, model := range models { 36 | //create a new variable to pass to function to avoid 37 | //passing pointer which will end up inserting the same 38 | //model multiple times 39 | m := model 40 | f := func() error { return g.Insert(m) } 41 | tx.AddOperation(f) 42 | } 43 | return tx.Commit() 44 | } 45 | 46 | func (g *gitdb) insert(m Model) error { 47 | if !g.isRegistered(m.GetSchema().dataset) { 48 | return ErrInvalidDataset 49 | } 50 | 51 | if _, err := os.Stat(g.fullPath(m)); err != nil { 52 | err := os.MkdirAll(g.fullPath(m), 0755) 53 | if err != nil { 54 | return fmt.Errorf("failed to make dir %s: %w", g.fullPath(m), err) 55 | } 56 | } 57 | 58 | schema := m.GetSchema() 59 | blockFilePath := g.blockFilePath(schema.name(), schema.block) 60 | dataBlock, err := g.loadBlock(blockFilePath) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | log.Test(fmt.Sprintf("Size of block before write - %d", dataBlock.Len())) 66 | 67 | //...append new record to block 68 | newRecordBytes, err := json.Marshal(m) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | mID := ID(m) 74 | 75 | //construct a commit message 76 | commitMsg := "Inserting " + mID 77 | if _, err := dataBlock.Get(mID); err == nil { 78 | commitMsg = "Updating " + mID 79 | } 80 | 81 | newRecordStr := string(newRecordBytes) 82 | //encrypt data if need be 83 | if m.ShouldEncrypt() { 84 | newRecordStr = crypto.Encrypt(g.config.EncryptionKey, newRecordStr) 85 | } 86 | 87 | dataBlock.Add(mID, newRecordStr) 88 | 89 | g.events <- newWriteBeforeEvent("...", mID) 90 | if err := g.writeBlock(blockFilePath, dataBlock); err != nil { 91 | return err 92 | } 93 | 94 | log.Info(fmt.Sprintf("autoCommit: %v", g.autoCommit)) 95 | 96 | g.commit.Add(1) 97 | g.events <- newWriteEvent(commitMsg, blockFilePath, g.autoCommit) 98 | log.Test("sent write event to loop") 99 | g.updateIndexes(dataBlock) 100 | 101 | //block here until write has been committed 102 | g.waitForCommit() 103 | 104 | return nil 105 | } 106 | 107 | func (g *gitdb) waitForCommit() { 108 | if g.autoCommit { 109 | log.Test("waiting for gitdb to commit changes") 110 | g.commit.Wait() 111 | } 112 | } 113 | 114 | func (g *gitdb) writeBlock(blockFile string, block *db.Block) error { 115 | g.writeMu.Lock() 116 | defer g.writeMu.Unlock() 117 | 118 | blockBytes, fmtErr := json.MarshalIndent(block, "", "\t") 119 | if fmtErr != nil { 120 | return fmtErr 121 | } 122 | 123 | //update cache 124 | if g.loadedBlocks != nil { 125 | g.loadedBlocks[blockFile] = block 126 | } 127 | return ioutil.WriteFile(blockFile, blockBytes, 0744) 128 | } 129 | 130 | func (g *gitdb) Delete(id string) error { 131 | return g.doDelete(id, false) 132 | } 133 | 134 | func (g *gitdb) DeleteOrFail(id string) error { 135 | return g.doDelete(id, true) 136 | } 137 | 138 | func (g *gitdb) doDelete(id string, failNotFound bool) error { 139 | 140 | dataset, block, _, err := ParseID(id) 141 | if err != nil { 142 | return err 143 | } 144 | 145 | blockFilePath := g.blockFilePath(dataset, block) 146 | err = g.delByID(id, blockFilePath, failNotFound) 147 | 148 | if err == nil { 149 | log.Test("sending delete event to loop") 150 | g.commit.Add(1) 151 | g.events <- newDeleteEvent(fmt.Sprintf("Deleting %s", id), blockFilePath, g.autoCommit) 152 | g.waitForCommit() 153 | } 154 | 155 | return err 156 | } 157 | 158 | func (g *gitdb) delByID(id string, blockFile string, failIfNotFound bool) error { 159 | 160 | if _, err := os.Stat(blockFile); err != nil { 161 | if failIfNotFound { 162 | return errors.New("Could not delete [" + id + "]: record does not exist") 163 | } 164 | return nil 165 | } 166 | 167 | dataBlock := db.LoadBlock(blockFile, g.config.EncryptionKey) 168 | if err := dataBlock.Delete(id); err != nil { 169 | if failIfNotFound { 170 | return errors.New("Could not delete [" + id + "]: record does not exist") 171 | } 172 | return nil 173 | } 174 | 175 | //write undeleted records back to block file 176 | return g.writeBlock(blockFile, dataBlock) 177 | } 178 | -------------------------------------------------------------------------------- /write_test.go: -------------------------------------------------------------------------------- 1 | package gitdb_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gogitdb/gitdb/v2" 7 | ) 8 | 9 | func TestInsert(t *testing.T) { 10 | teardown := setup(t, nil) 11 | defer teardown(t) 12 | m := getTestMessage() 13 | err := insert(m, false) 14 | if err != nil { 15 | t.Errorf(err.Error()) 16 | } 17 | } 18 | 19 | func TestInsertMany(t *testing.T) { 20 | teardown := setup(t, nil) 21 | defer teardown(t) 22 | defer testDb.Close() 23 | msgs := []gitdb.Model{} 24 | for i := 0; i < 10; i++ { 25 | m := getTestMessage() 26 | msgs = append(msgs, m) 27 | } 28 | 29 | err := testDb.InsertMany(msgs) 30 | if err != nil { 31 | t.Errorf(err.Error()) 32 | } 33 | } 34 | 35 | func BenchmarkInsert(b *testing.B) { 36 | teardown := setup(b, nil) 37 | defer teardown(b) 38 | b.ReportAllocs() 39 | 40 | var m gitdb.Model 41 | for i := 0; i <= b.N; i++ { 42 | m = getTestMessage() 43 | err := insert(m, true) 44 | if err != nil { 45 | b.Errorf(err.Error()) 46 | } 47 | } 48 | } 49 | 50 | func TestDelete(t *testing.T) { 51 | teardown := setup(t, nil) 52 | defer teardown(t) 53 | 54 | m := getTestMessageWithId(0) 55 | if err := insert(m, flagFakeRemote); err != nil { 56 | t.Errorf("Error: %s", err.Error()) 57 | } 58 | 59 | if err := testDb.Delete(gitdb.ID(m)); err != nil { 60 | t.Errorf("Error: %s", err.Error()) 61 | } 62 | } 63 | 64 | func TestDeleteOrFail(t *testing.T) { 65 | teardown := setup(t, nil) 66 | defer teardown(t) 67 | 68 | m := getTestMessageWithId(0) 69 | if err := insert(m, flagFakeRemote); err != nil { 70 | t.Errorf("Error: %s", err.Error()) 71 | } 72 | 73 | if err := testDb.DeleteOrFail("non_existent_id"); err == nil { 74 | t.Errorf("Error: %s", err.Error()) 75 | } 76 | 77 | if err := testDb.DeleteOrFail("Message/b0/1"); err == nil { 78 | t.Errorf("Error: %s", err.Error()) 79 | } 80 | } 81 | 82 | func TestLockUnlock(t *testing.T) { 83 | teardown := setup(t, nil) 84 | defer teardown(t) 85 | 86 | m := getTestMessage() 87 | if err := testDb.Lock(m); err != nil { 88 | t.Errorf("testDb.Lock returned - %s", err) 89 | } 90 | 91 | if err := testDb.Unlock(m); err != nil { 92 | t.Errorf("testDb.Unlock returned - %s", err) 93 | } 94 | } 95 | 96 | func TestGetLockFileNames(t *testing.T) { 97 | m := getTestMessage() 98 | locks := m.GetLockFileNames() 99 | if len(locks) != 1 { 100 | t.Errorf("testMessage return %d lock files", len(locks)) 101 | } 102 | } 103 | --------------------------------------------------------------------------------