├── .github ├── ISSUE_TEMPLATE │ └── bug_report.md └── workflows │ └── ci.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── ToDo.md ├── cache ├── v1 │ └── v1.go ├── v2 │ └── v2.go └── v3 │ └── v3.go ├── db ├── constants.go ├── convert.go ├── sql.go ├── state.go ├── store_lib.go ├── utils.go └── wrappers.go ├── docs └── examples │ ├── kubernetes-labels-update.md │ └── manifests │ └── deployment.yaml ├── errors └── errors.go ├── go.mod ├── go.sum └── tests ├── delete_test.go ├── generic_test.go ├── get_first_test.go ├── get_path_test.go ├── get_test.go ├── multi_doc_test.go ├── storage_test.go └── upsert_test.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Template for bug reporting 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behaviour: 15 | 16 | **Expected behaviour** 17 | 18 | **Additional context** 19 | Add any other context about the problem here. 20 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | 11 | Lint: 12 | name: Lint 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: ^1.13 20 | 21 | - name: Check out code into the Go module directory 22 | uses: actions/checkout@v2 23 | 24 | - name: Get dependencies 25 | run: | 26 | go get -v -t -d ./... 27 | 28 | - name: Get Lint 29 | run: go get -u golang.org/x/lint/golint 30 | 31 | - name: Lint 32 | run: golint -set_exit_status ./... 33 | 34 | - name: Test 35 | run: make lint 36 | 37 | Test: 38 | name: Test 39 | runs-on: ubuntu-latest 40 | steps: 41 | 42 | - name: Set up Go 1.x 43 | uses: actions/setup-go@v2 44 | with: 45 | go-version: ^1.13 46 | 47 | - name: Check out code into the Go module directory 48 | uses: actions/checkout@v2 49 | 50 | - name: Get dependencies 51 | run: | 52 | go get -v -t -d ./... 53 | 54 | - name: Test 55 | run: make test 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | # local dev 18 | main.go 19 | local 20 | ops 21 | test.* 22 | 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Christos Kotsis 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL:=/bin/bash 2 | 3 | .PHONY: clean 4 | clean: 5 | rm -rf tests/.test; 6 | 7 | .PHONY: lint 8 | lint: 9 | @if ! command -v golint; then \ 10 | go get -u golang.org/x/lint/golint; \ 11 | fi 12 | golint -set_exit_status ./... 13 | 14 | 15 | .PHONY: test 16 | test: clean 17 | @set -e; \ 18 | hasErr=0; \ 19 | for i in {0..10}; do \ 20 | go test -v ./tests/; \ 21 | if [[ "$${?}" != 0 ]]; then \ 22 | hasErr=1; break; \ 23 | fi; \ 24 | done; \ 25 | rmdir ./tests/.test 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## DB Yaml 2 | 3 | Simple DB using yaml. A project for managing the content of yaml files. 4 | 5 | Table of Contents 6 | ================= 7 | - [DB Yaml](#db-yaml) 8 | - [Features](#features) 9 | - [Usage](#usage) 10 | * [Write to DB](#write-to-db) 11 | * [Query DB](#query-db) 12 | + [Get First Key](#get-first-key) 13 | + [Search for Keys](#search-for-keys) 14 | * [Query Path](#query-path) 15 | + [Query Path with Arrays](#query-path-with-arrays) 16 | - [Without trailing array](#without-trailing-array) 17 | - [With trailing array](#with-trailing-array) 18 | * [Delete Key By Path](#delete-key-by-path) 19 | * [Document Management](#document-management) 20 | + [Add a new doc](#add-a-new-doc) 21 | + [Switch Doc](#switch-doc) 22 | + [Document names](#document-names) 23 | - [Name documents manually](#name-documents-manually) 24 | - [Name all documents automatically](#name-all-documents-automatically) 25 | - [Switch between docs by name](#switch-between-docs-by-name) 26 | + [Import Docs](#import-docs) 27 | + [Global Commands](#global-commands) 28 | - [Global Upsert](#global-upsert) 29 | - [Global Update](#global-update) 30 | - [Global GetFirst](#global-getfirst) 31 | - [Global FindKeys](#global-findkeys) 32 | - [Global GetPath](#global-getpath) 33 | - [Global Delete](#global-delete) 34 | * [Convert Utils](#convert-utils) 35 | + [Get map of strings from interface](#get-map-of-strings-from-interface) 36 | - [Get map directly from a GetPath object](#get-map-directly-from-a-getpath-object) 37 | - [Get map manually](#get-map-manually) 38 | + [Get array of string from interface](#get-array-of-string-from-interface) 39 | - [Get array directly from a GetPath object](#get-array-directly-from-a-getpath-object) 40 | - [Get array manually](#get-array-manually) 41 | 42 | 43 | ## Features 44 | 45 | The module can do 46 | 47 | - Create/Load yaml files 48 | - Update content 49 | - Get values from keys 50 | - Query for keys 51 | - Delete keys 52 | - Merge content 53 | 54 | ## Usage 55 | 56 | Simple examples for working with yaml files as db 57 | 58 | ### Initiate a new stateful DB 59 | 60 | Create a new local DB 61 | 62 | ```go 63 | package main 64 | 65 | import ( 66 | "github.com/sirupsen/logrus" 67 | "github.com/ulfox/dby/db" 68 | ) 69 | 70 | func main() { 71 | logger := logrus.New() 72 | 73 | state, err := db.NewStorageFactory("local/db.yaml") 74 | if err != nil { 75 | logger.Fatalf(err.Error()) 76 | } 77 | } 78 | ``` 79 | 80 | The code above will create a new yaml file under **local** directory. 81 | 82 | ### Initiate a new stateless DB 83 | 84 | ```go 85 | package main 86 | 87 | import ( 88 | "github.com/sirupsen/logrus" 89 | "github.com/ulfox/dby/db" 90 | ) 91 | 92 | func main() { 93 | logger := logrus.New() 94 | 95 | state, err := db.NewStorageFactory() 96 | if err != nil { 97 | logger.Fatalf(err.Error()) 98 | } 99 | } 100 | ``` 101 | 102 | Initiating a db without arguments will not create/write/read from a file. All operations 103 | will be done in memory and unless the caller saves the data externally, all data will be lose 104 | on termination 105 | 106 | 107 | ### Write to DB 108 | 109 | Insert a map to the local yaml file. 110 | 111 | ```go 112 | err = state.Upsert( 113 | "some.path", 114 | map[string]string{ 115 | "key-1": "value-1", 116 | "key-2": "value-2", 117 | }, 118 | ) 119 | 120 | if err != nil { 121 | logger.Fatalf(err.Error()) 122 | } 123 | ``` 124 | 125 | ### Query DB 126 | 127 | #### Get First Key 128 | 129 | Get the value of the first key in the hierarchy (if any) 130 | 131 | ```go 132 | val, err := state.GetFirst("key-1") 133 | if err != nil { 134 | logger.Fatalf(err.Error()) 135 | } 136 | logger.Info(val) 137 | ``` 138 | 139 | For example if we have the following structure 140 | 141 | ```yaml 142 | key-1: 143 | key-2: 144 | key-3: "1" 145 | key-3: "2" 146 | ``` 147 | 148 | And we query for `key-3`, then we will get back **"2"** and not **"1"** 149 | since `key-3` appears first on a higher layer with a value of **2** 150 | 151 | #### Search for keys 152 | 153 | Get all they keys (if any). This returns the full path for the key, 154 | not the key values. To get the values check the next section **GetPath** 155 | 156 | ```go 157 | keys, err := state.FindKeys("key-1") 158 | if err != nil { 159 | logger.Fatalf(err.Error()) 160 | } 161 | logger.Info(keys) 162 | ``` 163 | 164 | From the previous example, this query would have returned 165 | 166 | ```yaml 167 | ["key-1.key-2.key-3", "key-1.key-3"] 168 | ``` 169 | 170 | ### Query Path 171 | 172 | Get the value from a given path (if any) 173 | 174 | For example if we have in yaml file the following key-path 175 | 176 | ```yaml 177 | key-1: 178 | key-2: 179 | key-3: someValue 180 | ``` 181 | 182 | Then to get someValue, issue 183 | 184 | ```go 185 | keyPath, err := state.GetPath("key-1.key-2.key-3") 186 | if err != nil { 187 | logger.Fatalf(err.Error()) 188 | } 189 | logger.Info(keyPath) 190 | ``` 191 | 192 | #### Query Path with Arrays 193 | 194 | We can also query paths that have arrays. 195 | 196 | ##### Without trailing array 197 | 198 | ```yaml 199 | key-1: 200 | key-2: 201 | - key-3: 202 | key-4: value-1 203 | ``` 204 | 205 | To get the value of `key-4`, issue 206 | 207 | ```go 208 | keyPath, err := state.GetPath("key-1.key-2.[0].key-3.key-4") 209 | if err != nil { 210 | logger.Fatalf(err.Error()) 211 | } 212 | logger.Info(keyPath) 213 | ``` 214 | 215 | ##### With trailing array 216 | 217 | ```yaml 218 | key-1: 219 | key-2: 220 | - value-1 221 | - value-2 222 | - value-3 223 | ``` 224 | 225 | To get the first index of `key-2`, issue 226 | 227 | ```go 228 | keyPath, err := state.GetPath("key-1.key-2.[0]") 229 | if err != nil { 230 | logger.Fatalf(err.Error()) 231 | } 232 | logger.Info(keyPath) 233 | ``` 234 | 235 | ### Delete Key By Path 236 | 237 | To delete a single key for a given path, e.g. key-2 238 | from the example above, issue 239 | 240 | ```go 241 | err = state.Delete("key-1.key-2") 242 | if err != nil { 243 | logger.Fatalf(err.Error()) 244 | } 245 | ``` 246 | 247 | 248 | ### Document Management 249 | 250 | DBy creates by default an array of documents called library. That is in fact an array of interfaces 251 | 252 | When initiating DBy, document 0 (index 0) is creatd by default and any action is done to that document, unless we switch to a new one 253 | 254 | #### Add a new doc 255 | 256 | To add a new doc, issue 257 | 258 | ```go 259 | err = state.AddDoc() 260 | if err != nil { 261 | logger.Fatal(err) 262 | } 263 | 264 | ``` 265 | 266 | **Note: Adding a new doc also switches the pointer to that doc. Any action will write/read from the new doc by default** 267 | 268 | #### Switch Doc 269 | 270 | To switch a different document, we can use **Switch** method that takes as an argument an index 271 | 272 | For example to switch to doc 1 (second doc), issue 273 | 274 | ```go 275 | err = state.Switch(1) 276 | if err != nil { 277 | logger.Fatal(err) 278 | } 279 | ``` 280 | 281 | #### Document names 282 | 283 | When we work with more than 1 document, we may want to set names in order to easily switch between docs 284 | 285 | We have 2 ways to name our documents 286 | 287 | - Add a name to each document manually 288 | - Add a name providing a path that exists in all documents 289 | 290 | ##### Name documents manually 291 | 292 | To name a document manually, we can use the **SetName** method which takes 2 arguments 293 | 294 | - name 295 | - doc index 296 | 297 | For example to name document with index 0, as myDoc 298 | 299 | ```go 300 | err := state.SetName("myDoc", 0) 301 | if err != nil { 302 | logger.Fatal(err) 303 | } 304 | ``` 305 | 306 | ##### Name all documents automatically 307 | 308 | To name all documents automatically we need to ensure that the same path exists in all documents. 309 | 310 | The method for updating all documents is called **SetNames** and takes 2 arguments 311 | 312 | - Prefix: A path in the documents that will be used for the first name 313 | - Suffix: A path in the documents that will be used for the last name 314 | 315 | **Note: Docs that do not have the paths that are queried will not get a name** 316 | 317 | This method best works with **Kubernetes** manifests, where all docs have a common set of fields. 318 | 319 | For example 320 | 321 | ```yaml 322 | apiVersion: someApi-0 323 | kind: someKind-0 324 | metadata: 325 | ... 326 | name: someName-0 327 | ... 328 | --- 329 | apiVersion: someApi-1 330 | kind: someKind-1 331 | metadata: 332 | ... 333 | name: someName-1 334 | ... 335 | --- 336 | ``` 337 | 338 | From above we could give a name for all our documents if we use **kind** + **metadata.name** for the name. 339 | 340 | ```go 341 | err := state.SetNames("kind", "metadata.name") 342 | if err != nil { 343 | logger.Fatal(err) 344 | } 345 | ``` 346 | 347 | ###### List all doc names 348 | 349 | To get the name of all named docs, issue 350 | 351 | ```go 352 | for i, j := range state.ListDocs() { 353 | fmt.Println(i, j) 354 | } 355 | ``` 356 | Example output based on the previous **SetNames** example 357 | 358 | ```bash 359 | 0 service/listener-svc 360 | 1 poddisruptionbudget/listener-svc 361 | 2 horizontalpodautoscaler/caller-svc 362 | 3 deployment/caller-svc 363 | 4 service/caller-svc 364 | 5 poddisruptionbudget/caller-svc 365 | 6 horizontalpodautoscaler/listener-svc 366 | 7 deployment/listener-svc 367 | ``` 368 | 369 | ##### Switch between docs by name 370 | 371 | To switch to a doc by using the doc's name, issue 372 | 373 | ```go 374 | err = state.SwitchDoc("PodDisruptionBudget/caller-svc") 375 | if err != nil { 376 | logger.Fatal(err) 377 | } 378 | ``` 379 | 380 | #### Import Docs 381 | 382 | We can import a set of docs with **ImportDocs** method. For example if we have the following yaml 383 | 384 | ```yaml 385 | apiVersion: someApi-0 386 | kind: someKind-0 387 | metadata: 388 | ... 389 | name: someName-0 390 | ... 391 | --- 392 | apiVersion: someApi-1 393 | kind: someKind-1 394 | metadata: 395 | ... 396 | name: someName-1 397 | ... 398 | --- 399 | ``` 400 | 401 | We can import it by giving the path of the file 402 | 403 | ```go 404 | err = state.ImportDocs("file-name.yaml") 405 | if err != nil { 406 | logger.Fatal(err) 407 | } 408 | ``` 409 | 410 | #### Global Commands 411 | 412 | Wrappers for working with all documents 413 | 414 | ##### Global Upsert 415 | 416 | We can use upsert to update or create keys on all documents 417 | 418 | ```go 419 | err = state.UpsertGlobal( 420 | "some.path", 421 | "v0.3.0", 422 | ) 423 | if err != nil { 424 | logger.Fatal(err) 425 | } 426 | 427 | ``` 428 | 429 | ##### Global Update 430 | 431 | Global update works as **GlobalUpsert** but it skips documents that 432 | miss a path rather than creating the path on those docs. 433 | 434 | ##### Global GetFirst 435 | 436 | To get the value of the first key in the hierarchy for each document, issue 437 | 438 | ```go 439 | valueOfDocs, err := state.GetFirstGlobal("keyName") 440 | if err != nil { 441 | logger.Fatalf(err.Error()) 442 | } 443 | logger.Info(valueOfDocs) 444 | ``` 445 | 446 | This returns a `map[int]interface{}` object. The key is the index of each document and it's value 447 | is the value of the first key in the hierarchy in that document 448 | 449 | ##### Global FindKeys 450 | 451 | To get all the paths for a given from all documents, issue 452 | 453 | ```go 454 | mapOfPaths, err := state.FindKeysGlobal("keyName") 455 | if err != nil { 456 | logger.Fatalf(err.Error()) 457 | } 458 | logger.Info(mapOfPaths) 459 | ``` 460 | 461 | This returns a `map[int][]string` object. The key is the index of each document and it's value 462 | is a list of paths that have the queried key 463 | 464 | ##### Global GetPath 465 | 466 | To get a path that exists in all documents, issue 467 | 468 | ```go 469 | valueOfDocs, err := state.GetPathGlobal("key-1.key-2.key-3") 470 | if err != nil { 471 | logger.Fatalf(err.Error()) 472 | } 473 | logger.Info(valueOfDocs) 474 | ``` 475 | 476 | This returns a `map[int]interface{}` object. The key is the index of each document and it's value 477 | is the value of the specific key in that document 478 | 479 | ##### Global Delete 480 | 481 | To delete a path from all documents, issue 482 | 483 | ```go 484 | err := state.DeleteGlobal("key-1.key-2.key-3") 485 | if err != nil { 486 | logger.Fatalf(err.Error()) 487 | } 488 | ``` 489 | 490 | The above will delete all the paths that match the queried path from each doc 491 | 492 | ### Convert Utils 493 | 494 | Convert simply automate the need to 495 | explicitly do assertion each time we need to access 496 | an interface object. 497 | 498 | Let us assume we have the following YAML structure 499 | 500 | ```yaml 501 | 502 | to: 503 | array-1: 504 | key-1: 505 | - key-2: 2 506 | - key-3: 3 507 | - key-4: 4 508 | array-2: 509 | - 1 510 | - 2 511 | - 3 512 | - 4 513 | - 5 514 | array-3: 515 | - key-1: 1 516 | - key-2: 2 517 | 518 | ``` 519 | 520 | #### Get map of strings from interface 521 | 522 | We can do this in two ways, get object by giving a path and assert the interface to `map[string]string`, or work manually our way to the object 523 | 524 | ##### Get map directly from a GetPath object 525 | 526 | To get map **key-2: 2**, first get object via GetPath 527 | 528 | ```go 529 | 530 | obj, err := state.GetPath("to.array-1.key-1.[0]") 531 | if err != nil { 532 | logger.Fatalf(err.Error()) 533 | } 534 | logger.Info(val) 535 | 536 | ``` 537 | 538 | Next, assert **obj** as `map[string]string` 539 | 540 | ```go 541 | 542 | assertData := db.NewConvertFactory() 543 | 544 | assertData.Input(val) 545 | if assertData.GetError() != nil { 546 | logger.Fatal(assertData.GetError()) 547 | } 548 | vMap, err := assertData.GetMap() 549 | if err != nil { 550 | logger.Fatal(err) 551 | } 552 | logger.Info(vMap["key-2"]) 553 | 554 | ``` 555 | 556 | ##### Get map manually 557 | 558 | We can get the map manually by using only **Convert** operations 559 | 560 | ```go 561 | 562 | assertData := db.NewConvertFactory() 563 | 564 | assertData.Input(state.Data). 565 | Key("to"). 566 | Key("array-1"). 567 | Key("key-1").Index(0) 568 | if assertData.GetError() != nil { 569 | logger.Fatal(assertData.GetError()) 570 | } 571 | vMap, err := assertData.GetMap() 572 | if err != nil { 573 | logger.Fatal(err) 574 | } 575 | logger.Info(vMap["key-2"]) 576 | 577 | ``` 578 | 579 | #### Get array of string from interface 580 | 581 | Again here we can do it two ways as with the map example 582 | 583 | ##### Get array directly from a GetPath object 584 | 585 | To get **array-2** as **[]string**, first get object via GetPath 586 | 587 | ```go 588 | 589 | obj, err = state.GetPath("to.array-2") 590 | if err != nil { 591 | logger.Fatalf(err.Error()) 592 | } 593 | logger.Info(obj) 594 | 595 | ``` 596 | 597 | Next, assert **obj** as `[]string` 598 | 599 | ```go 600 | 601 | assertData := db.NewConvertFactory() 602 | 603 | assertData.Input(obj) 604 | if assertData.GetError() != nil { 605 | logger.Fatal(assertData.GetError()) 606 | } 607 | vArray, err := assertData.GetArray() 608 | if err != nil { 609 | logger.Fatal(err) 610 | } 611 | logger.Info(vArray) 612 | 613 | ``` 614 | 615 | ##### Get array manually 616 | 617 | We can get the array manually by using only **Convert** operations 618 | 619 | ```go 620 | 621 | assertData.Input(state.Data). 622 | Key("to"). 623 | Key("array-2") 624 | if assertData.GetError() != nil { 625 | logger.Fatal(assertData.GetError()) 626 | } 627 | vArray, err := assertData.GetArray() 628 | if err != nil { 629 | logger.Fatal(err) 630 | } 631 | logger.Info(vArray) 632 | 633 | ``` 634 | -------------------------------------------------------------------------------- /ToDo.md: -------------------------------------------------------------------------------- 1 | ## ToDo 2 | 3 | ### Features 4 | 5 | - Add multiple keys 6 | - Remote backends: Allow to work with yaml files that are on a remote location 7 | - S3 8 | - Google Storage 9 | - Export to STDOUT method 10 | 11 | ### Improvements 12 | 13 | ### Remove nested loops/ifs for better readability 14 | 15 | Description: Some blocks have nested loops/ifs (e.g. for{if{if{}}}) which are not good 16 | either for maintenace or readability. 17 | 18 | #### Introduce double linked lists for path seeking 19 | 20 | Description: When we want to find valid paths for a given key, or we want to get 21 | the content of a path, we use getPath() method which is a loop that calls itself 22 | until it finds a path that leads to the desired key. 23 | 24 | Linked lists would be a better solution for storing our path since we can easily 25 | know our full path and also have easier access to the content for any given branch. 26 | 27 | #### Optimize more the cache 28 | 29 | Description: Methods like getPath(), get(), deletePath() can be further optimized by 30 | using cache to avoid multiple copies and loops. 31 | 32 | #### Reduce write operations 33 | 34 | Description: Currently we do a write on Upsert() and Delete() methods. We could instead 35 | work in ram and write periodically in the persistant storage. 36 | 37 | #### MergeDBs should not shadow target 38 | 39 | Description: Currently DBy supports yaml merges. That is we can merge a source file with 40 | the content of the DBy yaml file. The issue we currently have is that the merge replaces 41 | same paths with the path from the source file rather than merging both paths. 42 | 43 | ##### Example 44 | 45 | DBy yaml content 46 | 47 | ``` 48 | key-1: 49 | key-2: 50 | key-3: value-3 51 | 52 | some: 53 | other: 54 | path: test 55 | ``` 56 | 57 | Some local yaml file 58 | 59 | ``` 60 | key-1: 61 | key-2: 62 | key-4: value-4 63 | 64 | ``` 65 | 66 | Current Merged DBy 67 | 68 | ``` 69 | key-1: 70 | key-2: 71 | key-4: value-4 72 | 73 | some: 74 | other: 75 | path: test 76 | ``` 77 | 78 | Expected Merge 79 | 80 | ``` 81 | key-1: 82 | key-2: 83 | key-4: value-4 84 | key-3: value-3 85 | 86 | some: 87 | other: 88 | path: test 89 | ``` 90 | 91 | ### Tests 92 | 93 | - Cover cache state during operations 94 | -------------------------------------------------------------------------------- /cache/v1/v1.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | // Cache for easily sharing state between map operations and methods 4 | // V{1,2} are common interface{} placeholders, C{1,2,3} are counters used 5 | // for condition, while []Keys is used by path discovery methods 6 | // to keep track and derive the right path. 7 | type Cache struct { 8 | v1, v2 interface{} 9 | v3 []interface{} 10 | b []byte 11 | e error 12 | c1, c2, c3 int 13 | keys []string 14 | } 15 | 16 | // NewCacheFactory for creating a new Cache 17 | func NewCacheFactory() Cache { 18 | cache := Cache{ 19 | v3: make([]interface{}, 0), 20 | b: make([]byte, 0), 21 | keys: make([]string, 0), 22 | } 23 | return cache 24 | } 25 | 26 | // E can be used to set or get e. If no argument is passed 27 | // the method returns e value. If an argument is passed, the method 28 | // sets e's value to that of the arguments 29 | func (c *Cache) E(e ...error) error { 30 | if len(e) > 0 { 31 | c.e = e[0] 32 | } 33 | return c.e 34 | } 35 | 36 | // Clear for clearing the cache and setting all 37 | // content to nil 38 | func (c *Cache) Clear() { 39 | c.v1, c.v2 = nil, nil 40 | c.v3 = make([]interface{}, 0) 41 | c.c1, c.c2, c.c3 = 0, 0, 0 42 | c.b = make([]byte, 0) 43 | c.e = nil 44 | c.keys = make([]string, 0) 45 | } 46 | 47 | // DropLastKey removes the last key added from the list 48 | func (c *Cache) DropLastKey() { 49 | if len(c.keys) > 0 { 50 | c.keys = c.keys[:len(c.keys)-1] 51 | } 52 | } 53 | 54 | // DropKeys removes all keys from the list 55 | func (c *Cache) DropKeys() { 56 | c.keys = make([]string, 0) 57 | } 58 | 59 | // AddKey for appending a new Key to cache 60 | func (c *Cache) AddKey(k string) { 61 | c.keys = append(c.keys, k) 62 | } 63 | 64 | // GetKeys returns Cache.Keys 65 | func (c *Cache) GetKeys() []string { 66 | return c.keys 67 | } 68 | 69 | // DropV3 removes all keys from the list 70 | func (c *Cache) DropV3() { 71 | c.v3 = make([]interface{}, 0) 72 | } 73 | 74 | // V3 returns Cache.V3 75 | func (c *Cache) V3(v ...interface{}) []interface{} { 76 | if len(v) > 0 { 77 | c.v3 = append(c.v3, v[0]) 78 | } 79 | return c.v3 80 | } 81 | 82 | // C1 can be used to set or get c1. If no argument is passed 83 | // the method returns c1 value. If an argument is passed, the method 84 | // sets c1's value to that of the arguments 85 | func (c *Cache) C1(i ...int) int { 86 | if len(i) > 0 { 87 | c.c1 = i[0] 88 | } 89 | return c.c1 90 | } 91 | 92 | // C2 can be used to set or get c2. If no argument is passed 93 | // the method returns c2 value. If an argument is passed, the method 94 | // sets c2's value to that of the arguments 95 | func (c *Cache) C2(i ...int) int { 96 | if len(i) > 0 { 97 | c.c2 = i[0] 98 | } 99 | return c.c2 100 | } 101 | 102 | // C3 can be used to set or get c3. If no argument is passed 103 | // the method returns c3 value. If an argument is passed, the method 104 | // sets c3's value to that of the arguments 105 | func (c *Cache) C3(i ...int) int { 106 | if len(i) > 0 { 107 | c.c3 = i[0] 108 | } 109 | return c.c3 110 | } 111 | 112 | // V1 can be used to set or get v1. If no argument is passed 113 | // the method returns v1 value. If an argument is passed, the method 114 | // sets v1's value to that of the arguments 115 | func (c *Cache) V1(v ...interface{}) interface{} { 116 | if len(v) > 0 { 117 | c.v1 = v[0] 118 | } 119 | return c.v1 120 | } 121 | 122 | // V2 can be used to set or get v2. If no argument is passed 123 | // the method returns v2 value. If an argument is passed, the method 124 | // sets v2's value to that of the arguments 125 | func (c *Cache) V2(v ...interface{}) interface{} { 126 | if len(v) > 0 { 127 | c.v2 = v[0] 128 | } 129 | 130 | return c.v2 131 | } 132 | 133 | // B can be used to set or get b. If no argument is passed 134 | // the method returns b value. If an argument is passed, the method 135 | // sets b's value to that of the arguments 136 | func (c *Cache) B(b ...[]byte) []byte { 137 | if len(b) > 0 { 138 | c.b = b[0] 139 | } 140 | 141 | return c.b 142 | } 143 | 144 | // BE expects 2 inputs, a byte array and an error. 145 | // If byte array is not nil, b will be set to the 146 | // value of the new byte array. Error is returned 147 | // with the new/old byte array at the end always 148 | func (c *Cache) BE(b []byte, e error) error { 149 | if b != nil { 150 | c.b = b 151 | } 152 | 153 | return e 154 | } 155 | 156 | // V1E expects 2 inputs, a byte array and an error. 157 | // If byte array is not nil, b will be set to the 158 | // value of the new byte array. Error is returned 159 | // with the new/old byte array at the end always 160 | func (c *Cache) V1E(v interface{}, e error) error { 161 | if v != nil { 162 | c.v1 = v 163 | } 164 | 165 | return e 166 | } 167 | -------------------------------------------------------------------------------- /cache/v2/v2.go: -------------------------------------------------------------------------------- 1 | package v2 2 | 3 | // Query hosts results from SQL methods 4 | type Query struct { 5 | keys []string 6 | } 7 | 8 | // NewQueryFactory for creating a new Query 9 | func NewQueryFactory() Query { 10 | Query := Query{ 11 | keys: make([]string, 0), 12 | } 13 | return Query 14 | } 15 | 16 | // Clear for clearing the Query 17 | func (c *Query) Clear() { 18 | c.keys = make([]string, 0) 19 | } 20 | 21 | // AddKey for appending a new Key to Query 22 | func (c *Query) AddKey(k string) { 23 | c.keys = append(c.keys, k) 24 | } 25 | 26 | // GetKeys returns Query.KeysFound 27 | func (c *Query) GetKeys() []string { 28 | return c.keys 29 | } 30 | -------------------------------------------------------------------------------- /cache/v3/v3.go: -------------------------------------------------------------------------------- 1 | package v3 2 | -------------------------------------------------------------------------------- /db/constants.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import "fmt" 4 | 5 | type objectType int 6 | 7 | const ( 8 | unknownObj objectType = iota 9 | mapObj 10 | arrayObj 11 | arrayMapObj 12 | mapStringString 13 | mapStringInterface 14 | mapStringArrayString 15 | arrayMapStringArrayString 16 | arrayMapStringString 17 | arrayMapStringInterface 18 | arrayMapStringArrayInterface 19 | ) 20 | 21 | // Informational Error constants. Used during a return err 22 | const ( 23 | notAMap = "target object is not a map" 24 | notArrayObj = "received a non array object but expected []interface{}" 25 | keyDoesNotExist = "the given key [%s] does not exist" 26 | fileNotExist = "the given file [%s] does not exist" 27 | dictNotFile = "can not create file [%s], a directory exists with that name" 28 | notAnIndex = "object (%s) is not an index. Index example: some.path.[someInteger].someKey" 29 | arrayOutOfRange = "index value (%s) is bigger than the length (%s) of the array to be indexed" 30 | invalidKeyPath = "the key||path [%s] that was given is not valid" 31 | emptyKey = "path [%s] contains an empty key" 32 | libOutOfIndex = "lib out of index" 33 | docNotExists = "doc [%s] does not exist in lib" 34 | fieldNotString = "[%s] with value [%s] is not a string" 35 | notAType = "value is not a %s" 36 | ) 37 | 38 | // Warnings 39 | const ( 40 | deprecatedFeature = "Warn: Deprecated is [%s]. Will be replaced by [%s] in the future" 41 | ) 42 | 43 | func issueWarning(s string, o ...interface{}) { 44 | warn := fmt.Sprintf(s, o...) 45 | fmt.Println(warn) 46 | } 47 | 48 | func getObjectType(o interface{}) objectType { 49 | _, isMap := o.(map[interface{}]interface{}) 50 | if isMap { 51 | return 1 52 | } 53 | 54 | _, isArray := o.([]interface{}) 55 | if isArray { 56 | return 2 57 | } 58 | 59 | _, isMapStringInterface := o.(map[string]interface{}) 60 | if isMapStringInterface { 61 | return 5 62 | } 63 | 64 | return unknownObj 65 | } 66 | -------------------------------------------------------------------------------- /db/convert.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | v1 "github.com/ulfox/dby/cache/v1" 5 | "gopkg.in/yaml.v2" 6 | ) 7 | 8 | // AssertData is used to for converting interface objects to 9 | // map of interfaces or array of interfaces 10 | type AssertData struct { 11 | d0 map[string]string 12 | s0 *string 13 | s1 []string 14 | i0 *int 15 | i1 []int 16 | cache v1.Cache 17 | } 18 | 19 | // NewConvertFactory for initializing AssertData 20 | func NewConvertFactory() *AssertData { 21 | ad := &AssertData{ 22 | d0: make(map[string]string), 23 | s1: make([]string, 0), 24 | cache: v1.NewCacheFactory(), 25 | } 26 | return ad 27 | } 28 | 29 | // Clear for resetting AssertData 30 | func (a *AssertData) Clear() { 31 | a.cache.Clear() 32 | a.d0 = make(map[string]string) 33 | a.s1 = make([]string, 0) 34 | a.i1 = make([]int, 0) 35 | a.i0 = nil 36 | a.s0 = nil 37 | } 38 | 39 | // GetError returns the any error set to AssertData 40 | func (a *AssertData) GetError() error { 41 | return a.cache.E() 42 | } 43 | 44 | func (a *AssertData) setErr(e ...error) *AssertData { 45 | if len(e) > 0 { 46 | a.cache.E(e[0]) 47 | } 48 | return a 49 | } 50 | 51 | // Input sets a data source that can be used for assertion 52 | func (a *AssertData) Input(o interface{}) *AssertData { 53 | a.Clear() 54 | a.cache.V1(o) 55 | return a 56 | } 57 | 58 | // GetString asserts the input as string 59 | func (a *AssertData) GetString() (string, error) { 60 | if a.GetError() != nil { 61 | return "", a.GetError() 62 | } 63 | 64 | s, isString := a.cache.V1().(string) 65 | if !isString { 66 | a.setErr(wrapErr(notAType, "string")) 67 | return "", a.GetError() 68 | } 69 | 70 | return s, nil 71 | } 72 | 73 | // GetInt asserts the input as int 74 | func (a *AssertData) GetInt() (int, error) { 75 | if a.GetError() != nil { 76 | return 0, a.GetError() 77 | } 78 | 79 | i, isInt := a.cache.V1().(int) 80 | if !isInt { 81 | a.setErr(wrapErr(notAType, "int")) 82 | return 0, a.GetError() 83 | } 84 | 85 | return i, nil 86 | } 87 | 88 | // GetMap for converting a map[interface{}]interface{} into a map[string]string 89 | func (a *AssertData) GetMap() (map[string]string, error) { 90 | if a.GetError() != nil { 91 | return nil, a.GetError() 92 | } 93 | 94 | a.cache.E(a.cache.BE(yaml.Marshal(a.cache.V1()))) 95 | if a.GetError() != nil { 96 | return nil, a.GetError() 97 | } 98 | 99 | a.cache.E(yaml.Unmarshal(a.cache.B(), &a.d0)) 100 | if a.GetError() != nil { 101 | return nil, a.GetError() 102 | } 103 | return a.d0, nil 104 | } 105 | 106 | // GetArray for converting a []interface{} to []string 107 | func (a *AssertData) GetArray() ([]string, error) { 108 | if a.GetError() != nil { 109 | return nil, a.GetError() 110 | } 111 | 112 | _, isArray := a.cache.V1().([]interface{}) 113 | if !isArray { 114 | a.setErr(wrapErr(notArrayObj)) 115 | return nil, a.GetError() 116 | } 117 | 118 | a.cache.E(a.cache.BE(yaml.Marshal(a.cache.V1()))) 119 | if a.GetError() != nil { 120 | return nil, a.GetError() 121 | } 122 | 123 | a.cache.E(yaml.Unmarshal(a.cache.B(), &a.s1)) 124 | if a.GetError() != nil { 125 | return nil, a.GetError() 126 | } 127 | 128 | return a.s1, nil 129 | } 130 | 131 | // Key copies initial interface object and returns a map of interfaces{} 132 | // Used to easily pipe interfaces 133 | func (a *AssertData) Key(k string) *AssertData { 134 | if a.GetError() != nil { 135 | return a 136 | } 137 | 138 | _, isMap := a.cache.V1().(map[interface{}]interface{}) 139 | if !isMap { 140 | return a.setErr(wrapErr(notAMap)) 141 | } 142 | 143 | a.cache.V1(a.cache.V1().(map[interface{}]interface{})[k]) 144 | 145 | return a 146 | } 147 | 148 | // Index getting an interface{} from a []interface{} 149 | func (a *AssertData) Index(i int) *AssertData { 150 | if a.GetError() != nil { 151 | return a 152 | } 153 | 154 | _, isArray := a.cache.V1().([]interface{}) 155 | if !isArray { 156 | return a.setErr(wrapErr(notArrayObj)) 157 | } 158 | a.cache.V1(a.cache.V1().([]interface{})[i]) 159 | 160 | return a 161 | } 162 | -------------------------------------------------------------------------------- /db/sql.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "io/ioutil" 5 | "strconv" 6 | "strings" 7 | 8 | v1 "github.com/ulfox/dby/cache/v1" 9 | v2 "github.com/ulfox/dby/cache/v2" 10 | "gopkg.in/yaml.v2" 11 | ) 12 | 13 | // SQL is the core struct for working with maps. 14 | type SQL struct { 15 | v1.Cache 16 | v2.Query 17 | } 18 | 19 | // NewSQLFactory creates a new empty SQL 20 | func NewSQLFactory() *SQL { 21 | sql := &SQL{ 22 | Query: v2.NewQueryFactory(), 23 | Cache: v1.NewCacheFactory(), 24 | } 25 | return sql 26 | } 27 | 28 | // Clear deletes all objects from Query and Cache structures 29 | func (s *SQL) Clear() *SQL { 30 | s.Query.Clear() 31 | s.Cache.Clear() 32 | 33 | return s 34 | } 35 | 36 | func (s *SQL) getObj(k string, o *interface{}) (*interface{}, bool) { 37 | _, isMap := (*o).(map[interface{}]interface{}) 38 | if !isMap { 39 | return s.getArrayObject(k, o) 40 | } 41 | 42 | for thisKey, thisObj := range (*o).(map[interface{}]interface{}) { 43 | s.Cache.AddKey(thisKey.(string)) 44 | if thisKey == k { 45 | return &thisObj, true 46 | } 47 | 48 | if objFinal, found := s.getObj(k, &thisObj); found { 49 | return objFinal, found 50 | } 51 | s.Cache.DropLastKey() 52 | } 53 | 54 | return nil, false 55 | } 56 | 57 | func (s *SQL) getArrayObject(k string, o *interface{}) (*interface{}, bool) { 58 | if o == nil { 59 | return nil, false 60 | } 61 | _, isArray := (*o).([]interface{}) 62 | if !isArray { 63 | return nil, false 64 | } 65 | 66 | for i, thisArrayObj := range (*o).([]interface{}) { 67 | s.Cache.AddKey("[" + strconv.Itoa(i) + "]") 68 | arrayObjFinal, found := s.getObj(k, &thisArrayObj) 69 | if found { 70 | return arrayObjFinal, found 71 | } 72 | 73 | s.Cache.DropLastKey() 74 | } 75 | 76 | return nil, false 77 | } 78 | 79 | func (s *SQL) getIndex(k string) (int, error) { 80 | if !strings.HasPrefix(k, "[") || !strings.HasSuffix(k, "]") { 81 | return 0, wrapErr(notAnIndex, k) 82 | } 83 | 84 | intVar, err := strconv.Atoi(k[1 : len(k)-1]) 85 | if err != nil { 86 | return 0, wrapErr(err) 87 | } 88 | return intVar, nil 89 | } 90 | 91 | func (s *SQL) getFromIndex(k []string, o *interface{}) (*interface{}, error) { 92 | _, isArray := (*o).([]interface{}) 93 | if !isArray { 94 | return nil, wrapErr(notArrayObj) 95 | } 96 | v := (*o).([]interface{}) 97 | 98 | i, err := s.getIndex(k[0]) 99 | if err != nil { 100 | return nil, wrapErr(err) 101 | } 102 | 103 | if i > len((*o).([]interface{}))-1 { 104 | return nil, wrapErr( 105 | arrayOutOfRange, 106 | strconv.Itoa(i), 107 | strconv.Itoa(len((*o).([]interface{}))-1), 108 | ) 109 | } 110 | 111 | if len(k) > 1 { 112 | return s.getPath(k[1:], &v[i]) 113 | } 114 | 115 | return &v[i], nil 116 | } 117 | 118 | func (s *SQL) getPath(k []string, o *interface{}) (*interface{}, error) { 119 | if err := checkKeyPath(k); err != nil { 120 | return nil, wrapErr(err) 121 | } 122 | 123 | _, ok := (*o).(map[interface{}]interface{}) 124 | if !ok { 125 | return s.getFromIndex(k, o) 126 | } 127 | 128 | if len(k) == 0 { 129 | return nil, wrapErr(keyDoesNotExist, k[0]) 130 | } 131 | 132 | for thisKey, thisObj := range (*o).(map[interface{}]interface{}) { 133 | if thisKey != k[0] { 134 | continue 135 | } 136 | s.Cache.AddKey(k[0]) 137 | if len(k) == 1 { 138 | return &thisObj, nil 139 | } 140 | 141 | objFinal, err := s.getPath(k[1:], &thisObj) 142 | if err != nil { 143 | return nil, wrapErr(err) 144 | } 145 | return objFinal, nil 146 | } 147 | 148 | return nil, wrapErr(keyDoesNotExist, k[0]) 149 | } 150 | 151 | func (s *SQL) deleteArrayItem(k string, o *interface{}) error { 152 | if o == nil { 153 | return wrapErr(notArrayObj) 154 | } 155 | 156 | i, err := s.getIndex(k) 157 | if err != nil { 158 | return wrapErr(err) 159 | } 160 | 161 | (*o).([]interface{})[i] = (*o).([]interface{})[len((*o).([]interface{}))-1] 162 | (*o).([]interface{})[len((*o).([]interface{}))-1] = "" 163 | *o = (*o).([]interface{})[:len((*o).([]interface{}))-1] 164 | 165 | return nil 166 | } 167 | 168 | func (s *SQL) deleteItem(k string, o *interface{}) error { 169 | _, ok := (*o).(map[interface{}]interface{}) 170 | if !ok { 171 | return s.deleteArrayItem(k, o) 172 | } 173 | 174 | for kn := range (*o).(map[interface{}]interface{}) { 175 | if kn.(string) == k { 176 | delete((*o).(map[interface{}]interface{}), kn) 177 | return nil 178 | } 179 | } 180 | return wrapErr(keyDoesNotExist, k) 181 | } 182 | 183 | func (s *SQL) delPath(k string, o *interface{}) error { 184 | keys := strings.Split(k, ".") 185 | if err := checkKeyPath(keys); err != nil { 186 | return wrapErr(err) 187 | } 188 | 189 | if len(keys) == 0 { 190 | return wrapErr(invalidKeyPath, k) 191 | } 192 | 193 | if len(keys) == 1 { 194 | if err := s.deleteItem(keys[0], o); err != nil { 195 | return wrapErr(keyDoesNotExist, k) 196 | } 197 | return nil 198 | } 199 | 200 | s.Cache.DropKeys() 201 | obj, err := s.getPath(keys[:len(keys)-1], o) 202 | if err != nil { 203 | return wrapErr(err) 204 | } 205 | 206 | s.Cache.DropKeys() 207 | if err := s.deleteItem(keys[len(keys)-1], obj); err != nil { 208 | return wrapErr(keyDoesNotExist, k) 209 | } 210 | 211 | return nil 212 | } 213 | 214 | func (s *SQL) findKeys(k string, o *interface{}) ([]string, error) { 215 | var err error 216 | var key string 217 | 218 | s.Clear() 219 | err = s.V1E(copyMap(*o)) 220 | if err != nil { 221 | return nil, wrapErr(err) 222 | } 223 | 224 | for { 225 | obj := s.V1() 226 | if _, found := s.getObj(k, &obj); !found { 227 | break 228 | } 229 | 230 | key = strings.Join(s.Cache.GetKeys(), ".") 231 | s.Query.AddKey(key) 232 | 233 | if err := s.delPath(key, &obj); err != nil { 234 | return s.Query.GetKeys(), wrapErr(err) 235 | } 236 | s.Cache.DropKeys() 237 | } 238 | 239 | return s.Query.GetKeys(), nil 240 | } 241 | 242 | func (s *SQL) getFirst(k string, o *interface{}) (*interface{}, error) { 243 | s.Clear() 244 | 245 | keys, err := s.findKeys(k, o) 246 | if err != nil { 247 | return nil, wrapErr(err) 248 | } 249 | 250 | if len(keys) == 0 { 251 | return nil, wrapErr(keyDoesNotExist, k) 252 | } 253 | 254 | keySlice := strings.Split(keys[0], ".") 255 | if err := checkKeyPath(keySlice); err != nil { 256 | return nil, wrapErr(err) 257 | } 258 | 259 | s.Cache.C1(len(keySlice)) 260 | if len(keys) == 1 { 261 | path, err := s.getPath(keySlice, o) 262 | return path, wrapErr(err) 263 | } 264 | 265 | for i, key := range keys[1:] { 266 | if len(strings.Split(key, ".")) < s.Cache.C1() { 267 | s.Cache.C1(len(strings.Split(key, "."))) 268 | s.Cache.C2(i + 1) 269 | } 270 | } 271 | 272 | path, err := s.getPath(strings.Split(keys[s.Cache.C2()], "."), o) 273 | return path, wrapErr(err) 274 | } 275 | 276 | func (s *SQL) toInterfaceMap(v interface{}) (interface{}, error) { 277 | var dataNew interface{} 278 | 279 | if v == nil { 280 | return make(map[interface{}]interface{}), nil 281 | } 282 | 283 | dataBytes, err := yaml.Marshal(&v) 284 | if err != nil { 285 | return nil, wrapErr(err) 286 | } 287 | err = yaml.Unmarshal(dataBytes, &dataNew) 288 | if err != nil { 289 | return nil, wrapErr(err) 290 | } 291 | 292 | return dataNew, nil 293 | } 294 | 295 | func (s *SQL) upsertRecursive(k []string, o, v interface{}) error { 296 | s.Clear() 297 | if err := checkKeyPath(k); err != nil { 298 | return wrapErr(err) 299 | } 300 | 301 | obj, err := interfaceToMap(o) 302 | if err != nil { 303 | return wrapErr(err) 304 | } 305 | 306 | for thisKey, thisObj := range obj { 307 | if thisKey != k[0] { 308 | continue 309 | } 310 | 311 | if len(k) > 1 { 312 | return wrapErr(s.upsertRecursive(k[1:], thisObj, v)) 313 | } 314 | 315 | switch getObjectType(thisObj) { 316 | case mapObj: 317 | deleteMap(thisObj) 318 | case arrayObj: 319 | thisObj = nil 320 | } 321 | 322 | break 323 | } 324 | 325 | obj[k[0]] = emptyMap() 326 | 327 | if len(k) > 1 { 328 | return wrapErr(s.upsertRecursive(k[1:], obj[k[0]], v)) 329 | } 330 | 331 | obj[k[0]] = v 332 | 333 | return nil 334 | } 335 | 336 | func (s *SQL) mergeDBs(path string, o interface{}) error { 337 | var dataNew interface{} 338 | 339 | ok, err := fileExists(path) 340 | if err != nil { 341 | return wrapErr(err) 342 | } 343 | 344 | if !ok { 345 | return wrapErr(fileNotExist, path) 346 | } 347 | 348 | f, err := ioutil.ReadFile(path) 349 | if err != nil { 350 | return wrapErr(err) 351 | } 352 | 353 | yaml.Unmarshal(f, &dataNew) 354 | 355 | obj, err := interfaceToMap(dataNew) 356 | if err != nil { 357 | return wrapErr(err) 358 | } 359 | 360 | for kn, vn := range obj { 361 | err = s.upsertRecursive(strings.Split(kn.(string), "."), o, vn) 362 | if err != nil { 363 | return wrapErr(err) 364 | } 365 | } 366 | return nil 367 | } 368 | -------------------------------------------------------------------------------- /db/state.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | const ( 9 | ire = "index error" 10 | ) 11 | 12 | // state struct used by dby storage 13 | type state struct { 14 | data []interface{} 15 | buffer []*interface{} 16 | lib map[string]int 17 | ad int 18 | } 19 | 20 | // newStateFactory for creating a new v3 State 21 | func newStateFactory() *state { 22 | s := state{ 23 | data: make([]interface{}, 0), 24 | buffer: make([]*interface{}, 0), 25 | lib: make(map[string]int), 26 | } 27 | return &s 28 | } 29 | 30 | // Clear for clearing the v3 state 31 | func (c *state) Clear() { 32 | c.data, c.buffer, c.lib = nil, nil, nil 33 | 34 | c.data = make([]interface{}, 0) 35 | c.buffer = make([]*interface{}, 0) 36 | c.lib = make(map[string]int) 37 | } 38 | 39 | // SetAD for setting new Active Document index 40 | func (c *state) SetAD(i int) error { 41 | if err := c.IndexInRange(i); err != nil { 42 | return wrapErr(err) 43 | } 44 | c.ad = i 45 | return nil 46 | } 47 | 48 | // GetAD returns the current active document index 49 | func (c *state) GetAD() int { 50 | return c.ad 51 | } 52 | 53 | // PushData for appending data to the data array 54 | func (c *state) PushData(d interface{}) { 55 | c.data = append(c.data, d) 56 | } 57 | 58 | // PushBuffer for appending data to the buffer array 59 | func (c *state) PushBuffer(d interface{}) { 60 | c.buffer = append(c.buffer, &d) 61 | } 62 | 63 | // GetAllData returns the data array 64 | func (c *state) GetAllData() []interface{} { 65 | return c.data 66 | } 67 | 68 | // GetAllBuffer returns the buffer array 69 | func (c *state) GetAllBuffer() []*interface{} { 70 | return c.buffer 71 | } 72 | 73 | // GetData returns the data in the c.ad index from the data array 74 | func (c *state) GetData() interface{} { 75 | data, _ := c.GetDataFromIndex(c.GetAD()) 76 | return data 77 | } 78 | 79 | // GetDataFromIndex returns the i'th element from the data array 80 | func (c *state) GetDataFromIndex(i int) (interface{}, error) { 81 | if err := c.IndexInRange(i); err != nil { 82 | return nil, wrapErr(err) 83 | } 84 | return c.data[i], nil 85 | } 86 | 87 | // SetData sets to input value the data in the c.ad index from the data array 88 | func (c *state) SetData(v interface{}) error { 89 | return c.SetDataFromIndex(v, c.GetAD()) 90 | } 91 | 92 | // SetDataFromIndex sets to input value the i'th element from the data array 93 | func (c *state) SetDataFromIndex(v interface{}, i int) error { 94 | if err := c.IndexInRange(i); err != nil { 95 | return wrapErr(err) 96 | } 97 | c.data[i] = v 98 | return nil 99 | } 100 | 101 | // GetBufferFromIndex returns the i'th element from the buffer array 102 | func (c *state) GetBufferFromIndex(i int) (*interface{}, error) { 103 | if len(c.buffer)-1 >= i { 104 | return c.buffer[i], nil 105 | } 106 | return nil, fmt.Errorf(ire) 107 | } 108 | 109 | // SetDataFromIndex sets to input value the i'th element from the data array 110 | func (c *state) SetBufferFromIndex(v interface{}, i int) error { 111 | if len(c.buffer)-1 >= i { 112 | c.buffer[i] = &v 113 | return nil 114 | } 115 | return fmt.Errorf(ire) 116 | 117 | } 118 | 119 | // IndexInRange check if index is within data array range 120 | func (c *state) IndexInRange(i int) error { 121 | if len(c.data)-1 >= i { 122 | return nil 123 | } 124 | return fmt.Errorf(ire) 125 | } 126 | 127 | // Lib returns the lib map 128 | func (c *state) Lib() map[string]int { 129 | return c.lib 130 | } 131 | 132 | // addDoc for adding a document to the lib map 133 | func (c *state) addDoc(k string, i int) error { 134 | if err := c.IndexInRange(i); err != nil { 135 | return wrapErr(err) 136 | } 137 | c.lib[k] = i 138 | return nil 139 | } 140 | 141 | // LibIndex returns the index for a given doc name 142 | func (c *state) LibIndex(doc string) (int, bool) { 143 | i, exists := c.lib[strings.ToLower(doc)] 144 | return i, exists 145 | } 146 | 147 | // RemoveDocName removes a doc from the lib 148 | func (c *state) RemoveDocName(i int) error { 149 | if err := c.IndexInRange(i); err != nil { 150 | return wrapErr(err) 151 | } 152 | for k, v := range c.lib { 153 | if v == i { 154 | delete(c.lib, k) 155 | } 156 | } 157 | 158 | return nil 159 | } 160 | 161 | // DeleteData for deleting the i'th element from the data array 162 | func (c *state) DeleteData(i int) error { 163 | if err := c.IndexInRange(i); err != nil { 164 | return wrapErr(err) 165 | } 166 | 167 | if err := c.RemoveDocName(i); err != nil { 168 | return wrapErr(err) 169 | } 170 | 171 | c.data[i] = nil 172 | c.data = append(c.data[:i], c.data[i+1:]...) 173 | 174 | if c.ad == i { 175 | if c.ad > 0 { 176 | c.ad = c.ad - 1 177 | } else { 178 | c.ad = 0 179 | } 180 | } 181 | 182 | return nil 183 | } 184 | 185 | // // CopyBufferToData for copying buffer array over data array 186 | // func (c *state) CopyBufferToData() { 187 | // copy(c.data, c.buffer) 188 | // } 189 | 190 | // UnsetDataArray for deleting all data. This sets data = nil 191 | func (c *state) UnsetDataArray() { 192 | c.data = nil 193 | } 194 | 195 | // DeleteAllData calls PurgeAllData first and then creates a new empty array 196 | func (c *state) DeleteAllData() { 197 | c.UnsetDataArray() 198 | c.data = make([]interface{}, 0) 199 | } 200 | 201 | // UnsetBufferArray This sets buffer = nil 202 | func (c *state) UnsetBufferArray() { 203 | c.buffer = nil 204 | } 205 | 206 | // DeleteBuffer deletes the data from the buffer array 207 | func (c *state) DeleteBuffer() { 208 | c.UnsetBufferArray() 209 | c.buffer = make([]*interface{}, 0) 210 | } 211 | 212 | // ClearLib removes all keys from the lib map 213 | func (c *state) ClearLib() { 214 | c.lib = nil 215 | c.lib = make(map[string]int) 216 | } 217 | -------------------------------------------------------------------------------- /db/store_lib.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "path/filepath" 10 | "strings" 11 | "sync" 12 | 13 | e "github.com/ulfox/dby/errors" 14 | "gopkg.in/yaml.v2" 15 | ) 16 | 17 | type erf = func(e interface{}, p ...interface{}) error 18 | 19 | var wrapErr erf = e.WrapErr 20 | 21 | // Storage is the main object exported by DBy. It consolidates together 22 | // the Yaml Data and SQL 23 | type Storage struct { 24 | sync.Mutex 25 | *state 26 | SQL *SQL 27 | Path string 28 | mem bool 29 | } 30 | 31 | // NewStorageFactory for creating a new Storage 32 | func NewStorageFactory(p ...interface{}) (*Storage, error) { 33 | var path string = "local/dby.yaml" 34 | var inMem bool = true 35 | 36 | if len(p) > 0 { 37 | switch i := p[0].(type) { 38 | case string: 39 | path = i 40 | inMem = false 41 | case bool: 42 | inMem = i 43 | } 44 | } 45 | 46 | state := &Storage{ 47 | SQL: NewSQLFactory(), 48 | state: newStateFactory(), 49 | Path: path, 50 | mem: inMem, 51 | } 52 | 53 | err := state.dbinit() 54 | if err != nil { 55 | return nil, wrapErr(err) 56 | } 57 | 58 | return state, nil 59 | } 60 | 61 | // Close method will do a write if InMem is false 62 | // and then clear cache and buffers 63 | func (s *Storage) Close() error { 64 | if !s.mem { 65 | err := s.Write() 66 | if err != nil { 67 | return wrapErr(err) 68 | } 69 | } 70 | 71 | s.Clear() 72 | s.SQL.Clear() 73 | return nil 74 | } 75 | 76 | func (s *Storage) dbinit() error { 77 | if s.mem { 78 | s.PushData(emptyMap()) 79 | s.SetAD(0) 80 | return nil 81 | } 82 | 83 | stateDir := filepath.Dir(s.Path) 84 | err := makeDirs(stateDir, 0700) 85 | if err != nil { 86 | return wrapErr(err) 87 | } 88 | 89 | stateExists, err := fileExists(s.Path) 90 | if err != nil { 91 | return wrapErr(err) 92 | } 93 | 94 | if !stateExists { 95 | s.PushData(emptyMap()) 96 | s.SetAD(0) 97 | err = s.Write() 98 | if err != nil { 99 | return wrapErr(err) 100 | } 101 | } 102 | 103 | err = s.Read() 104 | if err != nil { 105 | return wrapErr(err) 106 | } 107 | 108 | return nil 109 | } 110 | 111 | // SetNames can set names automatically to the documents 112 | // that have the queried paths. 113 | // input(f) is the first path that will be quieried 114 | // input(l) is the last path 115 | // 116 | // If a document has both paths, a name will be generated 117 | // and will be mapped with the document's index 118 | func (s *Storage) SetNames(f, l string) error { 119 | for i := range s.GetAllData() { 120 | s.SetAD(i) 121 | kind, err := s.GetPath(strings.ToLower(f)) 122 | if err != nil { 123 | continue 124 | } 125 | name, err := s.GetPath(strings.ToLower(l)) 126 | if err != nil { 127 | continue 128 | } 129 | 130 | sKind, ok := kind.(string) 131 | if !ok { 132 | wrapErr(fieldNotString, strings.ToLower(f), kind) 133 | } 134 | 135 | sName, ok := name.(string) 136 | if !ok { 137 | wrapErr(fieldNotString, strings.ToLower(l), name) 138 | } 139 | err = s.addDoc( 140 | fmt.Sprintf( 141 | "%s/%s", 142 | strings.ToLower(sKind), 143 | strings.ToLower(sName), 144 | ), 145 | i, 146 | ) 147 | if err != nil { 148 | return wrapErr(err) 149 | } 150 | } 151 | 152 | return nil 153 | } 154 | 155 | // SetName adds a name for a document and maps with it the given doc index 156 | func (s *Storage) SetName(n string, i int) error { 157 | err := s.Switch(i) 158 | if err != nil { 159 | return wrapErr(err) 160 | } 161 | err = s.addDoc(n, i) 162 | if err != nil { 163 | return wrapErr(err) 164 | } 165 | 166 | return nil 167 | } 168 | 169 | // DeleteDoc will the document with the given index 170 | func (s *Storage) DeleteDoc(i int) error { 171 | err := s.DeleteData(i) 172 | if err != nil { 173 | return wrapErr(err) 174 | } 175 | 176 | return nil 177 | } 178 | 179 | // Switch will change Active Document (AD) to the given index 180 | func (s *Storage) Switch(i int) error { 181 | err := s.SetAD(i) 182 | if err != nil { 183 | return wrapErr(err) 184 | } 185 | return nil 186 | } 187 | 188 | // AddDoc will add a new document to the stack and will switch 189 | // Active Document index to that document 190 | func (s *Storage) AddDoc() error { 191 | s.PushData(emptyMap()) 192 | s.SetAD(len(s.GetAllData()) - 1) 193 | return s.stateReload() 194 | } 195 | 196 | // ListDocs will return an array with all docs names 197 | func (s *Storage) ListDocs() []string { 198 | var docs []string 199 | for i := range s.Lib() { 200 | docs = append(docs, i) 201 | } 202 | return docs 203 | } 204 | 205 | // SwitchDoc for switching to a document using the documents name (if any) 206 | func (s *Storage) SwitchDoc(n string) error { 207 | i, exists := s.LibIndex(n) 208 | if !exists { 209 | return wrapErr(docNotExists, strings.ToLower(n)) 210 | } 211 | s.SetAD(i) 212 | return nil 213 | } 214 | 215 | // DeleteAll for removing all docs 216 | func (s *Storage) DeleteAll(delete bool) *Storage { 217 | if delete { 218 | s.DeleteAllData() 219 | s.ClearLib() 220 | } 221 | return s 222 | } 223 | 224 | // ImportDocs for importing documents 225 | func (s *Storage) ImportDocs(path string, o ...bool) error { 226 | impf, err := ioutil.ReadFile(path) 227 | if err != nil { 228 | return wrapErr(err) 229 | } 230 | 231 | var counter int 232 | var data interface{} 233 | s.UnsetBufferArray() 234 | 235 | dec := yaml.NewDecoder(bytes.NewReader(impf)) 236 | for { 237 | err = dec.Decode(&data) 238 | if err == nil { 239 | s.PushBuffer(data) 240 | data = nil 241 | 242 | counter++ 243 | continue 244 | } 245 | 246 | if err.Error() == "EOF" { 247 | break 248 | } 249 | s.UnsetBufferArray() 250 | return wrapErr(err) 251 | } 252 | 253 | if len(o) > 0 { 254 | issueWarning(deprecatedFeature, "ImportDocs(string, bool)", "Storage.DeleteAll(true).ImportDocs(path)") 255 | if o[0] { 256 | s.UnsetDataArray() 257 | s.ClearLib() 258 | } 259 | } 260 | 261 | for _, j := range s.GetAllBuffer() { 262 | if j == nil { 263 | continue 264 | } 265 | if len((*j).(map[interface{}]interface{})) == 0 { 266 | continue 267 | } 268 | s.PushData(*j) 269 | } 270 | s.UnsetBufferArray() 271 | return s.stateReload() 272 | } 273 | 274 | // InMem for configuring db to write only in memory 275 | func (s *Storage) InMem(m bool) *Storage { 276 | s.mem = m 277 | return s 278 | } 279 | 280 | // Read for reading the local yaml file and importing it 281 | // in memory 282 | func (s *Storage) Read() error { 283 | f, err := ioutil.ReadFile(s.Path) 284 | if err != nil { 285 | return wrapErr(err) 286 | } 287 | 288 | s.Lock() 289 | defer s.Unlock() 290 | 291 | s.UnsetBufferArray() 292 | 293 | var data interface{} 294 | dec := yaml.NewDecoder(bytes.NewReader(f)) 295 | for { 296 | err := dec.Decode(&data) 297 | if err == nil { 298 | s.PushBuffer(data) 299 | data = nil 300 | continue 301 | } 302 | 303 | if err.Error() == "EOF" { 304 | break 305 | } 306 | s.UnsetBufferArray() 307 | return wrapErr(err) 308 | } 309 | 310 | s.UnsetDataArray() 311 | 312 | for _, j := range s.GetAllBuffer() { 313 | if j == nil { 314 | continue 315 | } 316 | s.PushData(*j) 317 | } 318 | s.UnsetBufferArray() 319 | return nil 320 | } 321 | 322 | // Write for writing memory content to the local yaml file 323 | func (s *Storage) Write() error { 324 | s.Lock() 325 | defer s.Unlock() 326 | 327 | wrkDir := path.Dir(s.Path) 328 | f, err := ioutil.TempFile(wrkDir, ".tx.*") 329 | if err != nil { 330 | return wrapErr(err) 331 | } 332 | 333 | var buf bytes.Buffer 334 | enc := yaml.NewEncoder(&buf) 335 | 336 | for _, j := range s.GetAllData() { 337 | if j == nil { 338 | continue 339 | } 340 | 341 | err := enc.Encode(j) 342 | if err != nil { 343 | return wrapErr(err) 344 | } 345 | } 346 | 347 | _, err = f.Write(buf.Bytes()) 348 | if err != nil { 349 | return wrapErr(err) 350 | } 351 | err = f.Close() 352 | if err != nil { 353 | return wrapErr(err) 354 | } 355 | 356 | return wrapErr(os.Rename(f.Name(), s.Path)) 357 | } 358 | 359 | func (s *Storage) stateReload() error { 360 | if s.mem { 361 | return nil 362 | } 363 | 364 | err := s.Write() 365 | if err != nil { 366 | return wrapErr(err) 367 | } 368 | 369 | return wrapErr(s.Read()) 370 | } 371 | -------------------------------------------------------------------------------- /db/utils.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | "gopkg.in/yaml.v2" 9 | ) 10 | 11 | func checkKeyPath(k []string) error { 12 | for _, j := range k { 13 | if j == "" { 14 | return fmt.Errorf(emptyKey, strings.Join(k, ".")) 15 | } 16 | } 17 | return nil 18 | } 19 | 20 | func copyMap(o interface{}) (interface{}, error) { 21 | obj, err := interfaceToMap(o) 22 | if err != nil { 23 | return nil, wrapErr(err) 24 | } 25 | 26 | var cache interface{} 27 | 28 | data, err := yaml.Marshal(&obj) 29 | if err != nil { 30 | return nil, wrapErr(err) 31 | } 32 | 33 | err = yaml.Unmarshal(data, &cache) 34 | if err != nil { 35 | return nil, wrapErr(err) 36 | } 37 | 38 | return cache, nil 39 | } 40 | 41 | func emptyMap() map[interface{}]interface{} { 42 | return make(map[interface{}]interface{}) 43 | } 44 | 45 | func interfaceToMap(o interface{}) (map[interface{}]interface{}, error) { 46 | obj, isMap := o.(map[interface{}]interface{}) 47 | if !isMap { 48 | if o != nil { 49 | return nil, wrapErr(notAMap) 50 | } 51 | obj = emptyMap() 52 | } 53 | return obj, nil 54 | } 55 | 56 | func deleteMap(o interface{}) { 57 | for kn := range o.(map[interface{}]interface{}) { 58 | delete(o.(map[interface{}]interface{}), kn) 59 | } 60 | } 61 | 62 | // makeDirs create directories if they do not exist 63 | func makeDirs(p string, m os.FileMode) error { 64 | if p == "/" { 65 | // Nothing to do here 66 | return nil 67 | } 68 | 69 | p = strings.TrimSuffix(p, "/") 70 | if _, err := os.Stat(p); os.IsNotExist(err) { 71 | err = os.MkdirAll(p, m) 72 | if err != nil { 73 | return wrapErr(err) 74 | } 75 | } 76 | return nil 77 | } 78 | 79 | // fileExists for checking if a file exists 80 | func fileExists(filepath string) (bool, error) { 81 | f, err := os.Stat(filepath) 82 | if os.IsNotExist(err) { 83 | return false, nil 84 | } else if f.IsDir() { 85 | return false, wrapErr(dictNotFile, filepath) 86 | } 87 | 88 | return true, nil 89 | } 90 | -------------------------------------------------------------------------------- /db/wrappers.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | // Upsert is a SQL wrapper for adding/updating map structures 8 | func (s *Storage) Upsert(k string, i interface{}) error { 9 | data, err := s.SQL.toInterfaceMap(i) 10 | if err != nil { 11 | return wrapErr(err) 12 | } 13 | err = s.SQL.upsertRecursive(strings.Split(k, "."), s.GetData(), data) 14 | if err != nil { 15 | return wrapErr(err) 16 | } 17 | 18 | return s.stateReload() 19 | } 20 | 21 | // UpsertGlobal is a SQL wrapper for adding/updating map structures 22 | // in all documents. This will change all existing paths to the given 23 | // structure and add new if the path is missing for a document 24 | func (s *Storage) UpsertGlobal(k string, i interface{}) error { 25 | data, err := s.SQL.toInterfaceMap(i) 26 | if err != nil { 27 | return wrapErr(err) 28 | } 29 | 30 | c := s.GetAD() 31 | for j := range s.GetAllData() { 32 | s.SetAD(j) 33 | err := s.SQL.upsertRecursive(strings.Split(k, "."), s.GetData(), data) 34 | if err != nil { 35 | return wrapErr(err) 36 | } 37 | } 38 | 39 | s.SetAD(c) 40 | 41 | return s.stateReload() 42 | } 43 | 44 | // UpdateGlobal is a SQL wrapper for adding/updating map structures 45 | // in all documents. This will change all existing paths to the given 46 | // structure (if any) 47 | func (s *Storage) UpdateGlobal(k string, i interface{}) error { 48 | data, err := s.SQL.toInterfaceMap(i) 49 | if err != nil { 50 | return wrapErr(err) 51 | } 52 | 53 | c := s.GetAD() 54 | for j := range s.GetAllData() { 55 | s.SetAD(j) 56 | 57 | if _, err := s.GetPath(k); err != nil { 58 | continue 59 | } 60 | 61 | err := s.SQL.upsertRecursive(strings.Split(k, "."), s.GetData(), data) 62 | if err != nil { 63 | return wrapErr(err) 64 | } 65 | } 66 | 67 | s.SetAD(c) 68 | 69 | return s.stateReload() 70 | } 71 | 72 | // GetFirst is a SQL wrapper for finding the first key in the 73 | // yaml hierarchy. If two keys are on the same level but under 74 | // different paths, then the selection will be random 75 | func (s *Storage) GetFirst(k string) (interface{}, error) { 76 | dat := s.GetData() 77 | obj, err := s.SQL.getFirst(k, &dat) 78 | if err != nil { 79 | return nil, wrapErr(err) 80 | } 81 | 82 | return *obj, nil 83 | } 84 | 85 | // GetFirstGlobal does the same as GetFirst but for all docs. 86 | // Instead of returning an interface it returns a map with keys 87 | // the index of the doc that a key was found and value the value of the key 88 | func (s *Storage) GetFirstGlobal(k string) map[int]interface{} { 89 | found := make(map[int]interface{}) 90 | 91 | c := s.GetAD() 92 | for j := range s.GetAllData() { 93 | s.SetAD(j) 94 | dat := s.GetData() 95 | 96 | obj, err := s.SQL.getFirst(k, &dat) 97 | if err != nil { 98 | continue 99 | } 100 | found[j] = *obj 101 | } 102 | 103 | s.SetAD(c) 104 | 105 | return found 106 | } 107 | 108 | // Get is alias of FindKeys. This function will be replaced 109 | // by FindKeys in the future. 110 | // For now we keep both for compatibility 111 | func (s *Storage) Get(k string) ([]string, error) { 112 | issueWarning(deprecatedFeature, "Get()", "FindKeys()") 113 | dat := s.GetData() 114 | 115 | obj, err := s.SQL.findKeys(k, &dat) 116 | if err != nil { 117 | return nil, wrapErr(err) 118 | } 119 | 120 | return obj, nil 121 | } 122 | 123 | // FindKeys is a SQL wrapper that finds all the paths for a given 124 | // e.g. ["key-1.test", "key-2.key-3.test"] will be returned 125 | func (s *Storage) FindKeys(k string) ([]string, error) { 126 | dat := s.GetData() 127 | obj, err := s.SQL.findKeys(k, &dat) 128 | if err != nil { 129 | return nil, wrapErr(err) 130 | } 131 | 132 | return obj, nil 133 | } 134 | 135 | // FindKeysGlobal does the same as FindKeys but for all docs. 136 | // Instead of returning a list of keys it returns a map with indexes 137 | // from the docs and value an array of paths that was found 138 | func (s *Storage) FindKeysGlobal(k string) map[int][]string { 139 | found := make(map[int][]string) 140 | 141 | c := s.GetAD() 142 | for j := range s.GetAllData() { 143 | s.SetAD(j) 144 | dat := s.GetData() 145 | 146 | obj, err := s.SQL.findKeys(k, &dat) 147 | if err != nil || len(obj) == 0 { 148 | continue 149 | } 150 | found[j] = obj 151 | } 152 | 153 | s.SetAD(c) 154 | 155 | return found 156 | } 157 | 158 | // GetPath is a SQL wrapper that returns the value for a given 159 | // path. Example, it would return "value-1" if "key-1.key-2" was 160 | // the path asked from the following yaml 161 | // --------- 162 | // key-1: 163 | // key-2: value-1 164 | // 165 | func (s *Storage) GetPath(k string) (interface{}, error) { 166 | keys := strings.Split(k, ".") 167 | dat := s.GetData() 168 | obj, err := s.SQL.getPath(keys, &dat) 169 | if err != nil { 170 | return nil, wrapErr(err) 171 | } 172 | 173 | return *obj, nil 174 | } 175 | 176 | // GetPathGlobal does the same as GetPath but globally for all 177 | // docs 178 | func (s *Storage) GetPathGlobal(k string) map[int]interface{} { 179 | found := make(map[int]interface{}) 180 | keys := strings.Split(k, ".") 181 | 182 | c := s.GetAD() 183 | for j := range s.GetAllData() { 184 | s.SetAD(j) 185 | dat := s.GetData() 186 | obj, err := s.SQL.getPath(keys, &dat) 187 | if err != nil { 188 | continue 189 | } 190 | found[j] = *obj 191 | } 192 | 193 | s.SetAD(c) 194 | 195 | return found 196 | } 197 | 198 | // Delete is a SQL wrapper that deletes the last key from a given 199 | // path. For example, Delete("key-1.key-2.key-3") would first 200 | // validate that the path exists, then it would export the value of 201 | // GetPath("key-1.key-2") and delete the object that matches key-3 202 | func (s *Storage) Delete(k string) error { 203 | dat := s.GetData() 204 | 205 | err := s.SQL.delPath(k, &dat) 206 | if err != nil { 207 | return wrapErr(err) 208 | } 209 | 210 | return s.stateReload() 211 | } 212 | 213 | // DeleteGlobal is the same as Delete but will try to delete 214 | // the path on all docs (if found) 215 | func (s *Storage) DeleteGlobal(k string) error { 216 | dat := s.GetData() 217 | err := s.SQL.delPath(k, &dat) 218 | if err != nil { 219 | return wrapErr(err) 220 | } 221 | 222 | return s.stateReload() 223 | } 224 | 225 | // MergeDBs is a SQL wrapper that merges a source yaml file 226 | // with the DBy local yaml file. 227 | func (s *Storage) MergeDBs(path string) error { 228 | err := s.SQL.mergeDBs(path, s.GetData()) 229 | if err != nil { 230 | return wrapErr(err) 231 | } 232 | 233 | return s.stateReload() 234 | } 235 | -------------------------------------------------------------------------------- /docs/examples/kubernetes-labels-update.md: -------------------------------------------------------------------------------- 1 | ### Update labels for all kubernetes manifests 2 | 3 | In the **manifests directory** we have a **deployment.yaml** that we will import and update 4 | 5 | In this example we will import the manifest and update the version for all documents from **v0.2.0** to **v0.3.0** 6 | 7 | ```go 8 | 9 | package main 10 | 11 | import ( 12 | "github.com/sirupsen/logrus" 13 | "github.com/ulfox/dby/db" 14 | ) 15 | 16 | func main() { 17 | logger := logrus.New() 18 | state, err := db.NewStorageFactory("local/db.yaml") 19 | if err != nil { 20 | logger.Fatal(err) 21 | } 22 | 23 | // passing true is optional. When we pass it, we instruct the import method 24 | // overwrite the content of the db (local/db.yaml). If we do not pass true, 25 | // the local/db.yaml file will also have one additional document which is 26 | // an empty {} doc that is created during init. 27 | err = state.ImportDocs("manifests/deployment.yaml", true) 28 | if err != nil { 29 | logger.Fatal(err) 30 | } 31 | 32 | // Automatically update all document names based on "kind/metadata.name" values 33 | state.SetNames("kind", "metadata.name") 34 | 35 | // Set the paths we want to update 36 | paths := []string{ 37 | "spec.selector.matchLabels.version", 38 | "metadata.labels.version", 39 | "spec.selector.version", 40 | "spec.template.selector.matchLabels.version", 41 | "spec.template.metadata.labels.version", 42 | } 43 | 44 | // UpdateGlobal is a global command that updates all fields 45 | // that match the given path. Documents that do not have the 46 | // specific path will not be updated 47 | // 48 | // If we wanted to update or create the path then we could issue 49 | // UpsertGlobal() instead. Using that command however for Kubernetes 50 | // manifests is not recommended since you may end up having 51 | // manifests with fields that are not supported by the resource API 52 | for _, j := range paths { 53 | err = state.UpdateGlobal( 54 | j, 55 | "v0.3.0", 56 | ) 57 | if err != nil { 58 | logger.Fatal(err) 59 | } 60 | 61 | } 62 | 63 | // List Docs by name 64 | for _, j := range state.ListDocs() { 65 | // Switch to a doc by name 66 | err = state.SwitchDoc(j) 67 | if err != nil { 68 | logger.Fatal(err) 69 | } 70 | 71 | // Get the metadata 72 | val, err := state.GetPath("metadata.labels.version") 73 | if err != nil { 74 | // We use continue here because HorizontalPodAutoscaler does not have labels set 75 | // so no update was done and no path exists for us to get 76 | continue 77 | } 78 | logger.Infof("%s has version: %s", j, val) 79 | } 80 | } 81 | 82 | ``` 83 | 84 | 85 | Example output 86 | 87 | 88 | ```bash 89 | INFO[0000] poddisruptionbudget/caller-svc has version: v0.3.0 90 | INFO[0000] deployment/listener-svc has version: v0.3.0 91 | INFO[0000] service/listener-svc has version: v0.3.0 92 | INFO[0000] poddisruptionbudget/listener-svc has version: v0.3.0 93 | INFO[0000] deployment/caller-svc has version: v0.3.0 94 | INFO[0000] service/caller-svc has version: v0.3.0 95 | ``` -------------------------------------------------------------------------------- /docs/examples/manifests/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: autoscaling/v2beta2 3 | kind: HorizontalPodAutoscaler 4 | metadata: 5 | name: listener-svc 6 | namespace: echoserver 7 | spec: 8 | scaleTargetRef: 9 | apiVersion: apps/v1 10 | kind: Deployment 11 | name: listener-svc 12 | minReplicas: 3 13 | maxReplicas: 10 14 | metrics: 15 | - type: Resource 16 | resource: 17 | name: cpu 18 | target: 19 | type: Utilization 20 | averageUtilization: 30 21 | --- 22 | apiVersion: apps/v1 23 | kind: Deployment 24 | metadata: 25 | name: listener-svc 26 | namespace: echoserver 27 | # annotations: 28 | # sidecar.istio.io/extraStatTags: destination_port,request_host 29 | labels: 30 | app: listener-svc 31 | version: v0.1.1 32 | spec: 33 | strategy: 34 | rollingUpdate: 35 | maxSurge: "100%" 36 | maxUnavailable: 3 37 | type: "RollingUpdate" 38 | replicas: 1 39 | selector: 40 | matchLabels: 41 | app: listener-svc 42 | version: v0.1.1 43 | template: 44 | metadata: 45 | # annotations: 46 | # sidecar.istio.io/extraStatTags: destination_port,request_host 47 | labels: 48 | app: listener-svc 49 | version: v0.1.1 50 | spec: 51 | containers: 52 | - image: gcr.io/google_containers/echoserver:1.9 53 | imagePullPolicy: Always 54 | name: listener-svc 55 | ports: 56 | - containerPort: 8080 57 | --- 58 | apiVersion: v1 59 | kind: Service 60 | metadata: 61 | name: listener-svc 62 | namespace: echoserver 63 | annotations: 64 | prometheus.io/scrape: 'true' 65 | prometheus.io/port: '15090' 66 | prometheus.io/path: '/stats/prometheus' 67 | labels: 68 | app: listener-svc 69 | version: v0.1.1 70 | spec: 71 | ports: 72 | - port: 80 73 | targetPort: 8080 74 | protocol: TCP 75 | name: tcp-web 76 | selector: 77 | app: listener-svc 78 | version: v0.1.1 79 | --- 80 | apiVersion: policy/v1beta1 81 | kind: PodDisruptionBudget 82 | metadata: 83 | name: listener-svc 84 | namespace: echoserver 85 | labels: 86 | app: listener-svc 87 | version: v0.1.1 88 | spec: 89 | maxUnavailable: 3 90 | selector: 91 | matchLabels: 92 | app: listener-svc 93 | version: v0.1.1 94 | --- 95 | apiVersion: autoscaling/v2beta2 96 | kind: HorizontalPodAutoscaler 97 | metadata: 98 | name: caller-svc 99 | namespace: sysdebug 100 | spec: 101 | scaleTargetRef: 102 | apiVersion: apps/v1 103 | kind: Deployment 104 | name: caller-svc 105 | minReplicas: 3 106 | maxReplicas: 10 107 | metrics: 108 | - type: Resource 109 | resource: 110 | name: cpu 111 | target: 112 | type: Utilization 113 | averageUtilization: 30 114 | --- 115 | apiVersion: apps/v1 116 | kind: Deployment 117 | metadata: 118 | name: caller-svc 119 | namespace: sysdebug 120 | # annotations: 121 | # sidecar.istio.io/extraStatTags: destination_port,request_host 122 | labels: 123 | app: caller-svc 124 | version: v0.2.0 125 | spec: 126 | strategy: 127 | rollingUpdate: 128 | maxSurge: "100%" 129 | maxUnavailable: 3 130 | type: "RollingUpdate" 131 | replicas: 1 132 | selector: 133 | matchLabels: 134 | app: caller-svc 135 | version: v0.2.0 136 | template: 137 | metadata: 138 | # annotations: 139 | # sidecar.istio.io/extraStatTags: destination_port,request_host 140 | labels: 141 | app: caller-svc 142 | version: v0.2.0 143 | spec: 144 | # serviceAccount: sysdebug 145 | containers: 146 | - name: caller-svc 147 | ports: 148 | - containerPort: 8080 149 | image: gcr.io/google_containers/echoserver:1.9 150 | imagePullPolicy: IfNotPresent 151 | readinessProbe: 152 | timeoutSeconds: 7 153 | exec: 154 | command: 155 | - curl 156 | - -sS 157 | - --fail 158 | - --connect-timeout 159 | - "5" 160 | - -o 161 | - /dev/null 162 | - listener-svc.echoserver.svc.primef.org 163 | livenessProbe: 164 | timeoutSeconds: 7 165 | exec: 166 | command: 167 | - curl 168 | - -sS 169 | - --fail 170 | - --connect-timeout 171 | - "5" 172 | - -o 173 | - /dev/null 174 | - listener-svc.echoserver.svc.primef.org 175 | --- 176 | apiVersion: v1 177 | kind: Service 178 | metadata: 179 | name: caller-svc 180 | namespace: sysdebug 181 | annotations: 182 | prometheus.io/scrape: 'true' 183 | prometheus.io/port: '15090' 184 | prometheus.io/path: '/stats/prometheus' 185 | labels: 186 | app: caller-svc 187 | version: v0.2.0 188 | spec: 189 | ports: 190 | - port: 80 191 | targetPort: 8080 192 | protocol: TCP 193 | name: tcp-web 194 | type: ClusterIP 195 | selector: 196 | app: caller-svc 197 | version: v0.2.0 198 | 199 | --- 200 | apiVersion: policy/v1beta1 201 | kind: PodDisruptionBudget 202 | metadata: 203 | name: caller-svc 204 | namespace: sysdebug 205 | labels: 206 | app: caller-svc 207 | version: v0.2.0 208 | spec: 209 | maxUnavailable: 3 210 | selector: 211 | matchLabels: 212 | app: caller-svc 213 | version: v0.2.0 214 | -------------------------------------------------------------------------------- /errors/errors.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "strconv" 7 | 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | // WrapErr for creating errors and wrapping them along 12 | // with callers info 13 | func WrapErr(e interface{}, p ...interface{}) error { 14 | if e == nil { 15 | return nil 16 | } 17 | 18 | var err error 19 | 20 | switch e := e.(type) { 21 | case string: 22 | err = fmt.Errorf(e, p...) 23 | case error: 24 | err = e 25 | } 26 | 27 | pc, _, no, ok := runtime.Caller(1) 28 | details := runtime.FuncForPC(pc) 29 | if ok && details != nil { 30 | return errors.Wrap(err, fmt.Sprintf("%s#%s\n", details.Name(), strconv.Itoa(no))) 31 | } 32 | 33 | return err 34 | } 35 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ulfox/dby 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/likexian/gokit v0.25.2 7 | github.com/pkg/errors v0.9.1 8 | gopkg.in/yaml.v2 v2.4.0 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/likexian/gokit v0.25.2 h1:PHZc/uZr0RZXTKPIV1GzmkDC0x3NvJImsGyziR2F4H0= 2 | github.com/likexian/gokit v0.25.2/go.mod h1:NCv1RDZK5kR0T2SfAl/vjIO6rsjszt2C/25TKxJalhs= 3 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 4 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 5 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 6 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 7 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 8 | -------------------------------------------------------------------------------- /tests/delete_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/likexian/gokit/assert" 8 | "github.com/ulfox/dby/db" 9 | ) 10 | 11 | // TestDelete run unit tests for deleting objects 12 | // from a given path 13 | func TestDelete(t *testing.T) { 14 | t.Parallel() 15 | 16 | path := ".test/db-delete-key.yaml" 17 | storage, err := db.NewStorageFactory(path) 18 | assert.Equal(t, err, nil) 19 | 20 | err = storage.Upsert( 21 | "test.path", 22 | map[string]string{ 23 | "key-1": "value-1", 24 | "key-2": "value-2", 25 | "key-3": "value-3", 26 | "key-4": "value-4", 27 | }, 28 | ) 29 | 30 | assert.Equal(t, err, nil) 31 | 32 | err = storage.Delete("test.path.key-1") 33 | assert.Equal(t, err, nil) 34 | 35 | val, err := storage.GetPath("test.path.key-1") 36 | assert.NotEqual(t, err, nil) 37 | assert.Equal(t, val, nil) 38 | 39 | err = storage.Delete("test.path.key-12") 40 | assert.NotEqual(t, err, nil) 41 | 42 | err = storage.Upsert( 43 | "key-33", 44 | map[string][]string{ 45 | "key-5": {"value-5"}, 46 | "key-6": {"value-6"}, 47 | }, 48 | ) 49 | 50 | assert.Equal(t, err, nil) 51 | err = storage.Delete("key-33.key-5") 52 | assert.Equal(t, err, nil) 53 | err = storage.Delete("key-33.key-6.[0]") 54 | assert.Equal(t, err, nil) 55 | 56 | err = os.Remove(path) 57 | assert.Equal(t, err, nil) 58 | } 59 | -------------------------------------------------------------------------------- /tests/generic_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/likexian/gokit/assert" 8 | "github.com/ulfox/dby/db" 9 | ) 10 | 11 | // TestGeneric run generic tests for all scenarios 12 | func TestGeneric(t *testing.T) { 13 | t.Parallel() 14 | 15 | path := ".test/db-generic.yaml" 16 | storage, err := db.NewStorageFactory(path) 17 | assert.Equal(t, err, nil) 18 | storage.InMem(true) 19 | 20 | err = storage.Upsert( 21 | ".someKey", 22 | map[string]string{ 23 | "key-1": "value-1", 24 | "key-2": "value-2", 25 | }, 26 | ) 27 | 28 | assert.NotEqual(t, err, nil) 29 | 30 | err = storage.Upsert( 31 | ".", 32 | map[string]string{ 33 | "key-1": "value-1", 34 | "key-2": "value-2", 35 | }, 36 | ) 37 | 38 | assert.NotEqual(t, err, nil) 39 | 40 | err = storage.Upsert( 41 | "k01", 42 | nil, 43 | ) 44 | 45 | assert.Equal(t, err, nil) 46 | 47 | err = storage.Upsert( 48 | "k", 49 | []map[string][]map[string]string{ 50 | { 51 | "0": { 52 | {"1": "v03"}, 53 | }, 54 | "2": { 55 | {"03": "v05"}, 56 | }, 57 | }, 58 | { 59 | "3": { 60 | {"2": "v11"}, 61 | }, 62 | "4": { 63 | {"3": "v12"}, 64 | }, 65 | }, 66 | }, 67 | ) 68 | assert.Equal(t, err, nil) 69 | 70 | val, err := storage.GetFirst("1") 71 | assert.Equal(t, err, nil) 72 | assert.Equal(t, val, "v03") 73 | assertData := db.NewConvertFactory() 74 | 75 | val, err = storage.GetFirst("03") 76 | assert.Equal(t, err, nil) 77 | 78 | assertData.Input(val) 79 | s, err := assertData.GetString() 80 | assert.Equal(t, assertData.GetError(), nil) 81 | assert.Equal(t, err, nil) 82 | assert.Equal(t, s, "v05") 83 | 84 | keys, err := storage.Get("1") 85 | assert.Equal(t, err, nil) 86 | assert.Equal(t, len(keys), 1) 87 | 88 | err = storage.Upsert( 89 | "i", 90 | []int{ 91 | 1, 92 | 2, 93 | 3, 94 | }, 95 | ) 96 | assert.Equal(t, err, nil) 97 | 98 | val, err = storage.GetFirst("i") 99 | assert.Equal(t, err, nil) 100 | assertData.Input(val) 101 | 102 | i, err := assertData.GetArray() 103 | assert.Equal(t, assertData.GetError(), nil) 104 | assert.Equal(t, err, nil) 105 | assert.Equal(t, len(i), 3) 106 | 107 | err = os.Remove(path) 108 | assert.Equal(t, err, nil) 109 | } 110 | -------------------------------------------------------------------------------- /tests/get_first_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/likexian/gokit/assert" 7 | "github.com/ulfox/dby/db" 8 | ) 9 | 10 | // TestGetFirst run unit tests on GetSingle key 11 | func TestGetFirst(t *testing.T) { 12 | t.Parallel() 13 | 14 | storage, err := db.NewStorageFactory() 15 | assert.Equal(t, err, nil) 16 | 17 | err = storage.Upsert( 18 | "test.path", 19 | map[string]string{ 20 | "key-1": "value-1", 21 | "key-2": "value-2", 22 | }, 23 | ) 24 | 25 | assert.Equal(t, err, nil) 26 | 27 | val, err := storage.GetFirst("key-1") 28 | assert.Equal(t, err, nil) 29 | assert.Equal(t, val, "value-1") 30 | 31 | val, err = storage.GetFirst("key-2") 32 | assert.Equal(t, err, nil) 33 | assert.Equal(t, val, "value-2") 34 | 35 | err = storage.Upsert( 36 | "path-1", 37 | map[string][]string{ 38 | "key-3": {"value-3"}, 39 | "key-4": {"value-4"}, 40 | }, 41 | ) 42 | 43 | assert.Equal(t, err, nil) 44 | 45 | val, err = storage.GetFirst("key-3") 46 | assert.Equal(t, err, nil) 47 | assert.Equal(t, val, []interface{}{"value-3"}) 48 | 49 | val, err = storage.GetFirst("key-4") 50 | assert.Equal(t, err, nil) 51 | assert.Equal(t, val, []interface{}{"value-4"}) 52 | err = storage.Upsert( 53 | "key-3", 54 | map[string][]string{ 55 | "key-5": {"value-5"}, 56 | "key-6": {"value-6"}, 57 | }, 58 | ) 59 | 60 | assert.Equal(t, err, nil) 61 | 62 | val, err = storage.GetFirst("key-3") 63 | assert.Equal(t, err, nil) 64 | assert.Equal(t, val.(map[interface{}]interface{})["key-5"].([]interface{})[0], "value-5") 65 | assert.Equal(t, val.(map[interface{}]interface{})["key-6"].([]interface{})[0], "value-6") 66 | 67 | err = storage.Upsert( 68 | "test", 69 | map[string]string{}, 70 | ) 71 | assert.Equal(t, err, nil) 72 | err = storage.Upsert( 73 | "key-3", 74 | map[string][]string{}, 75 | ) 76 | assert.Equal(t, err, nil) 77 | err = storage.Upsert( 78 | "path-1", 79 | map[string][]string{}, 80 | ) 81 | assert.Equal(t, err, nil) 82 | err = storage.Upsert( 83 | "to.array-10", 84 | map[string][]map[string]int{ 85 | "key-10": { 86 | {"key-20": 20}, 87 | {"key-30": 30}, 88 | {"key-40": 40}, 89 | }, 90 | }, 91 | ) 92 | assert.Equal(t, err, nil) 93 | 94 | val, err = storage.GetFirst("key-30") 95 | assert.Equal(t, err, nil) 96 | assert.Equal(t, val, 30) 97 | } 98 | -------------------------------------------------------------------------------- /tests/get_path_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/likexian/gokit/assert" 7 | "github.com/ulfox/dby/db" 8 | ) 9 | 10 | // TestGetPath run unit tests on Get object from path 11 | func TestGetPath(t *testing.T) { 12 | t.Parallel() 13 | 14 | storage, err := db.NewStorageFactory() 15 | assert.Equal(t, err, nil) 16 | 17 | err = storage.Upsert( 18 | "test.path", 19 | map[string]string{ 20 | "key-1": "value-1", 21 | "key-2": "value-2", 22 | }, 23 | ) 24 | 25 | assert.Equal(t, err, nil) 26 | 27 | val, err := storage.GetPath("test.path.key-1") 28 | assert.Equal(t, err, nil) 29 | assert.Equal(t, val, "value-1") 30 | 31 | err = storage.Upsert( 32 | "some", 33 | []map[string][]string{ 34 | { 35 | "array": { 36 | "value-3", 37 | "value-4", 38 | }, 39 | }, 40 | }, 41 | ) 42 | 43 | assert.Equal(t, err, nil) 44 | 45 | val, err = storage.GetPath("some.[0].array") 46 | assert.Equal(t, err, nil) 47 | assert.Equal(t, val, []interface{}{"value-3", "value-4"}) 48 | 49 | err = storage.Upsert( 50 | "array", 51 | []string{ 52 | "value-5", 53 | "value-6", 54 | }, 55 | ) 56 | 57 | assert.Equal(t, err, nil) 58 | 59 | val, err = storage.GetPath("array.[0]") 60 | assert.Equal(t, err, nil) 61 | assert.Equal(t, val, "value-5") 62 | 63 | val, err = storage.GetPath("array.[1]") 64 | assert.Equal(t, err, nil) 65 | assert.Equal(t, val, "value-6") 66 | 67 | val, err = storage.GetPath("array.[2]") 68 | assert.NotEqual(t, err, nil) 69 | assert.Equal(t, val, nil) 70 | 71 | val, err = storage.GetPath("some.[2].array") 72 | assert.NotEqual(t, err, nil) 73 | assert.Equal(t, val, nil) 74 | } 75 | -------------------------------------------------------------------------------- /tests/get_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/likexian/gokit/assert" 7 | "github.com/ulfox/dby/db" 8 | ) 9 | 10 | // TestGet run unit tests for searching for keys 11 | func TestGet(t *testing.T) { 12 | t.Parallel() 13 | 14 | storage, err := db.NewStorageFactory() 15 | assert.Equal(t, err, nil) 16 | 17 | err = storage.Upsert( 18 | "path-1", 19 | []map[string][]map[string]string{ 20 | { 21 | "subpath-01": { 22 | {"k01": "v01"}, 23 | }, 24 | "subpath-02": { 25 | {"k02": "v02"}, 26 | }, 27 | }, 28 | { 29 | "subpath-11": { 30 | {"k11": "v11"}, 31 | }, 32 | "subpath-12": { 33 | {"k12": "v12"}, 34 | }, 35 | }, 36 | }, 37 | ) 38 | assert.Equal(t, err, nil) 39 | 40 | assertData := db.NewConvertFactory() 41 | assertData.Input(storage.GetData()) 42 | 43 | assertData. 44 | Key("path-1"). 45 | Index(0). 46 | Key("subpath-01"). 47 | Index(0) 48 | 49 | assert.Equal(t, assertData.GetError(), nil) 50 | m, err := assertData.GetMap() 51 | assert.Equal(t, assertData.GetError(), nil) 52 | assert.Equal(t, err, nil) 53 | 54 | assert.Equal(t, m["k01"], "v01") 55 | 56 | obj, err := storage.GetPath("path-1.[1].subpath-11.[0]") 57 | assert.Equal(t, err, nil) 58 | assertData.Input(obj) 59 | m, err = assertData.GetMap() 60 | assert.Equal(t, assertData.GetError(), nil) 61 | assert.Equal(t, err, nil) 62 | assert.Equal(t, m["k11"], "v11") 63 | 64 | assert.Equal(t, assertData.GetError(), nil) 65 | } 66 | -------------------------------------------------------------------------------- /tests/multi_doc_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/likexian/gokit/assert" 8 | "github.com/ulfox/dby/db" 9 | ) 10 | 11 | // TestMultiDoc run tests for docs 12 | func TestMultiDoc(t *testing.T) { 13 | t.Parallel() 14 | 15 | storage, err := db.NewStorageFactory() 16 | assert.Equal(t, err, nil) 17 | assert.Equal(t, len(storage.GetAllData()), 1) 18 | 19 | err = storage.AddDoc() 20 | assert.Equal(t, err, nil) 21 | assert.Equal(t, len(storage.GetAllData()), 2) 22 | 23 | assert.Equal(t, storage.GetAD(), 1) 24 | err = storage.Switch(0) 25 | assert.Equal(t, err, nil) 26 | assert.Equal(t, storage.GetAD(), 0) 27 | 28 | err = storage.DeleteDoc(1) 29 | assert.Equal(t, err, nil) 30 | assert.Equal(t, len(storage.GetAllData()), 1) 31 | 32 | err = storage.DeleteDoc(0) 33 | assert.Equal(t, err, nil) 34 | assert.Equal(t, len(storage.GetAllData()), 0) 35 | 36 | err = storage.AddDoc() 37 | assert.Equal(t, err, nil) 38 | assert.Equal(t, len(storage.GetAllData()), 1) 39 | 40 | err = storage.DeleteAll(true). 41 | ImportDocs("../docs/examples/manifests/deployment.yaml", true) 42 | assert.Equal(t, err, nil) 43 | 44 | assert.Equal(t, len(storage.GetAllData()), 8) 45 | 46 | err = storage.SetNames("kind", "metadata.name") 47 | assert.Equal(t, err, nil) 48 | assert.Equal(t, len(storage.ListDocs()), 8) 49 | 50 | for _, j := range []string{ 51 | "spec.selector.matchLabels.version", 52 | "metadata.labels.version", 53 | "spec.selector.version", 54 | "spec.template.selector.matchLabels.version", 55 | "spec.template.metadata.labels.version", 56 | } { 57 | err = storage.UpdateGlobal( 58 | j, 59 | "v0.3.0", 60 | ) 61 | assert.Equal(t, err, nil) 62 | } 63 | 64 | for _, j := range storage.ListDocs() { 65 | if strings.HasPrefix(j, "horizontalpodautoscaler/") { 66 | continue 67 | } 68 | err = storage.SwitchDoc(j) 69 | assert.Equal(t, err, nil) 70 | 71 | val, err := storage.GetPath("metadata.labels.version") 72 | assert.Equal(t, err, nil) 73 | assert.Equal(t, val, "v0.3.0") 74 | } 75 | 76 | err = storage.AddDoc() 77 | assert.Equal(t, err, nil) 78 | assert.Equal(t, len(storage.GetAllData()), 9) 79 | err = storage.SetNames("kind", "metadata.name") 80 | assert.Equal(t, err, nil) 81 | assert.Equal(t, len(storage.ListDocs()), 8) 82 | assert.Equal(t, len(storage.GetAllData()), 9) 83 | for i, j := range storage.Lib() { 84 | err = storage.Switch(j) 85 | assert.Equal(t, err, nil) 86 | 87 | kind, err := storage.GetPath("kind") 88 | assert.Equal(t, err, nil) 89 | 90 | name, err := storage.GetPath("metadata.name") 91 | assert.Equal(t, err, nil) 92 | 93 | sKind, ok := kind.(string) 94 | assert.Equal(t, ok, true) 95 | 96 | sName, ok := name.(string) 97 | assert.Equal(t, ok, true) 98 | assert.Equal(t, i, strings.ToLower(sKind)+"/"+strings.ToLower(sName)) 99 | } 100 | 101 | c0 := 0 102 | err = storage.AddDoc() 103 | assert.Equal(t, err, nil) 104 | assert.Equal(t, len(storage.GetAllData()), 10) 105 | err = storage.AddDoc() 106 | assert.Equal(t, err, nil) 107 | assert.Equal(t, len(storage.GetAllData()), 11) 108 | err = storage.SetNames("kind", "metadata.name") 109 | assert.Equal(t, err, nil) 110 | 111 | for i := range storage.GetAllData() { 112 | err = storage.Switch(i) 113 | assert.Equal(t, err, nil) 114 | 115 | kind, err := storage.GetPath("kind") 116 | if err != nil { 117 | c0++ 118 | continue 119 | } 120 | 121 | name, err := storage.GetPath("metadata.name") 122 | assert.Equal(t, err, nil) 123 | 124 | sKind, ok := kind.(string) 125 | assert.Equal(t, ok, true) 126 | 127 | sName, ok := name.(string) 128 | assert.Equal(t, ok, true) 129 | 130 | doc, ok := storage.LibIndex(strings.ToLower(sKind) + "/" + strings.ToLower(sName)) 131 | assert.Equal(t, ok, true) 132 | assert.Equal(t, doc, i) 133 | } 134 | assert.Equal(t, c0, 3) 135 | 136 | data := storage.GetPathGlobal("metadata.name") 137 | assert.Equal(t, len(data), 8) 138 | 139 | dataMap := map[int][]string{ 140 | 0: {"metadata.name", "spec.metrics.[0].resource.name", "spec.scaleTargetRef.name"}, 141 | 1: {"metadata.name", "spec.template.spec.containers.[0].name"}, 142 | 2: {"spec.ports.[0].name", "metadata.name"}, 143 | 3: {"metadata.name"}, 144 | 4: {"metadata.name", "spec.metrics.[0].resource.name", "spec.scaleTargetRef.name"}, 145 | 5: {"metadata.name", "spec.template.spec.containers.[0].name"}, 146 | 6: {"metadata.name", "spec.ports.[0].name"}, 147 | 7: {"metadata.name"}, 148 | } 149 | 150 | for i, j := range storage.FindKeysGlobal("name") { 151 | err = storage.Switch(i) 152 | assert.Equal(t, err, nil) 153 | 154 | assert.Equal(t, len(j), len(dataMap[i])) 155 | 156 | for _, m := range dataMap[i] { 157 | assert.Equal(t, checkArrays(m, j), true) 158 | } 159 | 160 | } 161 | } 162 | 163 | func checkArrays(v string, l []string) bool { 164 | for _, j := range l { 165 | if j == v { 166 | return true 167 | } 168 | } 169 | return false 170 | } 171 | -------------------------------------------------------------------------------- /tests/storage_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/likexian/gokit/assert" 8 | "github.com/ulfox/dby/db" 9 | ) 10 | 11 | // fileExists for checking if a file exists 12 | func fileExists(filepath string) bool { 13 | f, err := os.Stat(filepath) 14 | if os.IsNotExist(err) { 15 | return false 16 | } else if f.IsDir() { 17 | return true 18 | } 19 | 20 | return true 21 | } 22 | 23 | // TestStorage run unit tests on storage 24 | func TestStorage(t *testing.T) { 25 | t.Parallel() 26 | 27 | path := ".test/db-storage.yaml" 28 | assert.Equal(t, fileExists(path), false) 29 | 30 | empty, err := db.NewStorageFactory(path) 31 | assert.Equal(t, err, nil) 32 | 33 | assert.Equal(t, len(empty.GetAllData()), 1) 34 | 35 | assertData := db.NewConvertFactory() 36 | assertData.Input(empty.GetData()) 37 | 38 | emptyMap, err := assertData.GetMap() 39 | assert.Equal(t, err, nil) 40 | assert.Equal(t, len(emptyMap), 0) 41 | 42 | err = empty.Upsert( 43 | "test.path", 44 | map[string]string{ 45 | "key-1": "value-1", 46 | "key-2": "value-2", 47 | }, 48 | ) 49 | assert.Equal(t, err, nil) 50 | 51 | empty = nil 52 | 53 | data, err := db.NewStorageFactory(path) 54 | assert.Equal(t, err, nil) 55 | 56 | assert.Equal(t, len(data.GetAllData()), 1) 57 | assertData.Input(data.GetData()) 58 | 59 | assertData.Key("test") 60 | assert.Equal(t, assertData.GetError(), nil) 61 | assertData.Key("path") 62 | assert.Equal(t, assertData.GetError(), nil) 63 | 64 | dataMap, err := assertData.GetMap() 65 | assert.Equal(t, err, nil) 66 | assert.Equal(t, len(dataMap), 2) 67 | 68 | data.InMem(true) 69 | data.DeleteAll(true) 70 | assert.Equal(t, len(data.GetAllData()), 0) 71 | 72 | data, dataMap = nil, nil 73 | 74 | dataDue, err := db.NewStorageFactory(path) 75 | assert.Equal(t, err, nil) 76 | assert.Equal(t, len(dataDue.GetAllData()), 1) 77 | assertData.Input(dataDue.GetData()) 78 | 79 | assertData.Key("test") 80 | assert.Equal(t, assertData.GetError(), nil) 81 | assertData.Key("path") 82 | assert.Equal(t, assertData.GetError(), nil) 83 | 84 | dataMap, err = assertData.GetMap() 85 | assert.Equal(t, err, nil) 86 | assert.Equal(t, len(dataMap), 2) 87 | 88 | memData, err := db.NewStorageFactory() 89 | assert.Equal(t, err, nil) 90 | assert.Equal(t, len(memData.GetAllData()), 1) 91 | 92 | assertData.Input(memData.GetData()) 93 | 94 | emptyMap, err = assertData.GetMap() 95 | assert.Equal(t, err, nil) 96 | assert.Equal(t, len(emptyMap), 0) 97 | 98 | err = memData.Upsert( 99 | "test.path", 100 | map[string]string{ 101 | "key-1": "value-1", 102 | "key-2": "value-2", 103 | }, 104 | ) 105 | assert.Equal(t, err, nil) 106 | assertData.Input(memData.GetData()) 107 | 108 | assertData.Key("test") 109 | assert.Equal(t, assertData.GetError(), nil) 110 | assertData.Key("path") 111 | assert.Equal(t, assertData.GetError(), nil) 112 | 113 | emptyMap, err = assertData.GetMap() 114 | assert.Equal(t, err, nil) 115 | assert.Equal(t, len(emptyMap), 2) 116 | 117 | err = os.Remove(path) 118 | assert.Equal(t, err, nil) 119 | } 120 | -------------------------------------------------------------------------------- /tests/upsert_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | "github.com/likexian/gokit/assert" 10 | "github.com/ulfox/dby/db" 11 | "gopkg.in/yaml.v2" 12 | ) 13 | 14 | // TestUpsert run unit tests on Upsert 15 | func TestUpsert(t *testing.T) { 16 | t.Parallel() 17 | 18 | path := ".test/db-upsert.yaml" 19 | storage, err := db.NewStorageFactory(path) 20 | assert.Equal(t, err, nil) 21 | 22 | err = storage.Upsert( 23 | "test.path", 24 | map[string]string{ 25 | "key-1": "value-1", 26 | "key-2": "value-2", 27 | }, 28 | ) 29 | assert.Equal(t, err, nil) 30 | 31 | err = storage.Upsert( 32 | "path-1.sub-path-1", 33 | map[string][]string{ 34 | "sub-path-2": {"value-1", "value-2"}, 35 | "sub-path-3": {"value-3", "value-4"}, 36 | }, 37 | ) 38 | assert.Equal(t, err, nil) 39 | 40 | err = storage.Upsert( 41 | "path-2", 42 | []map[string][]string{ 43 | { 44 | "sub-path-1": {"value-1", "value-2"}, 45 | }, 46 | { 47 | "sub-path-2": {"value-3", "value-4"}, 48 | }, 49 | }, 50 | ) 51 | assert.Equal(t, err, nil) 52 | 53 | err = storage.Upsert( 54 | "path-3", 55 | []map[string]string{ 56 | { 57 | "sub-path-1": "value-1", 58 | }, 59 | { 60 | "sub-path-2": "value-2", 61 | }, 62 | }, 63 | ) 64 | assert.Equal(t, err, nil) 65 | 66 | err = storage.Upsert( 67 | "path-4", 68 | map[string]int{ 69 | "sub-path-1": 0, 70 | "sub-path-2": 1, 71 | }, 72 | ) 73 | assert.Equal(t, err, nil) 74 | 75 | f, err := ioutil.ReadFile(path) 76 | assert.Equal(t, err, nil) 77 | 78 | v := storage.GetData() 79 | yaml.Unmarshal(f, &v) 80 | 81 | testUpsert := []struct { 82 | Key string 83 | Value string 84 | }{ 85 | {"key-1", "value-1"}, 86 | {"key-2", "value-2"}, 87 | } 88 | 89 | data, ok := storage.GetData().(map[interface{}]interface{}) 90 | assert.Equal(t, ok, true) 91 | for _, testCase := range testUpsert { 92 | 93 | assert.Equal( 94 | t, 95 | data["test"].(map[interface{}]interface{})["path"].(map[interface{}]interface{})[testCase.Key], 96 | testCase.Value, 97 | fmt.Sprintf("Expected: %v", testCase.Value), 98 | ) 99 | } 100 | 101 | assert.Equal( 102 | t, 103 | data["path-2"].([]interface{})[0].(map[interface{}]interface{})["sub-path-1"].([]interface{})[0], 104 | "value-1", 105 | ) 106 | assert.Equal( 107 | t, 108 | data["path-2"].([]interface{})[0].(map[interface{}]interface{})["sub-path-1"].([]interface{})[1], 109 | "value-2", 110 | ) 111 | assert.Equal( 112 | t, 113 | data["path-2"].([]interface{})[1].(map[interface{}]interface{})["sub-path-2"].([]interface{})[0], 114 | "value-3", 115 | ) 116 | assert.Equal( 117 | t, 118 | data["path-2"].([]interface{})[1].(map[interface{}]interface{})["sub-path-2"].([]interface{})[1], 119 | "value-4", 120 | ) 121 | 122 | assert.Equal( 123 | t, 124 | data["path-3"].([]interface{})[0].(map[interface{}]interface{})["sub-path-1"], 125 | "value-1", 126 | ) 127 | assert.Equal( 128 | t, 129 | data["path-3"].([]interface{})[1].(map[interface{}]interface{})["sub-path-2"], 130 | "value-2", 131 | ) 132 | 133 | assert.Equal(t, data["path-4"].(map[interface{}]interface{})["sub-path-1"], 0) 134 | assert.Equal(t, data["path-4"].(map[interface{}]interface{})["sub-path-2"], 1) 135 | 136 | err = os.Remove(path) 137 | assert.Equal(t, err, nil) 138 | } 139 | --------------------------------------------------------------------------------