├── .gitignore ├── .golangci.yaml ├── LICENSE ├── README.md ├── doc.go ├── examples └── main_sort.go ├── fastdb.go ├── fastdb_test.go ├── go.mod ├── go.sum └── persist ├── aof.go ├── aof_internal_test.go ├── aof_test.go └── doc.go /.gitignore: -------------------------------------------------------------------------------- 1 | # IGNORED DIRECTORIES 2 | 3 | /.idea/* 4 | /.vscode/* 5 | /vendor 6 | 7 | # IGNORED FILES 8 | .dccache 9 | .DS_Store 10 | *workspace 11 | *.prof 12 | *.bak 13 | fastdb_fuzzset.db 14 | # Test binary, built with go test -c 15 | *.test 16 | *.out 17 | coverage.* 18 | debug.* 19 | __debug_bin 20 | debug 21 | Makefile 22 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | ## see : https://raw.githubusercontent.com/golangci/golangci-lint/master/.golangci.example.yml 2 | ## see : https://gist.github.com/maratori/47a4d00457a92aa426dbd48a18776322gocritic 3 | 4 | # This file contains all available configuration options 5 | # with their default values (in comments). 6 | # 7 | # This file is not a configuration example, 8 | # it contains the exhaustive configuration with explanations of the options. 9 | 10 | # Options for analysis running. 11 | run: 12 | # The default concurrency value is the number of available CPU. 13 | concurrency: 8 14 | 15 | # Timeout for analysis, e.g. 30s, 5m. 16 | # Default: 1m 17 | timeout: 3m 18 | 19 | # Exit code when at least one issue was found. 20 | # Default: 1 21 | issues-exit-code: 1 22 | 23 | # Include test files or not. 24 | # Default: true 25 | tests: true 26 | 27 | # List of build tags, all linters use it. 28 | # Default: []. 29 | # build-tags: 30 | # - mytag 31 | 32 | # Enables skipping of directories: 33 | # - vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ 34 | # Default: true 35 | skip-dirs-use-default: true 36 | 37 | # If set we pass it to "go list -mod={option}". From "go help modules": 38 | # If invoked with -mod=readonly, the go command is disallowed from the implicit 39 | # automatic updating of go.mod described above. Instead, it fails when any changes 40 | # to go.mod are needed. This setting is most useful to check that go.mod does 41 | # not need updates, such as in a continuous integration and testing system. 42 | # If invoked with -mod=vendor, the go command assumes that the vendor 43 | # directory holds the correct copies of dependencies and ignores 44 | # the dependency descriptions in go.mod. 45 | # 46 | # Allowed values: readonly|vendor|mod 47 | # By default, it isn't set. 48 | # modules-download-mode: vendor 49 | 50 | # Allow multiple parallel golangci-lint instances running. 51 | # If false (default) - golangci-lint acquires file lock on start. 52 | allow-parallel-runners: false 53 | 54 | # Define the Go version limit. 55 | # Mainly related to generics support since go1.18. 56 | # Default: use Go version from the go.mod file, fallback on the env var `GOVERSION`, fallback on 1.18 57 | # go: '1.19' 58 | 59 | # output configuration options 60 | output: 61 | # Format: colored-line-number|line-number|json|tab|checkstyle|code-climate|junit-xml|github-actions 62 | # 63 | # Multiple can be specified by separating them by comma, output can be provided 64 | # for each of them by separating format name and path by colon symbol. 65 | # Output path can be either `stdout`, `stderr` or path to the file to write to. 66 | # Example: "checkstyle:report.json,colored-line-number" 67 | # 68 | # Default: colored-line-number 69 | formats: colored-line-number 70 | 71 | # Print lines of code with issue. 72 | # Default: true 73 | print-issued-lines: true 74 | 75 | # Print linter name in the end of issue text. 76 | # Default: true 77 | print-linter-name: true 78 | 79 | # Make issues output unique by line. 80 | # Default: true 81 | uniq-by-line: true 82 | 83 | # Add a prefix to the output file references. 84 | # Default is no prefix. 85 | path-prefix: "" 86 | 87 | # Sort results by: filepath, line and column. 88 | sort-results: true 89 | 90 | # All available settings of specific linters. 91 | linters-settings: 92 | asasalint: 93 | # To specify a set of function names to exclude. 94 | # The values are merged with the builtin exclusions. 95 | # The builtin exclusions can be disabled by setting `use-builtin-exclusions` to `false`. 96 | # Default: ["^(fmt|log|logger|t|)\.(Print|Fprint|Sprint|Fatal|Panic|Error|Warn|Warning|Info|Debug|Log)(|f|ln)$"] 97 | exclude: 98 | - Append 99 | - \.Wrapf 100 | # To enable/disable the asasalint builtin exclusions of function names. 101 | # See the default value of `exclude` to get the builtin exclusions. 102 | # Default: true 103 | use-builtin-exclusions: false 104 | # Ignore *_test.go files. 105 | # Default: false 106 | ignore-test: true 107 | 108 | bidichk: 109 | # The following configurations check for all mentioned invisible unicode runes. 110 | # All runes are enabled by default. 111 | left-to-right-embedding: false 112 | right-to-left-embedding: false 113 | pop-directional-formatting: false 114 | left-to-right-override: false 115 | right-to-left-override: false 116 | left-to-right-isolate: false 117 | right-to-left-isolate: false 118 | first-strong-isolate: false 119 | pop-directional-isolate: false 120 | 121 | cyclop: 122 | # The maximal code complexity to report. 123 | # Default: 10 124 | max-complexity: 10 125 | # The maximal average package complexity. 126 | # If it's higher than 0.0 (float) the check is enabled 127 | # Default: 0.0 128 | package-average: 6.0 129 | # Should ignore tests. 130 | # Default: false 131 | skip-tests: true 132 | 133 | decorder: 134 | # Required order of `type`, `const`, `var` and `func` declarations inside a file. 135 | # Default: types before constants before variables before functions. 136 | dec-order: 137 | - const 138 | - type 139 | - var 140 | - func 141 | 142 | # If true, order of declarations is not checked at all. 143 | # Default: true (disabled) 144 | disable-dec-order-check: false 145 | 146 | # If true, `init` func can be anywhere in file (does not have to be declared before all other functions). 147 | # Default: true (disabled) 148 | disable-init-func-first-check: false 149 | 150 | # If true, multiple global `type`, `const` and `var` declarations are allowed. 151 | # Default: true (disabled) 152 | disable-dec-num-check: true 153 | 154 | depguard: 155 | # Rules to apply. 156 | # 157 | # Variables: 158 | # - File Variables 159 | # you can still use and exclamation mark ! in front of a variable to say not to use it. 160 | # Example !$test will match any file that is not a go test file. 161 | # 162 | # `$all` - matches all go files 163 | # `$test` - matches all go test files 164 | # 165 | # - Package Variables 166 | # 167 | # `$gostd` - matches all of go's standard library (Pulled from `GOROOT`) 168 | # 169 | # Default: no rules. 170 | rules: 171 | # Name of a rule. 172 | main: 173 | # List of file globs that will match this list of settings to compare against. 174 | # Default: $all 175 | files: 176 | - "$all" 177 | - "!$test" 178 | # List of allowed packages. 179 | allow: 180 | - $gostd 181 | - dev.local/marcelloh 182 | - github.com/marcelloh/fastdb 183 | - github.com/a-h/templ 184 | - github.com/aead/chacha20poly1305 185 | - github.com/bytedance/sonic 186 | - github.com/go-playground/validator 187 | - github.com/google/uuid 188 | - github.com/icza/session 189 | - github.com/jmoiron/sqlx 190 | - github.com/knadh/koanf 191 | - github.com/lmittmann/tint 192 | - github.com/mattn/go-sqlite3 193 | - github.com/nutsdb/nutsdb 194 | - github.com/plandem/xlsx 195 | - github.com/sugawarayuuta/sonnet 196 | - github.com/tidwall/buntdb 197 | - github.com/uptrace/bunrouter 198 | - github.com/vk-rv/pvx 199 | # Packages that are not allowed where the value is a suggestion. 200 | deny: 201 | - pkg: "github.com/sirupsen/logrus" 202 | desc: not allowed - logging is allowed only by logutils.Log 203 | - pkg: "github.com/pkg/errors" 204 | desc: Should be replaced by standard lib errors package 205 | 206 | dogsled: 207 | # Checks assignments with too many blank identifiers. 208 | # Default: 2 209 | max-blank-identifiers: 2 210 | dupl: 211 | # Tokens count to trigger issue. 212 | # Default: 150 213 | threshold: 100 214 | 215 | dupword: 216 | # Keywords for detecting duplicate words. 217 | # If this list is not empty, only the words defined in this list will be detected. 218 | # Default: [] 219 | keywords: 220 | - "the" 221 | - "and" 222 | - "a" 223 | 224 | errcheck: 225 | # Report about not checking of errors in type assertions: `a := b.(MyStruct)`. 226 | # Such cases aren't reported by default. 227 | # Default: false 228 | check-type-assertions: true 229 | 230 | # Report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`. 231 | # Such cases aren't reported by default. 232 | # Default: false 233 | check-blank: false 234 | 235 | # DEPRECATED comma-separated list of pairs of the form pkg:regex 236 | # 237 | # the regex is used to ignore names within pkg. (default "fmt:.*"). 238 | # see https://github.com/kisielk/errcheck#the-deprecated-method for details 239 | # ignore: fmt:.*,io/ioutil:^Read.* 240 | 241 | # To disable the errcheck built-in exclude list. 242 | # See `-excludeonly` option in https://github.com/kisielk/errcheck#excluding-functions for details. 243 | # Default: false 244 | disable-default-exclusions: true 245 | 246 | # DEPRECATED use exclude-functions instead. 247 | # 248 | # Path to a file containing a list of functions to exclude from checking. 249 | # See https://github.com/kisielk/errcheck#excluding-functions for details. 250 | exclude: #/path/to/file.txt 251 | 252 | # List of functions to exclude from checking, where each entry is a single function to exclude. 253 | # See https://github.com/kisielk/errcheck#excluding-functions for details. 254 | exclude-functions: 255 | - io/ioutil.ReadFile 256 | - io.Copy(*bytes.Buffer) 257 | - io.Copy(os.Stdout) 258 | - fmt.Print 259 | - fmt.Printf 260 | - fmt.Println 261 | 262 | errchkjson: 263 | # With check-error-free-encoding set to true, errchkjson does warn about errors 264 | # from json encoding functions that are safe to be ignored, 265 | # because they are not possible to happen. 266 | # 267 | # if check-error-free-encoding is set to true and errcheck linter is enabled, 268 | # it is recommended to add the following exceptions to prevent from false positives: 269 | # 270 | # linters-settings: 271 | # errcheck: 272 | # exclude-functions: 273 | # - encoding/json.Marshal 274 | # - encoding/json.MarshalIndent 275 | # 276 | # Default: false 277 | check-error-free-encoding: true 278 | 279 | # Issue on struct encoding that doesn't have exported fields. 280 | # Default: false 281 | report-no-exported: false 282 | 283 | # path to a file containing a list of functions to exclude from checking 284 | # see https://github.com/kisielk/errcheck#excluding-functions for details 285 | # exclude: /path/to/file.txt 286 | errorlint: 287 | # Check whether fmt.Errorf uses the %w verb for formatting errors. 288 | # See the https://github.com/polyfloyd/go-errorlint for caveats. 289 | # Default: true 290 | errorf: true 291 | # Check for plain type assertions and type switches. 292 | # Default: true 293 | asserts: false 294 | # Check for plain error comparisons. 295 | # Default: true 296 | comparison: false 297 | 298 | exhaustive: 299 | # Program elements to check for exhaustiveness. 300 | # Default: [ switch ] 301 | # check switch statements in generated files also 302 | check: 303 | - switch 304 | - map 305 | 306 | # Check switch statements in generated files also. 307 | # Default: false 308 | check-generated: false 309 | 310 | # Presence of "default" case in switch statements satisfies exhaustiveness, 311 | # even if all enum members are not listed. 312 | # Default: false 313 | # indicates that switch statements are to be considered exhaustive if a 314 | default-signifies-exhaustive: false 315 | # Enum members matching the supplied regex do not have to be listed in 316 | # switch statements to satisfy exhaustiveness. 317 | # Default: "" 318 | ignore-enum-members: "Example.+" 319 | # Enum types matching the supplied regex do not have to be listed in 320 | # switch statements to satisfy exhaustiveness. 321 | # Default: "" 322 | ignore-enum-types: "Example.+" 323 | # Consider enums only in package scopes, not in inner scopes. 324 | # Default: false 325 | package-scope-only: true 326 | # Only run exhaustive check on switches with "//exhaustive:enforce" comment. 327 | # Default: false 328 | explicit-exhaustive-switch: true 329 | # Only run exhaustive check on map literals with "//exhaustive:enforce" comment. 330 | # Default: false 331 | explicit-exhaustive-map: true 332 | 333 | exhaustruct: 334 | # Struct Patterns is list of expressions to match struct packages and names. 335 | # The struct packages have the form `example.com/package.ExampleStruct`. 336 | # The matching patterns can use matching syntax from https://pkg.go.dev/path#Match. 337 | # If this list is empty, all structs are tested. 338 | # Default: [] 339 | include: 340 | # List of regular expressions to exclude struct packages and names from check. 341 | # Default: [] 342 | exclude: 343 | #- "*_test.go" # Skips all files that end with "_test.go". 344 | #- ".*\.Test" # Skips anything inside functions that start with Test 345 | forbidigo: 346 | # Forbid the following identifiers (list of regexp). 347 | # Default: ["^(fmt\\.Print(|f|ln)|print|println)$"] 348 | forbid: 349 | - ^print.*$ 350 | - 'fmt\.Print.*' 351 | # Optionally put comments at the end of the regex, surrounded by `(# )?` 352 | # Escape any special characters. 353 | - 'fmt\.Print.*(# Do not commit print statements\.)?' 354 | # Exclude godoc examples from forbidigo checks. 355 | # Default: true 356 | exclude_godoc_examples: false 357 | 358 | funlen: 359 | # Checks the number of lines in a function. 360 | # If lower than 0, disable the check. 361 | # Default: 60 362 | lines: 70 363 | # Checks the number of statements in a function. 364 | # If lower than 0, disable the check. 365 | # Default: 40 366 | statements: 45 367 | 368 | gci: 369 | # Section configuration to compare against. 370 | # Section names are case-insensitive and may contain parameters in (). 371 | # The default order of sections is `standard > default > custom > blank > dot`, 372 | # If `custom-order` is `true`, it follows the order of `sections` option. 373 | # Default: ["standard", "default"] 374 | sections: 375 | - standard # Captures all standard packages if they do not match another section. 376 | - default # Contains all imports that could not be matched to another section type. 377 | - prefix(dev.local/marcelloh/MHGoLib) # Groups all imports with the specified Prefix. 378 | - prefix(dev.local/marcelloh/SailboatCompanion) # Groups all imports with the specified Prefix. 379 | 380 | # Skip generated files. 381 | # Default: true 382 | skip-generated: false 383 | 384 | # Enable custom order of sections. 385 | # If `true`, make the section order the same as the order of `sections`. 386 | # Default: false 387 | custom-order: true 388 | 389 | ginkgolinter: 390 | # Suppress the wrong length assertion warning. 391 | # Default: false 392 | suppress-len-assertion: true 393 | 394 | # Suppress the wrong nil assertion warning. 395 | # Default: false 396 | suppress-nil-assertion: true 397 | 398 | # Suppress the wrong error assertion warning. 399 | # Default: false 400 | suppress-err-assertion: true 401 | 402 | gocognit: 403 | # Minimal code complexity to report. 404 | # Default: 30 (but we recommend 10-20) 405 | min-complexity: 10 406 | 407 | goconst: 408 | # Minimal length of string constant. 409 | # Default: 3 410 | min-len: 3 411 | # Minimum occurrences of constant string count to trigger issue. 412 | # Default: 3 413 | min-occurrences: 10 414 | # Ignore test files. 415 | # Default: false 416 | ignore-tests: false 417 | # Look for existing constants matching the values. 418 | # Default: true 419 | match-constant: false 420 | # Search also for duplicated numbers. 421 | # Default: false 422 | numbers: true 423 | # Minimum value, only works with goconst.numbers 424 | # Default: 3 425 | min: 2 426 | # Maximum value, only works with goconst.numbers 427 | # Default: 3 428 | max: 2 429 | # Ignore when constant is not used as function argument. 430 | # Default: true 431 | ignore-calls: false 432 | 433 | gocritic: 434 | # Which checks should be enabled; can't be combined with 'disabled-checks'. 435 | # See https://go-critic.github.io/overview#checks-overview. 436 | # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run`. 437 | # By default, list of stable checks is used. 438 | enabled-checks: 439 | 440 | # Which checks should be disabled; can't be combined with 'enabled-checks'. 441 | # Default: [] 442 | disabled-checks: 443 | #- regexpMust 444 | #- dupImport # https://github.com/go-critic/go-critic/issues/845 445 | #- ifElseChain 446 | #- octalLiteral 447 | #- whyNoLint 448 | #- wrapperFunc 449 | #- performance 450 | 451 | # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint run` to see all tags and checks. 452 | # See https://github.com/go-critic/go-critic#usage -> section "Tags". 453 | # Default: [] 454 | enabled-tags: 455 | - diagnostic 456 | - style 457 | - performance 458 | - experimental 459 | - opinionated 460 | disabled-tags: 461 | # Settings passed to gocritic. 462 | # The settings key is the name of a supported gocritic checker. 463 | # The list of supported checkers can be find in https://go-critic.github.io/overview. 464 | settings: 465 | # Must be valid enabled check name. 466 | captLocal: 467 | # Whether to restrict checker to params only. 468 | # Default: true 469 | paramsOnly: false 470 | commentedOutCode: 471 | # Min length of the comment that triggers a warning. 472 | # Default: 15 473 | minLength: 80 474 | elseif: 475 | # Whether to skip balanced if-else pairs. 476 | # Default: true 477 | skipBalanced: false 478 | hugeParam: 479 | # Size in bytes that makes the warning trigger. 480 | # Default: 80 481 | sizeThreshold: 70 482 | nestingReduce: 483 | # Min number of statements inside a branch to trigger a warning. 484 | # Default: 5 485 | bodyWidth: 4 486 | rangeExprCopy: 487 | # Size in bytes that makes the warning trigger. 488 | # Default: 512 489 | sizeThreshold: 516 490 | # Whether to check test functions 491 | # Default: true 492 | skipTestFuncs: false 493 | rangeValCopy: 494 | # Size in bytes that makes the warning trigger. 495 | # Default: 128 496 | sizeThreshold: 32 497 | # Whether to check test functions. 498 | # Default: true 499 | skipTestFuncs: false 500 | ruleguard: 501 | # Enable debug to identify which 'Where' condition was rejected. 502 | # The value of the parameter is the name of a function in a ruleguard file. 503 | # 504 | # When a rule is evaluated: 505 | # If: 506 | # The Match() clause is accepted; and 507 | # One of the conditions in the Where() clause is rejected, 508 | # Then: 509 | # ruleguard prints the specific Where() condition that was rejected. 510 | # 511 | # The flag is passed to the ruleguard 'debug-group' argument. 512 | # Default: "" 513 | debug: 'emptyDecl' 514 | # Deprecated, use 'failOn' param. 515 | # If set to true, identical to failOn='all', otherwise failOn='' 516 | failOnError: false 517 | # Determines the behavior when an error occurs while parsing ruleguard files. 518 | # If flag is not set, log error and skip rule files that contain an error. 519 | # If flag is set, the value must be a comma-separated list of error conditions. 520 | # - 'all': fail on all errors. 521 | # - 'import': ruleguard rule imports a package that cannot be found. 522 | # - 'dsl': gorule file does not comply with the ruleguard DSL. 523 | # Default: "" 524 | failOn: dsl 525 | # Comma-separated list of file paths containing ruleguard rules. 526 | # If a path is relative, it is relative to the directory where the golangci-lint command is executed. 527 | # The special '${configDir}' variable is substituted with the absolute directory containing the golangci config file. 528 | # Glob patterns such as 'rules-*.go' may be specified. 529 | # Default: "" 530 | rules: #'${configDir}/ruleguard/rules-*.go,${configDir}/myrule1.go' 531 | # Comma-separated list of enabled groups or skip empty to enable everything. 532 | # Tags can be defined with # character prefix. 533 | # Default: "" 534 | enable: "" #"myGroupName,#myTagName" 535 | # Comma-separated list of disabled groups or skip empty to enable everything. 536 | # Tags can be defined with # character prefix. 537 | # Default: "" 538 | disable: "" #"myGroupName,#myTagName" 539 | tooManyResultsChecker: 540 | # Maximum number of results. 541 | # Default: 5 542 | maxResults: 10 543 | truncateCmp: 544 | # Whether to skip int/uint/uintptr types. 545 | # Default: true 546 | skipArchDependent: false 547 | underef: 548 | # Whether to skip (*x).method() calls where x is a pointer receiver. 549 | # Default: true 550 | skipRecvDeref: false 551 | unnamedResult: 552 | # Whether to check exported functions. 553 | # Default: false 554 | checkExported: true 555 | 556 | gocyclo: 557 | # Minimal code complexity to report. 558 | # Default: 30 (but we recommend 10-20) 559 | min-complexity: 9 560 | godot: 561 | # Comments to be checked: `declarations`, `toplevel`, or `all`. 562 | # Default: declarations 563 | scope: toplevel 564 | # List of regexps for excluding particular comment lines from check. 565 | # Default: [] 566 | exclude: 567 | # Exclude todo and fixme comments. 568 | - "^fixme:" 569 | - "^todo:" 570 | # Check that each sentence ends with a period. 571 | # Default: true 572 | period: false 573 | # Check that each sentence starts with a capital letter. 574 | # Default: false 575 | capital: true 576 | 577 | godox: 578 | # Report any comments starting with keywords, this is useful for TODO or FIXME comments that 579 | # might be left in the code accidentally and should be resolved before merging. 580 | # Default: ["TODO", "BUG", "FIXME"] 581 | keywords: 582 | # - TODO 583 | # - OPTIMIZE # marks code that should be optimized before merging 584 | # - HACK # marks hack-arounds that should be removed before merging 585 | 586 | gofmt: 587 | # Simplify code: gofmt with `-s` option. 588 | # Default: true 589 | simplify: true 590 | # Apply the rewrite rules to the source before reformatting. 591 | # https://pkg.go.dev/cmd/gofmt 592 | # Default: [] 593 | rewrite-rules: 594 | - pattern: 'interface{}' 595 | replacement: 'any' 596 | - pattern: 'a[b:len(a)]' 597 | replacement: 'a[b:]' 598 | 599 | gofumpt: 600 | # Select the Go version to target. 601 | # Default: "1.15" 602 | # Deprecated: use the global `run.go` instead. 603 | 604 | # Module path which contains the source code being formatted. 605 | # Default: "" 606 | 607 | # Choose whether to use the extra rules. 608 | # Default: false 609 | extra-rules: false 610 | goheader: 611 | # Supports two types 'const` and `regexp`. 612 | # Values can be used recursively. 613 | # Default: {} 614 | values: 615 | const: 616 | # Define here const type values in format k:v. 617 | # For example: 618 | COMPANY: MY COMPANY 619 | regexp: 620 | # Define here regexp type values. 621 | # for example: 622 | AUTHOR: .*@mycompany\.com 623 | # The template use for checking. 624 | # Default: "" 625 | template: |- 626 | # Put here copyright header template for source code files 627 | # For example: 628 | # Note: {{ YEAR }} is a builtin value that returns the year relative to the current machine time. 629 | # 630 | # {{ AUTHOR }} {{ COMPANY }} {{ YEAR }} 631 | # SPDX-License-Identifier: Apache-2.0 632 | 633 | # Licensed under the Apache License, Version 2.0 (the "License"); 634 | # you may not use this file except in compliance with the License. 635 | # You may obtain a copy of the License at: 636 | 637 | # http://www.apache.org/licenses/LICENSE-2.0 638 | 639 | # Unless required by applicable law or agreed to in writing, software 640 | # distributed under the License is distributed on an "AS IS" BASIS, 641 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 642 | # See the License for the specific language governing permissions and 643 | # limitations under the License. 644 | # As alternative of directive 'template', you may put the path to file with the template source. 645 | # Useful if you need to load the template from a specific file. 646 | # Default: "" 647 | 648 | goimports: 649 | # A comma-separated list of prefixes, which, if set, checks import paths 650 | # with the given prefixes are grouped after 3rd-party packages. 651 | # Default: "" 652 | local-prefixes: dev.local/marcelloh/SailboatCompanion 653 | golint: 654 | # Minimal confidence for issues. 655 | # Default: 0.8 656 | min-confidence: 0.7 657 | 658 | gomoddirectives: 659 | # Allow local `replace` directives. 660 | # Default: false 661 | replace-local: false 662 | # List of allowed `replace` directives. 663 | # Default: [] 664 | replace-allow-list: 665 | - launchpad.net/gocheck 666 | # Allow to not explain why the version has been retracted in the `retract` directives. 667 | # Default: false 668 | retract-allow-no-explanation: false 669 | # Forbid the use of the `exclude` directives. 670 | # Default: false 671 | exclude-forbidden: false 672 | 673 | gomodguard: 674 | allowed: 675 | # List of allowed modules. 676 | # Default: [] 677 | modules: 678 | 679 | # List of allowed module domains. 680 | # Default: [] 681 | domains: 682 | blocked: 683 | # List of blocked modules. 684 | # Default: [] 685 | modules: 686 | - github.com/golang/protobuf: 687 | recommendations: 688 | - google.golang.org/protobuf 689 | # Reason why the recommended module should be used. (Optional) 690 | reason: "see https://developers.google.com/protocol-buffers/docs/reference/go/faq#modules" 691 | - github.com/satori/go.uuid: 692 | recommendations: 693 | - github.com/google/uuid 694 | reason: "satori's package is not maintained" 695 | - github.com/gofrs/uuid: 696 | recommendations: 697 | - github.com/google/uuid 698 | reason: "see recommendation from dev-infra team: https://confluence.gtforge.com/x/gQI6Aw" 699 | # List of blocked module version constraints. 700 | # Default: [] 701 | versions: 702 | 703 | # Set to true to raise lint issues for packages that are loaded from a local path via replace directive. 704 | # Default: false 705 | local_replace_directives: true # default false 706 | gosec: 707 | # To select a subset of rules to run. 708 | # Available rules: https://github.com/securego/gosec#available-rules 709 | # Default: [] - means include all rules 710 | includes: 711 | 712 | # To specify a set of rules to explicitly exclude. 713 | # Available rules: https://github.com/securego/gosec#available-rules 714 | # Default: [] 715 | excludes: 716 | 717 | # Exclude generated files 718 | # Default: false 719 | exclude-generated: true 720 | 721 | # Filter out the issues with a lower severity than the given value. 722 | # Valid options are: low, medium, high. 723 | # Default: low 724 | severity: medium 725 | 726 | # Filter out the issues with a lower confidence than the given value. 727 | # Valid options are: low, medium, high. 728 | # Default: low 729 | confidence: medium 730 | 731 | # Concurrency value. 732 | # Default: the number of logical CPUs usable by the current process. 733 | #concurrency: 12 734 | 735 | gosimple: 736 | # Select the Go version to target. 737 | # Default: 1.13 738 | # Deprecated: use the global `run.go` instead. 739 | # go: "1.15" 740 | # Sxxxx checks in https://staticcheck.io/docs/configuration/options/#checks 741 | # Default: ["*"] 742 | checks: [ "*" ] 743 | 744 | govet: 745 | settings: 746 | printf: 747 | funcs: 748 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof 749 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf 750 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf 751 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf 752 | enable: 753 | - nilness 754 | - shadow 755 | 756 | grouper: 757 | # Require the use of a single global 'const' declaration only. 758 | # Default: false 759 | const-require-single-const: false 760 | # Require the use of grouped global 'const' declarations. 761 | # Default: false 762 | const-require-grouping: false 763 | 764 | # Require the use of a single 'import' declaration only. 765 | # Default: false 766 | import-require-single-import: true 767 | # Require the use of grouped 'import' declarations. 768 | # Default: false 769 | import-require-grouping: false 770 | 771 | # Require the use of a single global 'type' declaration only. 772 | # Default: false 773 | type-require-single-type: false 774 | # Require the use of grouped global 'type' declarations. 775 | # Default: false 776 | type-require-grouping: false 777 | 778 | # Require the use of a single global 'var' declaration only. 779 | # Default: false 780 | var-require-single-var: false 781 | # Require the use of grouped global 'var' declarations. 782 | # Default: false 783 | var-require-grouping: false 784 | 785 | ifshort: 786 | # Maximum length of variable declaration measured in number of lines, after which linter won't suggest using short syntax. 787 | # Has higher priority than max-decl-chars. 788 | # Default: 1 789 | max-decl-lines: 2 790 | # Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax. 791 | # Default: 30 792 | max-decl-chars: 40 793 | 794 | importas: 795 | # Do not allow unaliased imports of aliased packages. 796 | # Default: false 797 | no-unaliased: false 798 | # Do not allow non-required aliases. 799 | # Default: false 800 | no-extra-aliases: true 801 | # List of aliases 802 | # Default: [] 803 | alias: 804 | # Using `servingv1` alias for `knative.dev/serving/pkg/apis/serving/v1` package. 805 | - pkg: knative.dev/serving/pkg/apis/serving/v1 806 | alias: servingv1 807 | # Using `autoscalingv1alpha1` alias for `knative.dev/serving/pkg/apis/autoscaling/v1alpha1` package. 808 | - pkg: knative.dev/serving/pkg/apis/autoscaling/v1alpha1 809 | alias: autoscalingv1alpha1 810 | # You can specify the package path by regular expression, 811 | # and alias by regular expression expansion syntax like below. 812 | # see https://github.com/julz/importas#use-regular-expression for details 813 | - pkg: knative.dev/serving/pkg/apis/(\w+)/(v[\w\d]+) 814 | alias: $1$2 815 | 816 | interfacebloat: 817 | # The maximum number of methods allowed for an interface. 818 | # Default: 10 819 | max: 8 820 | 821 | ireturn: 822 | # ireturn allows using `allow` and `reject` settings at the same time. 823 | # Both settings are lists of the keywords and regular expressions matched to interface or package names. 824 | # keywords: 825 | # - `empty` for `interface{}` 826 | # - `error` for errors 827 | # - `stdlib` for standard library 828 | # - `anon` for anonymous interfaces 829 | 830 | # By default, it allows using errors, empty interfaces, anonymous interfaces, 831 | # and interfaces provided by the standard library. 832 | allow: 833 | - anon 834 | - error 835 | - empty 836 | - stdlib 837 | # You can specify idiomatic endings for interface 838 | - (or|er)$ 839 | 840 | # reject-list of interfaces 841 | reject: 842 | - github.com\/user\/package\/v4\.Type 843 | 844 | lll: 845 | # Max line length, lines longer will be reported. 846 | # '\t' is counted as 1 character by default, and can be changed with the tab-width option. 847 | # Default: 120. 848 | line-length: 130 849 | # Tab width in spaces. 850 | # Default: 1 851 | tab-width: 1 852 | maintidx: 853 | # Show functions with maintainability index lower than N. 854 | # A high index indicates better maintainability (it's kind of the opposite of complexity). 855 | # Default: 20 856 | under: 10 857 | makezero: 858 | # Allow only slices initialized with a length of zero. 859 | # Default: false 860 | always: false 861 | 862 | maligned: 863 | # Print struct with more effective memory layout or not. 864 | # Default: false 865 | suggest-new: false 866 | 867 | misspell: 868 | # Correct spellings using locale preferences for US or UK. 869 | # Setting locale to US will correct the British spelling of 'colour' to 'color'. 870 | # Default is to use a neutral variety of English. 871 | locale: UK 872 | # Default: [] 873 | ignore-words: 874 | - catalogs 875 | - catalog 876 | 877 | mnd: 878 | # List of enabled checks, see https://github.com/tommy-muehle/go-mnd/#checks for description. 879 | # Default: ["argument", "case", "condition", "operation", "return", "assign"] 880 | checks: 881 | - argument 882 | - case 883 | - condition 884 | - operation 885 | - return 886 | - assign 887 | # List of numbers to exclude from analysis. 888 | # The numbers should be written as string. 889 | # Values always ignored: "1", "1.0", "0" and "0.0" 890 | # Default: [] 891 | ignored-numbers: 892 | - '2' 893 | - '3' 894 | - '5' 895 | - '10' 896 | - '51' 897 | - '64' 898 | - '0600' 899 | - '0666' 900 | # List of file patterns to exclude from analysis. 901 | # Values always ignored: `.+_test.go` 902 | # Default: [] 903 | ignored-files: 904 | 905 | # List of function patterns to exclude from analysis. 906 | # Following functions are always ignored: `time.Date`, 907 | # `strconv.FormatInt`, `strconv.FormatUint`, `strconv.FormatFloat`, 908 | # `strconv.ParseInt`, `strconv.ParseUint`, `strconv.ParseFloat`. 909 | # Default: [] 910 | ignored-functions: 911 | 912 | nakedret: 913 | # Make an issue if func has more lines of code than this setting, and it has naked returns. 914 | # Default: 30 915 | max-func-lines: 30 916 | nestif: 917 | # Minimal complexity of if statements to report. 918 | # Default: 5 919 | min-complexity: 4 920 | 921 | nilnil: 922 | # Checks that there is no simultaneous return of `nil` error and an invalid value. 923 | # Default: ["ptr", "func", "iface", "map", "chan"] 924 | checked-types: 925 | - ptr 926 | - func 927 | - iface 928 | - map 929 | - chan 930 | 931 | nlreturn: 932 | # Size of the block (including return statement that is still "OK") 933 | # so no return split required. 934 | # Default: 1 935 | block-size: 3 936 | 937 | nolintlint: 938 | # Disable to ensure that all nolint directives actually have an effect. 939 | # Default: false 940 | allow-unused: false 941 | # Exclude following linters from requiring an explanation. 942 | # Default: [] 943 | allow-no-explanation: [ ] 944 | # Enable to require an explanation of nonzero length after each nolint directive. 945 | # Default: false 946 | require-explanation: true 947 | # Enable to require nolint directives to mention the specific linter being suppressed. 948 | # Default: false 949 | require-specific: true 950 | # Disable to ensure that nolint directives don't have a leading space. Default is true. 951 | allow-leading-space: true 952 | nonamedreturns: 953 | # Report named error if it is assigned inside defer. 954 | # Default: false 955 | report-error-in-defer: true 956 | 957 | paralleltest: 958 | # Ignore missing calls to `t.Parallel()` and only report incorrect uses of it. 959 | # Default: false 960 | ignore-missing: true 961 | prealloc: 962 | # IMPORTANT: we don't recommend using this linter before doing performance profiling. 963 | # For most programs usage of prealloc will be a premature optimization. 964 | 965 | # Report pre-allocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. 966 | # Default: true 967 | simple: true 968 | # Report pre-allocation suggestions on range loops. 969 | # Default: true 970 | range-loops: true 971 | # Report pre-allocation suggestions on for loops. 972 | # Default: false 973 | for-loops: true 974 | 975 | predeclared: 976 | # Comma-separated list of predeclared identifiers to not report on. 977 | # Default: "" 978 | ignore: "new,int" 979 | # Include method names and field names (i.e., qualified names) in checks. 980 | # Default: false 981 | q: true 982 | 983 | promlinter: 984 | # Promlinter cannot infer all metrics name in static analysis. 985 | # Enable strict mode will also include the errors caused by failing to parse the args. 986 | # Default: false 987 | strict: true 988 | # Please refer to https://github.com/yeya24/promlinter#usage for detailed usage. 989 | # Default: [] 990 | disabled-linters: 991 | - Help 992 | - MetricUnits 993 | - Counter 994 | - HistogramSummaryReserved 995 | - MetricTypeInName 996 | - ReservedChars 997 | - CamelCase 998 | - UnitAbbreviations 999 | 1000 | reassign: 1001 | # Patterns for global variable names that are checked for reassignment. 1002 | # See https://github.com/curioswitch/go-reassign#usage 1003 | # Default: ["EOF", "Err.*"] 1004 | patterns: 1005 | - ".*" 1006 | 1007 | revive: 1008 | # Maximum number of open files at the same time. 1009 | # See https://github.com/mgechev/revive#command-line-flags 1010 | # Defaults to unlimited. 1011 | max-open-files: 2048 1012 | 1013 | # When set to false, ignores files with "GENERATED" header, similar to golint. 1014 | # See https://github.com/mgechev/revive#available-rules for details. 1015 | # Default: false 1016 | ignore-generated-header: true 1017 | 1018 | # Sets the default severity. 1019 | # See https://github.com/mgechev/revive#configuration 1020 | # Default: warning 1021 | severity: error 1022 | 1023 | # Enable all available rules. 1024 | # Default: false 1025 | enable-all-rules: true 1026 | 1027 | # Sets the default failure confidence. 1028 | # This means that linting errors with less than 0.8 confidence will be ignored. 1029 | # Default: 0.8 1030 | confidence: 0.1 1031 | 1032 | rules: 1033 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#add-constant 1034 | - name: add-constant 1035 | severity: warning 1036 | disabled: false 1037 | arguments: 1038 | - maxLitCount: "5" 1039 | allowStrs: '"%v",""' 1040 | allowInts: "0,1,2,3,4,5,6,10,60,600" 1041 | allowFloats: "0.0,0.,1.0,1.,2.0,2." 1042 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#argument-limit 1043 | - name: argument-limit 1044 | severity: warning 1045 | disabled: false 1046 | arguments: [ 6 ] 1047 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#atomic 1048 | - name: atomic 1049 | severity: warning 1050 | disabled: false 1051 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#banned-characters 1052 | - name: banned-characters 1053 | severity: warning 1054 | disabled: false 1055 | arguments: [ "Ω","Σ","σ" ] 1056 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#bare-return 1057 | - name: bare-return 1058 | severity: warning 1059 | disabled: false 1060 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#blank-imports 1061 | - name: blank-imports 1062 | severity: warning 1063 | disabled: false 1064 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#bool-literal-in-expr 1065 | - name: bool-literal-in-expr 1066 | severity: warning 1067 | disabled: false 1068 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#call-to-gc 1069 | - name: call-to-gc 1070 | severity: warning 1071 | disabled: false 1072 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#cognitive-complexity 1073 | - name: cognitive-complexity 1074 | severity: warning 1075 | disabled: false 1076 | arguments: [ 10 ] 1077 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#confusing-naming 1078 | - name: confusing-naming 1079 | severity: warning 1080 | disabled: false 1081 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#confusing-results 1082 | - name: confusing-results 1083 | severity: warning 1084 | disabled: false 1085 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#comment-spacings 1086 | - name: comment-spacings 1087 | severity: warning 1088 | arguments: 1089 | - nolint 1090 | disabled: false 1091 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#constant-logical-expr 1092 | - name: constant-logical-expr 1093 | severity: warning 1094 | disabled: false 1095 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#context-as-argument 1096 | - name: context-as-argument 1097 | severity: warning 1098 | disabled: false 1099 | arguments: 1100 | - allowTypesBefore: "*testing.T,*github.com/user/repo/testing.Harness" 1101 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#context-keys-type 1102 | - name: context-keys-type 1103 | severity: warning 1104 | disabled: false 1105 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#cyclomatic 1106 | - name: cyclomatic 1107 | severity: warning 1108 | disabled: false 1109 | arguments: [ 10 ] 1110 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#datarace 1111 | - name: datarace 1112 | severity: warning 1113 | disabled: false 1114 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#deep-exit 1115 | - name: deep-exit 1116 | severity: warning 1117 | disabled: true 1118 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#defer 1119 | - name: defer 1120 | severity: warning 1121 | disabled: false 1122 | arguments: 1123 | # - [ "call-chain", "loop" ] 1124 | - [ "loop" ] 1125 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#dot-imports 1126 | - name: dot-imports 1127 | severity: warning 1128 | disabled: false 1129 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#duplicated-imports 1130 | - name: duplicated-imports 1131 | severity: warning 1132 | disabled: false 1133 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#early-return 1134 | - name: early-return 1135 | severity: warning 1136 | disabled: false 1137 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#empty-block 1138 | - name: empty-block 1139 | severity: warning 1140 | disabled: false 1141 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#empty-lines 1142 | - name: empty-lines 1143 | severity: warning 1144 | disabled: false 1145 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#error-naming 1146 | - name: error-naming 1147 | severity: warning 1148 | disabled: false 1149 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#error-return 1150 | - name: error-return 1151 | severity: warning 1152 | disabled: false 1153 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#error-strings 1154 | - name: error-strings 1155 | severity: warning 1156 | disabled: true 1157 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#errorf 1158 | - name: errorf 1159 | severity: warning 1160 | disabled: false 1161 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#exported 1162 | - name: exported 1163 | severity: warning 1164 | disabled: false 1165 | arguments: 1166 | - "checkPrivateReceivers" 1167 | - "sayRepetitiveInsteadOfStutters" 1168 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#file-header 1169 | - name: file-header 1170 | severity: warning 1171 | disabled: true 1172 | arguments: 1173 | - This is the text that must appear at the top of source files. 1174 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#flag-parameter 1175 | - name: flag-parameter 1176 | severity: warning 1177 | disabled: true 1178 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#function-result-limit 1179 | - name: function-result-limit 1180 | severity: warning 1181 | disabled: false 1182 | arguments: [ 3 ] 1183 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#function-length 1184 | - name: function-length 1185 | severity: warning 1186 | disabled: false 1187 | arguments: [ 40, 60 ] 1188 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#get-return 1189 | - name: get-return 1190 | severity: warning 1191 | disabled: true 1192 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#identical-branches 1193 | - name: identical-branches 1194 | severity: warning 1195 | disabled: false 1196 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#if-return 1197 | - name: if-return 1198 | severity: warning 1199 | disabled: false 1200 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#increment-decrement 1201 | - name: increment-decrement 1202 | severity: warning 1203 | disabled: false 1204 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#indent-error-flow 1205 | - name: indent-error-flow 1206 | severity: warning 1207 | disabled: false 1208 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#import-alias-naming 1209 | - name: import-alias-naming 1210 | arguments: "^[a-z][a-zA-Z0-9]*$" 1211 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#imports-blacklist 1212 | - name: imports-blacklist 1213 | severity: warning 1214 | disabled: false 1215 | arguments: 1216 | - "crypto/md5" 1217 | - "crypto/sha1" 1218 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#import-shadowing 1219 | - name: import-shadowing 1220 | severity: warning 1221 | disabled: false 1222 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#line-length-limit 1223 | - name: line-length-limit 1224 | severity: warning 1225 | disabled: false 1226 | arguments: [ 135 ] 1227 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#max-public-structs 1228 | - name: max-public-structs 1229 | severity: warning 1230 | disabled: false 1231 | arguments: [ 4 ] 1232 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#modifies-parameter 1233 | - name: modifies-parameter 1234 | severity: warning 1235 | disabled: false 1236 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#modifies-value-receiver 1237 | - name: modifies-value-receiver 1238 | severity: warning 1239 | disabled: false 1240 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#nested-structs 1241 | - name: nested-structs 1242 | severity: warning 1243 | disabled: false 1244 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#optimize-operands-order 1245 | - name: optimize-operands-order 1246 | severity: warning 1247 | disabled: true 1248 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#package-comments 1249 | - name: package-comments 1250 | severity: warning 1251 | disabled: false 1252 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#range 1253 | - name: range 1254 | severity: warning 1255 | disabled: false 1256 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#range-val-in-closure 1257 | - name: range-val-in-closure 1258 | severity: warning 1259 | disabled: false 1260 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#range-val-address 1261 | - name: range-val-address 1262 | severity: warning 1263 | disabled: false 1264 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#receiver-naming 1265 | - name: receiver-naming 1266 | severity: warning 1267 | disabled: false 1268 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#redefines-builtin-id 1269 | - name: redefines-builtin-id 1270 | severity: warning 1271 | disabled: false 1272 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#string-of-int 1273 | - name: string-of-int 1274 | severity: warning 1275 | disabled: false 1276 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#string-format 1277 | - name: string-format 1278 | severity: warning 1279 | disabled: false 1280 | arguments: 1281 | - - 'core.WriteError[1].Message' 1282 | - '/^([^A-Z]|$)/' 1283 | - must not start with a capital letter 1284 | - - 'fmt.Errorf[0]' 1285 | - '/(^|[^\.!?])$/' 1286 | - must not end in punctuation 1287 | - - panic 1288 | - '/^[^\n]*$/' 1289 | - must not contain line breaks 1290 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#struct-tag 1291 | - name: struct-tag 1292 | severity: warning 1293 | disabled: false 1294 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#superfluous-else 1295 | - name: superfluous-else 1296 | severity: warning 1297 | disabled: false 1298 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#time-equal 1299 | - name: time-equal 1300 | severity: warning 1301 | disabled: false 1302 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#time-naming 1303 | - name: time-naming 1304 | severity: warning 1305 | disabled: false 1306 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#var-naming 1307 | - name: var-naming 1308 | severity: warning 1309 | disabled: false 1310 | arguments: 1311 | - [ "ID" ] # AllowList 1312 | - [ "VM" ] # DenyList 1313 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#var-declaration 1314 | - name: var-declaration 1315 | severity: warning 1316 | disabled: false 1317 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unconditional-recursion 1318 | - name: unconditional-recursion 1319 | severity: warning 1320 | disabled: false 1321 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unexported-naming 1322 | - name: unexported-naming 1323 | severity: warning 1324 | disabled: false 1325 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unexported-return 1326 | - name: unexported-return 1327 | severity: warning 1328 | disabled: false 1329 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unhandled-error 1330 | - name: unhandled-error 1331 | severity: warning 1332 | disabled: false 1333 | arguments: 1334 | - "fmt.Print" 1335 | - "fmt.Printf" 1336 | - "fmt.Println" 1337 | - "myFunction" 1338 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unnecessary-stmt 1339 | - name: unnecessary-stmt 1340 | severity: warning 1341 | disabled: false 1342 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unreachable-code 1343 | - name: unreachable-code 1344 | severity: warning 1345 | disabled: false 1346 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unused-parameter 1347 | - name: unused-parameter 1348 | severity: warning 1349 | disabled: false 1350 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unused-receiver 1351 | - name: unused-receiver 1352 | severity: warning 1353 | disabled: false 1354 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#useless-break 1355 | - name: useless-break 1356 | severity: warning 1357 | disabled: false 1358 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#waitgroup-by-value 1359 | - name: waitgroup-by-value 1360 | severity: warning 1361 | disabled: false 1362 | 1363 | rowserrcheck: 1364 | # database/sql is always checked 1365 | # Default: [] 1366 | packages: 1367 | - github.com/jmoiron/sqlx 1368 | 1369 | staticcheck: 1370 | # Select the Go version to target. 1371 | # Default: "1.13" 1372 | # Deprecated: use the global `run.go` instead. 1373 | # go: "1.15" 1374 | # SAxxxx checks in https://staticcheck.io/docs/configuration/options/#checks 1375 | # Default: ["*"] 1376 | checks: [ "*" ] 1377 | stylecheck: 1378 | # Select the Go version to target. 1379 | # Default: 1.13 1380 | # Deprecated: use the global `run.go` instead. 1381 | # go: "1.15" 1382 | # STxxxx checks in https://staticcheck.io/docs/configuration/options/#checks 1383 | # Default: ["*"] 1384 | checks: ["*"] 1385 | # https://staticcheck.io/docs/configuration/options/#dot_import_whitelist 1386 | # Default: ["github.com/mmcloughlin/avo/build", "github.com/mmcloughlin/avo/operand", "github.com/mmcloughlin/avo/reg"] 1387 | dot-import-whitelist: 1388 | - fmt 1389 | # https://staticcheck.io/docs/configuration/options/#initialisms 1390 | # Default: ["ACL", "API", "ASCII", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", "ID", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS", "SIP", "RTP", "AMQP", "DB", "TS"] 1391 | initialisms: [ "ACL", "API", "ASCII", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", "ID", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS", "SIP", "RTP", "AMQP", "DB", "TS" ] 1392 | # https://staticcheck.io/docs/configuration/options/#http_status_code_whitelist 1393 | # Default: ["200", "400", "404", "500"] 1394 | http-status-code-whitelist: [ "200", "400", "404", "500" ] 1395 | 1396 | tagalign: 1397 | # Align and sort can be used together or separately. 1398 | # 1399 | # Whether enable align. If true, the struct tags will be aligned. 1400 | # eg: 1401 | # type FooBar struct { 1402 | # Bar string `json:"bar" validate:"required"` 1403 | # FooFoo int8 `json:"foo_foo" validate:"required"` 1404 | # } 1405 | # will be formatted to: 1406 | # type FooBar struct { 1407 | # Bar string `json:"bar" validate:"required"` 1408 | # FooFoo int8 `json:"foo_foo" validate:"required"` 1409 | # } 1410 | # Default: true. 1411 | align: false 1412 | # Whether enable tags sort. 1413 | # If true, the tags will be sorted by name in ascending order. 1414 | # eg: `xml:"bar" json:"bar" validate:"required"` -> `json:"bar" validate:"required" xml:"bar"` 1415 | # Default: true 1416 | sort: false 1417 | # Specify the order of tags, the other tags will be sorted by name. 1418 | # This option will be ignored if `sort` is false. 1419 | # Default: [] 1420 | order: 1421 | - json 1422 | - yaml 1423 | - yml 1424 | - toml 1425 | - mapstructure 1426 | - binding 1427 | - validate 1428 | tagliatelle: 1429 | # Check the struct tag name case. 1430 | case: 1431 | # Use the struct field name to check the name of the struct tag. 1432 | # Default: false 1433 | use-field-name: true 1434 | # `camel` is used for `json` and `yaml` (can be overridden) 1435 | # Default: {} 1436 | rules: 1437 | # Any struct tag type can be used. 1438 | # Support string case: `camel`, `pascal`, `kebab`, `snake`, `goCamel`, `goPascal`, `goKebab`, `goSnake`, `upper`, `lower` 1439 | json: camel 1440 | yaml: camel 1441 | xml: camel 1442 | bson: camel 1443 | avro: snake 1444 | mapstructure: kebab 1445 | 1446 | tenv: 1447 | # The option `all` will run against whole test files (`_test.go`) regardless of method/function signatures. 1448 | # Otherwise, only methods that take `*testing.T`, `*testing.B`, and `testing.TB` as arguments are checked. 1449 | # Default: false 1450 | all: false 1451 | 1452 | testpackage: 1453 | # Regexp pattern to skip files. 1454 | # Default: "(export|internal)_test\\.go" 1455 | skip-regexp: (export|internal)_test\.go 1456 | # List of packages that don't end with _test that tests are allowed to be in. 1457 | # Default: "main" 1458 | allow-packages: 1459 | - example 1460 | - main 1461 | 1462 | thelper: 1463 | test: 1464 | # Check *testing.T is first param (or after context.Context) of helper function. 1465 | # Default: true 1466 | first: true 1467 | # Check *testing.T param has name t. 1468 | # Default: true 1469 | name: true 1470 | # Check t.Helper() begins helper function. 1471 | # Default: true 1472 | begin: true 1473 | benchmark: 1474 | # Check *testing.B is first param (or after context.Context) of helper function. 1475 | # Default: true 1476 | first: true 1477 | # Check *testing.B param has name b. 1478 | # Default: true 1479 | name: true 1480 | # Check b.Helper() begins helper function. 1481 | # Default: true 1482 | begin: true 1483 | tb: 1484 | # Check *testing.TB is first param (or after context.Context) of helper function. 1485 | # Default: true 1486 | first: false 1487 | # Check *testing.TB param has name tb. 1488 | # Default: true 1489 | name: false 1490 | # Check tb.Helper() begins helper function. 1491 | # Default: true 1492 | begin: false 1493 | fuzz: 1494 | # Check *testing.F is first param (or after context.Context) of helper function. 1495 | # Default: true 1496 | first: false 1497 | # Check *testing.F param has name f. 1498 | # Default: true 1499 | name: false 1500 | # Check f.Helper() begins helper function. 1501 | # Default: true 1502 | begin: false 1503 | 1504 | usestdlibvars: 1505 | # Suggest the use of http.MethodXX. 1506 | # Default: true 1507 | http-method: true 1508 | # Suggest the use of http.StatusXX. 1509 | # Default: true 1510 | http-status-code: true 1511 | # Suggest the use of time.Weekday.String(). 1512 | # Default: true 1513 | time-weekday: true 1514 | # Suggest the use of time.Month.String(). 1515 | # Default: false 1516 | time-month: true 1517 | # Suggest the use of time.Layout. 1518 | # Default: false 1519 | time-layout: true 1520 | # Suggest the use of crypto.Hash.String(). 1521 | # Default: false 1522 | crypto-hash: true 1523 | # Suggest the use of rpc.DefaultXXPath. 1524 | # Default: false 1525 | default-rpc-path: true 1526 | # # Suggest the use of os.DevNull. 1527 | # # Default: false 1528 | # os-dev-null: true 1529 | # Suggest the use of sql.LevelXX.String(). 1530 | # Default: false 1531 | sql-isolation-level: true 1532 | # Suggest the use of tls.SignatureScheme.String(). 1533 | # Default: false 1534 | tls-signature-scheme: true 1535 | # Suggest the use of constant.Kind.String(). 1536 | # Default: false 1537 | constant-kind: true 1538 | # # Suggest the use of syslog.Priority. 1539 | # # Default: false 1540 | # syslog-priority: true 1541 | 1542 | unparam: 1543 | # Inspect exported functions. 1544 | # 1545 | # Set to true if no external program/library imports your code. 1546 | # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: 1547 | # if it's called for subdir of a project it can't find external interfaces. All text editor integrations 1548 | # with golangci-lint call it on a directory with the changed file. 1549 | # 1550 | # Default: false 1551 | check-exported: false 1552 | 1553 | unused: 1554 | # treat code as a program (not a library) and report unused exported identifiers; default is false. 1555 | # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: 1556 | # if it's called for subdir of a project it can't find funcs usages. All text editor integrations 1557 | # with golangci-lint call it on a directory with the changed file. 1558 | check-exported: false 1559 | 1560 | varcheck: 1561 | # Check usage of exported fields and variables. 1562 | # Default: false 1563 | exported-fields: true 1564 | 1565 | varnamelen: 1566 | # The longest distance, in source lines, that is being considered a "small scope". 1567 | # Variables used in at most this many lines will be ignored. 1568 | # Default: 5 1569 | max-distance: 5 1570 | # The minimum length of a variable's name that is considered "long". 1571 | # Variable names that are at least this long will be ignored. 1572 | # Default: 3 1573 | min-name-length: 3 1574 | # Check method receivers. 1575 | # Default: false 1576 | check-receiver: true 1577 | # Check named return values. 1578 | # Default: false 1579 | check-return: false 1580 | # Check type parameters. 1581 | # Default: false 1582 | check-type-param: false 1583 | # Ignore "ok" variables that hold the bool return value of a type assertion. 1584 | # Default: false 1585 | ignore-type-assert-ok: true 1586 | # Ignore "ok" variables that hold the bool return value of a map index. 1587 | # Default: false 1588 | ignore-map-index-ok: false 1589 | # Ignore "ok" variables that hold the bool return value of a channel receive. 1590 | # Default: false 1591 | ignore-chan-recv-ok: false 1592 | # Optional list of variable names that should be ignored completely. 1593 | # Default: [] 1594 | ignore-names: 1595 | - err 1596 | - id 1597 | # Optional list of variable declarations that should be ignored completely. 1598 | # Entries must be in one of the following forms (see below for examples): 1599 | # - for variables, parameters, named return values, method receivers, or type parameters: 1600 | # ( can also be a pointer/slice/map/chan/...) 1601 | # - for constants: const 1602 | # 1603 | # Default: [] 1604 | ignore-decls: 1605 | - c echo.Context 1606 | - t testing.T 1607 | - f *foo.Bar 1608 | - e error 1609 | - i int 1610 | - const C 1611 | - T any 1612 | - m map[string]int 1613 | 1614 | whitespace: 1615 | # Enforces newlines (or comments) after every multi-line if statement. 1616 | # Default: false 1617 | multi-if: false 1618 | # Enforces newlines (or comments) after every multi-line function signature. 1619 | # Default: false 1620 | multi-func: false 1621 | 1622 | wrapcheck: 1623 | # An array of strings that specify substrings of signatures to ignore. 1624 | # If this set, it will override the default set of ignored signatures. 1625 | # See https://github.com/tomarrell/wrapcheck#configuration for more information. 1626 | # Default: [".Errorf(", "errors.New(", "errors.Unwrap(", ".Wrap(", ".Wrapf(", ".WithMessage(", ".WithMessagef(", ".WithStack("] 1627 | ignoreSigs: 1628 | - .Errorf( 1629 | - errors.New( 1630 | - errors.Unwrap( 1631 | - .Wrap( 1632 | - .Wrapf( 1633 | - .WithMessage( 1634 | - .WithMessagef( 1635 | - .WithStack( 1636 | - .WrapError( 1637 | # An array of strings that specify regular expressions of signatures to ignore. 1638 | # Default: [] 1639 | ignoreSigRegexps: 1640 | - \.New.*Error\( 1641 | # An array of strings that specify globs of packages to ignore. 1642 | # Default: [] 1643 | ignorePackageGlobs: 1644 | - encoding/* 1645 | - github.com/pkg/* 1646 | # An array of strings that specify regular expressions of interfaces to ignore. 1647 | # Default: [] 1648 | ignoreInterfaceRegexps: 1649 | - ^(?i)c(?-i)ach(ing|e) 1650 | 1651 | wsl: 1652 | # See https://github.com/bombsimon/wsl/blob/master/doc/configuration.md for documentation of available settings. 1653 | # These are the defaults for `golangci-lint`. 1654 | 1655 | # Do strict checking when assigning from append (x = append(x, y)). If 1656 | # this is set to true - the append call must append either a variable 1657 | # assigned, called or used on the line above. 1658 | strict-append: true 1659 | 1660 | # Allows assignments to be cuddled with variables used in calls on 1661 | # line above and calls to be cuddled with assignments of variables 1662 | # used in call on line above. 1663 | allow-assign-and-call: true 1664 | 1665 | # Allows assignments to be cuddled with anything. 1666 | allow-assign-and-anything: false 1667 | 1668 | # Allows cuddling to assignments even if they span over multiple lines. 1669 | allow-multiline-assign: true 1670 | 1671 | # If the number of lines in a case block is equal to or lager than this 1672 | # number, the case *must* end white a newline. 1673 | force-case-trailing-whitespace: 0 1674 | 1675 | # Allow blocks to end with comments. 1676 | allow-trailing-comment: true 1677 | 1678 | # Allow multiple comments in the beginning of a block separated with newline. 1679 | allow-separated-leading-comment: true 1680 | 1681 | # Allow multiple var/declaration statements to be cuddled. 1682 | allow-cuddle-declarations: false 1683 | 1684 | # A list of call idents that everything can be cuddled with. 1685 | # Defaults to calls looking like locks. 1686 | allow-cuddle-with-calls: [ "Lock", "RLock" ] 1687 | 1688 | # AllowCuddleWithRHS is a list of right hand side variables that is allowed 1689 | # to be cuddled with anything. Defaults to assignments or calls looking 1690 | # like unlocks. 1691 | allow-cuddle-with-rhs: [ "Unlock", "RUnlock" ] 1692 | 1693 | # Causes an error when an If statement that checks an error variable doesn't 1694 | # cuddle with the assignment of that variable. 1695 | force-err-cuddling: true 1696 | 1697 | # When force-err-cuddling is enabled this is a list of names 1698 | # used for error variables to check for in the conditional. 1699 | error-variable-names: [ "err" ] 1700 | 1701 | # Causes an error if a short declaration (:=) cuddles with anything other than 1702 | # another short declaration. 1703 | # This logic overrides force-err-cuddling among others. 1704 | force-short-decl-cuddling: false 1705 | 1706 | linters: 1707 | # Disable all linters. 1708 | # Default: false 1709 | enable-all: true 1710 | #disable-all: true 1711 | disable: 1712 | - dupl 1713 | - err113 1714 | - errchkjson 1715 | - execinquery 1716 | - exportloopref 1717 | - exhaustruct 1718 | - forbidigo 1719 | - gochecknoglobals 1720 | - godox 1721 | - goheader 1722 | - gomnd 1723 | - gomodguard 1724 | - gosmopolitan 1725 | - importas 1726 | - mnd 1727 | - nonamedreturns 1728 | - paralleltest 1729 | - perfsprint 1730 | - thelper 1731 | # --- reactivate later --- 1732 | - gomoddirectives 1733 | - ireturn 1734 | - musttag 1735 | # --- deprecated --- 1736 | 1737 | # Disable specific linter 1738 | # Enable specific linter 1739 | # https://golangci-lint.run/usage/linters/#enabled-by-default 1740 | #enable: 1741 | 1742 | 1743 | issues: 1744 | # List of regexps of issue texts to exclude. 1745 | # 1746 | # But independently of this option we use default exclude patterns, 1747 | # it can be disabled by `exclude-use-default: false`. 1748 | # To list all excluded by default patterns execute `golangci-lint run --help` 1749 | # 1750 | # Default: https://golangci-lint.run/usage/false-positives/#default-exclusions 1751 | exclude: 1752 | - abcdef 1753 | 1754 | # Which dirs to skip: issues from them won't be reported. 1755 | # Can use regexp here: `generated.*`, regexp is applied on full path. 1756 | # Default value is empty list, 1757 | # but default dirs are skipped independently of this option's value (see skip-dirs-use-default). 1758 | # "/" will be replaced by current OS file path separator to properly work on Windows. 1759 | exclude-dirs: 1760 | - vendor 1761 | 1762 | # Which files to skip: they will be analyzed, but issues from them won't be reported. 1763 | # Default value is empty list, 1764 | # but there is no need to include all autogenerated files, 1765 | # we confidently recognize autogenerated files. 1766 | # If it's not please let us know. 1767 | # "/" will be replaced by current OS file path separator to properly work on Windows. 1768 | skip-files: 1769 | - ".generated\\.go" 1770 | # - lib/bad.go 1771 | 1772 | # Excluding configuration per-path, per-linter, per-text and per-source 1773 | exclude-rules: 1774 | - source: "// Noduplinspection" 1775 | linters: [ dupl ] 1776 | 1777 | # Exclude some linters from running on tests files. 1778 | - path: _test\.go 1779 | linters: 1780 | - depguard 1781 | - dupl 1782 | - exhaustruct 1783 | - funlen 1784 | - gocognit 1785 | - goconst 1786 | - mnd 1787 | - gosec 1788 | - importas 1789 | - lll 1790 | - maintidx 1791 | - revive 1792 | - varnamelen 1793 | - wrapcheck 1794 | 1795 | # we will need to tweak this later, but no point micro optimising atm 1796 | - path: \.go 1797 | linters: 1798 | - gocritic 1799 | text: "hugeParam" 1800 | 1801 | # https://github.com/go-critic/go-critic/issues/926 1802 | - linters: 1803 | - gocritic 1804 | text: "unnecessaryDefer:" 1805 | # Exclude lll issues for long lines with go:generate 1806 | - linters: 1807 | - lll 1808 | source: "^//go:generate " 1809 | 1810 | # Independently of option `exclude` we use default exclude patterns, 1811 | # it can be disabled by this option. 1812 | # To list all excluded by default patterns execute `golangci-lint run --help`. 1813 | # Default: true. 1814 | exclude-use-default: false 1815 | 1816 | # If set to true exclude and exclude-rules regular expressions become case-sensitive. 1817 | # Default: false 1818 | exclude-case-sensitive: false 1819 | 1820 | # The list of ids of default excludes to include or disable. 1821 | # https://golangci-lint.run/usage/false-positives/#default-exclusions 1822 | # Default: [] 1823 | include: 1824 | - EXC0002 # disable excluding of issues about comments from golint 1825 | 1826 | # Maximum issues count per one linter. 1827 | # Set to 0 to disable. 1828 | # Default: 50 1829 | max-issues-per-linter: 0 1830 | 1831 | # Maximum count of issues with the same text. 1832 | # Set to 0 to disable. 1833 | # Default: 3 1834 | max-same-issues: 0 1835 | 1836 | # Show only new issues: if there are unstaged changes or untracked files, 1837 | # only those changes are analyzed, else only changes in HEAD~ are analyzed. 1838 | # It's a super-useful option for integration of golangci-lint into existing large codebase. 1839 | # It's not practical to fix all existing issues at the moment of integration: 1840 | # much better don't allow issues in new code. 1841 | # 1842 | # Default: false. 1843 | new: false 1844 | 1845 | # Show only new issues created after git revision `REV`. 1846 | # new-from-rev: REV 1847 | 1848 | # Show only new issues created in git patch with set file path. 1849 | # new-from-patch: path/to/patch/file 1850 | 1851 | # Fix found issues (if it's supported by the linter). 1852 | fix: true 1853 | 1854 | severity: 1855 | # Set the default severity for issues. 1856 | # 1857 | # If severity rules are defined and the issues do not match or no severity is provided to the rule 1858 | # this will be the default severity applied. 1859 | # Severities should match the supported severity names of the selected out format. 1860 | # - Code climate: https://docs.codeclimate.com/docs/issues#issue-severity 1861 | # - Checkstyle: https://checkstyle.sourceforge.io/property_types.html#SeverityLevel 1862 | # - GitHub: https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-error-message 1863 | # 1864 | # Default value is an empty string. 1865 | default-severity: info 1866 | 1867 | # If set to true `severity-rules` regular expressions become case-sensitive. 1868 | # Default: false 1869 | case-sensitive: false 1870 | 1871 | # When a list of severity rules are provided, severity information will be added to lint issues. 1872 | # Severity rules have the same filtering capability as exclude rules 1873 | # except you are allowed to specify one matcher per severity rule. 1874 | # Only affects out formats that support setting severity information. 1875 | # 1876 | # Default: [] 1877 | #rules: 1878 | # - linters: 1879 | # - dupl 1880 | # severity: info 1881 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FastDB 2 | 3 | FastDB is an (persistent) in-memory key/value store in Go. 4 | 5 | ## Motivation 6 | 7 | I wanted to be able to access my data as fast as possible without it being persisted to disk. 8 | In the past I've used several other key-value solutions but as a challenge to myself, tried if I could make 9 | something that was faster than the fastest I've used. 10 | 11 | It is also important to know that memory access will always be faster than disk access, 12 | but it goes without saying that memory is more limited than disk storage. 13 | 14 | 15 | ## Semi-technical information 16 | 17 | This is in fact a key-value store, working with buckets. 18 | (Buckets are a kind of a "box" in which you store key-values of the same kind.) 19 | 20 | The "trick" behind this, is that the real key is a combination of bucket_key when 21 | the data is persisted to disk. 22 | 23 | When you open the database, you can set the timer (in milliseconds) which will be the 24 | trigger to persist to disk. A value of 100 should be okay. 25 | That means there is a tiny risk that data from within the last 100 milliseconds isn't 26 | written to disk when there is a power failure. 27 | 28 | If you want to minimize that risk, use a sync-time of 0. 29 | (but this will be slower!) 30 | 31 | ## How it works 32 | 33 | ### Set 34 | 35 | The way to store things: 36 | ``` 37 | err = store.Set(bucket, key, value) 38 | ``` 39 | bucket - string 40 | key - int 41 | value - []byte 42 | 43 | So it is ideal for storing JSON, for example: 44 | ``` 45 | record := &someRecord{ 46 | ID: 1, 47 | UUID: "UUIDtext", 48 | Text: "a text", 49 | } 50 | 51 | recordData, _ := json.Marshal(record) 52 | 53 | err = store.Set("texts", record.ID, recordData) 54 | ``` 55 | 56 | ### Get 57 | 58 | The way to retrieve 1 record: 59 | ``` 60 | value, ok := store.Get(bucket, key) 61 | ``` 62 | bucket - string 63 | key - int 64 | value - []byte 65 | 66 | ### GetAll 67 | 68 | The way to retrieve all the data from one bucket: 69 | ``` 70 | records, ok := store.GetAll(bucket) 71 | ``` 72 | bucket - string 73 | key - int 74 | records - map[int][]byte 75 | 76 | ### Info 77 | 78 | To get information about the storage: 79 | 80 | ``` 81 | info := store.Info() 82 | ``` 83 | Will show the number of buckets and the total of records. 84 | 85 | ### Del 86 | 87 | The way to delete 1 record: 88 | ``` 89 | ok, err := store.Del(bucket, key) 90 | ``` 91 | bucket - string 92 | key - int 93 | ok - bool (true: key was found and deleted) 94 | 95 | ### Defrag 96 | 97 | If overtime there are many deletions, the database could be compressed, 98 | so the next time you'll open and read the file, it will be faster. 99 | 100 | ``` 101 | err := store.Defrag() 102 | ``` 103 | if there's an error, the original file will exist as a.bak file. 104 | 105 | 106 | ## Some simple figures 107 | 108 | Done on my Macbook Pro M1. 109 | ``` 110 | created 100000 records in 62.726042ms 111 | read 100000 records in 250ns 112 | sort 100000 records by key in 23.126917ms 113 | sort 100000 records by UUID in 41.55275ms 114 | 115 | ``` 116 | Benchmarks 117 | ``` 118 | goos: darwin 119 | goarch: arm64 120 | pkg: github.com/marcelloh/fastdb 121 | Benchmark_Get_Memory_1000 122 | Benchmark_Get_Memory_1000-8 51353016 23.30 ns/op 0 B/op 0 allocs/op 123 | Benchmark_Set_File_NoSyncTime 124 | Benchmark_Set_File_NoSyncTime-8 157 7094052 ns/op 265 B/op 3 allocs/op 125 | Benchmark_Set_Memory 126 | Benchmark_Set_Memory-8 3586731 297.6 ns/op 93 B/op 1 allocs/op 127 | Benchmark_Set_File_WithSyncTime 128 | Benchmark_Set_File_WithSyncTime-8 468963 2407 ns/op 209 B/op 3 allocs/op 129 | Benchmark_Get_File_1000 130 | Benchmark_Get_File_1000-8 44613194 26.18 ns/op 0 B/op 0 allocs/op 131 | ``` 132 | 133 | ## Example(s) 134 | 135 | In the examples directory, you will find an example on how to sort the data. 136 | 137 | If more examples are needed, I will write them. -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package fastdb holds all the fastdb functionalities. 3 | */ 4 | package fastdb 5 | -------------------------------------------------------------------------------- /examples/main_sort.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package main holds some examples of the usage of the library. 3 | */ 4 | package main 5 | 6 | /* ------------------------------- Imports --------------------------- */ 7 | 8 | import ( 9 | "crypto/rand" 10 | "encoding/json" 11 | "fmt" 12 | "log" 13 | "sort" 14 | "strconv" 15 | "time" 16 | 17 | "github.com/marcelloh/fastdb" 18 | "github.com/tidwall/gjson" 19 | ) 20 | 21 | /* ---------------------- Constants/Types/Variables ------------------ */ 22 | 23 | type user struct { 24 | ID int 25 | UUID string 26 | Email string 27 | } 28 | 29 | type record struct { 30 | SortField any 31 | Data []byte 32 | } 33 | 34 | /* -------------------------- Methods/Functions ---------------------- */ 35 | 36 | /* 37 | main is the bootstrap of the application. 38 | */ 39 | func main() { 40 | store, err := fastdb.Open(":memory:", 100) 41 | if err != nil { 42 | log.Fatal(err) 43 | } 44 | 45 | defer func() { 46 | err = store.Close() 47 | if err != nil { 48 | log.Fatal(err) 49 | } 50 | }() 51 | 52 | total := 100000 // nr of records to work with 53 | 54 | start := time.Now() 55 | fillData(store, total) 56 | log.Printf("created %d records in %s", total, time.Since(start)) 57 | 58 | start = time.Now() 59 | dbRecords, err := store.GetAll("user") 60 | if err != nil { 61 | log.Panic(err) 62 | } 63 | 64 | log.Printf("read %d records in %s", total, time.Since(start)) 65 | 66 | sortByUUID(dbRecords) 67 | } 68 | 69 | /* 70 | sortByUUID sorts the records by UUID. 71 | */ 72 | func sortByUUID(dbRecords map[int][]byte) { 73 | start := time.Now() 74 | count := 0 75 | keys := make([]record, len(dbRecords)) 76 | 77 | for key := range dbRecords { 78 | json := string(dbRecords[key]) 79 | value := gjson.Get(json, "UUID").Str + strconv.Itoa(key) 80 | keys[count] = record{SortField: value, Data: dbRecords[key]} 81 | count++ 82 | } 83 | 84 | sort.Slice(keys, func(i, j int) bool { 85 | return keys[i].SortField.(string) < keys[j].SortField.(string) 86 | }) 87 | 88 | log.Printf("sort %d records by UUID in %s", count, time.Since(start)) 89 | 90 | for key, value := range keys { 91 | if key >= 15 { 92 | break 93 | } 94 | 95 | fmt.Printf("value : %v\n", string(value.Data)) 96 | } 97 | } 98 | 99 | func fillData(store *fastdb.DB, total int) { 100 | user := &user{ 101 | ID: 1, 102 | UUID: "UUIDtext_", 103 | Email: "test@example.com", 104 | } 105 | 106 | for i := 1; i <= total; i++ { 107 | user.ID = i 108 | user.UUID = "UUIDtext_" + generateRandomString(8) + strconv.Itoa(user.ID) 109 | 110 | userData, err := json.Marshal(user) 111 | if err != nil { 112 | log.Fatal(err) 113 | } 114 | 115 | err = store.Set("user", user.ID, userData) 116 | if err != nil { 117 | log.Fatal(err) 118 | } 119 | } 120 | } 121 | 122 | // generateRandomString creates a random string of the specified length 123 | func generateRandomString(length int) string { 124 | const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" 125 | 126 | b := make([]byte, length) 127 | 128 | _, err := rand.Read(b) 129 | if err != nil { 130 | log.Fatal(err) 131 | } 132 | 133 | for i := range b { 134 | b[i] = charset[int(b[i])%len(charset)] 135 | } 136 | 137 | return string(b) 138 | } 139 | -------------------------------------------------------------------------------- /fastdb.go: -------------------------------------------------------------------------------- 1 | package fastdb 2 | 3 | /* ------------------------------- Imports --------------------------- */ 4 | 5 | import ( 6 | "errors" 7 | "fmt" 8 | "maps" 9 | "slices" 10 | "strconv" 11 | "sync" 12 | 13 | "github.com/marcelloh/fastdb/persist" 14 | ) 15 | 16 | /* ---------------------- Constants/Types/Variables ------------------ */ 17 | 18 | // DB represents a collection of key-value pairs that persist on disk or memory. 19 | type DB struct { 20 | aof *persist.AOF 21 | keys map[string]map[int][]byte 22 | mu sync.RWMutex 23 | } 24 | 25 | // SortRecord represents a record from a sorted collection of sliced records 26 | type SortRecord struct { 27 | SortField any 28 | Data []byte 29 | } 30 | 31 | /* -------------------------- Methods/Functions ---------------------- */ 32 | 33 | /* 34 | Open opens a database at the provided path. 35 | If the file doesn't exist, it will be created automatically. 36 | If the path is ':memory:' then the database will be opened in memory only. 37 | */ 38 | func Open(path string, syncIime int) (*DB, error) { 39 | var ( 40 | aof *persist.AOF 41 | err error 42 | ) 43 | 44 | keys := map[string]map[int][]byte{} 45 | 46 | if path != ":memory:" { 47 | aof, keys, err = persist.OpenPersister(path, syncIime) 48 | } 49 | 50 | return &DB{aof: aof, keys: keys}, err //nolint:wrapcheck // it is already wrapped 51 | } 52 | 53 | /* 54 | Defrag optimises the file to reflect the latest state. 55 | */ 56 | func (fdb *DB) Defrag() error { 57 | defer fdb.lockUnlock()() 58 | 59 | var err error 60 | 61 | err = fdb.aof.Defrag(fdb.keys) 62 | if err != nil { 63 | err = fmt.Errorf("defrag error: %w", err) 64 | } 65 | 66 | return err 67 | } 68 | 69 | /* 70 | Del deletes one map value in a bucket. 71 | */ 72 | func (fdb *DB) Del(bucket string, key int) (bool, error) { 73 | defer fdb.lockUnlock()() 74 | 75 | var err error 76 | 77 | // bucket exists? 78 | _, found := fdb.keys[bucket] 79 | if !found { 80 | return found, nil 81 | } 82 | 83 | // key exists in bucket? 84 | _, found = fdb.keys[bucket][key] 85 | if !found { 86 | return found, nil 87 | } 88 | 89 | if fdb.aof != nil { 90 | lines := "del\n" + bucket + "_" + strconv.Itoa(key) + "\n" 91 | 92 | err = fdb.aof.Write(lines) 93 | if err != nil { 94 | return false, fmt.Errorf("del->write error: %w", err) 95 | } 96 | } 97 | 98 | delete(fdb.keys[bucket], key) 99 | 100 | if len(fdb.keys[bucket]) == 0 { 101 | delete(fdb.keys, bucket) 102 | } 103 | 104 | return true, nil 105 | } 106 | 107 | /* 108 | Get returns one map value from a bucket. 109 | */ 110 | func (fdb *DB) Get(bucket string, key int) ([]byte, bool) { 111 | fdb.mu.RLock() 112 | defer fdb.mu.RUnlock() 113 | 114 | data, ok := fdb.keys[bucket][key] 115 | 116 | return data, ok 117 | } 118 | 119 | /* 120 | GetAll returns all map values from a bucket in random order. 121 | */ 122 | func (fdb *DB) GetAll(bucket string) (map[int][]byte, error) { 123 | fdb.mu.RLock() 124 | defer fdb.mu.RUnlock() 125 | 126 | bmap, found := fdb.keys[bucket] 127 | if !found { 128 | return nil, fmt.Errorf("bucket (%s) not found", bucket) 129 | } 130 | 131 | return bmap, nil 132 | } 133 | 134 | /* 135 | GetAllSorted returns all map values from a bucket in Key sorted order. 136 | */ 137 | func (fdb *DB) GetAllSorted(bucket string) ([]*SortRecord, error) { 138 | memRecords, err := fdb.GetAll(bucket) 139 | if err != nil { 140 | return nil, err 141 | } 142 | 143 | sortedKeys := slices.Sorted(maps.Keys(memRecords)) 144 | 145 | sortedRecords := make([]*SortRecord, len(memRecords)) 146 | 147 | for count, key := range sortedKeys { 148 | sortedRecords[count] = &SortRecord{SortField: key, Data: memRecords[key]} 149 | // count++ 150 | } 151 | 152 | return sortedRecords, nil 153 | } 154 | 155 | /* 156 | GetNewIndex returns the next available index for a bucket. 157 | */ 158 | func (fdb *DB) GetNewIndex(bucket string) (newKey int) { 159 | memRecords, err := fdb.GetAll(bucket) 160 | if err != nil { 161 | return 1 162 | } 163 | 164 | lkey := 0 165 | for key := range memRecords { 166 | if key > lkey { 167 | lkey = key 168 | } 169 | } 170 | 171 | newKey = lkey + 1 172 | 173 | return newKey 174 | } 175 | 176 | /* 177 | Info returns info about the storage. 178 | */ 179 | func (fdb *DB) Info() string { 180 | count := 0 181 | for i := range fdb.keys { 182 | count += len(fdb.keys[i]) 183 | } 184 | 185 | return fmt.Sprintf("%d record(s) in %d bucket(s)", count, len(fdb.keys)) 186 | } 187 | 188 | /* 189 | Set stores one map value in a bucket. 190 | */ 191 | func (fdb *DB) Set(bucket string, key int, value []byte) error { 192 | defer fdb.lockUnlock()() 193 | 194 | if key < 0 { 195 | return errors.New("set->key should be positive") 196 | } 197 | 198 | if fdb.aof != nil { 199 | lines := "set\n" + bucket + "_" + strconv.Itoa(key) + "\n" + string(value) + "\n" 200 | 201 | err := fdb.aof.Write(lines) 202 | if err != nil { 203 | return fmt.Errorf("set->write error: %w", err) 204 | } 205 | } 206 | 207 | _, found := fdb.keys[bucket] 208 | if !found { 209 | fdb.keys[bucket] = map[int][]byte{} 210 | } 211 | 212 | fdb.keys[bucket][key] = value 213 | 214 | return nil 215 | } 216 | 217 | /* 218 | Close closes the database. 219 | */ 220 | func (fdb *DB) Close() error { 221 | if fdb.aof != nil { 222 | defer fdb.lockUnlock()() 223 | 224 | err := fdb.aof.Close() 225 | if err != nil { 226 | return fmt.Errorf("close error: %w", err) 227 | } 228 | } 229 | 230 | fdb.keys = map[string]map[int][]byte{} 231 | 232 | return nil 233 | } 234 | 235 | /* 236 | lockUnlock locks the database and unlocks it later 237 | 238 | if you call it like this: defer fdb.lockUnlock()() 239 | the first function call locks it and because it returns a function, 240 | that function will actually be called as the defer. 241 | */ 242 | func (fdb *DB) lockUnlock() func() { 243 | fdb.mu.Lock() 244 | //nolint:gocritic // leave it here 245 | // log.Println("> Locked") 246 | 247 | return func() { 248 | fdb.mu.Unlock() 249 | //nolint:gocritic // leave it here 250 | // log.Println("> Unlocked") 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /fastdb_test.go: -------------------------------------------------------------------------------- 1 | package fastdb_test 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "math/rand" 8 | "os" 9 | "path/filepath" 10 | "sync" 11 | "testing" 12 | "time" 13 | 14 | "github.com/marcelloh/fastdb" 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | const ( 20 | syncIime = 10 21 | dataDir = "./data" 22 | memory = ":memory:" 23 | ) 24 | 25 | type someRecord struct { 26 | UUID string 27 | Text string 28 | ID int 29 | } 30 | 31 | func Test_Open_File_noData(t *testing.T) { 32 | path := "data/fastdb_open_no_data.db" 33 | 34 | defer func() { 35 | filePath := filepath.Clean(path) 36 | err := os.Remove(filePath) 37 | require.NoError(t, err) 38 | }() 39 | 40 | store, err := fastdb.Open(path, syncIime) 41 | require.NoError(t, err) 42 | assert.NotNil(t, store) 43 | 44 | defer func() { 45 | err = store.Close() 46 | require.NoError(t, err) 47 | }() 48 | } 49 | 50 | func Test_Open_Memory(t *testing.T) { 51 | path := memory 52 | 53 | store, err := fastdb.Open(path, syncIime) 54 | require.NoError(t, err) 55 | assert.NotNil(t, store) 56 | 57 | defer func() { 58 | err = store.Close() 59 | require.NoError(t, err) 60 | }() 61 | } 62 | 63 | func Test_SetGetDel_oneRecord(t *testing.T) { 64 | path := "data/fastdb_set.db" 65 | filePath := filepath.Clean(path) 66 | 67 | store, err := fastdb.Open(filePath, syncIime) 68 | require.NoError(t, err) 69 | assert.NotNil(t, store) 70 | 71 | defer func() { 72 | err = store.Close() 73 | require.NoError(t, err) 74 | 75 | err = os.Remove(filePath) 76 | require.NoError(t, err) 77 | }() 78 | 79 | var newKey int 80 | 81 | newKey = store.GetNewIndex("texts") 82 | assert.Equal(t, 1, newKey) 83 | 84 | record := &someRecord{ 85 | ID: 1, 86 | UUID: "UUIDtext", 87 | Text: "a text", 88 | } 89 | 90 | recordData, err := json.Marshal(record) 91 | require.NoError(t, err) 92 | 93 | err = store.Set("texts", record.ID, recordData) 94 | require.NoError(t, err) 95 | 96 | newKey = store.GetNewIndex("texts") 97 | assert.Equal(t, 2, newKey) 98 | 99 | info := store.Info() 100 | assert.Equal(t, "1 record(s) in 1 bucket(s)", info) 101 | 102 | memData, ok := store.Get("texts", 1) 103 | assert.True(t, ok) 104 | 105 | memRecord := &someRecord{} 106 | err = json.Unmarshal(memData, &memRecord) 107 | require.NoError(t, err) 108 | assert.NotNil(t, memRecord) 109 | assert.Equal(t, record.UUID, memRecord.UUID) 110 | assert.Equal(t, record.Text, memRecord.Text) 111 | assert.Equal(t, record.ID, memRecord.ID) 112 | 113 | // delete in non existing bucket 114 | ok, err = store.Del("notexisting", 1) 115 | require.NoError(t, err) 116 | assert.False(t, ok) 117 | 118 | // delete non existing key 119 | ok, err = store.Del("texts", 123) 120 | require.NoError(t, err) 121 | assert.False(t, ok) 122 | 123 | ok, err = store.Del("texts", 1) 124 | require.NoError(t, err) 125 | assert.True(t, ok) 126 | 127 | newKey = store.GetNewIndex("texts") 128 | assert.Equal(t, 1, newKey) 129 | 130 | info = store.Info() 131 | assert.Equal(t, "0 record(s) in 0 bucket(s)", info) 132 | } 133 | 134 | func Fuzz_SetGetDel_oneRecord(f *testing.F) { 135 | path := "data/fastdb_fuzzset.db" 136 | 137 | filePath := filepath.Clean(path) 138 | _ = os.Remove(filePath) 139 | 140 | store, err := fastdb.Open(filePath, 1000) 141 | require.NoError(f, err) 142 | assert.NotNil(f, store) 143 | 144 | defer func() { 145 | err = store.Close() 146 | require.NoError(f, err) 147 | }() 148 | 149 | var newKey int 150 | 151 | newKey = store.GetNewIndex("texts") 152 | assert.Equal(f, 1, newKey) 153 | 154 | s1 := rand.NewSource(time.Now().UnixNano()) 155 | rdom := rand.New(s1) 156 | 157 | for range 50 { 158 | tc := rdom.Intn(10000) + 1 159 | f.Add(tc) // Use f.Add to provide a seed corpus 160 | } 161 | 162 | record := &someRecord{ 163 | ID: 1, 164 | UUID: "UUIDtext", 165 | Text: "a text", 166 | } 167 | 168 | counter := 0 169 | highest := 0 170 | 171 | f.Fuzz(func(t *testing.T, id int) { 172 | if id < 0 { 173 | return 174 | } 175 | 176 | record.ID = id 177 | 178 | if highest < id { 179 | highest = id 180 | } 181 | 182 | recordData, err := json.Marshal(record) 183 | require.NoError(t, err) 184 | 185 | _, ok := store.Get("texts", id) 186 | if !ok { 187 | counter++ 188 | } 189 | 190 | err = store.Set("texts", record.ID, recordData) 191 | require.NoError(t, err) 192 | 193 | newKey = store.GetNewIndex("texts") 194 | assert.Equal(t, highest+1, newKey, id) 195 | 196 | info := store.Info() 197 | text := fmt.Sprintf("%d record(s) in 1 bucket(s)", counter) 198 | assert.Equal(t, text, info, id) 199 | 200 | memData, ok := store.Get("texts", id) 201 | assert.True(t, ok) 202 | 203 | memRecord := &someRecord{} 204 | err = json.Unmarshal(memData, &memRecord) 205 | require.NoError(t, err) 206 | 207 | if id%5 == 0 { 208 | ok, err = store.Del("texts", id) 209 | require.NoError(t, err) 210 | assert.True(t, ok) 211 | 212 | counter-- 213 | 214 | newKey = store.GetNewIndex("texts") 215 | highest = newKey - 1 216 | } 217 | }) 218 | } 219 | 220 | func Test_Get_wrongRecord(t *testing.T) { 221 | path := memory 222 | 223 | store, err := fastdb.Open(path, syncIime) 224 | require.NoError(t, err) 225 | assert.NotNil(t, store) 226 | 227 | defer func() { 228 | err = store.Close() 229 | require.NoError(t, err) 230 | }() 231 | 232 | // store a record 233 | err = store.Set("bucket", 1, []byte("a text")) 234 | require.NoError(t, err) 235 | 236 | // get same record back 237 | memData, ok := store.Get("bucket", 1) 238 | assert.True(t, ok) 239 | assert.NotNil(t, memData) 240 | 241 | // get record back from wrong bucket 242 | memData, ok = store.Get("wrong_bucket", 1) 243 | assert.False(t, ok) 244 | assert.Nil(t, memData) 245 | 246 | // get record back from good bucket with wrong key 247 | memData, ok = store.Get("bucket", 2) 248 | assert.False(t, ok) 249 | assert.Nil(t, memData) 250 | } 251 | 252 | func Test_Defrag_1000lines(t *testing.T) { 253 | path := "data/fastdb_defrag1000.db" 254 | filePath := filepath.Clean(path) 255 | 256 | store, err := fastdb.Open(path, syncIime) 257 | require.NoError(t, err) 258 | assert.NotNil(t, store) 259 | 260 | defer func() { 261 | err = store.Close() 262 | require.NoError(t, err) 263 | 264 | err = os.Remove(filePath) 265 | require.NoError(t, err) 266 | 267 | _ = os.Remove(filePath + ".bak") 268 | }() 269 | 270 | record := &someRecord{ 271 | ID: 1, 272 | UUID: "UUIDtext", 273 | Text: "a text", 274 | } 275 | 276 | // create a lot of records first 277 | total := 1000 278 | 279 | s1 := rand.NewSource(time.Now().UnixNano()) 280 | rdom := rand.New(s1) 281 | 282 | var recordData []byte 283 | 284 | for range total { 285 | record.ID = rdom.Intn(10) + 1 286 | recordData, err = json.Marshal(record) 287 | require.NoError(t, err) 288 | 289 | err = store.Set("records", record.ID, recordData) 290 | require.NoError(t, err) 291 | } 292 | 293 | checkFileLines(t, filePath, total*3) 294 | 295 | err = store.Defrag() 296 | require.NoError(t, err) 297 | 298 | checkFileLines(t, filePath, 30) 299 | } 300 | 301 | func Test_Defrag_1000000lines(t *testing.T) { 302 | path := "data/fastdb_defrag1000000.db" 303 | filePath := filepath.Clean(path) 304 | 305 | store, err := fastdb.Open(filePath, 250) 306 | require.NoError(t, err) 307 | assert.NotNil(t, store) 308 | 309 | defer func() { 310 | err = store.Close() 311 | require.NoError(t, err) 312 | 313 | _ = os.Remove(filePath) 314 | 315 | _ = os.Remove(filePath + ".bak") 316 | }() 317 | 318 | record := &someRecord{ 319 | ID: 1, 320 | UUID: "UUIDtext", 321 | Text: "a text", 322 | } 323 | 324 | // create a lot of records first 325 | total := 1000000 326 | 327 | s1 := rand.NewSource(time.Now().UnixNano()) 328 | rdom := rand.New(s1) 329 | 330 | var recordData []byte 331 | 332 | for range total { 333 | record.ID = rdom.Intn(10) + 1 334 | recordData, err = json.Marshal(record) 335 | require.NoError(t, err) 336 | 337 | err = store.Set("records", record.ID, recordData) 338 | require.NoError(t, err) 339 | } 340 | 341 | checkFileLines(t, filePath, total*3) 342 | 343 | err = store.Defrag() 344 | require.NoError(t, err) 345 | 346 | checkFileLines(t, filePath, 30) 347 | } 348 | 349 | func Test_GetAllFromMemory_1000(t *testing.T) { 350 | total := 1000 351 | path := memory 352 | 353 | store, err := fastdb.Open(path, syncIime) 354 | require.NoError(t, err) 355 | assert.NotNil(t, store) 356 | 357 | defer func() { 358 | err = store.Close() 359 | require.NoError(t, err) 360 | }() 361 | 362 | record := &someRecord{ 363 | ID: 1, 364 | UUID: "UUIDtext", 365 | Text: "a text", 366 | } 367 | 368 | var recordData []byte 369 | 370 | for i := 1; i <= total; i++ { 371 | record.ID = i 372 | recordData, err = json.Marshal(record) 373 | require.NoError(t, err) 374 | 375 | err = store.Set("records", record.ID, recordData) 376 | require.NoError(t, err) 377 | } 378 | 379 | records, err := store.GetAll("records") 380 | require.NoError(t, err) 381 | assert.NotNil(t, records) 382 | assert.Len(t, records, total) 383 | 384 | records, err = store.GetAll("wrong_bucket") 385 | require.Error(t, err) 386 | assert.Nil(t, records) 387 | } 388 | 389 | func Test_GetAllFromFile_1000(t *testing.T) { 390 | total := 1000 391 | path := "data/fastdb_1000.db" 392 | 393 | filePath := filepath.Clean(path) 394 | _ = os.Remove(filePath) 395 | 396 | store, err := fastdb.Open(path, syncIime) 397 | require.NoError(t, err) 398 | assert.NotNil(t, store) 399 | 400 | defer func() { 401 | err = store.Close() 402 | require.NoError(t, err) 403 | 404 | err = os.Remove(filePath) 405 | require.NoError(t, err) 406 | }() 407 | 408 | record := &someRecord{ 409 | ID: 1, 410 | UUID: "UUIDtext", 411 | Text: "a text", 412 | } 413 | 414 | s1 := rand.NewSource(time.Now().UnixNano()) 415 | rdom := rand.New(s1) 416 | 417 | var recordData []byte 418 | 419 | for i := 1; i <= total; i++ { 420 | record.ID = rdom.Intn(1000000) 421 | recordData, err = json.Marshal(record) 422 | require.NoError(t, err) 423 | 424 | err = store.Set("user", record.ID, recordData) 425 | require.NoError(t, err) 426 | } 427 | 428 | records, err := store.GetAll("user") 429 | require.NoError(t, err) 430 | assert.NotNil(t, records) 431 | } 432 | 433 | func Test_GetAllSortedFromFile_10000(t *testing.T) { 434 | total := 10000 435 | path := "data/fastdb_1000.db" 436 | 437 | filePath := filepath.Clean(path) 438 | _ = os.Remove(filePath) 439 | 440 | defer func() { 441 | err := os.Remove(filePath) 442 | require.NoError(t, err) 443 | }() 444 | 445 | store, err := fastdb.Open(path, syncIime) 446 | require.NoError(t, err) 447 | assert.NotNil(t, store) 448 | 449 | defer func() { 450 | err = store.Close() 451 | require.NoError(t, err) 452 | }() 453 | 454 | record := &someRecord{ 455 | ID: 1, 456 | UUID: "UUIDtext", 457 | Text: "a text", 458 | } 459 | 460 | s1 := rand.NewSource(time.Now().UnixNano()) 461 | rdom := rand.New(s1) 462 | 463 | var recordData []byte 464 | 465 | for i := 1; i <= total; i++ { 466 | // while loop 467 | record.ID = rdom.Intn(1000000000) 468 | _, ok := store.Get("user", record.ID) 469 | 470 | for ok { 471 | record.ID = rdom.Intn(1000000000) 472 | _, ok = store.Get("user", record.ID) 473 | } 474 | 475 | recordData, err = json.Marshal(record) 476 | require.NoError(t, err) 477 | 478 | err = store.Set("user", record.ID, recordData) 479 | require.NoError(t, err) 480 | } 481 | 482 | records, err := store.GetAllSorted("user") 483 | require.NoError(t, err) 484 | assert.NotNil(t, records) 485 | assert.Len(t, records, total) 486 | } 487 | 488 | func Test_GetAllSortedFromMemory_10000(t *testing.T) { 489 | total := 10000 490 | path := memory 491 | 492 | store, err := fastdb.Open(path, syncIime) 493 | require.NoError(t, err) 494 | assert.NotNil(t, store) 495 | 496 | defer func() { 497 | err = store.Close() 498 | require.NoError(t, err) 499 | }() 500 | 501 | record := &someRecord{ 502 | ID: 1, 503 | UUID: "UUIDtext", 504 | Text: "a text", 505 | } 506 | 507 | s1 := rand.NewSource(time.Now().UnixNano()) 508 | rdom := rand.New(s1) 509 | 510 | var recordData []byte 511 | 512 | for i := 1; i <= total; i++ { 513 | // while loop 514 | record.ID = rdom.Intn(1000000000) 515 | _, ok := store.Get("sortedRecords", record.ID) 516 | 517 | for ok { 518 | record.ID = rdom.Intn(1000000000) 519 | _, ok = store.Get("sortedRecords", record.ID) 520 | } 521 | 522 | recordData, err = json.Marshal(record) 523 | require.NoError(t, err) 524 | err = store.Set("sortedRecords", record.ID, recordData) 525 | require.NoError(t, err) 526 | } 527 | 528 | records, err := store.GetAllSorted("sortedRecords") 529 | require.NoError(t, err) 530 | assert.NotNil(t, records) 531 | assert.Len(t, records, total) 532 | 533 | records, err = store.GetAllSorted("wrong_bucket") 534 | require.Error(t, err) 535 | assert.Nil(t, records) 536 | } 537 | 538 | func Test_Set_error(t *testing.T) { 539 | path := "data/fastdb_set_error.db" 540 | filePath := filepath.Clean(path) 541 | 542 | store, err := fastdb.Open(filePath, syncIime) 543 | require.NoError(t, err) 544 | assert.NotNil(t, store) 545 | 546 | defer func() { 547 | err = os.Remove(filePath) 548 | require.NoError(t, err) 549 | }() 550 | 551 | err = store.Close() 552 | require.NoError(t, err) 553 | 554 | // store a record 555 | err = store.Set("bucket", 1, []byte("a text")) 556 | require.Error(t, err) 557 | } 558 | 559 | func Test_Set_wrongBucket(t *testing.T) { 560 | path := "data/fastdb_set_bucket_error.db" 561 | filePath := filepath.Clean(path) 562 | _ = os.Remove(filePath) 563 | 564 | defer func() { 565 | err := os.Remove(filePath) 566 | require.NoError(t, err) 567 | }() 568 | 569 | store, err := fastdb.Open(path, syncIime) 570 | require.NoError(t, err) 571 | assert.NotNil(t, store) 572 | 573 | // store a record 574 | err = store.Set("under_score", 1, []byte("a text for key 1")) 575 | require.NoError(t, err) 576 | 577 | err = store.Set("under_score", 2, []byte("a text for key 2")) 578 | require.NoError(t, err) 579 | 580 | err = store.Close() 581 | require.NoError(t, err) 582 | 583 | store2, err := fastdb.Open(path, syncIime) 584 | require.NoError(t, err) 585 | assert.NotNil(t, store2) 586 | 587 | defer func() { 588 | err = store2.Close() 589 | require.NoError(t, err) 590 | }() 591 | } 592 | 593 | func TestConcurrentOperationsWithDelete(t *testing.T) { 594 | path := "testdb_concurrent_delete" 595 | filePath := filepath.Clean(path) 596 | _ = os.Remove(filePath) 597 | 598 | defer func() { 599 | err := os.Remove(filePath) 600 | require.NoError(t, err) 601 | }() 602 | 603 | store, err := fastdb.Open(path, syncIime) 604 | require.NoError(t, err) 605 | 606 | defer func() { 607 | err = store.Close() 608 | require.NoError(t, err) 609 | }() 610 | 611 | const ( 612 | numGoroutines = 100 613 | numOperations = 100 614 | bucket = "test" 615 | ) 616 | 617 | var wg sync.WaitGroup 618 | 619 | wg.Add(numGoroutines) 620 | 621 | for i := range numGoroutines { 622 | go func(id int) { 623 | defer wg.Done() 624 | 625 | for j := range numOperations { 626 | key := id*numOperations + j 627 | value := []byte(fmt.Sprintf("value_%d_%d", id, j)) 628 | 629 | // Set operation 630 | err := store.Set(bucket, key, value) 631 | assert.NoError(t, err) 632 | 633 | // Get operation 634 | retrievedValue, ok := store.Get(bucket, key) 635 | assert.True(t, ok) 636 | assert.Equal(t, value, retrievedValue) 637 | 638 | // Delete operation (delete every other entry) 639 | if j%2 == 0 { 640 | deleted, err := store.Del(bucket, key) 641 | assert.NoError(t, err) 642 | assert.True(t, deleted) 643 | 644 | // Verify deletion 645 | _, ok = store.Get(bucket, key) 646 | assert.False(t, ok) 647 | } 648 | } 649 | }(i) 650 | } 651 | 652 | wg.Wait() 653 | 654 | // Verify final state 655 | for i := range numGoroutines { 656 | for j := range numOperations { 657 | key := i*numOperations + j 658 | expectedValue := []byte(fmt.Sprintf("value_%d_%d", i, j)) 659 | 660 | retrievedValue, ok := store.Get(bucket, key) 661 | if j%2 == 0 { 662 | // Even entries should have been deleted 663 | assert.False(t, ok) 664 | } else { 665 | // Odd entries should still exist 666 | assert.True(t, ok) 667 | assert.Equal(t, expectedValue, retrievedValue) 668 | } 669 | } 670 | } 671 | } 672 | 673 | func Benchmark_Get_File_1000(b *testing.B) { 674 | path := "data/bench-get.db" 675 | total := 1000 676 | 677 | filePath := filepath.Clean(path) 678 | _ = os.Remove(filePath) 679 | 680 | defer func() { 681 | err := os.Remove(filePath) 682 | require.NoError(b, err) 683 | }() 684 | 685 | store, err := fastdb.Open(path, syncIime) 686 | require.NoError(b, err) 687 | assert.NotNil(b, store) 688 | 689 | x1 := rand.NewSource(time.Now().UnixNano()) 690 | _ = rand.New(x1) 691 | 692 | record := &someRecord{ 693 | ID: 1, 694 | UUID: "UUIDtext", 695 | Text: "a text", 696 | } 697 | 698 | s1 := rand.NewSource(time.Now().UnixNano()) 699 | rdom := rand.New(s1) 700 | 701 | var recordData []byte 702 | 703 | for i := 1; i <= total; i++ { 704 | record.ID = rdom.Intn(1000000) 705 | recordData, err = json.Marshal(record) 706 | require.NoError(b, err) 707 | 708 | err = store.Set("bench_bucket", record.ID, recordData) 709 | require.NoError(b, err) 710 | } 711 | 712 | b.ResetTimer() 713 | 714 | for i := 0; i < b.N; i++ { // use b.N for looping 715 | _, _ = store.Get("bench_bucket", rand.Intn(1000000)) 716 | } 717 | 718 | err = store.Close() 719 | require.NoError(b, err) 720 | } 721 | 722 | func Benchmark_Get_Memory_1000(b *testing.B) { 723 | path := memory 724 | total := 1000 725 | 726 | store, err := fastdb.Open(path, syncIime) 727 | require.NoError(b, err) 728 | assert.NotNil(b, store) 729 | 730 | x1 := rand.NewSource(time.Now().UnixNano()) 731 | _ = rand.New(x1) 732 | 733 | record := &someRecord{ 734 | ID: 1, 735 | UUID: "UUIDtext", 736 | Text: "a text", 737 | } 738 | 739 | var recordData []byte 740 | 741 | s1 := rand.NewSource(time.Now().UnixNano()) 742 | rdom := rand.New(s1) 743 | 744 | for i := 1; i <= total; i++ { 745 | record.ID = rdom.Intn(1000000) 746 | recordData, err = json.Marshal(record) 747 | require.NoError(b, err) 748 | 749 | err = store.Set("bench_bucket", record.ID, recordData) 750 | require.NoError(b, err) 751 | } 752 | 753 | b.ResetTimer() 754 | 755 | for i := 0; i < b.N; i++ { // use b.N for looping 756 | _, _ = store.Get("bench_bucket", rand.Intn(1000000)) 757 | } 758 | 759 | err = store.Close() 760 | require.NoError(b, err) 761 | } 762 | 763 | func Benchmark_Set_File_NoSyncTime(b *testing.B) { 764 | path := "data/bench-set.db" 765 | 766 | filePath := filepath.Clean(path) 767 | _ = os.Remove(filePath) 768 | 769 | defer func() { 770 | err := os.Remove(filePath) 771 | require.NoError(b, err) 772 | }() 773 | 774 | store, err := fastdb.Open(path, 0) 775 | require.NoError(b, err) 776 | assert.NotNil(b, store) 777 | 778 | x1 := rand.NewSource(time.Now().UnixNano()) 779 | _ = rand.New(x1) 780 | 781 | record := &someRecord{ 782 | ID: 1, 783 | UUID: "UUIDtext", 784 | Text: "a text", 785 | } 786 | 787 | var recordData []byte 788 | 789 | b.ResetTimer() 790 | 791 | for i := 0; i < b.N; i++ { // use b.N for looping 792 | record.ID = rand.Intn(1000000) 793 | recordData, err = json.Marshal(record) 794 | require.NoError(b, err) 795 | 796 | err = store.Set("user", record.ID, recordData) 797 | require.NoError(b, err) 798 | } 799 | 800 | err = store.Close() 801 | require.NoError(b, err) 802 | } 803 | 804 | func Benchmark_Set_File_WithSyncTime(b *testing.B) { 805 | path := "data/bench-set.db" 806 | 807 | filePath := filepath.Clean(path) 808 | _ = os.Remove(filePath) 809 | 810 | defer func() { 811 | err := os.Remove(filePath) 812 | require.NoError(b, err) 813 | }() 814 | 815 | store, err := fastdb.Open(path, syncIime) 816 | require.NoError(b, err) 817 | assert.NotNil(b, store) 818 | 819 | x1 := rand.NewSource(time.Now().UnixNano()) 820 | _ = rand.New(x1) 821 | 822 | record := &someRecord{ 823 | ID: 1, 824 | UUID: "UUIDtext", 825 | Text: "a text", 826 | } 827 | 828 | var recordData []byte 829 | 830 | b.ResetTimer() 831 | 832 | for i := 0; i < b.N; i++ { // use b.N for looping 833 | record.ID = rand.Intn(1000000) 834 | recordData, err = json.Marshal(record) 835 | require.NoError(b, err) 836 | 837 | err = store.Set("user", record.ID, recordData) 838 | require.NoError(b, err) 839 | } 840 | 841 | err = store.Close() 842 | require.NoError(b, err) 843 | } 844 | 845 | func checkFileLines(t *testing.T, filePath string, checkCount int) { 846 | readFile, err := os.Open(filePath) 847 | require.NoError(t, err) 848 | assert.NotNil(t, readFile) 849 | 850 | count := 0 851 | 852 | scanner := bufio.NewScanner(readFile) 853 | for scanner.Scan() { 854 | count++ 855 | } 856 | 857 | err = readFile.Close() 858 | require.NoError(t, err) 859 | 860 | assert.Equal(t, checkCount, count) 861 | } 862 | 863 | func Benchmark_Set_Memory(b *testing.B) { 864 | path := memory 865 | 866 | store, err := fastdb.Open(path, syncIime) 867 | require.NoError(b, err) 868 | assert.NotNil(b, store) 869 | 870 | x1 := rand.NewSource(time.Now().UnixNano()) 871 | _ = rand.New(x1) 872 | 873 | record := &someRecord{ 874 | ID: 1, 875 | UUID: "UUIDtext", 876 | Text: "a text", 877 | } 878 | 879 | var recordData []byte 880 | 881 | b.ResetTimer() 882 | 883 | for i := 0; i < b.N; i++ { // use b.N for looping 884 | record.ID = rand.Intn(1000000) 885 | recordData, err = json.Marshal(record) 886 | require.NoError(b, err) 887 | 888 | err = store.Set("user", record.ID, recordData) 889 | require.NoError(b, err) 890 | } 891 | 892 | err = store.Close() 893 | require.NoError(b, err) 894 | } 895 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/marcelloh/fastdb 2 | 3 | go 1.23.2 4 | 5 | require ( 6 | github.com/stretchr/testify v1.9.0 7 | github.com/tidwall/gjson v1.18.0 8 | ) 9 | 10 | require ( 11 | github.com/davecgh/go-spew v1.1.1 // indirect 12 | github.com/pmezard/go-difflib v1.0.0 // indirect 13 | github.com/tidwall/match v1.1.1 // indirect 14 | github.com/tidwall/pretty v1.2.1 // indirect 15 | gopkg.in/yaml.v3 v3.0.1 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 5 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 6 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 7 | github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= 8 | github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 9 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 10 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 11 | github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 12 | github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= 13 | github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 14 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 15 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 16 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 17 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 18 | -------------------------------------------------------------------------------- /persist/aof.go: -------------------------------------------------------------------------------- 1 | package persist 2 | 3 | /* ------------------------------- Imports --------------------------- */ 4 | 5 | import ( 6 | "bufio" 7 | "fmt" 8 | "io" 9 | "os" 10 | "path/filepath" 11 | "strconv" 12 | "strings" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | /* ---------------------- Constants/Types/Variables ------------------ */ 18 | 19 | const fileMode = 0o600 20 | 21 | // AOF is Append Only File. 22 | type AOF struct { 23 | file *os.File 24 | syncTime int 25 | mu sync.RWMutex 26 | } 27 | 28 | var ( 29 | lock = &sync.Mutex{} 30 | osCreate = os.O_CREATE 31 | ) 32 | 33 | /* -------------------------- Methods/Functions ---------------------- */ 34 | 35 | /* 36 | OpenPersister opens the append only file and reads in all the data. 37 | */ 38 | func OpenPersister(path string, syncIime int) (*AOF, map[string]map[int][]byte, error) { 39 | aof := &AOF{syncTime: syncIime} 40 | 41 | filePath := filepath.Clean(path) 42 | if filePath != path { 43 | return nil, nil, fmt.Errorf("openPersister error: invalid path '%s'", path) 44 | } 45 | 46 | _, err := os.Stat(filepath.Dir(filePath)) 47 | if err != nil { 48 | return nil, nil, fmt.Errorf("openPersister (%s) error: %w", path, err) 49 | } 50 | 51 | keys, err := aof.getData(filePath) 52 | if err != nil { 53 | return nil, nil, err 54 | } 55 | 56 | go aof.flush() 57 | 58 | return aof, keys, nil 59 | } 60 | 61 | /* 62 | getData opens a file and reads the data into the memory. 63 | */ 64 | func (aof *AOF) getData(path string) (map[string]map[int][]byte, error) { 65 | aof.mu.Lock() 66 | defer aof.mu.Unlock() 67 | 68 | var ( 69 | file *os.File 70 | err error 71 | ) 72 | 73 | file, err = os.OpenFile(path, os.O_RDWR|osCreate, fileMode) //nolint:gosec // path is clean 74 | if err != nil { 75 | return nil, fmt.Errorf("openfile (%s) error: %w", path, err) 76 | } 77 | 78 | aof.file = file 79 | 80 | return aof.readDataFromFile(path) 81 | } 82 | 83 | /* 84 | readDataFromFile reads the file and fills the keys map. 85 | Returns the keys map and an error if something went wrong. 86 | It also closes the file if there was an error, and returns 87 | an error with the close error if there is one. 88 | */ 89 | func (aof *AOF) readDataFromFile(path string) (map[string]map[int][]byte, error) { 90 | keys, err := aof.fileReader() 91 | if err != nil { 92 | closeErr := aof.file.Close() 93 | if closeErr != nil { 94 | return nil, fmt.Errorf("fileReader (%s) error: %w; close error: %w", path, err, closeErr) 95 | } 96 | 97 | return nil, fmt.Errorf("fileReader (%s) error: %w", path, err) 98 | } 99 | 100 | return keys, err 101 | } 102 | 103 | /* 104 | fileReader reads the file and fills the keys. 105 | */ 106 | func (aof *AOF) fileReader() (map[string]map[int][]byte, error) { 107 | var ( 108 | count int 109 | err error 110 | ) 111 | 112 | keys := make(map[string]map[int][]byte, 1) 113 | scanner := bufio.NewScanner(aof.file) 114 | scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024) // Increase buffer size 115 | 116 | for scanner.Scan() { 117 | count++ 118 | instruction := scanner.Text() 119 | 120 | count, err = aof.processInstruction(instruction, scanner, count, keys) 121 | if err != nil { 122 | return nil, err 123 | } 124 | } 125 | 126 | return keys, nil 127 | } 128 | 129 | /* 130 | processInstruction processes an instruction from the AOF file and fills the keys. 131 | */ 132 | func (aof *AOF) processInstruction( 133 | instruction string, 134 | scanner *bufio.Scanner, 135 | count int, 136 | keys map[string]map[int][]byte, 137 | ) (int, error) { 138 | switch instruction { 139 | case "set": 140 | return aof.handleSetInstruction(scanner, count, keys) 141 | case "del": 142 | return aof.handleDelInstruction(scanner, count, keys) 143 | default: 144 | return count, fmt.Errorf("file (%s) has wrong instruction format '%s' on line: %d", aof.file.Name(), instruction, count) 145 | } 146 | } 147 | 148 | /* 149 | handleSetInstruction handles the set instruction. 150 | */ 151 | func (aof *AOF) handleSetInstruction(scanner *bufio.Scanner, inpCount int, keys map[string]map[int][]byte) (int, error) { 152 | count := inpCount 153 | 154 | if !scanner.Scan() { 155 | return count, fmt.Errorf("file (%s) has incomplete set instruction on line: %d", aof.file.Name(), count) 156 | } 157 | 158 | key := scanner.Text() 159 | 160 | if !scanner.Scan() { 161 | return count, fmt.Errorf("file (%s) has incomplete set instruction on line: %d", aof.file.Name(), count) 162 | } 163 | 164 | line := scanner.Text() 165 | 166 | err := aof.setBucketAndKey(key, line, keys) 167 | if err != nil { 168 | return count, err 169 | } 170 | 171 | count += 2 172 | 173 | return count, nil 174 | } 175 | 176 | /* 177 | handleDelInstruction handles the del instruction. 178 | */ 179 | func (aof *AOF) handleDelInstruction(scanner *bufio.Scanner, inpCount int, keys map[string]map[int][]byte) (int, error) { 180 | count := inpCount 181 | 182 | if !scanner.Scan() { 183 | return count, fmt.Errorf("file (%s) has incomplete del instruction on line: %d", aof.file.Name(), count) 184 | } 185 | 186 | key := scanner.Text() 187 | 188 | bucket, keyID, ok := aof.parseBucketAndKey(key) 189 | if !ok { 190 | return count, fmt.Errorf("file (%s) has wrong key format: '%s' on line: %d", aof.file.Name(), key, count) 191 | } 192 | 193 | delete(keys[bucket], keyID) 194 | 195 | count++ 196 | 197 | return count, nil 198 | } 199 | 200 | /* 201 | setBucketAndKey sets a key-value pair in a bucket. 202 | */ 203 | func (aof *AOF) setBucketAndKey(key, value string, keys map[string]map[int][]byte) error { 204 | bucket, keyID, ok := aof.parseBucketAndKey(key) 205 | if !ok { 206 | return fmt.Errorf("file (%s) has wrong key format: %s", aof.file.Name(), key) 207 | } 208 | 209 | if _, found := keys[bucket]; !found { 210 | keys[bucket] = map[int][]byte{} 211 | } 212 | 213 | keys[bucket][keyID] = []byte(value) 214 | 215 | return nil 216 | } 217 | 218 | /* 219 | parseBucketAndKey parses a key in the format "bucket_keyid" and returns 220 | the bucket name, key id and true if the key is valid. 221 | Otherwise it returns empty string, 0 and false. 222 | */ 223 | func (*AOF) parseBucketAndKey(key string) (string, int, bool) { 224 | uPos := strings.LastIndex(key, "_") 225 | if uPos < 0 { 226 | return "", 0, false 227 | } 228 | 229 | bucket := key[:uPos] 230 | 231 | keyID, err := strconv.Atoi(key[uPos+1:]) 232 | if err != nil { 233 | return "", 0, false 234 | } 235 | 236 | return bucket, keyID, true 237 | } 238 | 239 | /* 240 | Write writes to the file. 241 | */ 242 | func (aof *AOF) Write(lines string) error { 243 | _, err := aof.file.WriteString(lines) 244 | if err == nil && aof.syncTime == 0 { 245 | err = aof.file.Sync() 246 | } 247 | 248 | if err != nil { 249 | err = fmt.Errorf("write error: %#v %w", aof.file.Name(), err) 250 | } 251 | 252 | return err 253 | } 254 | 255 | /* 256 | Flush starts a goroutine to sync the database. 257 | The routine will stop if the file is closed 258 | */ 259 | func (aof *AOF) flush() { 260 | if aof.syncTime == 0 { 261 | return 262 | } 263 | 264 | flushPause := time.Millisecond * time.Duration(aof.syncTime) 265 | tick := time.NewTicker(flushPause) 266 | 267 | defer func() { 268 | tick.Stop() 269 | }() 270 | 271 | for range tick.C { 272 | err := aof.file.Sync() 273 | if err != nil { 274 | break 275 | } 276 | } 277 | } 278 | 279 | /* 280 | Defrag will only store the last key information, so all the history is lost 281 | This can mean a smaller filesize, which is quicker to read. 282 | */ 283 | func (aof *AOF) Defrag(keys map[string]map[int][]byte) (err error) { 284 | lock.Lock() 285 | defer lock.Unlock() 286 | 287 | // close current file (to flush the last parts) 288 | err = aof.Close() 289 | if err != nil { 290 | return fmt.Errorf("defrag->close error: %w", err) 291 | } 292 | 293 | err = aof.makeBackup() 294 | if err != nil { 295 | return fmt.Errorf("defrag->makeBackup error: %w", err) 296 | } 297 | 298 | err = aof.writeFile(keys) 299 | if err != nil { 300 | return fmt.Errorf("defrag->writeFile error: %w", err) 301 | } 302 | 303 | return nil 304 | } 305 | 306 | /* 307 | Close stops the flush routine, flushes the last data to disk and closes the file. 308 | */ 309 | func (aof *AOF) Close() error { 310 | err := aof.file.Sync() 311 | if err != nil { 312 | return fmt.Errorf("close->Sync error: %s %w", aof.file.Name(), err) 313 | } 314 | 315 | err = aof.file.Close() 316 | if err != nil { 317 | return fmt.Errorf("close error: %s %w", aof.file.Name(), err) 318 | } 319 | 320 | // to be sure that the flushing is stopped 321 | flushPause := time.Millisecond * time.Duration(aof.syncTime) 322 | time.Sleep(flushPause) 323 | 324 | return nil 325 | } 326 | 327 | /* 328 | makeBackup creates a backup of the current file. 329 | */ 330 | func (aof *AOF) makeBackup() (err error) { 331 | path := filepath.Clean(aof.file.Name()) 332 | 333 | source, err := os.Open(path) 334 | if err != nil { 335 | return fmt.Errorf("defrag->open error: %w", err) 336 | } 337 | 338 | defer func() { 339 | err = source.Close() 340 | }() 341 | 342 | // copy current file to backup 343 | destination, err := os.Create(filepath.Clean(path + ".bak")) 344 | if err != nil { 345 | return fmt.Errorf("defrag->create error: %w", err) 346 | } 347 | 348 | defer func() { 349 | err = destination.Close() 350 | if err != nil { 351 | err = fmt.Errorf("defrag->close error: %w", err) 352 | } 353 | }() 354 | 355 | _, err = io.Copy(destination, source) 356 | if err != nil { 357 | return fmt.Errorf("defrag->copy error: %w", err) 358 | } 359 | 360 | return nil 361 | } 362 | 363 | func (aof *AOF) writeFile(keys map[string]map[int][]byte) error { 364 | var err error 365 | 366 | path := aof.file.Name() 367 | 368 | // create and open temp file 369 | err = os.Remove(path) 370 | if err != nil { 371 | return fmt.Errorf("writeFile->remove (%#v) error: %w", path, err) 372 | } 373 | 374 | _, err = aof.getData(path) 375 | if err != nil { 376 | return fmt.Errorf("writeFile->getData error: %w", err) 377 | } 378 | 379 | // write keys to file 380 | go aof.flush() 381 | 382 | for bucket := range keys { 383 | startLine := "set\n" + bucket + "_" 384 | for key := range keys[bucket] { 385 | lines := startLine + strconv.Itoa(key) + "\n" + string(keys[bucket][key]) + "\n" 386 | 387 | err = aof.Write(lines) 388 | if err != nil { 389 | return fmt.Errorf("write error:%w", err) 390 | } 391 | } 392 | } 393 | 394 | return nil 395 | } 396 | -------------------------------------------------------------------------------- /persist/aof_internal_test.go: -------------------------------------------------------------------------------- 1 | package persist 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func Test_OpenPersister(t *testing.T) { 13 | path := "../data/fast_persister_error.db" 14 | 15 | orgCreate := osCreate 16 | osCreate = os.O_RDONLY 17 | 18 | defer func() { 19 | osCreate = orgCreate 20 | filePath := filepath.Clean(path) 21 | _ = os.Remove(filePath) 22 | }() 23 | 24 | aof, keys, err := OpenPersister(path, 0) 25 | require.Error(t, err) 26 | assert.Nil(t, keys) 27 | assert.Nil(t, aof) 28 | } 29 | 30 | func Test_OpenPersister_closeError(t *testing.T) { 31 | path := "../data/fast_persister_close_error.db" 32 | 33 | defer func() { 34 | filePath := filepath.Clean(path) 35 | err := os.Remove(filePath) 36 | require.NoError(t, err) 37 | }() 38 | 39 | aof, keys, err := OpenPersister(path, 100) 40 | require.NoError(t, err) 41 | assert.NotNil(t, aof) 42 | assert.NotNil(t, keys) 43 | 44 | err = aof.file.Close() 45 | require.NoError(t, err) 46 | 47 | err = aof.Close() 48 | require.Error(t, err) 49 | } 50 | -------------------------------------------------------------------------------- /persist/aof_test.go: -------------------------------------------------------------------------------- 1 | package persist_test 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "sync" 9 | "testing" 10 | 11 | "github.com/marcelloh/fastdb/persist" 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | const ( 17 | syncIime = 100 18 | dataDir = "./../data" 19 | ) 20 | 21 | func Test_OpenPersister_noData(t *testing.T) { 22 | path := "../data/fast_nodata.db" 23 | 24 | defer func() { 25 | filePath := filepath.Clean(path) 26 | err := os.Remove(filePath) 27 | require.NoError(t, err) 28 | }() 29 | 30 | aof, keys, err := persist.OpenPersister(path, syncIime) 31 | require.NoError(t, err) 32 | assert.NotNil(t, aof) 33 | assert.NotNil(t, keys) 34 | 35 | defer func() { 36 | err = aof.Close() 37 | require.NoError(t, err) 38 | }() 39 | } 40 | 41 | func Test_OpenPersister_invalidPath(t *testing.T) { 42 | path := "../data/../fast.db" 43 | aof, keys, err := persist.OpenPersister(path, syncIime) 44 | require.Error(t, err) 45 | assert.Nil(t, aof) 46 | assert.Nil(t, keys) 47 | } 48 | 49 | func Test_OpenPersister_nonExistingPath(t *testing.T) { 50 | path := "../data/non_existent_dir/fast.db" 51 | aof, keys, err := persist.OpenPersister(path, syncIime) 52 | require.Error(t, err) 53 | assert.Nil(t, aof) 54 | assert.Nil(t, keys) 55 | } 56 | 57 | func Test_OpenPersister_withData(t *testing.T) { 58 | path := "../data/fast_persister.db" 59 | 60 | defer func() { 61 | filePath := filepath.Clean(path) 62 | err := os.Remove(filePath) 63 | require.NoError(t, err) 64 | }() 65 | 66 | aof, keys, err := persist.OpenPersister(path, syncIime) 67 | require.NoError(t, err) 68 | assert.NotNil(t, aof) 69 | assert.NotNil(t, keys) 70 | 71 | lines := "set\ntext_1\nvalue for key 1\n" 72 | err = aof.Write(lines) 73 | require.NoError(t, err) 74 | 75 | lines = "set\ntext_2\nvalue for key 2\n" 76 | err = aof.Write(lines) 77 | require.NoError(t, err) 78 | 79 | lines = "del\ntext_2\n" 80 | err = aof.Write(lines) 81 | require.NoError(t, err) 82 | 83 | err = aof.Close() 84 | require.NoError(t, err) 85 | 86 | // here's were we check the actual reading of the data 87 | 88 | aof, keys, err = persist.OpenPersister(path, 0) 89 | defer func() { 90 | err = aof.Close() 91 | require.NoError(t, err) 92 | }() 93 | 94 | require.NoError(t, err) 95 | assert.NotNil(t, aof) 96 | assert.NotNil(t, keys) 97 | assert.Len(t, keys, 1) 98 | bucketKeys := keys["text"] 99 | assert.NotNil(t, bucketKeys) 100 | assert.Len(t, bucketKeys, 1) 101 | } 102 | 103 | func Test_OpenPersister_withWeirdData(t *testing.T) { 104 | path := "../data/fast_persister_weird.db" 105 | 106 | defer func() { 107 | filePath := filepath.Clean(path) 108 | err := os.Remove(filePath) 109 | require.NoError(t, err) 110 | }() 111 | 112 | aof, keys, err := persist.OpenPersister(path, syncIime) 113 | require.NoError(t, err) 114 | assert.NotNil(t, aof) 115 | assert.NotNil(t, keys) 116 | 117 | lines := "set\nmyBucket_1\nvalue for key 1\nwith extra enter\n" 118 | err = aof.Write(lines) 119 | require.NoError(t, err) 120 | 121 | lines = "set\nmyBucket_2\nvalue for key 2\n" 122 | err = aof.Write(lines) 123 | require.NoError(t, err) 124 | 125 | err = aof.Close() 126 | require.NoError(t, err) 127 | 128 | // here's were we check the actual reading of the data 129 | 130 | aof, keys, err = persist.OpenPersister(path, 0) 131 | require.Error(t, err) 132 | assert.Nil(t, aof) 133 | assert.Empty(t, keys) 134 | } 135 | 136 | func Test_OpenPersister_IncompleteSetInstructionNoKey(t *testing.T) { 137 | path := "../data/fast_persister_weird.db" 138 | 139 | defer func() { 140 | filePath := filepath.Clean(path) 141 | err := os.Remove(filePath) 142 | require.NoError(t, err) 143 | }() 144 | 145 | aof, keys, err := persist.OpenPersister(path, syncIime) 146 | require.NoError(t, err) 147 | assert.NotNil(t, aof) 148 | assert.NotNil(t, keys) 149 | 150 | lines := "set\n" 151 | err = aof.Write(lines) 152 | require.NoError(t, err) 153 | 154 | err = aof.Close() 155 | require.NoError(t, err) 156 | 157 | // here's were we check the actual reading of the data 158 | 159 | aof, keys, err = persist.OpenPersister(path, 0) 160 | require.Error(t, err) 161 | assert.Nil(t, aof) 162 | assert.Empty(t, keys) 163 | } 164 | 165 | func Test_OpenPersister_IncompleteSetInstructionWithKey(t *testing.T) { 166 | path := "../data/fast_persister_weird.db" 167 | 168 | defer func() { 169 | filePath := filepath.Clean(path) 170 | err := os.Remove(filePath) 171 | require.NoError(t, err) 172 | }() 173 | 174 | aof, keys, err := persist.OpenPersister(path, syncIime) 175 | require.NoError(t, err) 176 | assert.NotNil(t, aof) 177 | assert.NotNil(t, keys) 178 | 179 | lines := "set\nmyBucket_2\n" 180 | err = aof.Write(lines) 181 | require.NoError(t, err) 182 | 183 | err = aof.Close() 184 | require.NoError(t, err) 185 | 186 | // here's were we check the actual reading of the data 187 | 188 | aof, keys, err = persist.OpenPersister(path, 0) 189 | require.Error(t, err) 190 | assert.Nil(t, aof) 191 | assert.Empty(t, keys) 192 | } 193 | 194 | func Test_OpenPersister_IncompleteDelInstructionNoKey(t *testing.T) { 195 | path := "../data/fast_persister_weird.db" 196 | 197 | defer func() { 198 | filePath := filepath.Clean(path) 199 | err := os.Remove(filePath) 200 | require.NoError(t, err) 201 | }() 202 | 203 | aof, keys, err := persist.OpenPersister(path, syncIime) 204 | require.NoError(t, err) 205 | assert.NotNil(t, aof) 206 | assert.NotNil(t, keys) 207 | 208 | lines := "del\n" 209 | err = aof.Write(lines) 210 | require.NoError(t, err) 211 | 212 | err = aof.Close() 213 | require.NoError(t, err) 214 | 215 | // here's were we check the actual reading of the data 216 | 217 | aof, keys, err = persist.OpenPersister(path, 0) 218 | require.Error(t, err) 219 | assert.Nil(t, aof) 220 | assert.Empty(t, keys) 221 | } 222 | 223 | func Test_OpenPersister_IncompleteDelInstructionWithKey(t *testing.T) { 224 | path := "../data/fast_persister_weird.db" 225 | 226 | defer func() { 227 | filePath := filepath.Clean(path) 228 | err := os.Remove(filePath) 229 | require.NoError(t, err) 230 | }() 231 | 232 | aof, keys, err := persist.OpenPersister(path, syncIime) 233 | require.NoError(t, err) 234 | assert.NotNil(t, aof) 235 | assert.NotNil(t, keys) 236 | 237 | lines := "del\nmyBucket_two\n" 238 | err = aof.Write(lines) 239 | require.NoError(t, err) 240 | 241 | err = aof.Close() 242 | require.NoError(t, err) 243 | 244 | // here's were we check the actual reading of the data 245 | 246 | aof, keys, err = persist.OpenPersister(path, 0) 247 | require.Error(t, err) 248 | assert.Nil(t, aof) 249 | assert.Empty(t, keys) 250 | } 251 | 252 | func Test_OpenPersister_writeError(t *testing.T) { 253 | path := "../data/fast_persister_write_error.db" 254 | 255 | defer func() { 256 | filePath := filepath.Clean(path) 257 | err := os.Remove(filePath) 258 | require.NoError(t, err) 259 | }() 260 | 261 | aof, keys, err := persist.OpenPersister(path, syncIime) 262 | require.NoError(t, err) 263 | assert.NotNil(t, aof) 264 | assert.NotNil(t, keys) 265 | 266 | err = aof.Close() 267 | require.NoError(t, err) 268 | 269 | lines := "set\ntext_1\na value\n" 270 | err = aof.Write(lines) 271 | require.Error(t, err) 272 | } 273 | 274 | func Test_OpenPersister_withNoUnderscoredKey(t *testing.T) { 275 | path := "../data/fast_persister_wrong_key1.db" 276 | 277 | defer func() { 278 | filePath := filepath.Clean(path) 279 | err := os.Remove(filePath) 280 | require.NoError(t, err) 281 | }() 282 | 283 | aof, keys, err := persist.OpenPersister(path, syncIime) 284 | require.NoError(t, err) 285 | assert.NotNil(t, aof) 286 | assert.NotNil(t, keys) 287 | 288 | lines := "set\ntextone\na value\n" 289 | err = aof.Write(lines) 290 | require.NoError(t, err) 291 | 292 | err = aof.Close() 293 | require.NoError(t, err) 294 | 295 | // here's were we check the actual reading of the data 296 | 297 | aof, keys, err = persist.OpenPersister(path, 0) 298 | require.Error(t, err) 299 | assert.Nil(t, aof) 300 | assert.Nil(t, keys) 301 | } 302 | 303 | func Test_OpenPersister_withNoNumericKey(t *testing.T) { 304 | path := "../data/fast_persister_wrong_key.db" 305 | 306 | defer func() { 307 | filePath := filepath.Clean(path) 308 | err := os.Remove(filePath) 309 | require.NoError(t, err) 310 | }() 311 | 312 | aof, keys, err := persist.OpenPersister(path, syncIime) 313 | require.NoError(t, err) 314 | assert.NotNil(t, aof) 315 | assert.NotNil(t, keys) 316 | 317 | lines := "set\nwrong_key\na value\n" 318 | err = aof.Write(lines) 319 | require.NoError(t, err) 320 | 321 | err = aof.Close() 322 | require.NoError(t, err) 323 | 324 | // here's were we check the actual reading of the data 325 | 326 | aof, keys, err = persist.OpenPersister(path, 0) 327 | require.Error(t, err) 328 | assert.Nil(t, aof) 329 | assert.Nil(t, keys) 330 | } 331 | 332 | func Test_OpenPersister_withWrongInstruction(t *testing.T) { 333 | path := "../data/fast_persister_wrong_instruction.db" 334 | 335 | filePath := filepath.Clean(path) 336 | _ = os.Remove(filePath) 337 | 338 | aof, keys, err := persist.OpenPersister(path, syncIime) 339 | require.NoError(t, err) 340 | assert.NotNil(t, aof) 341 | assert.NotNil(t, keys) 342 | 343 | lines := "wrong\ntext_1\na value\n" 344 | err = aof.Write(lines) 345 | require.NoError(t, err) 346 | 347 | err = aof.Close() 348 | require.NoError(t, err) 349 | 350 | // here's were we check the actual reading of the data 351 | 352 | aof, keys, err = persist.OpenPersister(path, 0) 353 | require.Error(t, err) 354 | assert.Nil(t, aof) 355 | assert.Nil(t, keys) 356 | 357 | defer func() { 358 | err = os.Remove(filePath) 359 | require.NoError(t, err) 360 | }() 361 | } 362 | 363 | func Test_OpenPersister_concurrentWrites(t *testing.T) { 364 | path := "../data/concurrent_write.db" 365 | 366 | defer func() { 367 | filePath := filepath.Clean(path) 368 | err := os.Remove(filePath) 369 | require.NoError(t, err) 370 | }() 371 | 372 | aof, _, err := persist.OpenPersister(path, syncIime) 373 | 374 | require.NoError(t, err) 375 | assert.NotNil(t, aof) 376 | 377 | var wg sync.WaitGroup 378 | for i := range 10 { 379 | wg.Add(1) 380 | 381 | go func(i int) { 382 | defer wg.Done() 383 | 384 | lines := fmt.Sprintf("set\nkey_%d\nvalue for key %d\n", i, i) 385 | 386 | err = aof.Write(lines) 387 | assert.NoError(t, err) 388 | }(i) 389 | } 390 | 391 | wg.Wait() 392 | 393 | // Check if all keys were written correctly 394 | aof, keys, err := persist.OpenPersister(path, 0) 395 | require.NoError(t, err) 396 | assert.Len(t, keys, 1) // Expecting 10 keys 397 | bucketKeys := keys["key"] 398 | assert.NotNil(t, bucketKeys) 399 | assert.Len(t, bucketKeys, 10) 400 | } 401 | 402 | func Test_OpenPersister_writeAfterClose(t *testing.T) { 403 | path := "../data/write_after_close.db" 404 | defer func() { 405 | filePath := filepath.Clean(path) 406 | err := os.Remove(filePath) 407 | require.NoError(t, err) 408 | }() 409 | 410 | aof, _, err := persist.OpenPersister(path, syncIime) 411 | require.NoError(t, err) 412 | assert.NotNil(t, aof) 413 | 414 | err = aof.Close() 415 | require.NoError(t, err) 416 | 417 | lines := "set\nkey_after_close\nvalue\n" 418 | err = aof.Write(lines) 419 | require.Error(t, err) // Expect an error since the file is closed 420 | } 421 | 422 | func Test_OpenPersister_invalidInstructionFormat(t *testing.T) { 423 | path := "../data/invalid_instruction_format.db" 424 | defer func() { 425 | filePath := filepath.Clean(path) 426 | err := os.Remove(filePath) 427 | require.NoError(t, err) 428 | }() 429 | 430 | lines := "invalid_instruction\nkey\nvalue\n" 431 | err := os.WriteFile(path, []byte(lines), 0o644) 432 | require.NoError(t, err) 433 | 434 | aof, keys, err := persist.OpenPersister(path, syncIime) 435 | require.Error(t, err) 436 | assert.Nil(t, aof) 437 | assert.Nil(t, keys) 438 | } 439 | 440 | func Test_Defrag(t *testing.T) { 441 | path := "../data/fastdb_defrag100.db" 442 | filePath := filepath.Clean(path) 443 | 444 | defer func() { 445 | err := os.Remove(filePath) 446 | require.NoError(t, err) 447 | 448 | _ = os.Remove(filePath + ".bak") 449 | }() 450 | 451 | total := 100 452 | 453 | aof, keys, err := persist.OpenPersister(path, syncIime) 454 | require.NoError(t, err) 455 | assert.NotNil(t, aof) 456 | assert.NotNil(t, keys) 457 | 458 | defer func() { 459 | err = aof.Close() 460 | require.NoError(t, err) 461 | }() 462 | 463 | for range total { 464 | lines := "set\ntext_1\na value for key 1\n" 465 | err = aof.Write(lines) 466 | require.NoError(t, err) 467 | } 468 | 469 | checkFileLines(t, filePath, total*3) 470 | 471 | keys["text"] = map[int][]byte{} 472 | keys["text"][1] = []byte("value for key 1") 473 | err = aof.Defrag(keys) 474 | require.NoError(t, err) 475 | 476 | checkFileLines(t, filePath, 3) 477 | } 478 | 479 | func Test_Defrag_AlreadyClosed(t *testing.T) { 480 | path := "../data/fastdb_defrag100.db" 481 | filePath := filepath.Clean(path) 482 | 483 | defer func() { 484 | err := os.Remove(filePath) 485 | require.NoError(t, err) 486 | 487 | _ = os.Remove(filePath + ".bak") 488 | }() 489 | 490 | aof, keys, err := persist.OpenPersister(path, syncIime) 491 | require.NoError(t, err) 492 | assert.NotNil(t, aof) 493 | assert.NotNil(t, keys) 494 | 495 | err = aof.Close() 496 | require.NoError(t, err) 497 | 498 | keys["text"] = map[int][]byte{} 499 | keys["text"][1] = []byte("value for key 1") 500 | err = aof.Defrag(keys) 501 | require.Error(t, err) 502 | } 503 | 504 | func checkFileLines(t *testing.T, filePath string, checkCount int) { 505 | readFile, err := os.Open(filePath) 506 | require.NoError(t, err) 507 | assert.NotNil(t, readFile) 508 | 509 | count := 0 510 | 511 | scanner := bufio.NewScanner(readFile) 512 | for scanner.Scan() { 513 | count++ 514 | } 515 | 516 | err = readFile.Close() 517 | require.NoError(t, err) 518 | assert.Equal(t, checkCount, count) 519 | } 520 | -------------------------------------------------------------------------------- /persist/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package persist holds persistence functionality. 3 | */ 4 | package persist 5 | --------------------------------------------------------------------------------