├── .github ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── lint.yml │ └── tests.yml ├── .gitignore ├── .golangci.yml ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── auth ├── README.md ├── access_token_credentials │ ├── README.md │ └── main.go ├── anonymous_credentials │ ├── README.md │ └── main.go ├── environ │ ├── README.md │ └── main.go ├── metadata_credentials │ ├── README.md │ └── main.go ├── service_account_credentials │ ├── README.md │ └── main.go └── static_credentials │ ├── README.md │ └── main.go ├── basic ├── database_sql │ ├── README.md │ ├── data.go │ ├── main.go │ └── series.go ├── gorm │ ├── README.md │ ├── data.go │ ├── main.go │ └── models.go └── native │ ├── README.md │ ├── data.go │ ├── main.go │ └── series.go ├── bulk_upsert ├── README.md ├── example.go └── main.go ├── containers ├── README.md ├── containers.go └── main.go ├── ddl ├── README.md ├── ddl.go └── main.go ├── decimal ├── README.md ├── decimal.go └── main.go ├── describe ├── README.md ├── main.go └── table.tpl ├── go.mod ├── go.sum ├── pagination ├── README.md ├── cities.go ├── data.go └── main.go ├── partitioning_policies ├── README.md ├── example.go └── main.go ├── read_table ├── README.md ├── main.go └── orders.go ├── serverless ├── healthcheck │ ├── README.md │ ├── main.go │ └── service.go └── url_shortener │ ├── README.md │ ├── main.go │ ├── service.go │ └── static │ └── index.html ├── topic ├── cdc-cache-bus-freeseats │ ├── .gitignore │ ├── balancer.go │ ├── cache.go │ ├── database.go │ ├── main.go │ ├── webserver.go │ └── webserver_cdc.go ├── cdc-fill-and-read │ ├── cdc-reader.go │ ├── main.go │ └── tables.go ├── topicreader │ ├── stubs.go │ ├── topicreader_advanced.go │ ├── topicreader_show.go │ ├── topicreader_simple.go │ └── topicreader_trace.go └── topicwriter │ └── topicwriter.go ├── ttl ├── README.md ├── main.go └── series.go └── ttl_readtable ├── main.go └── series.go /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla/?lang=en 2 | 3 | 4 | 5 | ## Pull request type 6 | 7 | 8 | 9 | Please check the type of change your PR introduces: 10 | 11 | - [ ] Bugfix 12 | - [ ] Feature 13 | - [ ] Code style update (formatting, renaming) 14 | - [ ] Refactoring (no functional changes, no api changes) 15 | - [ ] Build related changes 16 | - [ ] Documentation content changes 17 | - [ ] Other (please describe): 18 | 19 | ## What is the current behavior? 20 | 21 | 22 | 23 | Issue Number: N/A 24 | 25 | ## What is the new behavior? 26 | 27 | 28 | 29 | - 30 | - 31 | - 32 | 33 | ## Other information 34 | 35 | 36 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | workflow_dispatch: 8 | jobs: 9 | golangci: 10 | concurrency: 11 | group: lint-${{ github.ref }}-${{ matrix.os }}-${{ matrix.go-version }} 12 | cancel-in-progress: true 13 | name: golangci-lint 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: golangci-lint 18 | uses: golangci/golangci-lint-action@v2 19 | with: 20 | version: v1.50.1 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # options for analysis running 2 | run: 3 | # default concurrency is a available CPU number 4 | concurrency: 4 5 | 6 | # timeout for analysis, e.g. 30s, 5m, default is 1m 7 | deadline: 5m 8 | 9 | # exit code when at least one issue was found, default is 1 10 | issues-exit-code: 1 11 | 12 | # include test files or not, default is true 13 | tests: true 14 | 15 | # list of build tags, all linters use it. Default is empty list. 16 | #build-tags: 17 | # - mytag 18 | 19 | # which dirs to skip: they won't be analyzed; 20 | # can use regexp here: generated.*, regexp is applied on full path; 21 | # default value is empty list, but next dirs are always skipped independently 22 | # from this option's value: 23 | # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ 24 | # skip-dirs: 25 | 26 | # which files to skip: they will be analyzed, but issues from them 27 | # won't be reported. Default value is empty list, but there is 28 | # no need to include all autogenerated files, we confidently recognize 29 | # autogenerated files. If it's not please let us know. 30 | # skip-files: 31 | # - ".*\\.my\\.go$" 32 | # - lib/bad.go 33 | 34 | 35 | # output configuration options 36 | output: 37 | # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number" 38 | format: colored-line-number 39 | 40 | # print lines of code with issue, default is true 41 | print-issued-lines: true 42 | 43 | # print linter name in the end of issue text, default is true 44 | print-linter-name: true 45 | 46 | 47 | # all available settings of specific linters 48 | linters-settings: 49 | errcheck: 50 | # report about not checking of errors in type assetions: `a := b.(MyStruct)`; 51 | # default is false: such cases aren't reported by default. 52 | check-type-assertions: false 53 | 54 | # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; 55 | # default is false: such cases aren't reported by default. 56 | check-blank: false 57 | govet: 58 | # report about shadowed variables 59 | check-shadowing: true 60 | fieldalignment: true 61 | golint: 62 | # minimal confidence for issues, default is 0.8 63 | min-confidence: 0.8 64 | gofmt: 65 | # simplify code: gofmt with `-s` option, true by default 66 | simplify: true 67 | goimports: 68 | # put imports beginning with prefix after 3rd-party packages; 69 | # it's a comma-separated list of prefixes 70 | local-prefixes: github.com/ydb-platform/ydb-go-examples 71 | goconst: 72 | # minimal length of string constant, 3 by default 73 | min-len: 2 74 | # minimal occurrences count to trigger, 3 by default 75 | min-occurrences: 2 76 | maligned: 77 | # print struct with more effective memory layout or not, false by default 78 | suggest-new: true 79 | misspell: 80 | # Correct spellings using locale preferences for US or UK. 81 | # Default is to use a neutral variety of English. 82 | # Setting locale to US will correct the British spelling of 'colour' to 'color'. 83 | locale: US 84 | revive: 85 | rules: 86 | - name: package-comments 87 | disabled: true 88 | unused: 89 | # treat code as a program (not a library) and report unused exported identifiers; default is false. 90 | # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: 91 | # if it's called for subdir of a project it can't find funcs usages. All text editor integrations 92 | # with golangci-lint call it on a directory with the changed file. 93 | check-exported: false 94 | unparam: 95 | # call graph construction algorithm (cha, rta). In general, use cha for libraries, 96 | # and rta for programs with main packages. Default is cha. 97 | algo: cha 98 | 99 | # Inspect exported functions, default is false. Set to true if no external program/library imports your code. 100 | # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: 101 | # if it's called for subdir of a project it can't find external interfaces. All text editor integrations 102 | # with golangci-lint call it on a directory with the changed file. 103 | check-exported: false 104 | 105 | linters: 106 | disable-all: true 107 | enable: 108 | - deadcode 109 | - depguard 110 | - errcheck 111 | - goconst 112 | - gofmt # On why gofmt when goimports is enabled - https://github.com/golang/go/issues/21476 113 | - goimports 114 | - gosimple 115 | - govet 116 | - ineffassign 117 | - misspell 118 | - revive 119 | - staticcheck 120 | - structcheck 121 | - typecheck 122 | - unconvert 123 | - unparam 124 | - unused 125 | - varcheck 126 | 127 | issues: 128 | # List of regexps of issue texts to exclude, empty list by default. 129 | # But independently from this option we use default exclude patterns, 130 | # it can be disabled by `exclude-use-default: false`. To list all 131 | # excluded by default patterns execute `golangci-lint run --help` 132 | # exclude: 133 | 134 | # Independently from option `exclude` we use default exclude patterns, 135 | # it can be disabled by this option. To list all 136 | # excluded by default patterns execute `golangci-lint run --help`. 137 | # Default value for this option is true. 138 | exclude-use-default: false 139 | 140 | # Maximum issues count per one linter. Set to 0 to disable. Default is 50. 141 | max-per-linter: 0 142 | 143 | # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. 144 | max-same-issues: 0 145 | 146 | # Show only new issues: if there are unstaged changes or untracked files, 147 | # only those changes are analyzed, else only changes in HEAD~ are analyzed. 148 | # It's a super-useful option for integration of golangci-lint into existing 149 | # large codebase. It's not practical to fix all existing issues at the moment 150 | # of integration: much better don't allow issues in new code. 151 | # Default is false. 152 | new: false 153 | 154 | # Show only new issues created after git revision `REV` 155 | # new-from-rev: REV 156 | 157 | # Show only new issues created in git patch with set file path. 158 | # new-from-patch: path/to/patch/file 159 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ydb-platform/ydb-go-examples/9a6bae30f1343c84911ed0ff80b908f352b8da8f/CHANGELOG.md -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers. All 58 | complaints will be reviewed and investigated and will result in a response that 59 | is deemed necessary and appropriate to the circumstances. The project team is 60 | obligated to maintain confidentiality with regard to the reporter of an incident. 61 | Further details of specific enforcement policies may be posted separately. 62 | 63 | ## Attribution 64 | 65 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 66 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 67 | 68 | [homepage]: https://www.contributor-covenant.org 69 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Notice to external contributors 2 | 3 | 4 | ## General info 5 | 6 | Hello! In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the “**CLA**”). The current version of the CLA you may find here: 7 | 1) https://yandex.ru/legal/cla/?lang=en (in English) and 8 | 2) https://yandex.ru/legal/cla/?lang=ru (in Russian). 9 | 10 | By adopting the CLA, you state the following: 11 | 12 | * You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA, 13 | * You has read the terms and conditions of the CLA and agree with them in full, 14 | * You are legally able to provide and license your contributions as stated, 15 | * We may use your contributions for our open source projects and for any other our project too, 16 | * We rely on your assurances concerning the rights of third parties in relation to your contributes. 17 | 18 | If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you has already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA. 19 | 20 | ## Provide contributions 21 | 22 | If you have already adopted terms and conditions of the CLA, you are able to provide your contributes. When you submit your pull request, please add the following information into it: 23 | 24 | ``` 25 | I hereby agree to the terms of the CLA available at: [link]. 26 | ``` 27 | 28 | Replace the bracketed text as follows: 29 | * [link] is the link at the current version of the CLA (you may add here a link https://yandex.ru/legal/cla/?lang=en (in English) or a link https://yandex.ru/legal/cla/?lang=ru (in Russian). 30 | 31 | It is enough to provide us such notification at once. 32 | 33 | ## Other questions 34 | 35 | If you have any questions, please mail us at opensource@yandex-team.ru. 36 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: basic bulk_upsert containers ddl decimal serverless/healthcheck pagination partitioning_policies read_table ttl ttl_readtable serverless/url_shortener 2 | 3 | lint: 4 | golangci-lint run ./basic ./bulk_upsert ./containers ./ddl ./decimal ./healthcheck ./pagination ./partitioning_policies ./read_table ./ttl ./ttl_readtable ./url_shortener 5 | 6 | basic: 7 | go run ./basic/native -ydb=${YDB_CONNECTION_STRING} -prefix=basic 8 | 9 | database_sql: 10 | go run ./basic/database_sql -ydb=${YDB_CONNECTION_STRING} -prefix=database/sql 11 | 12 | bulk_upsert: 13 | go run ./bulk_upsert -ydb=${YDB_CONNECTION_STRING} -prefix=bulk_upsert -table=bulk_upsert 14 | 15 | containers: 16 | go run ./containers -ydb=${YDB_CONNECTION_STRING} -prefix=containers 17 | 18 | ddl: 19 | go run ./ddl -ydb=${YDB_CONNECTION_STRING} -prefix=ddl 20 | 21 | decimal: 22 | go run ./decimal -ydb=${YDB_CONNECTION_STRING} -prefix=decimal 23 | 24 | pagination: 25 | go run ./pagination -ydb=${YDB_CONNECTION_STRING} -prefix=pagination 26 | 27 | partitioning_policies: 28 | go run ./partitioning_policies -ydb=${YDB_CONNECTION_STRING} -prefix=partitioning_policies -table=partitioning_policies 29 | 30 | read_table: 31 | go run ./read_table -ydb=${YDB_CONNECTION_STRING} -prefix=read_table 32 | 33 | ttl: 34 | go run ./ttl -ydb=${YDB_CONNECTION_STRING} -prefix=ttl 35 | 36 | ttl_readtable: 37 | go run ./ttl_readtable -ydb=${YDB_CONNECTION_STRING} -prefix=ttl_readtable 38 | 39 | healthcheck: 40 | go run ./serverless/healthcheck -ydb=${YDB_CONNECTION_STRING} -prefix=healthcheck -url=ya.ru -url=google.com 41 | 42 | url_shortener: 43 | go run ./serverless/url_shortener -ydb=${YDB_CONNECTION_STRING} -prefix=url_shortener 44 | 45 | describe: 46 | go run ./describe -ydb=${YDB_CONNECTION_STRING} -prefix=/ -t="Optional" -t="Interval" 47 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2019 YANDEX LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ydb-go-examples 2 | 3 | > examples for work with YDB 4 | 5 | ## Navigation of examples 6 | 7 | | Example | Description | Run command | 8 | |------------------------------------|-----------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------| 9 | | `auth/access_token_credentials` | authenticate with access token credentials | see [README.md](https://github.com/ydb-platform/ydb-go-examples/tree/master/auth/access_token_credentials#readme) | 10 | | `auth/anonymous_credentials` | authenticate with anonymous credentials | see [README.md](https://github.com/ydb-platform/ydb-go-examples/tree/master/auth/anonymous_credentials#readme) | 11 | | `auth/metadata_credentials` | authenticate with metadata credentials | see [README.md](https://github.com/ydb-platform/ydb-go-examples/tree/master/auth/metadata_credentials#readme) | 12 | | `auth/service_account_credentials` | authenticate with service account credentials | see [README.md](https://github.com/ydb-platform/ydb-go-examples/tree/master/auth/service_account_credentials#readme) | 13 | | `auth/environ` | authenticate using environment variables | see [README.md](https://github.com/ydb-platform/ydb-go-examples/tree/master/auth/environ#readme) | 14 | | `basic/native` | store and read the series with native driver | `make basic` | 15 | | `basic/database_sql` | store and read the series with database/sql driver | `make database_sql` | 16 | | `serverless/healthcheck` | healthcheck site by URL (yandex function and local http-server) | `make healthcheck` | 17 | | `serverless/url_shortener` | URL shortener example (yandex function and local http-server) | `make url_shortener` | 18 | | `bulk_upsert` | bulk upserting data | `make bulk_upsert` | 19 | | `containers` | containers example | `make containers` | 20 | | `ddl` | DDL requests example | `make ddl` | 21 | | `decimal` | decimal store and read | `make decimal` | 22 | | `pagination` | pagination example | `make pagination` | 23 | | `partitioning_policies` | partitioning_policies example | `make partitioning_policies` | 24 | | `read_table` | read table example | `make read_table` | 25 | | `topic/cdc-cache-bus-freeseats` | example of use cdc for cache updates in web application | `go run topic/cdc-example-cache-freeseats/*.go` | 26 | | `topic/cdc-fill-and-read` | change table records and read cdc stream | `go run topic/cdc/*.go` | 27 | | `ttl` | TTL using example | `make ttl` | 28 | | `ttl_readtable` | TTL using example | `make ttl_readtable` | 29 | 30 | Run command needs prepared environ like this: 31 | ```bash 32 | export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=~/.ydb/SA.json 33 | export YDB_CONNECTION_STRING="grpcs://ydb.serverless.yandexcloud.net:2135/?database=/ru-central1/b1g8skpblkos03malf3s/etn02qhd0tfkrq4riqgd" 34 | ``` 35 | -------------------------------------------------------------------------------- /auth/README.md: -------------------------------------------------------------------------------- 1 | # auth examples 2 | 3 | Auth examples helps to understand YDB authentication: 4 | * `access_token_credentials` - example of use access token credentials 5 | * `anonymous_credentials` - example of use anonymous credentials 6 | * `metadata_credentials` - example of use metadata credentials 7 | * `service_account_credentials` - example of use service account key file credentials 8 | * `static_credentials` - example of use static credentials 9 | * `environ` - example of use environment variables to configure YDB authenticate 10 | 11 | By default, connect to YDB used environment variables unless otherwise specified 12 | -------------------------------------------------------------------------------- /auth/access_token_credentials/README.md: -------------------------------------------------------------------------------- 1 | # Authenticate with access token credentials 2 | 3 | `access_token_credentials` example provide code snippet for authenticate to YDB with access token credentials 4 | 5 | ## Runing code snippet 6 | ```bash 7 | access_token_credentials -ydb="grpcs://endpoint/?database=database" -token="mytoken" 8 | ``` 9 | 10 | -------------------------------------------------------------------------------- /auth/access_token_credentials/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | 9 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 10 | ) 11 | 12 | var ( 13 | dsn string 14 | token string 15 | ) 16 | 17 | func init() { 18 | required := []string{"ydb", "token"} 19 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 20 | flagSet.Usage = func() { 21 | out := flagSet.Output() 22 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 23 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 24 | flagSet.PrintDefaults() 25 | } 26 | flagSet.StringVar(&dsn, 27 | "ydb", "", 28 | "YDB connection string", 29 | ) 30 | flagSet.StringVar(&token, 31 | "token", "", 32 | "access token", 33 | ) 34 | if err := flagSet.Parse(os.Args[1:]); err != nil { 35 | flagSet.Usage() 36 | os.Exit(1) 37 | } 38 | flagSet.Visit(func(f *flag.Flag) { 39 | for i, arg := range required { 40 | if arg == f.Name { 41 | required = append(required[:i], required[i+1:]...) 42 | } 43 | } 44 | }) 45 | if len(required) > 0 { 46 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 47 | flagSet.Usage() 48 | os.Exit(1) 49 | } 50 | } 51 | 52 | func main() { 53 | ctx, cancel := context.WithCancel(context.Background()) 54 | defer cancel() 55 | db, err := ydb.Open(ctx, dsn, 56 | ydb.WithAccessTokenCredentials(token), 57 | ) 58 | if err != nil { 59 | panic(err) 60 | } 61 | defer func() { _ = db.Close(ctx) }() 62 | 63 | whoAmI, err := db.Discovery().WhoAmI(ctx) 64 | if err != nil { 65 | panic(err) 66 | } 67 | 68 | fmt.Println(whoAmI.String()) 69 | } 70 | -------------------------------------------------------------------------------- /auth/anonymous_credentials/README.md: -------------------------------------------------------------------------------- 1 | # Authenticate with anonymous credentials 2 | 3 | `anonymous_credentials` example provide code snippet for authenticate to YDB with anonymous credentials 4 | 5 | ## Runing code snippet 6 | ```bash 7 | anonymous_credentials -ydb="grpcs://endpoint/?database=database" 8 | ``` 9 | 10 | 11 | -------------------------------------------------------------------------------- /auth/anonymous_credentials/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | 9 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 10 | ) 11 | 12 | var ( 13 | dsn string 14 | ) 15 | 16 | func init() { 17 | required := []string{"ydb"} 18 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 19 | flagSet.Usage = func() { 20 | out := flagSet.Output() 21 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 22 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 23 | flagSet.PrintDefaults() 24 | } 25 | flagSet.StringVar(&dsn, 26 | "ydb", "", 27 | "YDB connection string", 28 | ) 29 | if err := flagSet.Parse(os.Args[1:]); err != nil { 30 | flagSet.Usage() 31 | os.Exit(1) 32 | } 33 | flagSet.Visit(func(f *flag.Flag) { 34 | for i, arg := range required { 35 | if arg == f.Name { 36 | required = append(required[:i], required[i+1:]...) 37 | } 38 | } 39 | }) 40 | if len(required) > 0 { 41 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 42 | flagSet.Usage() 43 | os.Exit(1) 44 | } 45 | } 46 | 47 | func main() { 48 | ctx, cancel := context.WithCancel(context.Background()) 49 | defer cancel() 50 | db, err := ydb.Open(ctx, dsn, 51 | ydb.WithAnonymousCredentials(), 52 | ) 53 | if err != nil { 54 | panic(err) 55 | } 56 | defer func() { _ = db.Close(ctx) }() 57 | 58 | whoAmI, err := db.Discovery().WhoAmI(ctx) 59 | if err != nil { 60 | panic(err) 61 | } 62 | 63 | fmt.Println(whoAmI.String()) 64 | } 65 | -------------------------------------------------------------------------------- /auth/environ/README.md: -------------------------------------------------------------------------------- 1 | # Authenticate with environ 2 | 3 | `environ` example provide code snippet for authenticate to YDB with environ variables 4 | 5 | Authenticate produced with one of next environment variables: 6 | 7 | * `YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=` — used service account key file by path 8 | * `YDB_ANONYMOUS_CREDENTIALS="1"` — used for authenticate with anonymous access. Anonymous access needs for connect to testing YDB installation 9 | * `YDB_METADATA_CREDENTIALS="1"` — used metadata service for authenticate to YDB from yandex cloud virtual machine or from yandex function 10 | * `YDB_ACCESS_TOKEN_CREDENTIALS=` — used for authenticate to YDB with short-life access token. For example, access token may be IAM token 11 | 12 | ## Runing code snippet 13 | ```bash 14 | export YDB_CONNECTION_STRING="grpcs://endpoint/?database=database" 15 | export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/Users/user/.ydb/sa.jsoon 16 | environ 17 | # or 18 | export YDB_CONNECTION_STRING="grpcs://endpoint/?database=database" 19 | export YDB_ANONYMOUS_CREDENTIALS="1" 20 | environ 21 | # or 22 | export YDB_CONNECTION_STRING="grpcs://endpoint/?database=database" 23 | export YDB_METADATA_CREDENTIALS="1" 24 | environ 25 | # or 26 | export YDB_CONNECTION_STRING="grpcs://endpoint/?database=database" 27 | export YDB_ACCESS_TOKEN_CREDENTIALS="YDB_ACCESS_TOKEN" 28 | environ 29 | ``` 30 | -------------------------------------------------------------------------------- /auth/environ/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 9 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 10 | ) 11 | 12 | func main() { 13 | ctx, cancel := context.WithCancel(context.Background()) 14 | defer cancel() 15 | db, err := ydb.Open(ctx, 16 | os.Getenv("YDB_CONNECTION_STRING"), 17 | environ.WithEnvironCredentials(ctx), 18 | ) 19 | if err != nil { 20 | panic(err) 21 | } 22 | defer func() { _ = db.Close(ctx) }() 23 | 24 | whoAmI, err := db.Discovery().WhoAmI(ctx) 25 | if err != nil { 26 | panic(err) 27 | } 28 | 29 | fmt.Println(whoAmI.String()) 30 | } 31 | -------------------------------------------------------------------------------- /auth/metadata_credentials/README.md: -------------------------------------------------------------------------------- 1 | # Authenticate with metadata credentials 2 | 3 | `metadata_credentials` example provide code snippet for authenticate to YDB with metadata credentials 4 | 5 | ## Runing code snippet 6 | ```bash 7 | metadata_credentials -ydb="grpcs://endpoint/?database=database" 8 | ``` 9 | -------------------------------------------------------------------------------- /auth/metadata_credentials/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | 9 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 10 | yc "github.com/ydb-platform/ydb-go-yc" 11 | ) 12 | 13 | var ( 14 | dsn string 15 | ) 16 | 17 | func init() { 18 | required := []string{"ydb"} 19 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 20 | flagSet.Usage = func() { 21 | out := flagSet.Output() 22 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 23 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 24 | flagSet.PrintDefaults() 25 | } 26 | flagSet.StringVar(&dsn, 27 | "ydb", "", 28 | "YDB connection string", 29 | ) 30 | if err := flagSet.Parse(os.Args[1:]); err != nil { 31 | flagSet.Usage() 32 | os.Exit(1) 33 | } 34 | flagSet.Visit(func(f *flag.Flag) { 35 | for i, arg := range required { 36 | if arg == f.Name { 37 | required = append(required[:i], required[i+1:]...) 38 | } 39 | } 40 | }) 41 | if len(required) > 0 { 42 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 43 | flagSet.Usage() 44 | os.Exit(1) 45 | } 46 | } 47 | 48 | func main() { 49 | ctx, cancel := context.WithCancel(context.Background()) 50 | defer cancel() 51 | db, err := ydb.Open(ctx, dsn, 52 | yc.WithMetadataCredentials(), 53 | yc.WithInternalCA(), // append Yandex Cloud certificates 54 | ) 55 | if err != nil { 56 | panic(err) 57 | } 58 | defer func() { _ = db.Close(ctx) }() 59 | 60 | whoAmI, err := db.Discovery().WhoAmI(ctx) 61 | if err != nil { 62 | panic(err) 63 | } 64 | 65 | fmt.Println(whoAmI.String()) 66 | } 67 | -------------------------------------------------------------------------------- /auth/service_account_credentials/README.md: -------------------------------------------------------------------------------- 1 | # Authenticate with service account key file credentials 2 | 3 | `service_account_credentials` example provide code snippet for authenticate to YDB with service account key file credentials 4 | 5 | ## Runing code snippet 6 | ```bash 7 | service_account_credentials -ydb="grpcs://endpoint/?database=database" -sa-file="/Users/user/.ydb/sa.json" 8 | ``` 9 | -------------------------------------------------------------------------------- /auth/service_account_credentials/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | 9 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 10 | yc "github.com/ydb-platform/ydb-go-yc" 11 | ) 12 | 13 | var ( 14 | dsn string 15 | saFile string 16 | ) 17 | 18 | func init() { 19 | required := []string{"ydb", "sa-file"} 20 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 21 | flagSet.Usage = func() { 22 | out := flagSet.Output() 23 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 24 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 25 | flagSet.PrintDefaults() 26 | } 27 | flagSet.StringVar(&dsn, 28 | "ydb", "", 29 | "YDB connection string", 30 | ) 31 | flagSet.StringVar(&saFile, 32 | "sa-file", "", 33 | "service account key file", 34 | ) 35 | if err := flagSet.Parse(os.Args[1:]); err != nil { 36 | flagSet.Usage() 37 | os.Exit(1) 38 | } 39 | flagSet.Visit(func(f *flag.Flag) { 40 | for i, arg := range required { 41 | if arg == f.Name { 42 | required = append(required[:i], required[i+1:]...) 43 | } 44 | } 45 | }) 46 | if len(required) > 0 { 47 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 48 | flagSet.Usage() 49 | os.Exit(1) 50 | } 51 | } 52 | 53 | func main() { 54 | ctx, cancel := context.WithCancel(context.Background()) 55 | defer cancel() 56 | db, err := ydb.Open(ctx, dsn, 57 | yc.WithInternalCA(), 58 | yc.WithServiceAccountKeyFileCredentials(saFile), 59 | ) 60 | if err != nil { 61 | panic(err) 62 | } 63 | defer func() { _ = db.Close(ctx) }() 64 | 65 | whoAmI, err := db.Discovery().WhoAmI(ctx) 66 | if err != nil { 67 | panic(err) 68 | } 69 | 70 | fmt.Println(whoAmI.String()) 71 | } 72 | -------------------------------------------------------------------------------- /auth/static_credentials/README.md: -------------------------------------------------------------------------------- 1 | # Authenticate with static credentials 2 | 3 | `user_password` example provide code snippet for authenticate to YDB with static credentials 4 | 5 | ## Runing code snippet 6 | ```bash 7 | static_credentials -ydb="grpcs://endpoint/?database=database" -user="user" -password="password" 8 | ``` 9 | 10 | -------------------------------------------------------------------------------- /auth/static_credentials/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | 9 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 10 | ) 11 | 12 | var ( 13 | dsn string 14 | user string 15 | password string 16 | ) 17 | 18 | func init() { 19 | required := []string{"ydb", "user", "password"} 20 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 21 | flagSet.Usage = func() { 22 | out := flagSet.Output() 23 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 24 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 25 | flagSet.PrintDefaults() 26 | } 27 | flagSet.StringVar(&dsn, 28 | "ydb", "", 29 | "YDB connection string", 30 | ) 31 | flagSet.StringVar(&user, 32 | "user", "", 33 | "username", 34 | ) 35 | flagSet.StringVar(&password, 36 | "password", "", 37 | "password", 38 | ) 39 | if err := flagSet.Parse(os.Args[1:]); err != nil { 40 | flagSet.Usage() 41 | os.Exit(1) 42 | } 43 | flagSet.Visit(func(f *flag.Flag) { 44 | for i, arg := range required { 45 | if arg == f.Name { 46 | required = append(required[:i], required[i+1:]...) 47 | } 48 | } 49 | }) 50 | if len(required) > 0 { 51 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 52 | flagSet.Usage() 53 | os.Exit(1) 54 | } 55 | } 56 | 57 | func main() { 58 | ctx, cancel := context.WithCancel(context.Background()) 59 | defer cancel() 60 | db, err := ydb.Open(ctx, dsn, 61 | ydb.WithStaticCredentials(user, password), 62 | ) 63 | if err != nil { 64 | panic(err) 65 | } 66 | defer func() { _ = db.Close(ctx) }() 67 | 68 | whoAmI, err := db.Discovery().WhoAmI(ctx) 69 | if err != nil { 70 | panic(err) 71 | } 72 | 73 | fmt.Println(whoAmI.String()) 74 | } 75 | -------------------------------------------------------------------------------- /basic/database_sql/README.md: -------------------------------------------------------------------------------- 1 | # Basic example via database/sql driver 2 | 3 | Basic example demonstrates the possibilities of `YDB` with `database/sql` driver: 4 | - create/drop tables with `scheme` query mode 5 | - upsert data with `data` query mode 6 | - select with `data` query mode (unary request `ExecuteDataQuery` in native `ydb-go-sdk`) 7 | - select with `scan` query mode (streaming request `StreamExecuteScanQuery` in native `ydb-go-sdk`) 8 | - `explain` query mode for getting AST and Plan of query processing 9 | - modify transaction control 10 | - different types of query args: 11 | - multiple `sql.NamedArg` (uniform `database/sql` arg) 12 | - multiple native (for `ydb-go-sdk`) `table.ParameterOption` which constructs from `table.ValueParam("name", value)` 13 | - single native (for `ydb-go-sdk`) `*table.QueryParameters` which constructs from `table.NewQueryParameters(parameterOptions...)` 14 | -------------------------------------------------------------------------------- /basic/database_sql/data.go: -------------------------------------------------------------------------------- 1 | /* 2 | database_sql 3 | */ 4 | 5 | package main 6 | 7 | import ( 8 | "time" 9 | 10 | "github.com/google/uuid" 11 | 12 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 13 | ) 14 | 15 | func seriesData(id string, released time.Time, title, info, comment string) types.Value { 16 | var commentv types.Value 17 | if comment == "" { 18 | commentv = types.NullValue(types.TypeUTF8) 19 | } else { 20 | commentv = types.OptionalValue(types.TextValue(comment)) 21 | } 22 | return types.StructValue( 23 | types.StructFieldValue("series_id", types.BytesValueFromString(id)), 24 | types.StructFieldValue("release_date", types.DateValueFromTime(released)), 25 | types.StructFieldValue("title", types.TextValue(title)), 26 | types.StructFieldValue("series_info", types.TextValue(info)), 27 | types.StructFieldValue("comment", commentv), 28 | ) 29 | } 30 | 31 | func seasonData(seriesID, seasonID string, title string, first, last time.Time) types.Value { 32 | return types.StructValue( 33 | types.StructFieldValue("series_id", types.BytesValueFromString(seriesID)), 34 | types.StructFieldValue("season_id", types.BytesValueFromString(seasonID)), 35 | types.StructFieldValue("title", types.TextValue(title)), 36 | types.StructFieldValue("first_aired", types.DateValueFromTime(first)), 37 | types.StructFieldValue("last_aired", types.DateValueFromTime(last)), 38 | ) 39 | } 40 | 41 | func episodeData(seriesID, seasonID, episodeID string, title string, date time.Time) types.Value { 42 | return types.StructValue( 43 | types.StructFieldValue("series_id", types.BytesValueFromString(seriesID)), 44 | types.StructFieldValue("season_id", types.BytesValueFromString(seasonID)), 45 | types.StructFieldValue("episode_id", types.BytesValueFromString(episodeID)), 46 | types.StructFieldValue("title", types.TextValue(title)), 47 | types.StructFieldValue("air_date", types.DateValueFromTime(date)), 48 | ) 49 | } 50 | 51 | func getData() (series []types.Value, seasons []types.Value, episodes []types.Value) { 52 | for seriesID, fill := range map[string]func(seriesID string) (seriesData types.Value, seasons []types.Value, episodes []types.Value){ 53 | uuid.New().String(): getDataForITCrowd, 54 | uuid.New().String(): getDataForSiliconValley, 55 | } { 56 | seriesData, seasonsData, episodesData := fill(seriesID) 57 | series = append(series, seriesData) 58 | seasons = append(seasons, seasonsData...) 59 | episodes = append(episodes, episodesData...) 60 | } 61 | return 62 | } 63 | 64 | func getDataForITCrowd(seriesID string) (series types.Value, seasons []types.Value, episodes []types.Value) { 65 | series = seriesData( 66 | seriesID, date("2006-02-03"), "IT Crowd", ""+ 67 | "The IT Crowd is a British sitcom produced by Channel 4, written by Graham Linehan, produced by "+ 68 | "Ash Atalla and starring Chris O'Dowd, Richard Ayoade, Katherine Parkinson, and Matt Berry.", 69 | "", // NULL comment. 70 | ) 71 | for _, season := range []struct { 72 | title string 73 | first time.Time 74 | last time.Time 75 | episodes map[string]time.Time 76 | }{ 77 | {"Season 1", date("2006-02-03"), date("2006-03-03"), map[string]time.Time{ 78 | "Yesterday's Jam": date("2006-02-03"), 79 | "Calamity Jen": date("2006-02-03"), 80 | "Fifty-Fifty": date("2006-02-10"), 81 | "The Red Door": date("2006-02-17"), 82 | "The Haunting of Bill Crouse": date("2006-02-24"), 83 | "Aunt Irma Visits": date("2006-03-03"), 84 | }}, 85 | {"Season 2", date("2007-08-24"), date("2007-09-28"), map[string]time.Time{ 86 | "The Work Outing": date("2006-08-24"), 87 | "Return of the Golden Child": date("2007-08-31"), 88 | "Moss and the German": date("2007-09-07"), 89 | "The Dinner Party": date("2007-09-14"), 90 | "Smoke and Mirrors": date("2007-09-21"), 91 | "Men Without Women": date("2007-09-28"), 92 | }}, 93 | {"Season 3", date("2008-11-21"), date("2008-12-26"), map[string]time.Time{ 94 | "From Hell": date("2008-11-21"), 95 | "Are We Not Men?": date("2008-11-28"), 96 | "Tramps Like Us": date("2008-12-05"), 97 | "The Speech": date("2008-12-12"), 98 | "Friendface": date("2008-12-19"), 99 | "Calendar Geeks": date("2008-12-26"), 100 | }}, 101 | {"Season 4", date("2010-06-25"), date("2010-07-30"), map[string]time.Time{ 102 | "Jen The Fredo": date("2010-06-25"), 103 | "The Final Countdown": date("2010-07-02"), 104 | "Something Happened": date("2010-07-09"), 105 | "Italian For Beginners": date("2010-07-16"), 106 | "Bad Boys": date("2010-07-23"), 107 | "Reynholm vs Reynholm": date("2010-07-30"), 108 | }}, 109 | } { 110 | seasonID := uuid.New().String() 111 | seasons = append(seasons, seasonData(seriesID, seasonID, season.title, season.first, season.last)) 112 | for title, date := range season.episodes { 113 | episodes = append(episodes, episodeData(seriesID, seasonID, uuid.New().String(), title, date)) 114 | } 115 | } 116 | return 117 | } 118 | 119 | func getDataForSiliconValley(seriesID string) (series types.Value, seasons []types.Value, episodes []types.Value) { 120 | series = seriesData( 121 | seriesID, date("2014-04-06"), "Silicon Valley", ""+ 122 | "Silicon Valley is an American comedy television series created by Mike Judge, John Altschuler and "+ 123 | "Dave Krinsky. The series focuses on five young men who founded a startup company in Silicon Valley.", 124 | "Some comment here", 125 | ) 126 | for _, season := range []struct { 127 | title string 128 | first time.Time 129 | last time.Time 130 | episodes map[string]time.Time 131 | }{ 132 | {"Season 1", date("2006-02-03"), date("2006-03-03"), map[string]time.Time{ 133 | "Minimum Viable Product": date("2014-04-06"), 134 | "The Cap Table": date("2014-04-13"), 135 | "Articles of Incorporation": date("2014-04-20"), 136 | "Fiduciary Duties": date("2014-04-27"), 137 | "Signaling Risk": date("2014-05-04"), 138 | "Third Party Insourcing": date("2014-05-11"), 139 | "Proof of Concept": date("2014-05-18"), 140 | "Optimal Tip-to-Tip Efficiency": date("2014-06-01"), 141 | }}, 142 | {"Season 2", date("2007-08-24"), date("2007-09-28"), map[string]time.Time{ 143 | "Sand Hill Shuffle": date("2015-04-12"), 144 | "Runaway Devaluation": date("2015-04-19"), 145 | "Bad Money": date("2015-04-26"), 146 | "The Lady": date("2015-05-03"), 147 | "Server Space": date("2015-05-10"), 148 | "Homicide": date("2015-05-17"), 149 | "Adult Content": date("2015-05-24"), 150 | "White Hat/Black Hat": date("2015-05-31"), 151 | "Binding Arbitration": date("2015-06-07"), 152 | "Two Days of the Condor": date("2015-06-14"), 153 | }}, 154 | {"Season 3", date("2008-11-21"), date("2008-12-26"), map[string]time.Time{ 155 | "Founder Friendly": date("2016-04-24"), 156 | "Two in the Box": date("2016-05-01"), 157 | "Meinertzhagen's Haversack": date("2016-05-08"), 158 | "Maleant Data Systems Solutions": date("2016-05-15"), 159 | "The Empty Chair": date("2016-05-22"), 160 | "Bachmanity Insanity": date("2016-05-29"), 161 | "To Build a Better Beta": date("2016-06-05"), 162 | "Bachman's Earnings Over-Ride": date("2016-06-12"), 163 | "Daily Active Users": date("2016-06-19"), 164 | "The Uptick": date("2016-06-26"), 165 | }}, 166 | {"Season 4", date("2010-06-25"), date("2010-07-30"), map[string]time.Time{ 167 | "Success Failure": date("2017-04-23"), 168 | "Terms of Service": date("2017-04-30"), 169 | "Intellectual Property": date("2017-05-07"), 170 | "Teambuilding Exercise": date("2017-05-14"), 171 | "The Blood Boy": date("2017-05-21"), 172 | "Customer Service": date("2017-05-28"), 173 | "The Patent Troll": date("2017-06-04"), 174 | "The Keenan Vortex": date("2017-06-11"), 175 | "Hooli-Con": date("2017-06-18"), 176 | "Server Error": date("2017-06-25"), 177 | }}, 178 | {"Season 5", date("2018-03-25"), date("2018-05-13"), map[string]time.Time{ 179 | "Grow Fast or Die Slow": date("2018-03-25"), 180 | "Reorientation": date("2018-04-01"), 181 | "Chief Operating Officer": date("2018-04-08"), 182 | "Tech Evangelist": date("2018-04-15"), 183 | "Facial Recognition": date("2018-04-22"), 184 | "Artificial Emotional Intelligence": date("2018-04-29"), 185 | "Initial Coin Offering": date("2018-05-06"), 186 | "Fifty-One Percent": date("2018-05-13"), 187 | }}, 188 | } { 189 | seasonID := uuid.New().String() 190 | seasons = append(seasons, seasonData(seriesID, seasonID, season.title, season.first, season.last)) 191 | for title, date := range season.episodes { 192 | episodes = append(episodes, episodeData(seriesID, seasonID, uuid.New().String(), title, date)) 193 | } 194 | } 195 | return 196 | } 197 | 198 | const dateISO8601 = "2006-01-02" 199 | 200 | func date(date string) time.Time { 201 | t, err := time.Parse(dateISO8601, date) 202 | if err != nil { 203 | panic(err) 204 | } 205 | return t 206 | } 207 | -------------------------------------------------------------------------------- /basic/database_sql/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "os" 10 | "path" 11 | "time" 12 | 13 | "github.com/ydb-platform/ydb-go-sdk/v3" 14 | "github.com/ydb-platform/ydb-go-sdk/v3/sugar" 15 | ) 16 | 17 | var ( 18 | dsn string 19 | prefix string 20 | ) 21 | 22 | func init() { 23 | required := []string{"ydb"} 24 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 25 | flagSet.Usage = func() { 26 | out := flagSet.Output() 27 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 28 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 29 | flagSet.PrintDefaults() 30 | } 31 | flagSet.StringVar(&dsn, 32 | "ydb", "", 33 | "YDB connection string", 34 | ) 35 | flagSet.StringVar(&prefix, 36 | "prefix", "", 37 | "tables prefix", 38 | ) 39 | if err := flagSet.Parse(os.Args[1:]); err != nil { 40 | flagSet.Usage() 41 | os.Exit(1) 42 | } 43 | flagSet.Visit(func(f *flag.Flag) { 44 | for i, arg := range required { 45 | if arg == f.Name { 46 | required = append(required[:i], required[i+1:]...) 47 | } 48 | } 49 | }) 50 | if len(required) > 0 { 51 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 52 | flagSet.Usage() 53 | os.Exit(1) 54 | } 55 | } 56 | 57 | func main() { 58 | db, err := sql.Open("ydb", dsn) 59 | if err != nil { 60 | log.Fatalf("connect error: %v", err) 61 | } 62 | defer func() { _ = db.Close() }() 63 | 64 | db.SetMaxOpenConns(50) 65 | db.SetMaxIdleConns(50) 66 | db.SetConnMaxIdleTime(time.Second) 67 | 68 | ctx, cancel := context.WithCancel(context.Background()) 69 | defer cancel() 70 | 71 | cc, err := ydb.Unwrap(db) 72 | if err != nil { 73 | log.Fatalf("unwrap failed: %v", err) 74 | } 75 | 76 | prefix = path.Join(cc.Name(), prefix) 77 | 78 | err = sugar.RemoveRecursive(ctx, cc, prefix) 79 | if err != nil { 80 | log.Fatalf("remove recursive failed: %v", err) 81 | } 82 | 83 | err = prepareSchema(ctx, db, prefix) 84 | if err != nil { 85 | log.Fatalf("create tables error: %v", err) 86 | } 87 | 88 | err = fillTablesWithData(ctx, db, prefix) 89 | if err != nil { 90 | log.Fatalf("fill tables with data error: %v", err) 91 | } 92 | 93 | err = selectDefault(ctx, db, prefix) 94 | if err != nil { 95 | log.Fatal(err) 96 | } 97 | 98 | err = selectScan(ctx, db, prefix) 99 | if err != nil { 100 | log.Fatal(err) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /basic/gorm/README.md: -------------------------------------------------------------------------------- 1 | # Basic example via YDB gorm driver 2 | 3 | Basic example demonstrates the possibilities of `YDB` gorm driver: 4 | - define scheme of tables 5 | - upsert data 6 | - select all data from database 7 | -------------------------------------------------------------------------------- /basic/gorm/data.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "time" 4 | 5 | var ( 6 | data = []Series{ 7 | { 8 | Title: "IT Crowd", 9 | Info: "" + 10 | "The IT Crowd is a British sitcom produced by Channel 4, written by Graham Linehan, produced by " + 11 | "Ash Atalla and starring Chris O'Dowd, Richard Ayoade, Katherine Parkinson, and Matt Berry.", 12 | ReleaseDate: date("2006-02-03"), 13 | Seasons: []Season{ 14 | { 15 | Title: "Season 1", 16 | FirstAired: date("2006-02-03"), 17 | LastAired: date("2006-03-03"), 18 | Episodes: []Episode{ 19 | { 20 | Title: "Yesterday's Jam", 21 | AirDate: date("2006-02-03"), 22 | }, 23 | { 24 | Title: "Calamity Jen", 25 | AirDate: date("2006-02-03"), 26 | }, 27 | { 28 | Title: "Fifty-Fifty", 29 | AirDate: date("2006-02-10"), 30 | }, 31 | { 32 | Title: "The Red Door", 33 | AirDate: date("2006-02-17"), 34 | }, 35 | { 36 | Title: "The Haunting of Bill Crouse", 37 | AirDate: date("2006-02-24"), 38 | }, 39 | { 40 | Title: "Aunt Irma Visits", 41 | AirDate: date("2006-03-03"), 42 | }, 43 | }, 44 | }, 45 | { 46 | Title: "Season 2", 47 | FirstAired: date("2007-08-24"), 48 | LastAired: date("2007-09-28"), 49 | Episodes: []Episode{ 50 | { 51 | Title: "The Work Outing", 52 | AirDate: date("2006-08-24"), 53 | }, 54 | { 55 | Title: "Return of the Golden Child", 56 | AirDate: date("2007-08-31"), 57 | }, 58 | { 59 | Title: "Moss and the German", 60 | AirDate: date("2007-09-07"), 61 | }, 62 | { 63 | Title: "The Dinner Party", 64 | AirDate: date("2007-09-14"), 65 | }, 66 | { 67 | Title: "Smoke and Mirrors", 68 | AirDate: date("2007-09-21"), 69 | }, 70 | { 71 | Title: "Men Without Women", 72 | AirDate: date("2007-09-28"), 73 | }, 74 | }, 75 | }, 76 | { 77 | Title: "Season 3", 78 | FirstAired: date("2008-11-21"), 79 | LastAired: date("2008-12-26"), 80 | Episodes: []Episode{ 81 | { 82 | Title: "From Hell", 83 | AirDate: date("2008-11-21"), 84 | }, 85 | { 86 | Title: "Are We Not Men?", 87 | AirDate: date("2008-11-28"), 88 | }, 89 | { 90 | Title: "Tramps Like Us", 91 | AirDate: date("2008-12-05"), 92 | }, 93 | { 94 | Title: "The Speech", 95 | AirDate: date("2008-12-12"), 96 | }, 97 | { 98 | Title: "Friendface", 99 | AirDate: date("2008-12-19"), 100 | }, 101 | { 102 | Title: "Calendar Geeks", 103 | AirDate: date("2008-12-26"), 104 | }, 105 | }, 106 | }, 107 | { 108 | Title: "Season 4", 109 | FirstAired: date("2010-06-25"), 110 | LastAired: date("2010-07-30"), 111 | Episodes: []Episode{ 112 | { 113 | Title: "Jen The Fredo", 114 | AirDate: date("2010-06-25"), 115 | }, 116 | { 117 | Title: "The Final Countdown", 118 | AirDate: date("2010-07-02"), 119 | }, 120 | { 121 | Title: "Something Happened", 122 | AirDate: date("2010-07-09"), 123 | }, 124 | { 125 | Title: "Italian For Beginners", 126 | AirDate: date("2010-07-16"), 127 | }, 128 | { 129 | Title: "Bad Boys", 130 | AirDate: date("2010-07-23"), 131 | }, 132 | { 133 | Title: "Reynholm vs Reynholm", 134 | AirDate: date("2010-07-30"), 135 | }, 136 | }, 137 | }, 138 | }, 139 | }, 140 | { 141 | Title: "Silicon Valley", 142 | Info: "" + 143 | "Silicon Valley is an American comedy television series created by Mike Judge, John Altschuler and " + 144 | "Dave Krinsky. The series focuses on five young men who founded a startup company in Silicon Valley.", 145 | ReleaseDate: date("2014-04-06"), 146 | Seasons: []Season{ 147 | { 148 | Title: "Season 1", 149 | FirstAired: date("2006-02-03"), 150 | LastAired: date("2006-03-03"), 151 | Episodes: []Episode{ 152 | { 153 | Title: "Minimum Viable Product", 154 | AirDate: date("2014-04-06"), 155 | }, 156 | { 157 | Title: "The Cap Table", 158 | AirDate: date("2014-04-13"), 159 | }, 160 | { 161 | Title: "Articles of Incorporation", 162 | AirDate: date("2014-04-20"), 163 | }, 164 | { 165 | Title: "Fiduciary Duties", 166 | AirDate: date("2014-04-27"), 167 | }, 168 | { 169 | Title: "Signaling Risk", 170 | AirDate: date("2014-05-04"), 171 | }, 172 | { 173 | Title: "Third Party Insourcing", 174 | AirDate: date("2014-05-11"), 175 | }, 176 | { 177 | Title: "Proof of Concept", 178 | AirDate: date("2014-05-18"), 179 | }, 180 | { 181 | Title: "Optimal Tip-to-Tip Efficiency", 182 | AirDate: date("2014-06-01"), 183 | }, 184 | }, 185 | }, 186 | { 187 | Title: "Season 2", 188 | FirstAired: date("2007-08-24"), 189 | LastAired: date("2007-09-28"), 190 | Episodes: []Episode{ 191 | { 192 | Title: "Sand Hill Shuffle", 193 | AirDate: date("2015-04-12"), 194 | }, 195 | { 196 | Title: "Runaway Devaluation", 197 | AirDate: date("2015-04-19"), 198 | }, 199 | { 200 | Title: "Bad Money", 201 | AirDate: date("2015-04-26"), 202 | }, 203 | { 204 | Title: "The Lady", 205 | AirDate: date("2015-05-03"), 206 | }, 207 | { 208 | Title: "Server Space", 209 | AirDate: date("2015-05-10"), 210 | }, 211 | { 212 | Title: "Homicide", 213 | AirDate: date("2015-05-17"), 214 | }, 215 | { 216 | Title: "Adult Content", 217 | AirDate: date("2015-05-24"), 218 | }, 219 | { 220 | Title: "White Hat/Black Hat", 221 | AirDate: date("2015-05-31"), 222 | }, 223 | { 224 | Title: "Binding Arbitration", 225 | AirDate: date("2015-06-07"), 226 | }, 227 | { 228 | Title: "Two Days of the Condor", 229 | AirDate: date("2015-06-14"), 230 | }, 231 | }, 232 | }, 233 | { 234 | Title: "Season 3", 235 | FirstAired: date("2008-11-21"), 236 | LastAired: date("2008-12-26"), 237 | Episodes: []Episode{ 238 | { 239 | Title: "Founder Friendly", 240 | AirDate: date("2016-04-24"), 241 | }, 242 | { 243 | Title: "Two in the Box", 244 | AirDate: date("2016-05-01"), 245 | }, 246 | { 247 | Title: "Meinertzhagen's Haversack", 248 | AirDate: date("2016-05-08"), 249 | }, 250 | { 251 | Title: "Maleant Data Systems Solutions", 252 | AirDate: date("2016-05-15"), 253 | }, 254 | { 255 | Title: "The Empty Chair", 256 | AirDate: date("2016-05-22"), 257 | }, 258 | { 259 | Title: "Bachmanity Insanity", 260 | AirDate: date("2016-05-29"), 261 | }, 262 | { 263 | Title: "To Build a Better Beta", 264 | AirDate: date("2016-06-05"), 265 | }, 266 | { 267 | Title: "Bachman's Earnings Over-Ride", 268 | AirDate: date("2016-06-12"), 269 | }, 270 | { 271 | Title: "Daily Active Users", 272 | AirDate: date("2016-06-19"), 273 | }, 274 | { 275 | Title: "The Uptick", 276 | AirDate: date("2016-06-26"), 277 | }, 278 | }, 279 | }, 280 | { 281 | Title: "Season 4", 282 | FirstAired: date("2010-06-25"), 283 | LastAired: date("2010-07-30"), 284 | Episodes: []Episode{ 285 | { 286 | Title: "Success Failure", 287 | AirDate: date("2017-04-23"), 288 | }, 289 | { 290 | Title: "Terms of Service", 291 | AirDate: date("2017-04-30"), 292 | }, 293 | { 294 | Title: "Intellectual Property", 295 | AirDate: date("2017-05-07"), 296 | }, 297 | { 298 | Title: "Teambuilding Exercise", 299 | AirDate: date("2017-05-14"), 300 | }, 301 | { 302 | Title: "The Blood Boy", 303 | AirDate: date("2017-05-21"), 304 | }, 305 | { 306 | Title: "Customer Service", 307 | AirDate: date("2017-05-28"), 308 | }, 309 | { 310 | Title: "The Patent Troll", 311 | AirDate: date("2017-06-04"), 312 | }, 313 | { 314 | Title: "The Keenan Vortex", 315 | AirDate: date("2017-06-11"), 316 | }, 317 | { 318 | Title: "Hooli-Con", 319 | AirDate: date("2017-06-18"), 320 | }, 321 | { 322 | Title: "Server Error", 323 | AirDate: date("2017-06-25"), 324 | }, 325 | }, 326 | }, 327 | { 328 | Title: "Season 5", 329 | FirstAired: date("2018-03-25"), 330 | LastAired: date("2018-05-13"), 331 | Episodes: []Episode{ 332 | { 333 | Title: "Grow Fast or Die Slow", 334 | AirDate: date("2018-03-25"), 335 | }, 336 | { 337 | Title: "Reorientation", 338 | AirDate: date("2018-04-01"), 339 | }, 340 | { 341 | Title: "Chief Operating Officer", 342 | AirDate: date("2018-04-08"), 343 | }, 344 | { 345 | Title: "Tech Evangelist", 346 | AirDate: date("2018-04-15"), 347 | }, 348 | { 349 | Title: "Facial Recognition", 350 | AirDate: date("2018-04-22"), 351 | }, 352 | { 353 | Title: "Artificial Emotional Intelligence", 354 | AirDate: date("2018-04-29"), 355 | }, 356 | { 357 | Title: "Initial Coin Offering", 358 | AirDate: date("2018-05-06"), 359 | }, 360 | { 361 | Title: "Fifty-One Percent", 362 | AirDate: date("2018-05-13"), 363 | }, 364 | }, 365 | }, 366 | }, 367 | }, 368 | } 369 | ) 370 | 371 | const dateISO8601 = "2006-01-02" 372 | 373 | func date(date string) time.Time { 374 | t, err := time.Parse(dateISO8601, date) 375 | if err != nil { 376 | panic(err) 377 | } 378 | return t 379 | } 380 | -------------------------------------------------------------------------------- /basic/gorm/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | "os" 7 | 8 | "gorm.io/driver/postgres" 9 | "gorm.io/driver/sqlite" 10 | "gorm.io/gorm" 11 | "gorm.io/gorm/clause" 12 | "gorm.io/gorm/logger" 13 | 14 | ydb "github.com/ydb-platform/gorm-driver" 15 | ) 16 | 17 | func initDB(cfg *gorm.Config) (*gorm.DB, error) { 18 | // docker run -it postgres psql -h 127.0.0.1 -p 5432 -U postgres -d postgres 19 | // POSTGRES_CONNECTION_STRING="user=postgres password=mysecretpassword dbname=postgres host=127.0.0.1 port=5432 sslmode=disable" 20 | if dsn, has := os.LookupEnv("POSTGRES_CONNECTION_STRING"); has { 21 | return gorm.Open(postgres.Open(dsn), cfg) 22 | } 23 | // SQLITE_CONNECTION_STRING=./test.db 24 | if dsn, has := os.LookupEnv("SQLITE_CONNECTION_STRING"); has { 25 | return gorm.Open(sqlite.Open(dsn), cfg) 26 | } 27 | if dsn, has := os.LookupEnv("YDB_CONNECTION_STRING"); has { 28 | return gorm.Open(ydb.Open(dsn, ydb.WithTablePathPrefix("gorm")), cfg) 29 | } 30 | return nil, errors.New("cannot initialize DB") 31 | } 32 | 33 | func main() { 34 | // connect 35 | db, err := initDB(&gorm.Config{ 36 | Logger: logger.Default.LogMode(logger.Info), 37 | }) 38 | if err != nil { 39 | panic(err) 40 | } 41 | 42 | // prepare scheme and migrations 43 | if err = prepareScheme(db); err != nil { 44 | panic(err) 45 | } 46 | 47 | // fill data 48 | if err = fillData(db); err != nil { 49 | panic(err) 50 | } 51 | 52 | // read all data 53 | if err = readAll(db); err != nil { 54 | panic(err) 55 | } 56 | 57 | // find by condition 58 | if err = findByTitle(db); err != nil { 59 | panic(err) 60 | } 61 | } 62 | 63 | func prepareScheme(db *gorm.DB) error { 64 | if err := db.Migrator().DropTable( 65 | &Series{}, 66 | &Season{}, 67 | &Episode{}, 68 | ); err != nil { 69 | return err 70 | } 71 | return db.AutoMigrate( 72 | &Series{}, 73 | &Season{}, 74 | &Episode{}, 75 | ) 76 | } 77 | 78 | func fillData(db *gorm.DB) error { 79 | return db.Create(data).Error 80 | } 81 | 82 | func readAll(db *gorm.DB) error { 83 | // get all series 84 | var series []Series 85 | if err := db.Preload("Seasons.Episodes").Find(&series).Error; err != nil { 86 | return err 87 | } 88 | log.Println("all known series:") 89 | for _, s := range series { 90 | log.Printf( 91 | " > [%s] %s (%s)\n", 92 | s.ID, s.Title, s.ReleaseDate.Format("2006"), 93 | ) 94 | for _, ss := range s.Seasons { 95 | log.Printf( 96 | " > [%s] %s\n", 97 | ss.ID, ss.Title, 98 | ) 99 | for _, e := range ss.Episodes { 100 | log.Printf( 101 | " > [%s] [%s] %s\n", 102 | e.ID, e.AirDate.Format(dateISO8601), e.Title, 103 | ) 104 | } 105 | } 106 | } 107 | return nil 108 | } 109 | 110 | func findByTitle(db *gorm.DB) error { 111 | var episodes []Episode 112 | if err := db.Find(&episodes, clause.Like{ 113 | Column: "title", 114 | Value: "%Bad%", 115 | }).Error; err != nil { 116 | return err 117 | } 118 | log.Println("all episodes with title with word 'bad':") 119 | for _, e := range episodes { 120 | ss := Season{ 121 | ID: e.SeasonID, 122 | } 123 | if err := db.Take(&ss).Error; err != nil { 124 | return err 125 | } 126 | s := Series{ 127 | ID: ss.SeriesID, 128 | } 129 | if err := db.Take(&s).Error; err != nil { 130 | return err 131 | } 132 | log.Printf( 133 | " > [%s] %s (%s)\n", 134 | s.ID, s.Title, s.ReleaseDate.Format("2006"), 135 | ) 136 | log.Printf( 137 | " > [%s] %s\n", 138 | ss.ID, ss.Title, 139 | ) 140 | log.Printf( 141 | " > [%s] [%s] %s\n", 142 | e.ID, e.AirDate.Format(dateISO8601), e.Title, 143 | ) 144 | } 145 | return nil 146 | } 147 | -------------------------------------------------------------------------------- /basic/gorm/models.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/google/uuid" 7 | "gorm.io/gorm" 8 | ) 9 | 10 | type Series struct { 11 | ID string `gorm:"column:series_id;primarykey;not null"` 12 | Title string `gorm:"column:title;not null"` 13 | Info string `gorm:"column:series_info"` 14 | Comment string `gorm:"column:comment"` 15 | ReleaseDate time.Time `gorm:"column:release_date;not null"` 16 | 17 | Seasons []Season 18 | } 19 | 20 | func (s *Series) BeforeCreate(tx *gorm.DB) (err error) { 21 | id, err := uuid.NewUUID() 22 | if err != nil { 23 | return err 24 | } 25 | s.ID = id.String() 26 | for _, season := range s.Seasons { 27 | season.SeriesID = s.ID 28 | } 29 | return 30 | } 31 | 32 | type Season struct { 33 | ID string `gorm:"column:season_id;primarykey"` 34 | SeriesID string `gorm:"column:series_id;index"` 35 | Title string `gorm:"column:title;not null"` 36 | FirstAired time.Time `gorm:"column:first_aired;not null"` 37 | LastAired time.Time `gorm:"column:last_aired;not null"` 38 | 39 | Episodes []Episode 40 | } 41 | 42 | func (s *Season) BeforeCreate(tx *gorm.DB) (err error) { 43 | id, err := uuid.NewUUID() 44 | if err != nil { 45 | return err 46 | } 47 | s.ID = id.String() 48 | for _, episode := range s.Episodes { 49 | episode.SeasonID = s.ID 50 | } 51 | return 52 | } 53 | 54 | type Episode struct { 55 | ID string `gorm:"column:episode_id;primarykey"` 56 | SeasonID string `gorm:"column:season_id;index;not null"` 57 | Title string `gorm:"column:title;not null"` 58 | AirDate time.Time `gorm:"column:air_date;not null"` 59 | } 60 | 61 | func (e *Episode) BeforeCreate(tx *gorm.DB) (err error) { 62 | id, err := uuid.NewUUID() 63 | if err != nil { 64 | return err 65 | } 66 | e.ID = id.String() 67 | return 68 | } 69 | -------------------------------------------------------------------------------- /basic/native/README.md: -------------------------------------------------------------------------------- 1 | # Basic example via native driver 2 | 3 | Basic example demonstrates the possibilities of `YDB`: 4 | - create/drop/describe tables 5 | - upsert data 6 | - select with data query (request-response API) 7 | - select with scan query (streaming API) 8 | - read table (streaming API) -------------------------------------------------------------------------------- /basic/native/data.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 7 | ) 8 | 9 | func seriesData(id uint64, released time.Time, title, info, comment string) types.Value { 10 | var commentv types.Value 11 | if comment == "" { 12 | commentv = types.NullValue(types.TypeUTF8) 13 | } else { 14 | commentv = types.OptionalValue(types.TextValue(comment)) 15 | } 16 | return types.StructValue( 17 | types.StructFieldValue("series_id", types.Uint64Value(id)), 18 | types.StructFieldValue("release_date", types.DateValueFromTime(released)), 19 | types.StructFieldValue("title", types.TextValue(title)), 20 | types.StructFieldValue("series_info", types.TextValue(info)), 21 | types.StructFieldValue("comment", commentv), 22 | ) 23 | } 24 | 25 | func seasonData(seriesID, seasonID uint64, title string, first, last time.Time) types.Value { 26 | return types.StructValue( 27 | types.StructFieldValue("series_id", types.Uint64Value(seriesID)), 28 | types.StructFieldValue("season_id", types.Uint64Value(seasonID)), 29 | types.StructFieldValue("title", types.TextValue(title)), 30 | types.StructFieldValue("first_aired", types.DateValueFromTime(first)), 31 | types.StructFieldValue("last_aired", types.DateValueFromTime(last)), 32 | ) 33 | } 34 | 35 | func episodeData(seriesID, seasonID, episodeID uint64, title string, date time.Time) types.Value { 36 | return types.StructValue( 37 | types.StructFieldValue("series_id", types.Uint64Value(seriesID)), 38 | types.StructFieldValue("season_id", types.Uint64Value(seasonID)), 39 | types.StructFieldValue("episode_id", types.Uint64Value(episodeID)), 40 | types.StructFieldValue("title", types.TextValue(title)), 41 | types.StructFieldValue("air_date", types.DateValueFromTime(date)), 42 | ) 43 | } 44 | 45 | func getSeriesData() types.Value { 46 | return types.ListValue( 47 | seriesData( 48 | 1, days("2006-02-03"), "IT Crowd", ""+ 49 | "The IT Crowd is a British sitcom produced by Channel 4, written by Graham Linehan, produced by "+ 50 | "Ash Atalla and starring Chris O'Dowd, Richard Ayoade, Katherine Parkinson, and Matt Berry.", 51 | "", // NULL comment. 52 | ), 53 | seriesData( 54 | 2, days("2014-04-06"), "Silicon Valley", ""+ 55 | "Silicon Valley is an American comedy television series created by Mike Judge, John Altschuler and "+ 56 | "Dave Krinsky. The series focuses on five young men who founded a startup company in Silicon Valley.", 57 | "Some comment here", 58 | ), 59 | ) 60 | } 61 | 62 | func getSeasonsData() types.Value { 63 | return types.ListValue( 64 | seasonData(1, 1, "Season 1", days("2006-02-03"), days("2006-03-03")), 65 | seasonData(1, 2, "Season 2", days("2007-08-24"), days("2007-09-28")), 66 | seasonData(1, 3, "Season 3", days("2008-11-21"), days("2008-12-26")), 67 | seasonData(1, 4, "Season 4", days("2010-06-25"), days("2010-07-30")), 68 | seasonData(2, 1, "Season 1", days("2014-04-06"), days("2014-06-01")), 69 | seasonData(2, 2, "Season 2", days("2015-04-12"), days("2015-06-14")), 70 | seasonData(2, 3, "Season 3", days("2016-04-24"), days("2016-06-26")), 71 | seasonData(2, 4, "Season 4", days("2017-04-23"), days("2017-06-25")), 72 | seasonData(2, 5, "Season 5", days("2018-03-25"), days("2018-05-13")), 73 | ) 74 | } 75 | 76 | func getEpisodesData() types.Value { 77 | return types.ListValue( 78 | episodeData(1, 1, 1, "Yesterday's Jam", days("2006-02-03")), 79 | episodeData(1, 1, 2, "Calamity Jen", days("2006-02-03")), 80 | episodeData(1, 1, 3, "Fifty-Fifty", days("2006-02-10")), 81 | episodeData(1, 1, 4, "The Red Door", days("2006-02-17")), 82 | episodeData(1, 1, 5, "The Haunting of Bill Crouse", days("2006-02-24")), 83 | episodeData(1, 1, 6, "Aunt Irma Visits", days("2006-03-03")), 84 | episodeData(1, 2, 1, "The Work Outing", days("2006-08-24")), 85 | episodeData(1, 2, 2, "Return of the Golden Child", days("2007-08-31")), 86 | episodeData(1, 2, 3, "Moss and the German", days("2007-09-07")), 87 | episodeData(1, 2, 4, "The Dinner Party", days("2007-09-14")), 88 | episodeData(1, 2, 5, "Smoke and Mirrors", days("2007-09-21")), 89 | episodeData(1, 2, 6, "Men Without Women", days("2007-09-28")), 90 | episodeData(1, 3, 1, "From Hell", days("2008-11-21")), 91 | episodeData(1, 3, 2, "Are We Not Men?", days("2008-11-28")), 92 | episodeData(1, 3, 3, "Tramps Like Us", days("2008-12-05")), 93 | episodeData(1, 3, 4, "The Speech", days("2008-12-12")), 94 | episodeData(1, 3, 5, "Friendface", days("2008-12-19")), 95 | episodeData(1, 3, 6, "Calendar Geeks", days("2008-12-26")), 96 | episodeData(1, 4, 1, "Jen The Fredo", days("2010-06-25")), 97 | episodeData(1, 4, 2, "The Final Countdown", days("2010-07-02")), 98 | episodeData(1, 4, 3, "Something Happened", days("2010-07-09")), 99 | episodeData(1, 4, 4, "Italian For Beginners", days("2010-07-16")), 100 | episodeData(1, 4, 5, "Bad Boys", days("2010-07-23")), 101 | episodeData(1, 4, 6, "Reynholm vs Reynholm", days("2010-07-30")), 102 | episodeData(2, 1, 1, "Minimum Viable Product", days("2014-04-06")), 103 | episodeData(2, 1, 2, "The Cap Table", days("2014-04-13")), 104 | episodeData(2, 1, 3, "Articles of Incorporation", days("2014-04-20")), 105 | episodeData(2, 1, 4, "Fiduciary Duties", days("2014-04-27")), 106 | episodeData(2, 1, 5, "Signaling Risk", days("2014-05-04")), 107 | episodeData(2, 1, 6, "Third Party Insourcing", days("2014-05-11")), 108 | episodeData(2, 1, 7, "Proof of Concept", days("2014-05-18")), 109 | episodeData(2, 1, 8, "Optimal Tip-to-Tip Efficiency", days("2014-06-01")), 110 | episodeData(2, 2, 1, "Sand Hill Shuffle", days("2015-04-12")), 111 | episodeData(2, 2, 2, "Runaway Devaluation", days("2015-04-19")), 112 | episodeData(2, 2, 3, "Bad Money", days("2015-04-26")), 113 | episodeData(2, 2, 4, "The Lady", days("2015-05-03")), 114 | episodeData(2, 2, 5, "Server Space", days("2015-05-10")), 115 | episodeData(2, 2, 6, "Homicide", days("2015-05-17")), 116 | episodeData(2, 2, 7, "Adult Content", days("2015-05-24")), 117 | episodeData(2, 2, 8, "White Hat/Black Hat", days("2015-05-31")), 118 | episodeData(2, 2, 9, "Binding Arbitration", days("2015-06-07")), 119 | episodeData(2, 2, 10, "Two Days of the Condor", days("2015-06-14")), 120 | episodeData(2, 3, 1, "Founder Friendly", days("2016-04-24")), 121 | episodeData(2, 3, 2, "Two in the Box", days("2016-05-01")), 122 | episodeData(2, 3, 3, "Meinertzhagen's Haversack", days("2016-05-08")), 123 | episodeData(2, 3, 4, "Maleant Data Systems Solutions", days("2016-05-15")), 124 | episodeData(2, 3, 5, "The Empty Chair", days("2016-05-22")), 125 | episodeData(2, 3, 6, "Bachmanity Insanity", days("2016-05-29")), 126 | episodeData(2, 3, 7, "To Build a Better Beta", days("2016-06-05")), 127 | episodeData(2, 3, 8, "Bachman's Earnings Over-Ride", days("2016-06-12")), 128 | episodeData(2, 3, 9, "Daily Active Users", days("2016-06-19")), 129 | episodeData(2, 3, 10, "The Uptick", days("2016-06-26")), 130 | episodeData(2, 4, 1, "Success Failure", days("2017-04-23")), 131 | episodeData(2, 4, 2, "Terms of Service", days("2017-04-30")), 132 | episodeData(2, 4, 3, "Intellectual Property", days("2017-05-07")), 133 | episodeData(2, 4, 4, "Teambuilding Exercise", days("2017-05-14")), 134 | episodeData(2, 4, 5, "The Blood Boy", days("2017-05-21")), 135 | episodeData(2, 4, 6, "Customer Service", days("2017-05-28")), 136 | episodeData(2, 4, 7, "The Patent Troll", days("2017-06-04")), 137 | episodeData(2, 4, 8, "The Keenan Vortex", days("2017-06-11")), 138 | episodeData(2, 4, 9, "Hooli-Con", days("2017-06-18")), 139 | episodeData(2, 4, 10, "Server Error", days("2017-06-25")), 140 | episodeData(2, 5, 1, "Grow Fast or Die Slow", days("2018-03-25")), 141 | episodeData(2, 5, 2, "Reorientation", days("2018-04-01")), 142 | episodeData(2, 5, 3, "Chief Operating Officer", days("2018-04-08")), 143 | episodeData(2, 5, 4, "Tech Evangelist", days("2018-04-15")), 144 | episodeData(2, 5, 5, "Facial Recognition", days("2018-04-22")), 145 | episodeData(2, 5, 6, "Artificial Emotional Intelligence", days("2018-04-29")), 146 | episodeData(2, 5, 7, "Initial Coin Offering", days("2018-05-06")), 147 | episodeData(2, 5, 8, "Fifty-One Percent", days("2018-05-13")), 148 | ) 149 | } 150 | 151 | const dateISO8601 = "2006-01-02" 152 | 153 | func days(date string) time.Time { 154 | t, err := time.Parse(dateISO8601, date) 155 | if err != nil { 156 | panic(err) 157 | } 158 | return t 159 | } 160 | -------------------------------------------------------------------------------- /basic/native/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path" 9 | 10 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 11 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 12 | "github.com/ydb-platform/ydb-go-sdk/v3/sugar" 13 | ) 14 | 15 | var ( 16 | dsn string 17 | prefix string 18 | ) 19 | 20 | func init() { 21 | required := []string{"ydb"} 22 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 23 | flagSet.Usage = func() { 24 | out := flagSet.Output() 25 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 26 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 27 | flagSet.PrintDefaults() 28 | } 29 | flagSet.StringVar(&dsn, 30 | "ydb", "", 31 | "YDB connection string", 32 | ) 33 | flagSet.StringVar(&prefix, 34 | "prefix", "", 35 | "tables prefix", 36 | ) 37 | if err := flagSet.Parse(os.Args[1:]); err != nil { 38 | flagSet.Usage() 39 | os.Exit(1) 40 | } 41 | flagSet.Visit(func(f *flag.Flag) { 42 | for i, arg := range required { 43 | if arg == f.Name { 44 | required = append(required[:i], required[i+1:]...) 45 | } 46 | } 47 | }) 48 | if len(required) > 0 { 49 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 50 | flagSet.Usage() 51 | os.Exit(1) 52 | } 53 | } 54 | 55 | func main() { 56 | ctx, cancel := context.WithCancel(context.Background()) 57 | defer cancel() 58 | opts := []ydb.Option{ 59 | environ.WithEnvironCredentials(ctx), 60 | } 61 | db, err := ydb.Open(ctx, dsn, opts...) 62 | if err != nil { 63 | panic(fmt.Errorf("connect error: %w", err)) 64 | } 65 | defer func() { _ = db.Close(ctx) }() 66 | 67 | prefix = path.Join(db.Name(), prefix) 68 | 69 | err = sugar.RemoveRecursive(ctx, db, prefix) 70 | if err != nil { 71 | panic(err) 72 | } 73 | 74 | err = sugar.MakeRecursive(ctx, db, prefix) 75 | if err != nil { 76 | panic(err) 77 | } 78 | 79 | err = describeTableOptions(ctx, db.Table()) 80 | if err != nil { 81 | panic(fmt.Errorf("describe table options error: %w", err)) 82 | } 83 | 84 | err = createTables(ctx, db.Table(), prefix) 85 | if err != nil { 86 | panic(fmt.Errorf("create tables error: %w", err)) 87 | } 88 | 89 | err = describeTable(ctx, db.Table(), path.Join( 90 | prefix, "series", 91 | )) 92 | if err != nil { 93 | panic(fmt.Errorf("describe table error: %w", err)) 94 | } 95 | 96 | err = fillTablesWithData(ctx, db.Table(), prefix) 97 | if err != nil { 98 | panic(fmt.Errorf("fill tables with data error: %w", err)) 99 | } 100 | 101 | err = selectSimple(ctx, db.Table(), prefix) 102 | if err != nil { 103 | panic(fmt.Errorf("select simple error: %w", err)) 104 | } 105 | 106 | err = scanQuerySelect(ctx, db.Table(), prefix) 107 | if err != nil { 108 | panic(fmt.Errorf("scan query select error: %w", err)) 109 | } 110 | 111 | err = readTable(ctx, db.Table(), path.Join( 112 | prefix, "series", 113 | )) 114 | if err != nil { 115 | panic(fmt.Errorf("read table error: %w", err)) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /basic/native/series.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "log" 7 | "path" 8 | "text/template" 9 | 10 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 11 | "github.com/ydb-platform/ydb-go-sdk/v3/table/options" 12 | "github.com/ydb-platform/ydb-go-sdk/v3/table/result" 13 | "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" 14 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 15 | ) 16 | 17 | type templateConfig struct { 18 | TablePathPrefix string 19 | } 20 | 21 | var fill = template.Must(template.New("fill database").Parse(` 22 | PRAGMA TablePathPrefix("{{ .TablePathPrefix }}"); 23 | 24 | DECLARE $seriesData AS List>>; 30 | 31 | DECLARE $seasonsData AS List>; 37 | 38 | DECLARE $episodesData AS List>; 44 | 45 | REPLACE INTO series 46 | SELECT 47 | series_id, 48 | title, 49 | series_info, 50 | CAST(release_date AS Uint64) AS release_date, 51 | comment 52 | FROM AS_TABLE($seriesData); 53 | 54 | REPLACE INTO seasons 55 | SELECT 56 | series_id, 57 | season_id, 58 | title, 59 | CAST(first_aired AS Uint64) AS first_aired, 60 | CAST(last_aired AS Uint64) AS last_aired 61 | FROM AS_TABLE($seasonsData); 62 | 63 | REPLACE INTO episodes 64 | SELECT 65 | series_id, 66 | season_id, 67 | episode_id, 68 | title, 69 | CAST(air_date AS Uint64) AS air_date 70 | FROM AS_TABLE($episodesData); 71 | `)) 72 | 73 | func readTable(ctx context.Context, c table.Client, path string) (err error) { 74 | var res result.StreamResult 75 | err = c.Do(ctx, 76 | func(ctx context.Context, s table.Session) (err error) { 77 | res, err = s.StreamReadTable(ctx, path, 78 | options.ReadOrdered(), 79 | options.ReadColumn("series_id"), 80 | options.ReadColumn("title"), 81 | options.ReadColumn("release_date"), 82 | ) 83 | return 84 | }, 85 | ) 86 | if err != nil { 87 | return err 88 | } 89 | defer func() { 90 | _ = res.Close() 91 | }() 92 | log.Printf("> read_table:") 93 | var ( 94 | id *uint64 95 | title *string 96 | date *uint64 97 | ) 98 | for res.NextResultSet(ctx) { 99 | for res.NextRow() { 100 | err = res.ScanNamed( 101 | named.Optional("series_id", &id), 102 | named.Optional("title", &title), 103 | named.Optional("release_date", &date), 104 | ) 105 | if err != nil { 106 | return err 107 | } 108 | log.Printf("# %d %s %d", *id, *title, *date) 109 | } 110 | } 111 | if err := res.Err(); err != nil { 112 | return err 113 | } 114 | if stats := res.Stats(); stats != nil { 115 | for i := 0; ; i++ { 116 | phase, ok := stats.NextPhase() 117 | if !ok { 118 | break 119 | } 120 | log.Printf( 121 | "# phase #%d: took %s", 122 | i, phase.Duration(), 123 | ) 124 | for { 125 | tbl, ok := phase.NextTableAccess() 126 | if !ok { 127 | break 128 | } 129 | log.Printf( 130 | "# accessed %s: read=(%drows, %dbytes)", 131 | tbl.Name, tbl.Reads.Rows, tbl.Reads.Bytes, 132 | ) 133 | } 134 | } 135 | } 136 | 137 | return res.Err() 138 | } 139 | 140 | func describeTableOptions(ctx context.Context, c table.Client) (err error) { 141 | var desc options.TableOptionsDescription 142 | err = c.Do(ctx, 143 | func(ctx context.Context, s table.Session) (err error) { 144 | desc, err = s.DescribeTableOptions(ctx) 145 | return 146 | }, 147 | ) 148 | if err != nil { 149 | return err 150 | } 151 | log.Println("> describe_table_options:") 152 | 153 | for i, p := range desc.TableProfilePresets { 154 | log.Printf("TableProfilePresets: %d/%d: %+v", i+1, len(desc.TableProfilePresets), p) 155 | } 156 | for i, p := range desc.StoragePolicyPresets { 157 | log.Printf("StoragePolicyPresets: %d/%d: %+v", i+1, len(desc.StoragePolicyPresets), p) 158 | } 159 | for i, p := range desc.CompactionPolicyPresets { 160 | log.Printf("CompactionPolicyPresets: %d/%d: %+v", i+1, len(desc.CompactionPolicyPresets), p) 161 | } 162 | for i, p := range desc.PartitioningPolicyPresets { 163 | log.Printf("PartitioningPolicyPresets: %d/%d: %+v", i+1, len(desc.PartitioningPolicyPresets), p) 164 | } 165 | for i, p := range desc.ExecutionPolicyPresets { 166 | log.Printf("ExecutionPolicyPresets: %d/%d: %+v", i+1, len(desc.ExecutionPolicyPresets), p) 167 | } 168 | for i, p := range desc.ReplicationPolicyPresets { 169 | log.Printf("ReplicationPolicyPresets: %d/%d: %+v", i+1, len(desc.ReplicationPolicyPresets), p) 170 | } 171 | for i, p := range desc.CachingPolicyPresets { 172 | log.Printf("CachingPolicyPresets: %d/%d: %+v", i+1, len(desc.CachingPolicyPresets), p) 173 | } 174 | 175 | return nil 176 | } 177 | 178 | func selectSimple(ctx context.Context, c table.Client, prefix string) (err error) { 179 | query := render( 180 | template.Must(template.New("").Parse(` 181 | PRAGMA TablePathPrefix("{{ .TablePathPrefix }}"); 182 | DECLARE $seriesID AS Uint64; 183 | $format = DateTime::Format("%Y-%m-%d"); 184 | SELECT 185 | series_id, 186 | title, 187 | $format(DateTime::FromSeconds(CAST(DateTime::ToSeconds(DateTime::IntervalFromDays(CAST(release_date AS Int16))) AS Uint32))) AS release_date 188 | FROM 189 | series 190 | WHERE 191 | series_id = $seriesID; 192 | `)), 193 | templateConfig{ 194 | TablePathPrefix: prefix, 195 | }, 196 | ) 197 | readTx := table.TxControl( 198 | table.BeginTx( 199 | table.WithOnlineReadOnly(), 200 | ), 201 | table.CommitTx(), 202 | ) 203 | var res result.Result 204 | err = c.Do(ctx, 205 | func(ctx context.Context, s table.Session) (err error) { 206 | _, res, err = s.Execute(ctx, readTx, query, 207 | table.NewQueryParameters( 208 | table.ValueParam("$seriesID", types.Uint64Value(1)), 209 | ), 210 | options.WithCollectStatsModeBasic(), 211 | ) 212 | return 213 | }, 214 | ) 215 | if err != nil { 216 | return err 217 | } 218 | 219 | defer func() { 220 | _ = res.Close() 221 | }() 222 | 223 | var ( 224 | id *uint64 225 | title *string 226 | date *[]byte 227 | ) 228 | for res.NextResultSet(ctx) { 229 | for res.NextRow() { 230 | err = res.ScanNamed( 231 | named.Optional("series_id", &id), 232 | named.Optional("title", &title), 233 | named.Optional("release_date", &date), 234 | ) 235 | if err != nil { 236 | return err 237 | } 238 | log.Printf( 239 | "> select_simple_transaction: %d %s %s", 240 | *id, *title, *date, 241 | ) 242 | } 243 | } 244 | return res.Err() 245 | } 246 | 247 | func scanQuerySelect(ctx context.Context, c table.Client, prefix string) (err error) { 248 | query := render( 249 | template.Must(template.New("").Parse(` 250 | PRAGMA TablePathPrefix("{{ .TablePathPrefix }}"); 251 | 252 | DECLARE $series AS List; 253 | 254 | SELECT series_id, season_id, title, CAST(CAST(first_aired AS Date) AS Bytes) AS first_aired 255 | FROM seasons 256 | WHERE series_id IN $series 257 | `)), 258 | templateConfig{ 259 | TablePathPrefix: prefix, 260 | }, 261 | ) 262 | 263 | return c.Do(ctx, 264 | func(ctx context.Context, s table.Session) error { 265 | res, err := s.StreamExecuteScanQuery(ctx, query, 266 | table.NewQueryParameters( 267 | table.ValueParam("$series", 268 | types.ListValue( 269 | types.Uint64Value(1), 270 | types.Uint64Value(10), 271 | ), 272 | ), 273 | ), 274 | ) 275 | if err != nil { 276 | return err 277 | } 278 | defer func() { 279 | _ = res.Close() 280 | }() 281 | var ( 282 | seriesID uint64 283 | seasonID uint64 284 | title string 285 | date string 286 | ) 287 | log.Print("> scan_query_select:") 288 | for res.NextResultSet(ctx) { 289 | for res.NextRow() { 290 | err = res.ScanNamed( 291 | named.OptionalWithDefault("series_id", &seriesID), 292 | named.OptionalWithDefault("season_id", &seasonID), 293 | named.OptionalWithDefault("title", &title), 294 | named.OptionalWithDefault("first_aired", &date), 295 | ) 296 | if err != nil { 297 | return err 298 | } 299 | log.Printf("# Season, SeriesId: %d, SeasonId: %d, Title: %s, Air date: %s", seriesID, seasonID, title, date) 300 | } 301 | } 302 | return res.Err() 303 | }, 304 | ) 305 | } 306 | 307 | func fillTablesWithData(ctx context.Context, c table.Client, prefix string) (err error) { 308 | writeTx := table.TxControl( 309 | table.BeginTx( 310 | table.WithSerializableReadWrite(), 311 | ), 312 | table.CommitTx(), 313 | ) 314 | err = c.Do(ctx, 315 | func(ctx context.Context, s table.Session) (err error) { 316 | _, _, err = s.Execute(ctx, writeTx, render(fill, templateConfig{ 317 | TablePathPrefix: prefix, 318 | }), table.NewQueryParameters( 319 | table.ValueParam("$seriesData", getSeriesData()), 320 | table.ValueParam("$seasonsData", getSeasonsData()), 321 | table.ValueParam("$episodesData", getEpisodesData()), 322 | )) 323 | return err 324 | }, 325 | ) 326 | return err 327 | } 328 | 329 | func createTables(ctx context.Context, c table.Client, prefix string) (err error) { 330 | err = c.Do(ctx, 331 | func(ctx context.Context, s table.Session) error { 332 | return s.CreateTable(ctx, path.Join(prefix, "series"), 333 | options.WithColumn("series_id", types.Optional(types.TypeUint64)), 334 | options.WithColumn("title", types.Optional(types.TypeUTF8)), 335 | options.WithColumn("series_info", types.Optional(types.TypeUTF8)), 336 | options.WithColumn("release_date", types.Optional(types.TypeUint64)), 337 | options.WithColumn("comment", types.Optional(types.TypeUTF8)), 338 | options.WithPrimaryKeyColumn("series_id"), 339 | ) 340 | }, 341 | ) 342 | if err != nil { 343 | return err 344 | } 345 | 346 | err = c.Do(ctx, 347 | func(ctx context.Context, s table.Session) error { 348 | return s.CreateTable(ctx, path.Join(prefix, "seasons"), 349 | options.WithColumn("series_id", types.Optional(types.TypeUint64)), 350 | options.WithColumn("season_id", types.Optional(types.TypeUint64)), 351 | options.WithColumn("title", types.Optional(types.TypeUTF8)), 352 | options.WithColumn("first_aired", types.Optional(types.TypeUint64)), 353 | options.WithColumn("last_aired", types.Optional(types.TypeUint64)), 354 | options.WithPrimaryKeyColumn("series_id", "season_id"), 355 | ) 356 | }, 357 | ) 358 | if err != nil { 359 | return err 360 | } 361 | 362 | err = c.Do(ctx, 363 | func(ctx context.Context, s table.Session) error { 364 | return s.CreateTable(ctx, path.Join(prefix, "episodes"), 365 | options.WithColumn("series_id", types.Optional(types.TypeUint64)), 366 | options.WithColumn("season_id", types.Optional(types.TypeUint64)), 367 | options.WithColumn("episode_id", types.Optional(types.TypeUint64)), 368 | options.WithColumn("title", types.Optional(types.TypeUTF8)), 369 | options.WithColumn("air_date", types.Optional(types.TypeUint64)), 370 | options.WithPrimaryKeyColumn("series_id", "season_id", "episode_id"), 371 | ) 372 | }, 373 | ) 374 | if err != nil { 375 | return err 376 | } 377 | 378 | return nil 379 | } 380 | 381 | func describeTable(ctx context.Context, c table.Client, path string) (err error) { 382 | err = c.Do(ctx, 383 | func(ctx context.Context, s table.Session) error { 384 | desc, err := s.DescribeTable(ctx, path) 385 | if err != nil { 386 | return err 387 | } 388 | log.Printf("> describe table: %s", path) 389 | for _, c := range desc.Columns { 390 | log.Printf("column, name: %s, %s", c.Type, c.Name) 391 | } 392 | return nil 393 | }, 394 | ) 395 | return 396 | } 397 | 398 | func render(t *template.Template, data interface{}) string { 399 | var buf bytes.Buffer 400 | err := t.Execute(&buf, data) 401 | if err != nil { 402 | panic(err) 403 | } 404 | return buf.String() 405 | } 406 | -------------------------------------------------------------------------------- /bulk_upsert/README.md: -------------------------------------------------------------------------------- 1 | # Bulk upsert example 2 | 3 | Bulk upsert example demonstrates bulk upsert YDB feature -------------------------------------------------------------------------------- /bulk_upsert/example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "time" 8 | 9 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 10 | "github.com/ydb-platform/ydb-go-sdk/v3/table/options" 11 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 12 | ) 13 | 14 | const batchSize = 1000 15 | 16 | type logMessage struct { 17 | App string 18 | Host string 19 | Timestamp time.Time 20 | HTTPCode uint32 21 | Message string 22 | } 23 | 24 | func wrap(err error, explanation string) error { 25 | if err != nil { 26 | return fmt.Errorf("%s: %w", explanation, err) 27 | } 28 | return err 29 | } 30 | 31 | func getLogBatch(logs []logMessage, offset int) []logMessage { 32 | logs = logs[:0] 33 | for i := 0; i < batchSize; i++ { 34 | message := logMessage{ 35 | App: fmt.Sprintf("App_%d", offset/256), 36 | Host: fmt.Sprintf("192.168.0.%d", offset%256), 37 | Timestamp: time.Now().Add(time.Millisecond * time.Duration(offset+i%1000)), 38 | HTTPCode: 200, 39 | } 40 | if i%2 == 0 { 41 | message.Message = "GET / HTTP/1.1" 42 | } else { 43 | message.Message = "GET /images/logo.png HTTP/1.1" 44 | } 45 | 46 | logs = append(logs, message) 47 | } 48 | return logs 49 | } 50 | 51 | func createLogTable(ctx context.Context, c table.Client, tablePath string) error { 52 | log.Printf("Create table: %v\n", tablePath) 53 | err := c.Do(ctx, 54 | func(ctx context.Context, session table.Session) error { 55 | return session.CreateTable(ctx, tablePath, 56 | options.WithColumn("App", types.Optional(types.TypeUTF8)), 57 | options.WithColumn("Timestamp", types.Optional(types.TypeTimestamp)), 58 | options.WithColumn("Host", types.Optional(types.TypeUTF8)), 59 | options.WithColumn("HTTPCode", types.Optional(types.TypeUint32)), 60 | options.WithColumn("Message", types.Optional(types.TypeUTF8)), 61 | options.WithPrimaryKeyColumn("App", "Timestamp", "Host"), 62 | ) 63 | }, 64 | ) 65 | return wrap(err, "failed to create table") 66 | } 67 | 68 | func writeLogBatch(ctx context.Context, c table.Client, tablePath string, logs []logMessage) error { 69 | err := c.Do(ctx, 70 | func(ctx context.Context, session table.Session) error { 71 | rows := make([]types.Value, 0, len(logs)) 72 | 73 | for _, msg := range logs { 74 | rows = append(rows, types.StructValue( 75 | types.StructFieldValue("App", types.TextValue(msg.App)), 76 | types.StructFieldValue("Host", types.TextValue(msg.Host)), 77 | types.StructFieldValue("Timestamp", types.TimestampValueFromTime(msg.Timestamp)), 78 | types.StructFieldValue("HTTPCode", types.Uint32Value(msg.HTTPCode)), 79 | types.StructFieldValue("Message", types.TextValue(msg.Message)), 80 | )) 81 | } 82 | 83 | return wrap(session.BulkUpsert(ctx, tablePath, types.ListValue(rows...)), 84 | "failed to perform bulk upsert") 85 | }) 86 | return wrap(err, "failed to write log batch") 87 | } 88 | -------------------------------------------------------------------------------- /bulk_upsert/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "path" 10 | 11 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 12 | "github.com/ydb-platform/ydb-go-sdk/v3/sugar" 13 | 14 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 15 | ) 16 | 17 | var ( 18 | dsn string 19 | prefix string 20 | tablePath string 21 | count int 22 | ) 23 | 24 | func init() { 25 | required := []string{"ydb", "table"} 26 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 27 | flagSet.Usage = func() { 28 | out := flagSet.Output() 29 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 30 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 31 | flagSet.PrintDefaults() 32 | } 33 | flagSet.StringVar(&dsn, 34 | "ydb", "", 35 | "YDB connection string", 36 | ) 37 | flagSet.StringVar(&prefix, 38 | "prefix", "", 39 | "tables prefix", 40 | ) 41 | flagSet.IntVar(&count, 42 | "count", 1000, 43 | "count requests", 44 | ) 45 | flagSet.StringVar(&tablePath, 46 | "table", "bulk_upsert_example", 47 | "Path for table", 48 | ) 49 | if err := flagSet.Parse(os.Args[1:]); err != nil { 50 | flagSet.Usage() 51 | os.Exit(1) 52 | } 53 | flagSet.Visit(func(f *flag.Flag) { 54 | for i, arg := range required { 55 | if arg == f.Name { 56 | required = append(required[:i], required[i+1:]...) 57 | } 58 | } 59 | }) 60 | if len(required) > 0 { 61 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 62 | flagSet.Usage() 63 | os.Exit(1) 64 | } 65 | } 66 | 67 | func main() { 68 | ctx, cancel := context.WithCancel(context.Background()) 69 | defer cancel() 70 | db, err := ydb.Open(ctx, dsn, 71 | environ.WithEnvironCredentials(ctx), 72 | ) 73 | 74 | if err != nil { 75 | panic(fmt.Errorf("connect error: %w", err)) 76 | } 77 | defer func() { _ = db.Close(ctx) }() 78 | 79 | tablePath = path.Join(db.Name(), prefix, tablePath) 80 | 81 | err = sugar.RemoveRecursive(ctx, db, prefix) 82 | if err != nil { 83 | panic(err) 84 | } 85 | err = sugar.MakeRecursive(ctx, db, prefix) 86 | if err != nil { 87 | panic(err) 88 | } 89 | 90 | if err := createLogTable(ctx, db.Table(), tablePath); err != nil { 91 | panic(wrap(err, "failed to create table")) 92 | } 93 | var logs []logMessage 94 | for offset := 0; offset < count; offset++ { 95 | logs = getLogBatch(logs, offset) 96 | if err := writeLogBatch(ctx, db.Table(), tablePath, logs); err != nil { 97 | panic(wrap(err, fmt.Sprintf("failed to write batch offset %d", offset))) 98 | } 99 | fmt.Print(".") 100 | } 101 | log.Print("Done.\n") 102 | } 103 | -------------------------------------------------------------------------------- /containers/README.md: -------------------------------------------------------------------------------- 1 | # Containers example 2 | 3 | Containers example demonstrates how to work with YDB container values such as `List`, `Tuple`, `Dict`, `Struct` and `Variant` -------------------------------------------------------------------------------- /containers/containers.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | "text/template" 7 | 8 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 9 | ) 10 | 11 | var query = template.Must(template.New("fill database").Parse(` 12 | DECLARE $var AS Variant; 13 | SELECT 14 | AsList("foo", "bar", "baz"); 15 | SELECT 16 | AsTuple(42, "foo", AsList(41, 42, 43)); 17 | SELECT 18 | AsDict( 19 | AsTuple("foo", 10), 20 | AsTuple("bar", 20), 21 | AsTuple("baz", 30), 22 | ); 23 | SELECT 24 | AsStruct( 25 | 41 AS foo, 26 | 42 AS bar, 27 | 43 AS baz, 28 | ); 29 | 30 | $struct = AsStruct( 31 | Uint32("0") as foo, 32 | UTF8("x") as bar, 33 | Int64("0") as baz, 34 | ); 35 | $variantStructType = VariantType(TypeOf($struct)); 36 | SELECT Variant(42, "baz", $variantStructType); 37 | 38 | $tuple = AsTuple( 39 | Uint32("0"), 40 | UTF8("x"), 41 | Int64("0"), 42 | ); 43 | $variantTupleType = VariantType(TypeOf($tuple)); 44 | SELECT Variant(42, "2", $variantTupleType); 45 | `)) 46 | 47 | type exampleStruct struct { 48 | } 49 | 50 | func (*exampleStruct) UnmarshalYDB(res types.RawValue) error { 51 | log.Printf("T: %s", res.Type()) 52 | for i, n := 0, res.StructIn(); i < n; i++ { 53 | name := res.StructField(i) 54 | val := res.Int32() 55 | log.Printf("(struct): %s: %d", name, val) 56 | } 57 | res.StructOut() 58 | return res.Err() 59 | } 60 | 61 | type exampleList struct { 62 | } 63 | 64 | func (*exampleList) UnmarshalYDB(res types.RawValue) error { 65 | log.Printf("T: %s", res.Type()) 66 | for i, n := 0, res.ListIn(); i < n; i++ { 67 | res.ListItem(i) 68 | log.Printf("(list) %q: %s", res.Path(), res.String()) 69 | } 70 | res.ListOut() 71 | return res.Err() 72 | } 73 | 74 | type exampleTuple struct { 75 | } 76 | 77 | func (*exampleTuple) UnmarshalYDB(res types.RawValue) error { 78 | log.Printf("T: %s", res.Type()) 79 | for i, n := 0, res.TupleIn(); i < n; i++ { 80 | res.TupleItem(i) 81 | switch i { 82 | case 0: 83 | log.Printf("(tuple) %q: %d", res.Path(), res.Int32()) 84 | case 1: 85 | log.Printf("(tuple) %q: %s", res.Path(), res.String()) 86 | case 2: 87 | n := res.ListIn() 88 | for j := 0; j < n; j++ { 89 | res.ListItem(j) 90 | log.Printf("(tuple) %q: %d", res.Path(), res.Int32()) 91 | } 92 | res.ListOut() 93 | } 94 | } 95 | res.TupleOut() 96 | return res.Err() 97 | } 98 | 99 | type exampleDict struct { 100 | } 101 | 102 | func (*exampleDict) UnmarshalYDB(res types.RawValue) error { 103 | log.Printf("T: %s", res.Type()) 104 | for i, n := 0, res.DictIn(); i < n; i++ { 105 | res.DictKey(i) 106 | key := res.String() 107 | 108 | res.DictPayload(i) 109 | val := res.Int32() 110 | 111 | log.Printf("(dict) %q: %s: %d", res.Path(), key, val) 112 | } 113 | res.DictOut() 114 | return res.Err() 115 | } 116 | 117 | type variantStruct struct { 118 | } 119 | 120 | func (*variantStruct) UnmarshalYDB(res types.RawValue) error { 121 | log.Printf("T: %s", res.Type()) 122 | name, index := res.Variant() 123 | var x interface{} 124 | switch name { 125 | case "foo": 126 | x = res.Uint32() 127 | case "bar": 128 | x = res.UTF8() 129 | case "baz": 130 | x = res.Int64() 131 | } 132 | log.Printf( 133 | "(struct variant): %s %s %q %d = %v", 134 | res.Path(), res.Type(), name, index, x, 135 | ) 136 | return res.Err() 137 | } 138 | 139 | type variantTuple struct { 140 | } 141 | 142 | func (*variantTuple) UnmarshalYDB(res types.RawValue) error { 143 | log.Printf("T: %s", res.Type()) 144 | name, index := res.Variant() 145 | var x interface{} 146 | switch index { 147 | case 0: 148 | x = res.Uint32() 149 | case 1: 150 | x = res.UTF8() 151 | case 2: 152 | x = res.Int64() 153 | } 154 | log.Printf( 155 | "(tuple variant): %s %s %q %d = %v", 156 | res.Path(), res.Type(), name, index, x, 157 | ) 158 | return res.Err() 159 | } 160 | 161 | func render(t *template.Template, data interface{}) string { 162 | var buf bytes.Buffer 163 | err := t.Execute(&buf, data) 164 | if err != nil { 165 | panic(err) 166 | } 167 | return buf.String() 168 | } 169 | -------------------------------------------------------------------------------- /containers/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | 9 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 10 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 11 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 12 | ) 13 | 14 | var ( 15 | dsn string 16 | prefix string 17 | ) 18 | 19 | func init() { 20 | required := []string{"ydb"} 21 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 22 | flagSet.Usage = func() { 23 | out := flagSet.Output() 24 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 25 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 26 | flagSet.PrintDefaults() 27 | } 28 | flagSet.StringVar(&dsn, 29 | "ydb", "", 30 | "YDB connection string", 31 | ) 32 | flagSet.StringVar(&prefix, 33 | "prefix", "", 34 | "tables prefix", 35 | ) 36 | if err := flagSet.Parse(os.Args[1:]); err != nil { 37 | flagSet.Usage() 38 | os.Exit(1) 39 | } 40 | flagSet.Visit(func(f *flag.Flag) { 41 | for i, arg := range required { 42 | if arg == f.Name { 43 | required = append(required[:i], required[i+1:]...) 44 | } 45 | } 46 | }) 47 | if len(required) > 0 { 48 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 49 | flagSet.Usage() 50 | os.Exit(1) 51 | } 52 | } 53 | 54 | func main() { 55 | ctx, cancel := context.WithCancel(context.Background()) 56 | defer cancel() 57 | db, err := ydb.Open(ctx, dsn, 58 | environ.WithEnvironCredentials(ctx), 59 | ) 60 | if err != nil { 61 | panic(fmt.Errorf("connect error: %w", err)) 62 | } 63 | defer func() { _ = db.Close(ctx) }() 64 | 65 | err = db.Table().DoTx(ctx, 66 | func(ctx context.Context, tx table.TransactionActor) (err error) { 67 | res, err := tx.Execute(ctx, render(query, nil), nil) 68 | if err != nil { 69 | return err 70 | } 71 | defer func() { 72 | _ = res.Close() 73 | }() 74 | 75 | parsers := [...]func() error{ 76 | func() error { 77 | return res.Scan(&exampleList{}) 78 | }, 79 | func() error { 80 | return res.Scan(&exampleTuple{}) 81 | }, 82 | func() error { 83 | return res.Scan(&exampleDict{}) 84 | }, 85 | func() error { 86 | return res.Scan(&exampleStruct{}) 87 | }, 88 | func() error { 89 | return res.Scan(&variantStruct{}) 90 | }, 91 | func() error { 92 | return res.Scan(&variantTuple{}) 93 | }, 94 | } 95 | 96 | for set := 0; res.NextResultSet(ctx); set++ { 97 | res.NextRow() 98 | err = parsers[set]() 99 | if err != nil { 100 | return err 101 | } 102 | } 103 | return res.Err() 104 | }) 105 | if err != nil { 106 | panic(err) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /ddl/README.md: -------------------------------------------------------------------------------- 1 | # DDL example 2 | 3 | DDL example demonstrates DDL (data definition language) support in YDB -------------------------------------------------------------------------------- /ddl/ddl.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 8 | ) 9 | 10 | var ( 11 | simpleCreateQuery = ` 12 | PRAGMA TablePathPrefix("%s"); 13 | CREATE TABLE small_table ( 14 | a Uint64, 15 | b Uint64, 16 | c Text, 17 | d Date, 18 | PRIMARY KEY (a, b) 19 | ); 20 | ` 21 | familyCreateQuery = ` 22 | PRAGMA TablePathPrefix("%s"); 23 | CREATE TABLE small_table2 ( 24 | a Uint64, 25 | b Uint64, 26 | c Text FAMILY family_large, 27 | d Date, 28 | PRIMARY KEY (a, b), 29 | FAMILY family_large ( 30 | COMPRESSION = "lz4" 31 | ) 32 | ); 33 | ` 34 | settingsCreateQuery = ` 35 | PRAGMA TablePathPrefix("%s"); 36 | CREATE TABLE small_table3 ( 37 | a Uint64, 38 | b Uint64, 39 | c Text, 40 | d Date, 41 | PRIMARY KEY (a, b) 42 | ) 43 | WITH ( 44 | AUTO_PARTITIONING_BY_SIZE = ENABLED, --Automatic positioning mode by the size of the partition 45 | AUTO_PARTITIONING_PARTITION_SIZE_MB = 512, --Preferred size of each partition in megabytes 46 | AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 32 --The minimum number of partitions when the automatic merging of partitions stops working 47 | ); 48 | ` 49 | dropQuery = ` 50 | PRAGMA TablePathPrefix("%s"); 51 | DROP TABLE small_table; 52 | DROP TABLE small_table2; 53 | DROP TABLE small_table3; 54 | ` 55 | alterQuery = ` 56 | PRAGMA TablePathPrefix("%s"); 57 | ALTER TABLE small_table ADD COLUMN e Uint64, DROP COLUMN c; 58 | ` 59 | alterSettingsQuery = ` 60 | PRAGMA TablePathPrefix("%s"); 61 | ALTER TABLE small_table2 SET (AUTO_PARTITIONING_BY_SIZE = DISABLED); 62 | ` 63 | alterTTLQuery = ` 64 | PRAGMA TablePathPrefix("%s"); 65 | ALTER TABLE small_table3 SET (TTL = Interval("PT3H") ON d); 66 | ` 67 | ) 68 | 69 | func executeQuery(ctx context.Context, c table.Client, prefix string, query string) (err error) { 70 | err = c.Do(ctx, 71 | func(ctx context.Context, s table.Session) error { 72 | err = s.ExecuteSchemeQuery(ctx, fmt.Sprintf(query, prefix)) 73 | return err 74 | }, 75 | ) 76 | if err != nil { 77 | return err 78 | } 79 | return nil 80 | } 81 | -------------------------------------------------------------------------------- /ddl/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path" 9 | 10 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 11 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 12 | ) 13 | 14 | var ( 15 | dsn string 16 | prefix string 17 | ) 18 | 19 | func init() { 20 | required := []string{"ydb"} 21 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 22 | flagSet.Usage = func() { 23 | out := flagSet.Output() 24 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 25 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 26 | flagSet.PrintDefaults() 27 | } 28 | flagSet.StringVar(&dsn, 29 | "ydb", "", 30 | "YDB connection string", 31 | ) 32 | flagSet.StringVar(&prefix, 33 | "prefix", "", 34 | "tables prefix", 35 | ) 36 | if err := flagSet.Parse(os.Args[1:]); err != nil { 37 | flagSet.Usage() 38 | os.Exit(1) 39 | } 40 | flagSet.Visit(func(f *flag.Flag) { 41 | for i, arg := range required { 42 | if arg == f.Name { 43 | required = append(required[:i], required[i+1:]...) 44 | } 45 | } 46 | }) 47 | if len(required) > 0 { 48 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 49 | flagSet.Usage() 50 | os.Exit(1) 51 | } 52 | } 53 | 54 | func main() { 55 | ctx, cancel := context.WithCancel(context.Background()) 56 | defer cancel() 57 | db, err := ydb.Open(ctx, dsn, 58 | environ.WithEnvironCredentials(ctx), 59 | ) 60 | 61 | if err != nil { 62 | panic(fmt.Errorf("connect error: %w", err)) 63 | } 64 | defer func() { _ = db.Close(ctx) }() 65 | 66 | prefix = path.Join(db.Name(), prefix) 67 | 68 | //simple creation with composite primary key 69 | err = executeQuery(ctx, db.Table(), prefix, simpleCreateQuery) 70 | if err != nil { 71 | panic(err) 72 | } 73 | 74 | //creation with column family 75 | err = executeQuery(ctx, db.Table(), prefix, familyCreateQuery) 76 | if err != nil { 77 | panic(err) 78 | } 79 | 80 | //creation with table settings 81 | err = executeQuery(ctx, db.Table(), prefix, settingsCreateQuery) 82 | if err != nil { 83 | panic(err) 84 | } 85 | 86 | //add column and drop column. 87 | err = executeQuery(ctx, db.Table(), prefix, alterQuery) 88 | if err != nil { 89 | panic(err) 90 | } 91 | 92 | //change AUTO_PARTITIONING_BY_SIZE setting. 93 | err = executeQuery(ctx, db.Table(), prefix, alterSettingsQuery) 94 | if err != nil { 95 | panic(err) 96 | } 97 | 98 | //add TTL. Clear the old data after the three-hour interval has expired. 99 | err = executeQuery(ctx, db.Table(), prefix, alterTTLQuery) 100 | if err != nil { 101 | panic(err) 102 | } 103 | 104 | //drop tables small_table,small_table2,small_table3. 105 | err = executeQuery(ctx, db.Table(), prefix, dropQuery) 106 | if err != nil { 107 | panic(err) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /decimal/README.md: -------------------------------------------------------------------------------- 1 | # Decimal example 2 | 3 | Decimal example demonstrates how to work with YDB decimal values -------------------------------------------------------------------------------- /decimal/decimal.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "text/template" 6 | ) 7 | 8 | type templateConfig struct { 9 | TablePathPrefix string 10 | } 11 | 12 | var writeQuery = template.Must(template.New("fill database").Parse(` 13 | PRAGMA TablePathPrefix("{{ .TablePathPrefix }}"); 14 | 15 | DECLARE $decimals AS List>; 18 | 19 | REPLACE INTO decimals 20 | SELECT 21 | id, 22 | value 23 | FROM AS_TABLE($decimals); 24 | `)) 25 | 26 | var readQuery = template.Must(template.New("fill database").Parse(` 27 | PRAGMA TablePathPrefix("{{ .TablePathPrefix }}"); 28 | SELECT value FROM decimals; 29 | `)) 30 | 31 | func render(t *template.Template, data interface{}) string { 32 | var buf bytes.Buffer 33 | err := t.Execute(&buf, data) 34 | if err != nil { 35 | panic(err) 36 | } 37 | return buf.String() 38 | } 39 | -------------------------------------------------------------------------------- /decimal/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "math/big" 8 | "os" 9 | "path" 10 | 11 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 12 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 13 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 14 | "github.com/ydb-platform/ydb-go-sdk/v3/table/options" 15 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 16 | ) 17 | 18 | var ( 19 | dsn string 20 | prefix string 21 | ) 22 | 23 | func init() { 24 | required := []string{"ydb"} 25 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 26 | flagSet.Usage = func() { 27 | out := flagSet.Output() 28 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 29 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 30 | flagSet.PrintDefaults() 31 | } 32 | flagSet.StringVar(&dsn, 33 | "ydb", "", 34 | "YDB connection string", 35 | ) 36 | flagSet.StringVar(&prefix, 37 | "prefix", "", 38 | "tables prefix", 39 | ) 40 | if err := flagSet.Parse(os.Args[1:]); err != nil { 41 | flagSet.Usage() 42 | os.Exit(1) 43 | } 44 | flagSet.Visit(func(f *flag.Flag) { 45 | for i, arg := range required { 46 | if arg == f.Name { 47 | required = append(required[:i], required[i+1:]...) 48 | } 49 | } 50 | }) 51 | if len(required) > 0 { 52 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 53 | flagSet.Usage() 54 | os.Exit(1) 55 | } 56 | } 57 | 58 | func main() { 59 | ctx, cancel := context.WithCancel(context.Background()) 60 | defer cancel() 61 | db, err := ydb.Open(ctx, dsn, 62 | environ.WithEnvironCredentials(ctx), 63 | ) 64 | 65 | if err != nil { 66 | panic(fmt.Errorf("connect error: %w", err)) 67 | } 68 | defer func() { _ = db.Close(ctx) }() 69 | 70 | prefix = path.Join(db.Name(), prefix) 71 | 72 | var ( 73 | tablePath = path.Join(prefix, "decimals") 74 | ) 75 | err = db.Table().Do(ctx, 76 | func(ctx context.Context, s table.Session) (err error) { 77 | return s.CreateTable(ctx, tablePath, 78 | options.WithColumn("id", types.Optional(types.TypeUint32)), 79 | options.WithColumn("value", types.Optional(types.DefaultDecimal)), 80 | options.WithPrimaryKeyColumn("id"), 81 | ) 82 | }, 83 | ) 84 | if err != nil { 85 | panic(err) 86 | } 87 | 88 | err = db.Table().Do(ctx, 89 | func(ctx context.Context, s table.Session) (err error) { 90 | txc := table.TxControl( 91 | table.BeginTx( 92 | table.WithSerializableReadWrite(), 93 | ), 94 | table.CommitTx(), 95 | ) 96 | 97 | x := big.NewInt(42 * 1000000000) 98 | x.Mul(x, big.NewInt(2)) 99 | 100 | _, _, err = s.Execute(ctx, txc, render(writeQuery, templateConfig{ 101 | TablePathPrefix: prefix, 102 | }), table.NewQueryParameters( 103 | table.ValueParam("$decimals", 104 | types.ListValue( 105 | types.StructValue( 106 | types.StructFieldValue("id", types.Uint32Value(42)), 107 | types.StructFieldValue("value", types.DecimalValueFromBigInt(x, 22, 9)), 108 | ), 109 | ), 110 | ), 111 | )) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | _, res, err := s.Execute(ctx, txc, render(readQuery, templateConfig{ 117 | TablePathPrefix: prefix, 118 | }), nil) 119 | if err != nil { 120 | return err 121 | } 122 | defer func() { 123 | _ = res.Close() 124 | }() 125 | var p *types.Decimal 126 | for res.NextResultSet(ctx) { 127 | for res.NextRow() { 128 | err = res.Scan(&p) 129 | if err != nil { 130 | return err 131 | } 132 | 133 | fmt.Println(p.String()) 134 | } 135 | } 136 | return res.Err() 137 | }, 138 | ) 139 | if err != nil { 140 | panic(err) 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /describe/README.md: -------------------------------------------------------------------------------- 1 | # Describe example 2 | 3 | Example contains listing database and describes of tables 4 | 5 | # Usage 6 | 7 | Describe application have flags: 8 | * `-ydb` for define connection string 9 | * `-prefix` for define root of listening database 10 | * `-t` for define allowed type of table column (if empty - prints all columns) -------------------------------------------------------------------------------- /describe/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | _ "embed" 7 | "flag" 8 | "fmt" 9 | "os" 10 | "path" 11 | "text/template" 12 | 13 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 14 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 15 | "github.com/ydb-platform/ydb-go-sdk/v3/retry" 16 | "github.com/ydb-platform/ydb-go-sdk/v3/scheme" 17 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 18 | "github.com/ydb-platform/ydb-go-sdk/v3/table/options" 19 | ) 20 | 21 | var ( 22 | required = []string{"ydb"} 23 | dsn string 24 | prefix string 25 | 26 | //go:embed table.tpl 27 | defaultTableTemplate string 28 | tableTemplate string 29 | 30 | ignoreDirs = map[string]struct{}{ 31 | ".sys": {}, 32 | ".sys_health": {}, 33 | } 34 | ) 35 | 36 | func parseFlags() { 37 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 38 | flagSet.Usage = func() { 39 | out := flagSet.Output() 40 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 41 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 42 | flagSet.PrintDefaults() 43 | } 44 | flagSet.StringVar(&dsn, 45 | "ydb", "", 46 | "YDB connection string", 47 | ) 48 | flagSet.StringVar(&prefix, 49 | "prefix", "", 50 | "tables prefix", 51 | ) 52 | flagSet.StringVar(&tableTemplate, 53 | "template", defaultTableTemplate, 54 | "template for print table", 55 | ) 56 | if err := flagSet.Parse(os.Args[1:]); err != nil { 57 | flagSet.Usage() 58 | os.Exit(1) 59 | } 60 | flagSet.Visit(func(f *flag.Flag) { 61 | for i, arg := range required { 62 | if arg == f.Name { 63 | required = append(required[:i], required[i+1:]...) 64 | } 65 | } 66 | }) 67 | if len(required) > 0 { 68 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 69 | flagSet.Usage() 70 | os.Exit(1) 71 | } 72 | } 73 | 74 | func main() { 75 | parseFlags() 76 | 77 | ctx, cancel := context.WithCancel(context.Background()) 78 | defer cancel() 79 | db, err := ydb.Open(ctx, dsn, 80 | environ.WithEnvironCredentials(ctx), 81 | ) 82 | if err != nil { 83 | panic(fmt.Errorf("connect error: %w", err)) 84 | } 85 | defer func() { _ = db.Close(ctx) }() 86 | 87 | if prefix == "" { 88 | prefix = db.Name() 89 | } 90 | 91 | t, err := template.New("").Parse(tableTemplate) 92 | if err != nil { 93 | panic(fmt.Errorf("template parse error: %w", err)) 94 | } 95 | 96 | list(ctx, db, t, prefix) 97 | } 98 | 99 | func list(ctx context.Context, db ydb.Connection, t *template.Template, p string) { 100 | var dir scheme.Directory 101 | var err error 102 | err = retry.Retry(ctx, func(ctx context.Context) (err error) { 103 | dir, err = db.Scheme().ListDirectory(ctx, p) 104 | return err 105 | }, retry.WithIdempotent(true)) 106 | if err != nil { 107 | fmt.Printf("list directory '%s' failed: %v\n", p, err) 108 | return 109 | } 110 | 111 | for _, child := range dir.Children { 112 | pt := path.Join(p, child.Name) 113 | switch child.Type { 114 | case scheme.EntryDirectory, scheme.EntryDatabase: 115 | if _, ok := ignoreDirs[child.Name]; ok { 116 | continue 117 | } 118 | list(ctx, db, t, pt) 119 | 120 | case scheme.EntryTable: 121 | var desc options.Description 122 | err = db.Table().Do(ctx, func(ctx context.Context, s table.Session) (err error) { 123 | desc, err = s.DescribeTable(ctx, pt) 124 | return err 125 | }, table.WithIdempotent()) 126 | if err != nil { 127 | fmt.Printf("describe '%s' failed: %v\n", pt, err) 128 | continue 129 | } 130 | desc.Name = pt 131 | var buf bytes.Buffer 132 | if err = t.Execute(&buf, desc); err == nil { 133 | fmt.Println(buf.String()) 134 | } else { 135 | fmt.Println(err) 136 | } 137 | 138 | default: 139 | } 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /describe/table.tpl: -------------------------------------------------------------------------------- 1 | {{- .Name}}{{":"}} 2 | {{range .Columns}} 3 | {{- " - "}}{{.Type}}:{{.Name}} 4 | {{end}} -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ydb-platform/ydb-go-examples 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/google/uuid v1.3.0 7 | github.com/gorilla/mux v1.8.0 8 | github.com/prometheus/client_golang v1.13.0 9 | github.com/rs/zerolog v1.27.0 10 | github.com/ydb-platform/gorm-driver v0.0.1 11 | github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 12 | github.com/ydb-platform/ydb-go-sdk-prometheus v0.11.10 13 | github.com/ydb-platform/ydb-go-sdk-zerolog v0.12.2 14 | github.com/ydb-platform/ydb-go-sdk/v3 v3.42.7 15 | github.com/ydb-platform/ydb-go-yc v0.9.1 16 | google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc 17 | gorm.io/driver/postgres v1.4.6 18 | gorm.io/driver/sqlite v1.4.4 19 | gorm.io/gorm v1.24.2 20 | ) 21 | 22 | require ( 23 | github.com/beorn7/perks v1.0.1 // indirect 24 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 25 | github.com/golang-jwt/jwt v3.2.2+incompatible // indirect 26 | github.com/golang-jwt/jwt/v4 v4.4.2 // indirect 27 | github.com/golang/protobuf v1.5.2 // indirect 28 | github.com/jackc/pgpassfile v1.0.0 // indirect 29 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect 30 | github.com/jackc/pgx/v5 v5.2.0 // indirect 31 | github.com/jinzhu/inflection v1.0.0 // indirect 32 | github.com/jinzhu/now v1.1.5 // indirect 33 | github.com/jonboulle/clockwork v0.3.0 // indirect 34 | github.com/mattn/go-colorable v0.1.13 // indirect 35 | github.com/mattn/go-isatty v0.0.16 // indirect 36 | github.com/mattn/go-sqlite3 v1.14.15 // indirect 37 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 38 | github.com/prometheus/client_model v0.2.0 // indirect 39 | github.com/prometheus/common v0.37.0 // indirect 40 | github.com/prometheus/procfs v0.8.0 // indirect 41 | github.com/yandex-cloud/go-genproto v0.0.0-20220815090733-4c139c0154e2 // indirect 42 | github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f // indirect 43 | github.com/ydb-platform/ydb-go-sdk-metrics v0.16.3 // indirect 44 | github.com/ydb-platform/ydb-go-yc-metadata v0.5.3 // indirect 45 | golang.org/x/crypto v0.4.0 // indirect 46 | golang.org/x/net v0.3.0 // indirect 47 | golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 // indirect 48 | golang.org/x/sys v0.3.0 // indirect 49 | golang.org/x/text v0.5.0 // indirect 50 | google.golang.org/grpc v1.49.0 // indirect 51 | google.golang.org/protobuf v1.28.1 // indirect 52 | ) 53 | -------------------------------------------------------------------------------- /pagination/README.md: -------------------------------------------------------------------------------- 1 | # Pagination example 2 | 3 | Pagination example demonstrates how to use pagination with YDB queries -------------------------------------------------------------------------------- /pagination/cities.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 8 | "github.com/ydb-platform/ydb-go-sdk/v3/table/options" 9 | "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" 10 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 11 | ) 12 | 13 | func selectPaging( 14 | ctx context.Context, 15 | c table.Client, 16 | prefix string, 17 | limit int, 18 | lastNum *uint, 19 | lastCity *string, 20 | ) ( 21 | empty bool, 22 | err error, 23 | ) { 24 | 25 | var query = fmt.Sprintf(` 26 | PRAGMA TablePathPrefix("%v"); 27 | 28 | DECLARE $limit AS Uint64; 29 | DECLARE $lastCity AS Text; 30 | DECLARE $lastNumber AS Uint32; 31 | 32 | $part1 = ( 33 | SELECT * FROM schools 34 | WHERE city = $lastCity AND number > $lastNumber 35 | ORDER BY city, number LIMIT $limit 36 | ); 37 | 38 | $part2 = ( 39 | SELECT * FROM schools 40 | WHERE city > $lastCity 41 | ORDER BY city, number LIMIT $limit 42 | ); 43 | 44 | $union = ( 45 | SELECT * FROM $part1 46 | UNION ALL 47 | SELECT * FROM $part2 48 | ); 49 | 50 | SELECT * FROM $union 51 | ORDER BY city, number LIMIT $limit; 52 | `, prefix) 53 | 54 | readTx := table.TxControl(table.BeginTx(table.WithOnlineReadOnly()), table.CommitTx()) 55 | 56 | err = c.Do(ctx, 57 | func(ctx context.Context, s table.Session) (err error) { 58 | _, res, err := s.Execute(ctx, readTx, query, 59 | table.NewQueryParameters( 60 | table.ValueParam("$limit", types.Uint64Value(uint64(limit))), 61 | table.ValueParam("$lastCity", types.TextValue(*lastCity)), 62 | table.ValueParam("$lastNumber", types.Uint32Value(uint32(*lastNum))), 63 | ), 64 | ) 65 | if err != nil { 66 | return err 67 | } 68 | defer func() { 69 | _ = res.Close() 70 | }() 71 | if !res.NextResultSet(ctx) || !res.HasNextRow() { 72 | empty = true 73 | return 74 | } 75 | var addr string 76 | for res.NextRow() { 77 | err = res.ScanNamed( 78 | named.Optional("city", &lastCity), 79 | named.Optional("number", &lastNum), 80 | named.OptionalWithDefault("address", &addr), 81 | ) 82 | if err != nil { 83 | return err 84 | } 85 | fmt.Printf("\t%v, School #%v, Address: %v\n", *lastCity, *lastNum, addr) 86 | } 87 | return res.Err() 88 | }, 89 | ) 90 | if err != nil { 91 | return 92 | } 93 | return empty, nil 94 | } 95 | 96 | func fillTableWithData(ctx context.Context, c table.Client, prefix string) (err error) { 97 | var query = fmt.Sprintf(` 98 | PRAGMA TablePathPrefix("%v"); 99 | 100 | DECLARE $schoolsData AS List>; 104 | 105 | REPLACE INTO schools 106 | SELECT 107 | city, 108 | number, 109 | address 110 | FROM AS_TABLE($schoolsData);`, prefix) 111 | 112 | writeTx := table.TxControl(table.BeginTx(table.WithSerializableReadWrite()), table.CommitTx()) 113 | 114 | err = c.Do(ctx, 115 | func(ctx context.Context, s table.Session) (err error) { 116 | _, _, err = s.Execute(ctx, writeTx, query, table.NewQueryParameters( 117 | table.ValueParam("$schoolsData", getSchoolData()), 118 | )) 119 | return err 120 | }) 121 | return err 122 | } 123 | 124 | func createTable(ctx context.Context, c table.Client, path string) (err error) { 125 | err = c.Do(ctx, 126 | func(ctx context.Context, s table.Session) error { 127 | return s.CreateTable(ctx, path, 128 | options.WithColumn("city", types.Optional(types.TypeUTF8)), 129 | options.WithColumn("number", types.Optional(types.TypeUint32)), 130 | options.WithColumn("address", types.Optional(types.TypeUTF8)), 131 | options.WithPrimaryKeyColumn("city", "number"), 132 | ) 133 | }, 134 | ) 135 | if err != nil { 136 | return err 137 | } 138 | 139 | return nil 140 | } 141 | -------------------------------------------------------------------------------- /pagination/data.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 5 | ) 6 | 7 | func schoolData(city string, num uint32, addr string) types.Value { 8 | return types.StructValue( 9 | types.StructFieldValue("city", types.TextValue(city)), 10 | types.StructFieldValue("number", types.Uint32Value(num)), 11 | types.StructFieldValue("address", types.TextValue(addr)), 12 | ) 13 | } 14 | 15 | func getSchoolData() types.Value { 16 | return types.ListValue( 17 | schoolData("Орлов", 1, "Ст.Халтурина, 2"), 18 | schoolData("Орлов", 2, "Свободы, 4"), 19 | schoolData("Яранск", 1, "Гоголя, 25"), 20 | schoolData("Яранск", 2, "Кирова, 18"), 21 | schoolData("Яранск", 3, "Некрасова, 59"), 22 | schoolData("Кирс", 3, "Кирова, 6"), 23 | schoolData("Нолинск", 1, "Коммуны, 4"), 24 | schoolData("Нолинск", 2, "Федосеева, 2Б"), 25 | schoolData("Котельнич", 1, "Урицкого, 21"), 26 | schoolData("Котельнич", 2, "Октябрьская, 109"), 27 | schoolData("Котельнич", 3, "Советская, 153"), 28 | schoolData("Котельнич", 5, "Школьная, 2"), 29 | schoolData("Котельнич", 15, "Октябрьская, 91"), 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /pagination/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path" 9 | 10 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 11 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 12 | "github.com/ydb-platform/ydb-go-sdk/v3/sugar" 13 | ) 14 | 15 | var ( 16 | dsn string 17 | prefix string 18 | ) 19 | 20 | func init() { 21 | required := []string{"ydb"} 22 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 23 | flagSet.Usage = func() { 24 | out := flagSet.Output() 25 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 26 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 27 | flagSet.PrintDefaults() 28 | } 29 | flagSet.StringVar(&dsn, 30 | "ydb", "", 31 | "YDB connection string", 32 | ) 33 | flagSet.StringVar(&prefix, 34 | "prefix", "", 35 | "tables prefix", 36 | ) 37 | if err := flagSet.Parse(os.Args[1:]); err != nil { 38 | flagSet.Usage() 39 | os.Exit(1) 40 | } 41 | flagSet.Visit(func(f *flag.Flag) { 42 | for i, arg := range required { 43 | if arg == f.Name { 44 | required = append(required[:i], required[i+1:]...) 45 | } 46 | } 47 | }) 48 | if len(required) > 0 { 49 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 50 | flagSet.Usage() 51 | os.Exit(1) 52 | } 53 | } 54 | 55 | func main() { 56 | ctx, cancel := context.WithCancel(context.Background()) 57 | defer cancel() 58 | db, err := ydb.Open(ctx, dsn, 59 | environ.WithEnvironCredentials(ctx), 60 | ) 61 | 62 | if err != nil { 63 | panic(fmt.Errorf("connect error: %w", err)) 64 | } 65 | defer func() { _ = db.Close(ctx) }() 66 | 67 | err = sugar.RemoveRecursive(ctx, db, prefix) 68 | if err != nil { 69 | panic(err) 70 | } 71 | err = sugar.MakeRecursive(ctx, db, prefix) 72 | if err != nil { 73 | panic(err) 74 | } 75 | 76 | prefix = path.Join(db.Name(), prefix) 77 | 78 | err = createTable(ctx, db.Table(), path.Join(prefix, "schools")) 79 | if err != nil { 80 | panic(fmt.Errorf("create tables error: %w", err)) 81 | } 82 | 83 | err = fillTableWithData(ctx, db.Table(), prefix) 84 | if err != nil { 85 | panic(fmt.Errorf("fill tables with data error: %w", err)) 86 | } 87 | 88 | var lastNum uint 89 | lastCity := "" 90 | limit := 3 91 | maxPages := 10 92 | for i, empty := 0, false; i < maxPages && !empty; i++ { 93 | fmt.Printf("> Page %v:\n", i+1) 94 | empty, err = selectPaging(ctx, db.Table(), prefix, limit, &lastNum, &lastCity) 95 | if err != nil { 96 | panic(fmt.Errorf("get page %v error: %w", i, err)) 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /partitioning_policies/README.md: -------------------------------------------------------------------------------- 1 | # Partitioning policies example 2 | 3 | Partitioning policies example demonstrates how to use uniform and explicit partitioning policies in YDB -------------------------------------------------------------------------------- /partitioning_policies/example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log" 8 | 9 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 10 | "github.com/ydb-platform/ydb-go-sdk/v3/table/options" 11 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 12 | ) 13 | 14 | func wrap(err error, explanation string) error { 15 | if err != nil { 16 | return fmt.Errorf("%s: %w", explanation, err) 17 | } 18 | return err 19 | } 20 | 21 | func testUniformPartitions(ctx context.Context, c table.Client, tablePath string) error { 22 | log.Printf("Create uniform partitions table: %v\n", tablePath) 23 | 24 | err := c.Do(ctx, 25 | func(ctx context.Context, session table.Session) error { 26 | err := session.CreateTable(ctx, tablePath, 27 | options.WithColumn("key", types.Optional(types.TypeUint64)), 28 | options.WithColumn("value", types.Optional(types.TypeJSON)), 29 | options.WithPrimaryKeyColumn("key"), 30 | 31 | options.WithProfile( 32 | options.WithPartitioningPolicy( 33 | options.WithPartitioningPolicyMode(options.PartitioningAutoSplitMerge), 34 | options.WithPartitioningPolicyUniformPartitions(4), 35 | ), 36 | ), 37 | ) 38 | if err != nil { 39 | return wrap(err, "failed to create table") 40 | } 41 | 42 | desc, err := session.DescribeTable(ctx, tablePath, options.WithShardKeyBounds()) 43 | if err != nil { 44 | return wrap(err, "failed to get table description") 45 | } 46 | if len(desc.KeyRanges) != 4 { 47 | return errors.New("key ranges len is not as expected") 48 | } 49 | 50 | return nil 51 | }, 52 | ) 53 | return wrap(err, "failed to execute operation") 54 | } 55 | 56 | func testExplicitPartitions(ctx context.Context, c table.Client, tablePath string) error { 57 | log.Printf("Create explicit partitions table: %v\n", tablePath) 58 | 59 | err := c.Do(ctx, 60 | func(ctx context.Context, session table.Session) error { 61 | err := session.CreateTable(ctx, tablePath, 62 | options.WithColumn("key", types.Optional(types.TypeUint64)), 63 | options.WithColumn("value", types.Optional(types.TypeJSON)), 64 | options.WithPrimaryKeyColumn("key"), 65 | 66 | options.WithProfile( 67 | options.WithPartitioningPolicy( 68 | options.WithPartitioningPolicyExplicitPartitions( 69 | types.TupleValue(types.OptionalValue(types.Uint64Value(100))), 70 | types.TupleValue(types.OptionalValue(types.Uint64Value(300))), 71 | types.TupleValue(types.OptionalValue(types.Uint64Value(400))), 72 | ), 73 | ), 74 | ), 75 | ) 76 | if err != nil { 77 | return wrap(err, "failed to create table") 78 | } 79 | 80 | desc, err := session.DescribeTable(ctx, tablePath, options.WithShardKeyBounds()) 81 | if err != nil { 82 | return wrap(err, "failed to get table description") 83 | } 84 | if len(desc.KeyRanges) != 4 { 85 | return errors.New("key ranges len is not as expected") 86 | } 87 | 88 | return nil 89 | }, 90 | ) 91 | return wrap(err, "failed to execute operation") 92 | } 93 | -------------------------------------------------------------------------------- /partitioning_policies/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path" 9 | 10 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 11 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 12 | "github.com/ydb-platform/ydb-go-sdk/v3/sugar" 13 | ) 14 | 15 | var ( 16 | dsn string 17 | prefix string 18 | tablePath string 19 | ) 20 | 21 | func init() { 22 | required := []string{"ydb", "table"} 23 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 24 | flagSet.Usage = func() { 25 | out := flagSet.Output() 26 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 27 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 28 | flagSet.PrintDefaults() 29 | } 30 | flagSet.StringVar(&dsn, 31 | "ydb", "", 32 | "YDB connection string", 33 | ) 34 | flagSet.StringVar(&prefix, 35 | "prefix", "", 36 | "tables prefix", 37 | ) 38 | flagSet.StringVar(&tablePath, 39 | "table", "explicit_partitions_example", 40 | "Path for table", 41 | ) 42 | if err := flagSet.Parse(os.Args[1:]); err != nil { 43 | flagSet.Usage() 44 | os.Exit(1) 45 | } 46 | flagSet.Visit(func(f *flag.Flag) { 47 | for i, arg := range required { 48 | if arg == f.Name { 49 | required = append(required[:i], required[i+1:]...) 50 | } 51 | } 52 | }) 53 | if len(required) > 0 { 54 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 55 | flagSet.Usage() 56 | os.Exit(1) 57 | } 58 | } 59 | 60 | func main() { 61 | ctx, cancel := context.WithCancel(context.Background()) 62 | defer cancel() 63 | db, err := ydb.Open(ctx, dsn, 64 | environ.WithEnvironCredentials(ctx), 65 | ) 66 | 67 | if err != nil { 68 | panic(fmt.Errorf("connect error: %w", err)) 69 | } 70 | defer func() { _ = db.Close(ctx) }() 71 | 72 | tablePath = path.Join(db.Name(), prefix, tablePath) 73 | 74 | err = sugar.RemoveRecursive(ctx, db, prefix) 75 | if err != nil { 76 | panic(err) 77 | } 78 | err = sugar.MakeRecursive(ctx, db, prefix) 79 | if err != nil { 80 | panic(err) 81 | } 82 | 83 | if err = testUniformPartitions(ctx, db.Table(), tablePath); err != nil { 84 | panic(wrap(err, "failed to test uniform partitions")) 85 | } 86 | 87 | err = sugar.RemoveRecursive(ctx, db, prefix) 88 | if err != nil { 89 | panic(err) 90 | } 91 | 92 | if err = testExplicitPartitions(ctx, db.Table(), tablePath); err != nil { 93 | panic(wrap(err, "failed to test explicit partitions")) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /read_table/README.md: -------------------------------------------------------------------------------- 1 | # Read table example 2 | 3 | Read table example demonstrates how to read all data from table using `table.Session.StreamReadTable` API method -------------------------------------------------------------------------------- /read_table/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "path" 10 | 11 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 12 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 13 | "github.com/ydb-platform/ydb-go-sdk/v3/table/options" 14 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 15 | ) 16 | 17 | var ( 18 | dsn string 19 | prefix string 20 | ) 21 | 22 | func init() { 23 | required := []string{"ydb"} 24 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 25 | flagSet.Usage = func() { 26 | out := flagSet.Output() 27 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 28 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 29 | flagSet.PrintDefaults() 30 | } 31 | flagSet.StringVar(&dsn, 32 | "ydb", "", 33 | "YDB connection string", 34 | ) 35 | flagSet.StringVar(&prefix, 36 | "prefix", "", 37 | "tables prefix", 38 | ) 39 | if err := flagSet.Parse(os.Args[1:]); err != nil { 40 | flagSet.Usage() 41 | os.Exit(1) 42 | } 43 | flagSet.Visit(func(f *flag.Flag) { 44 | for i, arg := range required { 45 | if arg == f.Name { 46 | required = append(required[:i], required[i+1:]...) 47 | } 48 | } 49 | }) 50 | if len(required) > 0 { 51 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 52 | flagSet.Usage() 53 | os.Exit(1) 54 | } 55 | } 56 | 57 | func main() { 58 | ctx, cancel := context.WithCancel(context.Background()) 59 | defer cancel() 60 | db, err := ydb.Open(ctx, dsn, 61 | environ.WithEnvironCredentials(ctx), 62 | ) 63 | 64 | if err != nil { 65 | panic(fmt.Errorf("connect error: %w", err)) 66 | } 67 | defer func() { _ = db.Close(ctx) }() 68 | 69 | prefix = path.Join(db.Name(), prefix) 70 | 71 | tableName := "orders" 72 | 73 | log.Println("Drop table (if exists)...") 74 | err = dropTableIfExists( 75 | ctx, 76 | db.Table(), 77 | path.Join(prefix, tableName), 78 | ) 79 | if err != nil { 80 | panic(fmt.Errorf("drop table error: %w", err)) 81 | } 82 | log.Println("Drop table done") 83 | 84 | log.Println("Create table...") 85 | err = createTable( 86 | ctx, 87 | db.Table(), 88 | path.Join(prefix, tableName), 89 | ) 90 | if err != nil { 91 | panic(fmt.Errorf("create table error: %w", err)) 92 | } 93 | log.Println("Create table done") 94 | 95 | log.Println("Fill table...") 96 | err = fillTable( 97 | ctx, 98 | db.Table(), 99 | prefix, 100 | ) 101 | if err != nil { 102 | panic(fmt.Errorf("fill table error: %w", err)) 103 | } 104 | log.Println("Fill table done") 105 | 106 | log.Println("Read whole table, unsorted:") 107 | err = readTable( 108 | ctx, 109 | db.Table(), 110 | path.Join(prefix, tableName), 111 | ) 112 | if err != nil { 113 | panic(fmt.Errorf("read table error: %w", err)) 114 | } 115 | 116 | log.Println("Sorted by composite primary key:") 117 | err = readTable( 118 | ctx, 119 | db.Table(), 120 | path.Join(prefix, tableName), 121 | options.ReadOrdered(), 122 | ) 123 | if err != nil { 124 | panic(fmt.Errorf("read table error: %w", err)) 125 | } 126 | 127 | log.Println("Any five rows:") 128 | err = readTable( 129 | ctx, 130 | db.Table(), 131 | path.Join(prefix, tableName), 132 | options.ReadRowLimit(5), 133 | ) 134 | if err != nil { 135 | panic(fmt.Errorf("read table error: %w", err)) 136 | } 137 | 138 | log.Println("First five rows by PK (ascending) with subset of columns:") 139 | err = readTable( 140 | ctx, 141 | db.Table(), 142 | path.Join(prefix, tableName), 143 | options.ReadRowLimit(5), 144 | options.ReadColumn("customer_id"), 145 | options.ReadColumn("order_id"), 146 | options.ReadColumn("order_date"), 147 | options.ReadOrdered(), 148 | ) 149 | if err != nil { 150 | panic(fmt.Errorf("read table error: %w", err)) 151 | } 152 | 153 | log.Println("Read all rows with first PK component (customer_id,) greater or equal than 2 and less then 3:") 154 | keyRange := options.KeyRange{ 155 | From: types.TupleValue( 156 | types.OptionalValue(types.Uint64Value(2)), 157 | ), 158 | To: types.TupleValue( 159 | types.OptionalValue(types.Uint64Value(3)), 160 | ), 161 | } 162 | err = readTable( 163 | ctx, 164 | db.Table(), 165 | path.Join(prefix, tableName), 166 | options.ReadKeyRange(keyRange), 167 | ) 168 | if err != nil { 169 | panic(fmt.Errorf("read table error: %w", err)) 170 | } 171 | 172 | log.Println("Read all rows with composite PK lexicographically less or equal than (1,4):") 173 | err = readTable( 174 | ctx, 175 | db.Table(), 176 | path.Join(prefix, tableName), 177 | options.ReadLessOrEqual( 178 | types.TupleValue( 179 | types.OptionalValue(types.Uint64Value(1)), 180 | types.OptionalValue(types.Uint64Value(4)), 181 | ), 182 | ), 183 | ) 184 | if err != nil { 185 | panic(fmt.Errorf("read table error: %w", err)) 186 | } 187 | 188 | log.Println("Read all rows with composite PK lexicographically greater or equal than (1,2) and less than (3,4):") 189 | keyRange = options.KeyRange{ 190 | From: types.TupleValue( 191 | types.OptionalValue(types.Uint64Value(1)), 192 | types.OptionalValue(types.Uint64Value(2)), 193 | ), 194 | To: types.TupleValue( 195 | types.OptionalValue(types.Uint64Value(3)), 196 | types.OptionalValue(types.Uint64Value(1)), 197 | ), 198 | } 199 | err = readTable( 200 | ctx, 201 | db.Table(), 202 | path.Join(prefix, tableName), 203 | options.ReadKeyRange(keyRange), 204 | ) 205 | if err != nil { 206 | panic(fmt.Errorf("read table error: %w", err)) 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /read_table/orders.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "log" 7 | "text/template" 8 | "time" 9 | 10 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 11 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 12 | "github.com/ydb-platform/ydb-go-sdk/v3/table/options" 13 | "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" 14 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 15 | ) 16 | 17 | type row struct { 18 | id uint64 19 | orderID uint64 20 | date time.Time 21 | description string 22 | } 23 | 24 | func dropTableIfExists(ctx context.Context, c table.Client, path string) (err error) { 25 | err = c.Do(ctx, 26 | func(ctx context.Context, s table.Session) error { 27 | return s.DropTable(ctx, path) 28 | }, 29 | table.WithIdempotent(), 30 | ) 31 | if !ydb.IsOperationErrorSchemeError(err) { 32 | return err 33 | } 34 | return nil 35 | } 36 | 37 | func createTable(ctx context.Context, c table.Client, path string) (err error) { 38 | return c.Do(ctx, 39 | func(ctx context.Context, s table.Session) error { 40 | return s.CreateTable(ctx, path, 41 | options.WithColumn("customer_id", types.Optional(types.TypeUint64)), 42 | options.WithColumn("order_id", types.Optional(types.TypeUint64)), 43 | options.WithColumn("order_date", types.Optional(types.TypeDate)), 44 | options.WithColumn("description", types.Optional(types.TypeUTF8)), 45 | options.WithPrimaryKeyColumn("customer_id", "order_id"), 46 | ) 47 | }, 48 | table.WithIdempotent(), 49 | ) 50 | } 51 | 52 | var ( 53 | fillQuery = template.Must(template.New("fill orders").Parse(` 54 | PRAGMA TablePathPrefix("{{ .TablePathPrefix }}"); 55 | 56 | DECLARE $ordersData AS List>; 61 | 62 | REPLACE INTO orders 63 | SELECT 64 | customer_id, 65 | order_id, 66 | order_date, 67 | description 68 | FROM AS_TABLE($ordersData); 69 | `)) 70 | dateLayout = "2006-01-02" 71 | ) 72 | 73 | func render(t *template.Template, data interface{}) string { 74 | var buf bytes.Buffer 75 | err := t.Execute(&buf, data) 76 | if err != nil { 77 | panic(err) 78 | } 79 | return buf.String() 80 | } 81 | 82 | type templateConfig struct { 83 | TablePathPrefix string 84 | } 85 | 86 | func fillTable(ctx context.Context, c table.Client, prefix string) (err error) { 87 | return c.Do(ctx, 88 | func(ctx context.Context, s table.Session) (err error) { 89 | _, _, err = s.Execute( 90 | ctx, 91 | table.TxControl( 92 | table.BeginTx( 93 | table.WithSerializableReadWrite(), 94 | ), 95 | table.CommitTx(), 96 | ), 97 | render(fillQuery, templateConfig{ 98 | TablePathPrefix: prefix, 99 | }), 100 | table.NewQueryParameters( 101 | table.ValueParam( 102 | "$ordersData", 103 | types.ListValue( 104 | order(1, 1, "Order 1", "2006-02-03"), 105 | order(1, 2, "Order 2", "2007-08-24"), 106 | order(1, 3, "Order 3", "2008-11-21"), 107 | order(1, 4, "Order 4", "2010-06-25"), 108 | order(2, 1, "Order 1", "2014-04-06"), 109 | order(2, 2, "Order 2", "2015-04-12"), 110 | order(2, 3, "Order 3", "2016-04-24"), 111 | order(2, 4, "Order 4", "2017-04-23"), 112 | order(2, 5, "Order 5", "2018-03-25"), 113 | order(3, 1, "Order 1", "2019-04-23"), 114 | order(3, 2, "Order 3", "2020-03-25"), 115 | ), 116 | ), 117 | ), 118 | ) 119 | return err 120 | }, 121 | ) 122 | } 123 | 124 | func order(customerID uint64, orderID uint64, description string, date string) types.Value { 125 | orderDate, err := time.Parse(dateLayout, date) 126 | if err != nil { 127 | panic(err) 128 | } 129 | return types.StructValue( 130 | types.StructFieldValue("customer_id", types.Uint64Value(customerID)), 131 | types.StructFieldValue("order_id", types.Uint64Value(orderID)), 132 | types.StructFieldValue("order_date", types.DateValueFromTime(orderDate)), 133 | types.StructFieldValue("description", types.TextValue(description)), 134 | ) 135 | } 136 | 137 | func readTable(ctx context.Context, c table.Client, path string, opts ...options.ReadTableOption) (err error) { 138 | err = c.Do(ctx, 139 | func(ctx context.Context, s table.Session) (err error) { 140 | res, err := s.StreamReadTable(ctx, path, opts...) 141 | if err != nil { 142 | return err 143 | } 144 | defer func() { 145 | _ = res.Close() 146 | }() 147 | r := row{} 148 | for res.NextResultSet(ctx) { 149 | for res.NextRow() { 150 | if res.CurrentResultSet().ColumnCount() == 4 { 151 | err = res.ScanNamed( 152 | named.OptionalWithDefault("customer_id", &r.id), 153 | named.OptionalWithDefault("order_id", &r.orderID), 154 | named.OptionalWithDefault("order_date", &r.date), 155 | named.OptionalWithDefault("description", &r.description), 156 | ) 157 | if err != nil { 158 | return err 159 | } 160 | log.Printf("# Order, CustomerId: %d, OrderId: %d, Description: %s, Order date: %s", r.id, r.orderID, r.description, r.date.Format("2006-01-02")) 161 | } else { 162 | err = res.ScanNamed( 163 | named.OptionalWithDefault("customer_id", &r.id), 164 | named.OptionalWithDefault("order_id", &r.orderID), 165 | named.OptionalWithDefault("order_date", &r.date), 166 | ) 167 | if err != nil { 168 | return err 169 | } 170 | log.Printf("# Order, CustomerId: %d, OrderId: %d, Order date: %s", r.id, r.orderID, r.date.Format("2006-01-02")) 171 | } 172 | } 173 | } 174 | return res.Err() 175 | }, 176 | ) 177 | return err 178 | } 179 | -------------------------------------------------------------------------------- /serverless/healthcheck/README.md: -------------------------------------------------------------------------------- 1 | # Healthcheck 2 | 3 | Healthcheck application provide check URLs and store results into YDB. 4 | 5 | ## Usage 6 | 7 | ### Running as application 8 | 9 | ```bash 10 | go get -u github.com/ydb-platform/ydb-go-examples/cmd/serverless/healthcheck 11 | YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/path/to/sa/key/file \ 12 | healthcheck \ 13 | -ydb=grpcs://types.serverless.yandexcloud.net:2135/ru-central1/b1g8skpblkos03malf3s/etn01f8gv9an9sedo9fu \ 14 | -url=www.ya.ru 15 | -url=google.com 16 | -url=rampler.ru 17 | ``` 18 | 19 | ### Running as serverless function 20 | Yandex function needs a go module project. First you must create go.mod file. 21 | ```bash 22 | go mod init example && go mod tidy 23 | zip archive.zip service.go go.mod go.sum 24 | yc sls fn version create \ 25 | --service-account-id=aje46n285h0re8nmm5u6 \ 26 | --runtime=golang118 \ 27 | --entrypoint=main.Serverless \ 28 | --memory=128m \ 29 | --execution-timeout=1s \ 30 | --environment YDB_METADATA_CREDENTIALS="1" \ 31 | --environment YDB="grpcs://ydb.serverless.yandexcloud.net:2135/ru-central1/b1g8skpblkos03malf3s/etnpa7o3qltdfgu9vsap" \ 32 | --environment URLS="https://ya.ru,https://google.com,https://rambler.ru" \ 33 | --source-path=./archive.zip \ 34 | --function-id=d4empp866m0b4m2gspu9 35 | ``` 36 | -------------------------------------------------------------------------------- /serverless/healthcheck/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "strconv" 9 | "time" 10 | 11 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 12 | ) 13 | 14 | var ( 15 | dsn string 16 | prefix string 17 | count int 18 | interval time.Duration 19 | urls = URLs{} 20 | ) 21 | 22 | // URLs is a flag.Value implementation which holds URL's as string slice 23 | type URLs struct { 24 | urls []string 25 | } 26 | 27 | // String returns string representation of URLs 28 | func (u *URLs) String() string { 29 | return fmt.Sprintf("%v", u.urls) 30 | } 31 | 32 | // Set appends new value to URLs holder 33 | func (u *URLs) Set(s string) error { 34 | u.urls = append(u.urls, s) 35 | return nil 36 | } 37 | 38 | func init() { 39 | required := []string{"ydb", "url"} 40 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 41 | flagSet.Usage = func() { 42 | out := flagSet.Output() 43 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options] -url=URL1 [-url=URL2 -url=URL3]\n", os.Args[0]) 44 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 45 | flagSet.PrintDefaults() 46 | } 47 | flagSet.StringVar(&dsn, 48 | "ydb", "", 49 | "YDB connection string", 50 | ) 51 | flagSet.StringVar(&prefix, 52 | "prefix", "", 53 | "tables prefix", 54 | ) 55 | flagSet.Var(&urls, 56 | "url", 57 | "url for health check", 58 | ) 59 | flagSet.IntVar(&count, 60 | "count", 0, 61 | "count of needed checks (one check per minute, -1 for busy loop, 0 for single shot)", 62 | ) 63 | flagSet.DurationVar(&interval, 64 | "interval", time.Minute, 65 | "interval between checks", 66 | ) 67 | if err := flagSet.Parse(os.Args[1:]); err != nil { 68 | flagSet.Usage() 69 | os.Exit(1) 70 | } 71 | flagSet.Visit(func(f *flag.Flag) { 72 | for i, arg := range required { 73 | if arg == f.Name { 74 | required = append(required[:i], required[i+1:]...) 75 | } 76 | } 77 | }) 78 | if len(required) > 0 { 79 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 80 | flagSet.Usage() 81 | os.Exit(1) 82 | } 83 | } 84 | 85 | func main() { 86 | ctx, cancel := context.WithCancel(context.Background()) 87 | defer cancel() 88 | s, err := getService(ctx, dsn, environ.WithEnvironCredentials(ctx)) 89 | if err != nil { 90 | panic(fmt.Errorf("error on create service: %w", err)) 91 | } 92 | defer s.Close(ctx) 93 | for i := 0; ; i++ { 94 | fmt.Println("check " + strconv.Itoa(i) + ":") 95 | if err := s.check(ctx, urls.urls); err != nil { 96 | panic(fmt.Errorf("error on check URLS: %w", err)) 97 | } 98 | if count >= 0 && i == count { 99 | return 100 | } 101 | select { 102 | case <-time.After(interval): 103 | continue 104 | case <-ctx.Done(): 105 | return 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /serverless/healthcheck/service.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "os" 10 | "path" 11 | "strconv" 12 | "strings" 13 | "sync" 14 | "time" 15 | 16 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 17 | "github.com/ydb-platform/ydb-go-sdk/v3" 18 | "github.com/ydb-platform/ydb-go-sdk/v3/sugar" 19 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 20 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 21 | ) 22 | 23 | type service struct { 24 | db ydb.Connection 25 | client *http.Client 26 | } 27 | 28 | var ( 29 | s *service 30 | once sync.Once 31 | ) 32 | 33 | func getService(ctx context.Context, dsn string, opts ...ydb.Option) (s *service, err error) { 34 | once.Do(func() { 35 | s = &service{ 36 | client: &http.Client{ 37 | Transport: &http.Transport{ 38 | TLSClientConfig: &tls.Config{ 39 | InsecureSkipVerify: true, 40 | }, 41 | }, 42 | Timeout: time.Second * 10, 43 | }, 44 | } 45 | s.db, err = ydb.Open(ctx, dsn, opts...) 46 | if err != nil { 47 | err = fmt.Errorf("connect error: %w", err) 48 | return 49 | } 50 | err = s.createTableIfNotExists(ctx) 51 | if err != nil { 52 | _ = s.db.Close(ctx) 53 | err = fmt.Errorf("error on create table: %w", err) 54 | } 55 | }) 56 | if err != nil { 57 | once = sync.Once{} 58 | return nil, err 59 | } 60 | return s, nil 61 | } 62 | 63 | func (s *service) Close(ctx context.Context) { 64 | defer func() { _ = s.db.Close(ctx) }() 65 | } 66 | 67 | func (s *service) createTableIfNotExists(ctx context.Context) error { 68 | exists, err := sugar.IsTableExists(ctx, s.db.Scheme(), path.Join(s.db.Name(), prefix, "healthchecks")) 69 | if err != nil { 70 | return err 71 | } 72 | if exists { 73 | return nil 74 | } 75 | query := fmt.Sprintf(` 76 | PRAGMA TablePathPrefix("%s"); 77 | 78 | CREATE TABLE healthchecks ( 79 | url Text, 80 | code Int32, 81 | ts DateTime, 82 | error Text, 83 | PRIMARY KEY (url, ts) 84 | ) WITH ( 85 | AUTO_PARTITIONING_BY_LOAD = ENABLED 86 | );`, path.Join(s.db.Name(), prefix), 87 | ) 88 | return s.db.Table().Do(ctx, 89 | func(ctx context.Context, s table.Session) error { 90 | return s.ExecuteSchemeQuery(ctx, query) 91 | }, 92 | ) 93 | } 94 | 95 | func (s *service) ping(path string) (code int32, err error) { 96 | uri, err := url.Parse(path) 97 | if err != nil { 98 | return -1, err 99 | } 100 | if uri.Scheme == "" { 101 | uri.Scheme = "http" 102 | } 103 | request, err := http.NewRequest(http.MethodGet, uri.String(), nil) 104 | if err != nil { 105 | return -1, err 106 | } 107 | response, err := s.client.Do(request) 108 | if err != nil { 109 | return -1, err 110 | } 111 | return int32(response.StatusCode), nil 112 | } 113 | 114 | type row struct { 115 | url string 116 | code int32 117 | err error 118 | } 119 | 120 | func (s *service) check(ctx context.Context, urls []string) error { 121 | if len(urls) == 0 { 122 | return nil 123 | } 124 | wg := &sync.WaitGroup{} 125 | rows := make([]row, len(urls)) 126 | for idx := range urls { 127 | for _, u := range strings.Split(urls[idx], " ") { 128 | wg.Add(1) 129 | go func(idx int) { 130 | defer wg.Done() 131 | out := " > '" + u + "' => " 132 | code, err := s.ping(u) 133 | if err != nil { 134 | fmt.Println(out + err.Error()) 135 | } else { 136 | fmt.Println(out + strconv.Itoa(int(code))) 137 | } 138 | rows[idx] = row{ 139 | url: urls[idx], 140 | code: code, 141 | err: err, 142 | } 143 | }(idx) 144 | } 145 | } 146 | wg.Wait() 147 | 148 | return s.upsertRows(ctx, rows) 149 | } 150 | 151 | func (s *service) upsertRows(ctx context.Context, rows []row) (err error) { 152 | values := make([]types.Value, len(rows)) 153 | for i, row := range rows { 154 | values[i] = types.StructValue( 155 | types.StructFieldValue("url", types.TextValue(row.url)), 156 | types.StructFieldValue("code", types.Int32Value(row.code)), 157 | types.StructFieldValue("ts", types.DatetimeValueFromTime(time.Now())), 158 | types.StructFieldValue("error", types.TextValue(func(err error) string { 159 | if err != nil { 160 | return err.Error() 161 | } 162 | return "" 163 | }(row.err))), 164 | ) 165 | } 166 | err = s.db.Table().Do(ctx, 167 | func(ctx context.Context, session table.Session) (err error) { 168 | _, _, err = session.Execute(ctx, 169 | table.SerializableReadWriteTxControl(table.CommitTx()), 170 | fmt.Sprintf(` 171 | PRAGMA TablePathPrefix("%s"); 172 | 173 | DECLARE $rows AS List>; 179 | 180 | UPSERT INTO healthchecks ( url, code, ts, error ) 181 | SELECT url, code, ts, error FROM AS_TABLE($rows);`, 182 | path.Join(s.db.Name(), prefix), 183 | ), 184 | table.NewQueryParameters( 185 | table.ValueParam("$rows", types.ListValue(values...)), 186 | ), 187 | ) 188 | return err 189 | }, 190 | ) 191 | if err != nil { 192 | return fmt.Errorf("error on upsert rows: %w", err) 193 | } 194 | return nil 195 | } 196 | 197 | // Serverless is an entrypoint for serverless yandex function 198 | // nolint:deadcode 199 | func Serverless(ctx context.Context) error { 200 | s, err := getService( 201 | ctx, 202 | os.Getenv("YDB"), 203 | environ.WithEnvironCredentials(ctx), 204 | ydb.WithDialTimeout(time.Second), 205 | ) 206 | if err != nil { 207 | return fmt.Errorf("error on create service: %w", err) 208 | } 209 | defer s.Close(ctx) 210 | return s.check(ctx, strings.Split(os.Getenv("URLS"), ",")) 211 | } 212 | -------------------------------------------------------------------------------- /serverless/url_shortener/README.md: -------------------------------------------------------------------------------- 1 | # URL shortener 2 | 3 | URL shortener is an application which provide make short URL and store results into YDB. 4 | 5 | ## Usage 6 | 7 | ### Running as http-server 8 | 9 | ```bash 10 | go get -u github.com/ydb-platform/ydb-go-sdk/v3/example/url_shortener 11 | YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/path/to/sa/key/file \ 12 | url_shortener \ 13 | -ydb=grpcs://ydb.serverless.yandexcloud.net:2135/ru-central1/b1g8skpblkos03malf3s/etn01f8gv9an9sedo9fu \ 14 | -port=80 15 | ``` 16 | Open http://localhost/ in browse and use URL shortener web interface 17 | 18 | ### Running as serverless function 19 | Yandex function needs a go module project. First you must create go.mod file. 20 | ```bash 21 | go mod init example && go mod tidy 22 | zip archive.zip service.go go.mod go.sum 23 | yc sls fn version create \ 24 | --service-account-id=aje46n285h0re8nmm5u6 \ 25 | --runtime=golang116 \ 26 | --entrypoint=main.Serverless \ 27 | --memory=128m \ 28 | --execution-timeout=1s \ 29 | --environment YDB="grpcs://ydb.serverless.yandexcloud.net:2135/ru-central1/b1g8skpblkos03malf3s/etnpa7o3qltdfgu9vsap" \ 30 | --source-path=./archive.zip \ 31 | --function-id=d4euc5gp5614b56crpnj 32 | ``` 33 | -------------------------------------------------------------------------------- /serverless/url_shortener/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "net/http" 8 | "os" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/rs/zerolog" 13 | 14 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 15 | ) 16 | 17 | var ( 18 | dsn string 19 | prefix string 20 | port int 21 | sessionPoolLimit int 22 | shutdownAfter time.Duration 23 | logLevel string 24 | 25 | log = zerolog.New(os.Stdout).With().Timestamp().Logger() 26 | ) 27 | 28 | func init() { 29 | required := []string{"ydb"} 30 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 31 | flagSet.Usage = func() { 32 | out := flagSet.Output() 33 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 34 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 35 | flagSet.PrintDefaults() 36 | } 37 | flagSet.StringVar(&dsn, 38 | "ydb", "", 39 | "YDB connection string", 40 | ) 41 | flagSet.StringVar(&prefix, 42 | "prefix", "", 43 | "tables prefix", 44 | ) 45 | flagSet.StringVar(&logLevel, 46 | "log-level", "info", 47 | "logging level", 48 | ) 49 | flagSet.IntVar(&port, 50 | "port", 80, 51 | "http port for web-server", 52 | ) 53 | flagSet.IntVar(&sessionPoolLimit, 54 | "session-pool-limit", 50, 55 | "session pool size limit", 56 | ) 57 | flagSet.DurationVar(&shutdownAfter, 58 | "shutdown-after", -1, 59 | "duration for shutdown after start", 60 | ) 61 | if err := flagSet.Parse(os.Args[1:]); err != nil { 62 | flagSet.Usage() 63 | os.Exit(1) 64 | } 65 | flagSet.Visit(func(f *flag.Flag) { 66 | for i, arg := range required { 67 | if arg == f.Name { 68 | required = append(required[:i], required[i+1:]...) 69 | } 70 | } 71 | }) 72 | if len(required) > 0 { 73 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 74 | flagSet.Usage() 75 | os.Exit(1) 76 | } 77 | if l, err := zerolog.ParseLevel(logLevel); err == nil { 78 | zerolog.SetGlobalLevel(l) 79 | } else { 80 | panic(err) 81 | } 82 | } 83 | 84 | func main() { 85 | var ( 86 | ctx context.Context 87 | cancel context.CancelFunc 88 | done = make(chan struct{}) 89 | ) 90 | if shutdownAfter > 0 { 91 | ctx, cancel = context.WithTimeout(context.Background(), shutdownAfter) 92 | } else { 93 | ctx, cancel = context.WithCancel(context.Background()) 94 | } 95 | defer cancel() 96 | 97 | s, err := getService( 98 | ctx, 99 | dsn, 100 | environ.WithEnvironCredentials(ctx), 101 | ) 102 | if err != nil { 103 | fmt.Println() 104 | fmt.Println("Create service failed. Re-run with flag '-log-level=warn' and see logs") 105 | os.Exit(1) 106 | } 107 | defer s.Close(ctx) 108 | 109 | server := &http.Server{ 110 | Addr: ":" + strconv.Itoa(port), 111 | Handler: s.router, 112 | } 113 | defer func() { 114 | _ = server.Shutdown(ctx) 115 | }() 116 | 117 | go func() { 118 | _ = server.ListenAndServe() 119 | close(done) 120 | }() 121 | 122 | select { 123 | case <-ctx.Done(): 124 | return 125 | case <-done: 126 | return 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /serverless/url_shortener/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | URL shortener 6 | 25 | 26 | 27 |
28 | 29 | 32 |
33 |
34 | 35 |
36 | 60 | 61 | -------------------------------------------------------------------------------- /topic/cdc-cache-bus-freeseats/.gitignore: -------------------------------------------------------------------------------- 1 | /cdc-example-cache-freeseats 2 | /cdc-example-cache-freeseats.exe 3 | 4 | -------------------------------------------------------------------------------- /topic/cdc-cache-bus-freeseats/balancer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "sync/atomic" 6 | ) 7 | 8 | type balancer struct { 9 | handlers []http.Handler 10 | counter int32 11 | } 12 | 13 | func newBalancer(handlers ...http.Handler) *balancer { 14 | return &balancer{ 15 | handlers: handlers, 16 | } 17 | } 18 | 19 | func (b *balancer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { 20 | counter := atomic.AddInt32(&b.counter, 1) 21 | if counter < 0 { 22 | counter = -counter 23 | } 24 | index := int(counter-1) % len(b.handlers) 25 | b.handlers[index].ServeHTTP(writer, request) 26 | } 27 | -------------------------------------------------------------------------------- /topic/cdc-cache-bus-freeseats/cache.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | type Cache struct { 9 | timeout time.Duration 10 | 11 | m sync.Mutex 12 | setCounter int 13 | values map[string]CacheItem 14 | } 15 | 16 | func NewCache(timeout time.Duration) *Cache { 17 | return &Cache{ 18 | timeout: timeout, 19 | values: make(map[string]CacheItem), 20 | } 21 | } 22 | 23 | func (c *Cache) Get(key string) (int64, bool) { 24 | if c.timeout == 0 { 25 | return 0, false 26 | } 27 | 28 | c.m.Lock() 29 | defer c.m.Unlock() 30 | 31 | item, ok := c.values[key] 32 | if !ok { 33 | return 0, false 34 | } 35 | 36 | if time.Now().After(item.ExpiresAt) { 37 | delete(c.values, key) 38 | return 0, false 39 | } 40 | 41 | return item.Value, true 42 | } 43 | 44 | func (c *Cache) Set(key string, value int64) { 45 | if c.timeout == 0 { 46 | return 47 | } 48 | 49 | c.set(time.Now(), key, value) 50 | } 51 | 52 | func (c *Cache) Delete(key string) { 53 | if c.timeout == 0 { 54 | return 55 | } 56 | 57 | c.m.Lock() 58 | defer c.m.Unlock() 59 | 60 | delete(c.values, key) 61 | } 62 | 63 | func (c *Cache) set(now time.Time, key string, value int64) { 64 | c.m.Lock() 65 | defer c.m.Unlock() 66 | 67 | c.setCounter++ 68 | if c.setCounter%1000 == 0 { 69 | c.cleanupNeedLock(now) 70 | } 71 | 72 | c.values[key] = CacheItem{ 73 | Value: value, 74 | ExpiresAt: now.Add(c.timeout), 75 | } 76 | } 77 | 78 | func (c *Cache) cleanupNeedLock(now time.Time) { 79 | for key, item := range c.values { 80 | if now.After(item.ExpiresAt) { 81 | delete(c.values, key) 82 | } 83 | } 84 | } 85 | 86 | type CacheItem struct { 87 | ExpiresAt time.Time 88 | Value int64 89 | } 90 | -------------------------------------------------------------------------------- /topic/cdc-cache-bus-freeseats/database.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "os" 8 | "path" 9 | "time" 10 | 11 | "github.com/ydb-platform/ydb-go-sdk/v3" 12 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 13 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicoptions" 14 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topictypes" 15 | ) 16 | 17 | func createTableAndCDC(ctx context.Context, db ydb.Connection, consumersCount int) { 18 | err := createTables(ctx, db) 19 | if err != nil { 20 | log.Fatalf("failed to create tables: %+v", err) 21 | } 22 | 23 | err = createCosumers(ctx, db, consumersCount) 24 | if err != nil { 25 | log.Fatalf("failed to create consumers: %+v", err) 26 | } 27 | } 28 | 29 | func createTables(ctx context.Context, db ydb.Connection) error { 30 | err := db.Table().Do(ctx, func(ctx context.Context, s table.Session) error { 31 | err := s.DropTable(ctx, path.Join(db.Name(), "bus")) 32 | if ydb.IsOperationErrorSchemeError(err) { 33 | err = nil 34 | } 35 | return err 36 | }) 37 | if err != nil { 38 | return fmt.Errorf("failed to drop table: %w", err) 39 | } 40 | 41 | _, err = db.Scripting().Execute(ctx, ` 42 | CREATE TABLE bus (id Text, freeSeats Int64, PRIMARY KEY(id)); 43 | 44 | ALTER TABLE 45 | bus 46 | ADD CHANGEFEED 47 | updates 48 | WITH ( 49 | FORMAT = 'JSON', 50 | MODE = 'UPDATES' 51 | ) 52 | `, nil) 53 | if err != nil { 54 | return fmt.Errorf("failed to create table: %w", err) 55 | } 56 | _, err = db.Scripting().Execute(ctx, ` 57 | UPSERT INTO bus (id, freeSeats) VALUES ("bus1", 40), ("bus2", 60); 58 | `, nil) 59 | if err != nil { 60 | return fmt.Errorf("failed insert rows: %w", err) 61 | } 62 | return nil 63 | } 64 | 65 | func createCosumers(ctx context.Context, db ydb.Connection, consumersCount int) error { 66 | for i := 0; i < consumersCount; i++ { 67 | err := db.Topic().Alter(ctx, "bus/updates", topicoptions.AlterWithAddConsumers(topictypes.Consumer{ 68 | Name: consumerName(i), 69 | })) 70 | if err != nil { 71 | return err 72 | } 73 | } 74 | 75 | return nil 76 | } 77 | 78 | func connect() ydb.Connection { 79 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) 80 | defer cancel() 81 | 82 | connectionString := os.Getenv("YDB_CONNECTION_STRING") 83 | if *ydbConnectionString != "" { 84 | connectionString = *ydbConnectionString 85 | } 86 | if connectionString == "" { 87 | connectionString = defaultConnectionString 88 | } 89 | 90 | token := os.Getenv("YDB_TOKEN") 91 | if *ydbToken != "" { 92 | token = *ydbToken 93 | } 94 | var ydbOptions []ydb.Option 95 | if token != "" { 96 | ydbOptions = append(ydbOptions, ydb.WithAccessTokenCredentials(token)) 97 | } 98 | 99 | if *ydbToken != "" { 100 | ydbOptions = append(ydbOptions, ydb.WithAccessTokenCredentials(*ydbToken)) 101 | } 102 | db, err := ydb.Open(ctx, connectionString, ydbOptions...) 103 | if err != nil { 104 | log.Fatalf("failed to create to ydb: %+v", err) 105 | } 106 | log.Printf("connected to database") 107 | return db 108 | } 109 | 110 | func consumerName(index int) string { 111 | return fmt.Sprintf("consumer-%v", index) 112 | } 113 | -------------------------------------------------------------------------------- /topic/cdc-cache-bus-freeseats/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "flag" 7 | "log" 8 | "net/http" 9 | "strconv" 10 | "time" 11 | ) 12 | 13 | const defaultConnectionString = "grpc://localhost:2136/local" 14 | 15 | var ( 16 | host = flag.String("listen-host", "localhost", "host/ip for start listener") 17 | port = flag.Int("port", 3619, "port to listen") 18 | cacheTimeout = flag.Duration("cache", time.Second*10, "cache timeout, 0 mean disable cache") 19 | disableCDC = flag.Bool("disable-cdc", false, "disable cdc") 20 | skipCreateTable = flag.Bool("skip-init", false, "skip recreate table and topic") 21 | ydbConnectionString = flag.String("ydb-connection-string", "", "ydb connection string, default "+defaultConnectionString) 22 | ydbToken = flag.String("ydb-token", "", "Auth token for ydb") 23 | backendCount = flag.Int("backend-count", 1, "count of backend servers") 24 | ) 25 | 26 | func main() { 27 | flag.Parse() 28 | 29 | ctx := context.Background() 30 | db := connect() 31 | 32 | if !*skipCreateTable { 33 | createTableAndCDC(ctx, db, *backendCount) 34 | } 35 | 36 | servers := make([]http.Handler, *backendCount) 37 | cdcEnabled := !*disableCDC 38 | for i := 0; i < *backendCount; i++ { 39 | servers[i] = newServer(i, db, *cacheTimeout, cdcEnabled) 40 | } 41 | log.Printf("servers count: %v", len(servers)) 42 | handler := newBalancer(servers...) 43 | 44 | addr := *host + ":" + strconv.Itoa(*port) 45 | log.Printf("Start listen http://%s\n", addr) 46 | err := http.ListenAndServe(addr, handler) 47 | if errors.Is(err, http.ErrServerClosed) { 48 | log.Fatalf("failed to listen and serve: %+v", err) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /topic/cdc-cache-bus-freeseats/webserver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "path" 10 | "sync/atomic" 11 | "time" 12 | 13 | "github.com/ydb-platform/ydb-go-sdk/v3" 14 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 15 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 16 | ) 17 | 18 | var errNotEnthoughtFreeSeats = errors.New("not enough free seats") 19 | 20 | type server struct { 21 | cache *Cache 22 | mux http.ServeMux 23 | db ydb.Connection 24 | dbCounter int64 25 | id int 26 | } 27 | 28 | func newServer(id int, db ydb.Connection, cacheTimeout time.Duration, useCDC bool) *server { 29 | res := &server{ 30 | cache: NewCache(cacheTimeout), 31 | db: db, 32 | id: id, 33 | } 34 | 35 | res.mux.HandleFunc("/", res.IndexPageHandler) 36 | res.mux.HandleFunc("/get/", res.GetFreeSeatsHandler) 37 | res.mux.HandleFunc("/buy/", res.BuyTicketHandler) 38 | 39 | if useCDC { 40 | go res.cdcLoop() 41 | } 42 | 43 | return res 44 | } 45 | 46 | func (s *server) ServeHTTP(writer http.ResponseWriter, request *http.Request) { 47 | s.mux.ServeHTTP(writer, request) 48 | } 49 | 50 | func (s *server) GetFreeSeatsHandler(writer http.ResponseWriter, request *http.Request) { 51 | ctx := request.Context() 52 | id := path.Base(request.URL.Path) 53 | 54 | start := time.Now() 55 | freeSeats, err := s.getFreeSeats(ctx, id) 56 | if err != nil { 57 | http.Error(writer, err.Error(), http.StatusInternalServerError) 58 | return 59 | } 60 | duration := time.Since(start) 61 | s.writeAnswer(writer, freeSeats, duration) 62 | } 63 | 64 | func (s *server) BuyTicketHandler(writer http.ResponseWriter, request *http.Request) { 65 | ctx := request.Context() 66 | id := path.Base(request.URL.Path) 67 | 68 | start := time.Now() 69 | freeSeats, err := s.sellTicket(ctx, id) 70 | if err != nil { 71 | if errors.Is(err, errNotEnthoughtFreeSeats) { 72 | http.Error(writer, "Not enough free seats", http.StatusPreconditionFailed) 73 | } else { 74 | http.Error(writer, err.Error(), http.StatusInternalServerError) 75 | } 76 | return 77 | } 78 | // s.cache.Delete(id) // used without cdc, for single-instance application 79 | duration := time.Since(start) 80 | s.writeAnswer(writer, freeSeats, duration) 81 | } 82 | 83 | func (s *server) writeAnswer(writer http.ResponseWriter, freeSeats int64, duration time.Duration) { 84 | _, _ = fmt.Fprintf(writer, "%v\n\nDuration: %v\n", freeSeats, duration) 85 | } 86 | 87 | func (s *server) getFreeSeats(ctx context.Context, id string) (int64, error) { 88 | if content, ok := s.cache.Get(id); ok { 89 | return content, nil 90 | } 91 | 92 | freeSeats, err := s.getContentFromDB(ctx, id) 93 | 94 | if err == nil { 95 | s.cache.Set(id, freeSeats) 96 | } 97 | 98 | return freeSeats, err 99 | } 100 | 101 | func (s *server) getContentFromDB(ctx context.Context, id string) (int64, error) { 102 | atomic.AddInt64(&s.dbCounter, 1) 103 | var freeSeats int64 104 | err := s.db.Table().DoTx(ctx, func(ctx context.Context, tx table.TransactionActor) error { 105 | var err error 106 | freeSeats, err = s.getFreeSeatsTx(ctx, tx, id) 107 | return err 108 | }) 109 | 110 | return freeSeats, err 111 | } 112 | 113 | func (s *server) getFreeSeatsTx(ctx context.Context, tx table.TransactionActor, id string) (int64, error) { 114 | var freeSeats int64 115 | res, err := tx.Execute(ctx, ` 116 | DECLARE $id AS Text; 117 | 118 | SELECT freeSeats FROM bus WHERE id=$id; 119 | `, table.NewQueryParameters(table.ValueParam("$id", types.UTF8Value(id)))) 120 | if err != nil { 121 | return 0, err 122 | } 123 | 124 | err = res.NextResultSetErr(ctx, "freeSeats") 125 | if err != nil { 126 | return 0, err 127 | } 128 | 129 | if !res.NextRow() { 130 | freeSeats = 0 131 | return 0, errors.New("not found") 132 | } 133 | 134 | err = res.ScanWithDefaults(&freeSeats) 135 | if err != nil { 136 | return 0, err 137 | } 138 | 139 | return freeSeats, nil 140 | } 141 | 142 | func (s *server) sellTicket(ctx context.Context, id string) (int64, error) { 143 | var freeSeats int64 144 | err := s.db.Table().DoTx(ctx, func(ctx context.Context, tx table.TransactionActor) error { 145 | var err error 146 | freeSeats, err = s.getFreeSeatsTx(ctx, tx, id) 147 | if err != nil { 148 | return err 149 | } 150 | if freeSeats < 0 { 151 | return fmt.Errorf("failed to sell ticket: %w", errNotEnthoughtFreeSeats) 152 | } 153 | 154 | _, err = tx.Execute(ctx, ` 155 | DECLARE $id AS Text; 156 | 157 | UPDATE bus SET freeSeats = freeSeats - 1 WHERE id=$id; 158 | `, table.NewQueryParameters(table.ValueParam("$id", types.UTF8Value(id)))) 159 | return err 160 | }) 161 | if err == nil { 162 | freeSeats-- 163 | } 164 | return freeSeats, err 165 | } 166 | 167 | func (s *server) IndexPageHandler(writer http.ResponseWriter, request *http.Request) { 168 | ctx := request.Context() 169 | 170 | var busIDs []string 171 | 172 | err := s.db.Table().DoTx(ctx, func(ctx context.Context, tx table.TransactionActor) error { 173 | res, err := tx.Execute(ctx, "SELECT id FROM bus ORDER BY id", nil) 174 | if err != nil { 175 | return err 176 | } 177 | 178 | res.NextResultSet(ctx, "id") 179 | 180 | for res.HasNextRow() { 181 | res.NextRow() 182 | var id string 183 | err = res.ScanWithDefaults(&id) 184 | if err != nil { 185 | return err 186 | } 187 | busIDs = append(busIDs, id) 188 | } 189 | 190 | return nil 191 | }) 192 | if err != nil { 193 | http.Error(writer, err.Error(), http.StatusInternalServerError) 194 | return 195 | } 196 | 197 | writer.Header().Set("Content-Type", "text/html") 198 | writer.WriteHeader(http.StatusOK) 199 | 200 | _, _ = io.WriteString(writer, `Bus table
201 |
202 | 203 | 204 | 205 | 206 | 207 | 208 | `) 209 | for _, id := range busIDs { 210 | _, _ = fmt.Fprintf(writer, ` 211 | 212 | 213 | 214 | `, id, id, id, id, id) 215 | } 216 | _, _ = io.WriteString(writer, "
IDGet free seats linkBuy ticket link
%v/get/%v/buy/%v
") 217 | } 218 | -------------------------------------------------------------------------------- /topic/cdc-cache-bus-freeseats/webserver_cdc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "time" 7 | 8 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicoptions" 9 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicsugar" 10 | ) 11 | 12 | func (s *server) cdcLoop() { 13 | ctx := context.Background() 14 | consumer := consumerName(s.id) 15 | reader, err := s.db.Topic().StartReader(consumer, topicoptions.ReadSelectors{ 16 | { 17 | Path: "bus/updates", 18 | ReadFrom: time.Now(), 19 | }, 20 | }, 21 | ) 22 | if err != nil { 23 | log.Fatalf("failed to start reader: %+v", err) 24 | } 25 | 26 | log.Printf("Start cdc listen for server: %v", s.id) 27 | for { 28 | msg, err := reader.ReadMessage(ctx) 29 | if err != nil { 30 | log.Fatalf("failed to read message: %+v", err) 31 | } 32 | 33 | var cdcEvent struct { 34 | Key []string 35 | Update struct { 36 | FreeSeats int64 37 | } 38 | Erase *struct{} 39 | } 40 | 41 | err = topicsugar.JSONUnmarshal(msg, &cdcEvent) 42 | if err != nil { 43 | log.Fatalf("failed to unmarshal message: %+v", err) 44 | } 45 | 46 | busID := cdcEvent.Key[0] 47 | // s.dropFromCache(busID) // used for clean cache and force database request 48 | if cdcEvent.Erase == nil { 49 | s.cache.Set(busID, cdcEvent.Update.FreeSeats) // used for direct update cache from cdc without database request 50 | log.Println("server-id:", s.id, "Update record: ", busID, "set freeseats:", cdcEvent.Update.FreeSeats) 51 | } else { 52 | log.Println("server-id:", s.id, "Remove record from cache: ", busID) 53 | s.cache.Delete(busID) 54 | } 55 | err = reader.Commit(ctx, msg) 56 | if err != nil { 57 | log.Printf("failed to commit message: %+v", err) 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /topic/cdc-fill-and-read/cdc-reader.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | 8 | "github.com/ydb-platform/ydb-go-sdk/v3" 9 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicoptions" 10 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicsugar" 11 | ) 12 | 13 | func cdcRead(ctx context.Context, db ydb.Connection, consumerName, topicPath string) { 14 | // Connect to changefeed 15 | 16 | log.Println("Start cdc read") 17 | reader, err := db.Topic().StartReader(consumerName, []topicoptions.ReadSelector{{Path: topicPath}}) 18 | if err != nil { 19 | log.Fatal("failed to start read feed", err) 20 | } 21 | 22 | for { 23 | msg, err := reader.ReadMessage(ctx) 24 | if err != nil { 25 | panic(fmt.Errorf("failed to read message: %+v", err)) 26 | } 27 | 28 | var event interface{} 29 | err = topicsugar.JSONUnmarshal(msg, &event) 30 | if err != nil { 31 | panic(fmt.Errorf("failed to unmarshal json cdc: %+v", err)) 32 | } 33 | log.Println("new cdc event:", event) 34 | err = reader.Commit(ctx, msg) 35 | if err != nil { 36 | panic(fmt.Errorf("failed to commit message: %+v", err)) 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /topic/cdc-fill-and-read/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "path" 10 | "time" 11 | 12 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 13 | 14 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 15 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicoptions" 16 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topictypes" 17 | ) 18 | 19 | var ( 20 | dsn string 21 | useEnvCredentials bool 22 | ) 23 | 24 | func main() { 25 | readFlags() 26 | ctx, cancel := context.WithCancel(context.Background()) 27 | defer cancel() 28 | 29 | var opts []ydb.Option 30 | if useEnvCredentials { 31 | opts = append(opts, environ.WithEnvironCredentials(ctx)) 32 | } 33 | 34 | db, err := ydb.Open( 35 | ctx, 36 | dsn, 37 | opts..., 38 | ) 39 | if err != nil { 40 | panic(fmt.Errorf("connect error: %w", err)) 41 | } 42 | defer func() { _ = db.Close(ctx) }() 43 | 44 | prefix := path.Join(db.Name()) 45 | tableName := "cdc" 46 | topicPath := tableName + "/feed" 47 | consumerName := "test-consumer" 48 | 49 | prepareTableWithCDC(ctx, db, prefix, tableName, topicPath, consumerName) 50 | 51 | go fillTable(ctx, db.Table(), prefix, tableName) 52 | go func() { 53 | time.Sleep(interval / 2) 54 | removeFromTable(ctx, db.Table(), prefix, tableName) 55 | }() 56 | 57 | cdcRead(ctx, db, consumerName, topicPath) 58 | } 59 | 60 | func readFlags() { 61 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 62 | flagSet.Usage = func() { 63 | out := flagSet.Output() 64 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 65 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 66 | flagSet.PrintDefaults() 67 | } 68 | flagSet.StringVar(&dsn, 69 | "ydb", "grpc://localhost:2136/local", 70 | "YDB connection string", 71 | ) 72 | flagSet.BoolVar(&useEnvCredentials, 73 | "use-env-credentials", false, 74 | "Use credentials from env variables", 75 | ) 76 | if err := flagSet.Parse(os.Args[1:]); err != nil { 77 | flagSet.Usage() 78 | os.Exit(1) 79 | } 80 | } 81 | 82 | func prepareTableWithCDC(ctx context.Context, db ydb.Connection, prefix, tableName, topicPath, consumerName string) { 83 | log.Println("Drop table (if exists)...") 84 | err := dropTableIfExists( 85 | ctx, 86 | db.Table(), 87 | path.Join(prefix, tableName), 88 | ) 89 | if err != nil { 90 | panic(fmt.Errorf("drop table error: %w", err)) 91 | } 92 | log.Println("Drop table done") 93 | 94 | log.Println("Create table...") 95 | err = createTable( 96 | ctx, 97 | db.Table(), 98 | prefix, tableName, 99 | ) 100 | if err != nil { 101 | panic(fmt.Errorf("create table error: %w", err)) 102 | } 103 | log.Println("Create table done") 104 | 105 | log.Println("Create consumer") 106 | err = db.Topic().Alter(ctx, topicPath, topicoptions.AlterWithAddConsumers(topictypes.Consumer{ 107 | Name: consumerName, 108 | })) 109 | if err != nil { 110 | panic(fmt.Errorf("failed to create feed consumer: %+v", err)) 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /topic/cdc-fill-and-read/tables.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math/rand" 7 | "path" 8 | "strconv" 9 | "time" 10 | 11 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 12 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 13 | "github.com/ydb-platform/ydb-go-sdk/v3/table/options" 14 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 15 | ) 16 | 17 | const ( 18 | maxID = 100 19 | interval = time.Second 20 | ) 21 | 22 | func dropTableIfExists(ctx context.Context, c table.Client, path string) (err error) { 23 | err = c.Do(ctx, 24 | func(ctx context.Context, s table.Session) error { 25 | return s.DropTable(ctx, path) 26 | }, 27 | table.WithIdempotent(), 28 | ) 29 | if !ydb.IsOperationErrorSchemeError(err) { 30 | return err 31 | } 32 | return nil 33 | } 34 | 35 | func createTable(ctx context.Context, c table.Client, prefix, tableName string) (err error) { 36 | err = c.Do(ctx, 37 | func(ctx context.Context, s table.Session) error { 38 | return s.CreateTable(ctx, path.Join(prefix, tableName), 39 | options.WithColumn("id", types.Optional(types.TypeUint64)), 40 | options.WithColumn("value", types.Optional(types.TypeUTF8)), 41 | options.WithPrimaryKeyColumn("id"), 42 | ) 43 | }, 44 | table.WithIdempotent(), 45 | ) 46 | if err != nil { 47 | return fmt.Errorf("failed to create table: %w", err) 48 | } 49 | 50 | err = c.Do(ctx, func(ctx context.Context, s table.Session) error { 51 | query := fmt.Sprintf(` 52 | PRAGMA TablePathPrefix("%v"); 53 | 54 | ALTER TABLE 55 | %v 56 | ADD CHANGEFEED 57 | feed 58 | WITH ( 59 | FORMAT = 'JSON', 60 | MODE = 'NEW_AND_OLD_IMAGES' 61 | ) 62 | `, prefix, tableName) 63 | return s.ExecuteSchemeQuery(ctx, query) 64 | }) 65 | if err != nil { 66 | return fmt.Errorf("failed to add changefeed to test table: %w", err) 67 | } 68 | return nil 69 | } 70 | 71 | func fillTable(ctx context.Context, c table.Client, prefix, tableName string) { 72 | query := fmt.Sprintf(` 73 | PRAGMA TablePathPrefix("%v"); 74 | 75 | DECLARE $id AS Uint64; 76 | DECLARE $value AS Text; 77 | 78 | UPSERT INTO 79 | %v 80 | (id, value) 81 | VALUES 82 | ($id, $value) 83 | `, prefix, tableName) 84 | for { 85 | id := uint64(rand.Intn(maxID)) 86 | val := "val-" + strconv.Itoa(rand.Intn(10)) 87 | params := table.NewQueryParameters( 88 | table.ValueParam("$id", types.Uint64Value(id)), 89 | table.ValueParam("$value", types.UTF8Value(val)), 90 | ) 91 | _ = c.DoTx(ctx, func(ctx context.Context, tx table.TransactionActor) error { 92 | _, err := tx.Execute(ctx, query, params) 93 | return err 94 | }) 95 | 96 | time.Sleep(interval) 97 | } 98 | } 99 | 100 | func removeFromTable(ctx context.Context, c table.Client, prefix, tableName string) { 101 | query := fmt.Sprintf(` 102 | PRAGMA TablePathPrefix("%v"); 103 | 104 | DECLARE $id AS Uint64; 105 | 106 | DELETE FROM 107 | %v 108 | WHERE id=$id 109 | `, prefix, tableName) 110 | for { 111 | id := uint64(rand.Intn(maxID)) 112 | params := table.NewQueryParameters( 113 | table.ValueParam("$id", types.Uint64Value(id)), 114 | ) 115 | _ = c.DoTx(ctx, func(ctx context.Context, tx table.TransactionActor) error { 116 | _, err := tx.Execute(ctx, query, params) 117 | return err 118 | }) 119 | 120 | time.Sleep(interval) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /topic/topicreader/stubs.go: -------------------------------------------------------------------------------- 1 | package topicreaderexamples 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicreader" 7 | ) 8 | 9 | func getEndOffset(b *topicreader.Batch) int64 { //nolint:unused 10 | panic("example stub") 11 | } 12 | 13 | func externalSystemCommit(ctx context.Context, topic string, partition int64, offset int64) error { //nolint:unused 14 | panic("example stub") 15 | } 16 | 17 | func externalSystemLock(ctx context.Context, topic string, partition int64) (err error) { 18 | panic("example stub") 19 | } 20 | 21 | func externalSystemUnlock(ctx context.Context, topic string, partition int64) error { 22 | panic("example stub") 23 | } 24 | 25 | func processBatch(ctx context.Context, batch *topicreader.Batch) { 26 | // recommend derive ctx from batch.Context() for handle signal about stop message processing 27 | panic("example stub") 28 | } 29 | 30 | func processMessage(ctx context.Context, m *topicreader.Message) { 31 | // recommend derive ctx from m.Context() for handle signal about stop message processing 32 | panic("example stub") 33 | } 34 | 35 | func readLastOffsetFromDB(ctx context.Context, topic string, partition int64) (int64, error) { 36 | panic("example stub") 37 | } 38 | -------------------------------------------------------------------------------- /topic/topicreader/topicreader_advanced.go: -------------------------------------------------------------------------------- 1 | package topicreaderexamples 2 | 3 | import ( 4 | "context" 5 | "encoding/binary" 6 | "errors" 7 | 8 | "github.com/ydb-platform/ydb-go-sdk/v3" 9 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicoptions" 10 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicreader" 11 | ) 12 | 13 | // ReadMessagesWithCustomBatching example of custom of readed message batch 14 | func ReadMessagesWithCustomBatching(ctx context.Context, db ydb.Connection) { 15 | reader, _ := db.Topic().StartReader("consumer", nil, 16 | topicoptions.WithBatchReadMinCount(1000), 17 | ) 18 | 19 | for { 20 | batch, _ := reader.ReadMessageBatch(ctx) 21 | processBatch(batch.Context(), batch) 22 | _ = reader.Commit(batch.Context(), batch) 23 | } 24 | } 25 | 26 | // MyMessage example type with own serialization 27 | type MyMessage struct { 28 | ID byte 29 | ChangeType byte 30 | Delta uint32 31 | } 32 | 33 | // UnmarshalYDBTopicMessage implements topicreader.MessageContentUnmarshaler interface 34 | func (m *MyMessage) UnmarshalYDBTopicMessage(data []byte) error { 35 | if len(data) != 6 { 36 | return errors.New("bad data len") 37 | } 38 | m.ID = data[0] 39 | m.ChangeType = data[1] 40 | m.Delta = binary.BigEndian.Uint32(data[2:]) 41 | return nil 42 | } 43 | 44 | // UnmarshalMessageContentToOwnType is example about effective unmarshal own format from message content 45 | func UnmarshalMessageContentToOwnType(ctx context.Context, reader *topicreader.Reader) { 46 | var v MyMessage 47 | mess, _ := reader.ReadMessage(ctx) 48 | _ = mess.UnmarshalTo(&v) 49 | } 50 | 51 | // ProcessMessagesWithSyncCommit example about guarantee wait for commit accepted by server 52 | func ProcessMessagesWithSyncCommit(ctx context.Context, db ydb.Connection) { 53 | reader, _ := db.Topic().StartReader("consumer", nil, 54 | topicoptions.WithCommitMode(topicoptions.CommitModeSync), 55 | ) 56 | defer func() { 57 | _ = reader.Close(ctx) 58 | }() 59 | 60 | for { 61 | batch, _ := reader.ReadMessageBatch(ctx) 62 | processBatch(batch.Context(), batch) 63 | _ = reader.Commit(ctx, batch) // will wait response about commit from server 64 | } 65 | } 66 | 67 | // OwnReadProgressStorage example about store reading progress in external system and don't use 68 | // commit messages to YDB 69 | func OwnReadProgressStorage(ctx context.Context, db ydb.Connection) { 70 | reader, _ := db.Topic().StartReader("consumer", topicoptions.ReadTopic("asd"), 71 | topicoptions.WithGetPartitionStartOffset( 72 | func( 73 | ctx context.Context, 74 | req topicoptions.GetPartitionStartOffsetRequest, 75 | ) ( 76 | res topicoptions.GetPartitionStartOffsetResponse, 77 | err error, 78 | ) { 79 | offset, err := readLastOffsetFromDB(ctx, req.Topic, req.PartitionID) 80 | res.StartFrom(offset) 81 | 82 | // Reader will stop if return err != nil 83 | return res, err 84 | }), 85 | ) 86 | 87 | for { 88 | batch, _ := reader.ReadMessageBatch(ctx) 89 | 90 | processBatch(batch.Context(), batch) 91 | _ = externalSystemCommit( 92 | batch.Context(), 93 | batch.Topic(), 94 | batch.PartitionID(), 95 | getEndOffset(batch), 96 | ) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /topic/topicreader/topicreader_show.go: -------------------------------------------------------------------------------- 1 | package topicreaderexamples 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ydb-platform/ydb-go-sdk/v3" 7 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicoptions" 8 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicreader" 9 | ) 10 | 11 | // PartitionStopHandled is example of sdk handle server signal about stop partition 12 | func PartitionStopHandled(ctx context.Context, reader *topicreader.Reader) { 13 | batch, _ := reader.ReadMessageBatch(ctx) 14 | if len(batch.Messages) == 0 { 15 | return 16 | } 17 | 18 | batchContext := batch.Context() // batch.Context() will cancel when partition revoke by server or connection broke 19 | processBatch(batchContext, batch) 20 | } 21 | 22 | // PartitionGracefulStopHandled is example of sdk handle server signal about graceful stop partition 23 | func PartitionGracefulStopHandled(ctx context.Context, db ydb.Connection) { 24 | reader, _ := db.Topic().StartReader("consumer", nil, 25 | topicoptions.WithBatchReadMinCount(1000), 26 | ) 27 | 28 | for { 29 | batch, _ := reader.ReadMessageBatch(ctx) // <- if partition soft stop batch can be less, then 1000 30 | processBatch(batch.Context(), batch) 31 | _ = reader.Commit(batch.Context(), batch) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /topic/topicreader/topicreader_simple.go: -------------------------------------------------------------------------------- 1 | package topicreaderexamples 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | firestore "google.golang.org/genproto/firestore/bundle" 9 | 10 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicreader" 11 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicsugar" 12 | ) 13 | 14 | // PrintMessageContent is simple example for easy start read messages 15 | // it is not recommend way for heavy-load processing, batch processing usually will faster 16 | func PrintMessageContent(ctx context.Context, reader *topicreader.Reader) { 17 | for { 18 | msg, _ := reader.ReadMessage(ctx) 19 | content, _ := io.ReadAll(msg) 20 | fmt.Println(string(content)) 21 | _ = reader.Commit(msg.Context(), msg) 22 | } 23 | } 24 | 25 | // ReadMessagesByBatch it is recommended way for process messages 26 | func ReadMessagesByBatch(ctx context.Context, reader *topicreader.Reader) { 27 | for { 28 | batch, _ := reader.ReadMessageBatch(ctx) 29 | processBatch(batch.Context(), batch) 30 | _ = reader.Commit(batch.Context(), batch) 31 | } 32 | } 33 | 34 | // UnmarshalMessageContentToJSONStruct is example for effective way for unmarshal json message content to value 35 | func UnmarshalMessageContentToJSONStruct(msg *topicreader.Message) { 36 | type S struct { 37 | MyField int `json:"my_field"` 38 | } 39 | 40 | var v S 41 | 42 | _ = topicsugar.JSONUnmarshal(msg, &v) 43 | } 44 | 45 | // UnmarshalMessageContentToProtobufStruct is example for effective way for unmarshal protobuf message content to value 46 | func UnmarshalMessageContentToProtobufStruct(msg *topicreader.Message) { 47 | v := &firestore.BundledDocumentMetadata{} // protobuf type 48 | 49 | _ = topicsugar.ProtoUnmarshal(msg, v) 50 | } 51 | -------------------------------------------------------------------------------- /topic/topicreader/topicreader_trace.go: -------------------------------------------------------------------------------- 1 | package topicreaderexamples 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/ydb-platform/ydb-go-sdk/v3" 8 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicoptions" 9 | "github.com/ydb-platform/ydb-go-sdk/v3/trace" 10 | ) 11 | 12 | // CommitNotify is example for receive commit notifications with async commit mode 13 | func CommitNotify(ctx context.Context, db ydb.Connection) { 14 | reader, _ := db.Topic().StartReader("consumer", topicoptions.ReadTopic("asd"), 15 | topicoptions.WithReaderTrace(trace.Topic{ 16 | OnReaderCommittedNotify: func(info trace.TopicReaderCommittedNotifyInfo) { 17 | // called when receive commit notify from server 18 | fmt.Println(info.Topic, info.PartitionID, info.CommittedOffset) 19 | }, 20 | }, 21 | ), 22 | ) 23 | 24 | for { 25 | msg, _ := reader.ReadMessage(ctx) 26 | processMessage(msg.Context(), msg) 27 | } 28 | } 29 | 30 | // ExplicitPartitionStartStopHandler is example for create own handler for stop partition event from server 31 | func ExplicitPartitionStartStopHandler(ctx context.Context, db ydb.Connection) { 32 | readContext, stopReader := context.WithCancel(context.Background()) 33 | defer stopReader() 34 | 35 | reader, _ := db.Topic().StartReader("consumer", topicoptions.ReadTopic("asd"), 36 | topicoptions.WithReaderTrace( 37 | trace.Topic{ 38 | OnReaderPartitionReadStartResponse: func( 39 | info trace.TopicReaderPartitionReadStartResponseStartInfo, 40 | ) func( 41 | trace.TopicReaderPartitionReadStartResponseDoneInfo, 42 | ) { 43 | err := externalSystemLock(info.PartitionContext, info.Topic, info.PartitionID) 44 | if err != nil { 45 | stopReader() 46 | } 47 | return nil 48 | }, 49 | OnReaderPartitionReadStopResponse: func( 50 | info trace.TopicReaderPartitionReadStopResponseStartInfo, 51 | ) func( 52 | trace.TopicReaderPartitionReadStopResponseDoneInfo, 53 | ) { 54 | if info.Graceful { 55 | err := externalSystemUnlock(ctx, info.Topic, info.PartitionID) 56 | if err != nil { 57 | stopReader() 58 | } 59 | } 60 | return nil 61 | }, 62 | }, 63 | ), 64 | ) 65 | 66 | go func() { 67 | <-readContext.Done() 68 | _ = reader.Close(ctx) 69 | }() 70 | 71 | for { 72 | batch, _ := reader.ReadMessageBatch(readContext) 73 | 74 | processBatch(batch.Context(), batch) 75 | _ = externalSystemCommit( 76 | batch.Context(), 77 | batch.Topic(), 78 | batch.PartitionID(), 79 | getEndOffset(batch), 80 | ) 81 | } 82 | } 83 | 84 | // PartitionStartStopHandlerAndOwnReadProgressStorage example of complex use explicit start/stop partition handler 85 | // and own progress storage in external system 86 | func PartitionStartStopHandlerAndOwnReadProgressStorage(ctx context.Context, db ydb.Connection) { 87 | readContext, stopReader := context.WithCancel(context.Background()) 88 | defer stopReader() 89 | 90 | readStartPosition := func( 91 | ctx context.Context, 92 | req topicoptions.GetPartitionStartOffsetRequest, 93 | ) (res topicoptions.GetPartitionStartOffsetResponse, err error) { 94 | offset, err := readLastOffsetFromDB(ctx, req.Topic, req.PartitionID) 95 | res.StartFrom(offset) 96 | 97 | // Reader will stop if return err != nil 98 | return res, err 99 | } 100 | 101 | onPartitionStart := func( 102 | info trace.TopicReaderPartitionReadStartResponseStartInfo, 103 | ) func( 104 | trace.TopicReaderPartitionReadStartResponseDoneInfo, 105 | ) { 106 | err := externalSystemLock(info.PartitionContext, info.Topic, info.PartitionID) 107 | if err != nil { 108 | stopReader() 109 | } 110 | return nil 111 | } 112 | 113 | onPartitionStop := func( 114 | info trace.TopicReaderPartitionReadStopResponseStartInfo, 115 | ) func( 116 | trace.TopicReaderPartitionReadStopResponseDoneInfo, 117 | ) { 118 | if info.Graceful { 119 | err := externalSystemUnlock(ctx, info.Topic, info.PartitionID) 120 | if err != nil { 121 | stopReader() 122 | } 123 | } 124 | return nil 125 | } 126 | 127 | r, _ := db.Topic().StartReader("consumer", topicoptions.ReadTopic("asd"), 128 | 129 | topicoptions.WithGetPartitionStartOffset(readStartPosition), 130 | topicoptions.WithReaderTrace( 131 | trace.Topic{ 132 | OnReaderPartitionReadStartResponse: onPartitionStart, 133 | OnReaderPartitionReadStopResponse: onPartitionStop, 134 | }, 135 | ), 136 | ) 137 | go func() { 138 | <-readContext.Done() 139 | _ = r.Close(ctx) 140 | }() 141 | 142 | for { 143 | batch, _ := r.ReadMessageBatch(readContext) 144 | 145 | processBatch(batch.Context(), batch) 146 | _ = externalSystemCommit(batch.Context(), batch.Topic(), batch.PartitionID(), getEndOffset(batch)) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /topic/topicwriter/topicwriter.go: -------------------------------------------------------------------------------- 1 | package topicwriter 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | 7 | "github.com/ydb-platform/ydb-go-sdk/v3" 8 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicoptions" 9 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topictypes" 10 | "github.com/ydb-platform/ydb-go-sdk/v3/topic/topicwriter" 11 | ) 12 | 13 | const groupID = "group-id" 14 | 15 | func ConnectSimple(ctx context.Context, db ydb.Connection) *topicwriter.Writer { 16 | producerAndGroupID := groupID 17 | writer, _ := db.Topic().StartWriter(producerAndGroupID, "topicName", 18 | topicoptions.WithMessageGroupID(producerAndGroupID), 19 | ) 20 | return writer 21 | } 22 | 23 | func ConnectWithSyncWrite(ctx context.Context, db ydb.Connection) *topicwriter.Writer { 24 | producerAndGroupID := groupID 25 | writer, _ := db.Topic().StartWriter(producerAndGroupID, "topicName", 26 | topicoptions.WithMessageGroupID(producerAndGroupID), 27 | topicoptions.WithSyncWrite(true), 28 | ) 29 | return writer 30 | } 31 | 32 | func ConnectSelectCodec(ctx context.Context, db ydb.Connection) *topicwriter.Writer { 33 | producerAndGroupID := groupID 34 | writer, _ := db.Topic().StartWriter(producerAndGroupID, "topicName", 35 | topicoptions.WithMessageGroupID(producerAndGroupID), 36 | topicoptions.WithCodec(topictypes.CodecGzip), 37 | ) 38 | return writer 39 | } 40 | 41 | func SendMessagesOneByOne(ctx context.Context, w *topicwriter.Writer) { 42 | data := []byte{1, 2, 3} 43 | mess := topicwriter.Message{Data: bytes.NewReader(data)} 44 | _ = w.Write(ctx, mess) 45 | } 46 | 47 | func SendGroupOfMessages(ctx context.Context, w *topicwriter.Writer) { 48 | data1 := []byte{1, 2, 3} 49 | data2 := []byte{4, 5, 6} 50 | mess1 := topicwriter.Message{Data: bytes.NewReader(data1)} 51 | mess2 := topicwriter.Message{Data: bytes.NewReader(data2)} 52 | 53 | _ = w.Write(ctx, mess1, mess2) 54 | } 55 | -------------------------------------------------------------------------------- /ttl/README.md: -------------------------------------------------------------------------------- 1 | # TTL example 2 | 3 | TTL example demonstrates how to read all data from table using `table.Session.StreamReadTable` API method -------------------------------------------------------------------------------- /ttl/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path" 9 | 10 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 11 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 12 | "github.com/ydb-platform/ydb-go-sdk/v3/sugar" 13 | ) 14 | 15 | var ( 16 | dsn string 17 | prefix string 18 | ) 19 | 20 | func init() { 21 | required := []string{"ydb"} 22 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 23 | flagSet.Usage = func() { 24 | out := flagSet.Output() 25 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 26 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 27 | flagSet.PrintDefaults() 28 | } 29 | flagSet.StringVar(&dsn, 30 | "ydb", "", 31 | "YDB connection string", 32 | ) 33 | flagSet.StringVar(&prefix, 34 | "prefix", "", 35 | "tables prefix", 36 | ) 37 | if err := flagSet.Parse(os.Args[1:]); err != nil { 38 | flagSet.Usage() 39 | os.Exit(1) 40 | } 41 | flagSet.Visit(func(f *flag.Flag) { 42 | for i, arg := range required { 43 | if arg == f.Name { 44 | required = append(required[:i], required[i+1:]...) 45 | } 46 | } 47 | }) 48 | if len(required) > 0 { 49 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 50 | flagSet.Usage() 51 | os.Exit(1) 52 | } 53 | } 54 | 55 | func main() { 56 | ctx, cancel := context.WithCancel(context.Background()) 57 | defer cancel() 58 | db, err := ydb.Open(ctx, dsn, 59 | environ.WithEnvironCredentials(ctx), 60 | ) 61 | 62 | if err != nil { 63 | panic(fmt.Errorf("connect error: %w", err)) 64 | } 65 | defer func() { _ = db.Close(ctx) }() 66 | 67 | err = sugar.RemoveRecursive(ctx, db, prefix) 68 | if err != nil { 69 | panic(err) 70 | } 71 | err = sugar.MakeRecursive(ctx, db, prefix) 72 | if err != nil { 73 | panic(err) 74 | } 75 | 76 | prefix = path.Join(db.Name(), prefix) 77 | 78 | err = createTables(ctx, db.Table(), prefix) 79 | if err != nil { 80 | panic(fmt.Errorf("create tables error: %w", err)) 81 | } 82 | 83 | err = addDocument(ctx, db.Table(), prefix, 84 | "https://yandex.ru/", 85 | "

Yandex

", 86 | 1) 87 | if err != nil { 88 | panic(fmt.Errorf("add document failed: %w", err)) 89 | } 90 | 91 | err = addDocument(ctx, db.Table(), prefix, 92 | "https://ya.ru/", 93 | "

Ya

", 94 | 2) 95 | if err != nil { 96 | panic(fmt.Errorf("add document failed: %w", err)) 97 | } 98 | 99 | err = readDocument(ctx, db.Table(), prefix, "https://yandex.ru/") 100 | if err != nil { 101 | panic(fmt.Errorf("read document failed: %w", err)) 102 | } 103 | err = readDocument(ctx, db.Table(), prefix, "https://ya.ru/") 104 | if err != nil { 105 | panic(fmt.Errorf("read document failed: %w", err)) 106 | } 107 | 108 | for i := uint64(0); i < expirationQueueCount; i++ { 109 | if err = deleteExpired(ctx, db.Table(), prefix, i, 1); err != nil { 110 | panic(fmt.Errorf("delete expired failed: %w", err)) 111 | } 112 | } 113 | 114 | err = readDocument(ctx, db.Table(), prefix, "https://ya.ru/") 115 | if err != nil { 116 | panic(fmt.Errorf("read document failed: %w", err)) 117 | } 118 | 119 | err = addDocument(ctx, db.Table(), prefix, 120 | "https://yandex.ru/", 121 | "

Yandex

", 122 | 2) 123 | if err != nil { 124 | panic(fmt.Errorf("add document failed: %w", err)) 125 | } 126 | 127 | err = addDocument(ctx, db.Table(), prefix, 128 | "https://yandex.ru/", 129 | "

Yandex

", 130 | 3) 131 | if err != nil { 132 | panic(fmt.Errorf("add document failed: %w", err)) 133 | } 134 | 135 | for i := uint64(0); i < expirationQueueCount; i++ { 136 | if err = deleteExpired(ctx, db.Table(), prefix, i, 2); err != nil { 137 | panic(fmt.Errorf("delete expired failed: %w", err)) 138 | } 139 | } 140 | 141 | err = readDocument(ctx, db.Table(), prefix, "https://yandex.ru/") 142 | if err != nil { 143 | panic(fmt.Errorf("read document failed: %w", err)) 144 | } 145 | err = readDocument(ctx, db.Table(), prefix, "https://ya.ru/") 146 | if err != nil { 147 | panic(fmt.Errorf("read document failed: %w", err)) 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /ttl/series.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math/rand" 7 | "path" 8 | 9 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 10 | "github.com/ydb-platform/ydb-go-sdk/v3/table/options" 11 | "github.com/ydb-platform/ydb-go-sdk/v3/table/result" 12 | "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" 13 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 14 | ) 15 | 16 | const ( 17 | docTablePartitionCount = 4 18 | expirationQueueCount = 4 19 | ) 20 | 21 | func readExpiredBatchTransaction(ctx context.Context, c table.Client, prefix string, queue, 22 | timestamp, prevTimestamp, prevDocID uint64) (result.Result, 23 | error) { 24 | 25 | query := fmt.Sprintf(` 26 | PRAGMA TablePathPrefix("%v"); 27 | 28 | DECLARE $timestamp AS Uint64; 29 | DECLARE $prev_timestamp AS Uint64; 30 | DECLARE $prev_doc_id AS Uint64; 31 | 32 | $data = ( 33 | SELECT * 34 | FROM expiration_queue_%v 35 | WHERE 36 | ts <= $timestamp 37 | AND 38 | ts > $prev_timestamp 39 | 40 | UNION ALL 41 | 42 | SELECT * 43 | FROM expiration_queue_%v 44 | WHERE 45 | ts = $prev_timestamp AND doc_id > $prev_doc_id 46 | ORDER BY ts, doc_id 47 | LIMIT 100 48 | ); 49 | 50 | SELECT ts, doc_id 51 | FROM $data 52 | ORDER BY ts, doc_id 53 | LIMIT 100;`, prefix, queue, queue) 54 | 55 | readTx := table.TxControl(table.BeginTx(table.WithOnlineReadOnly()), table.CommitTx()) 56 | 57 | var res result.Result 58 | err := c.Do(ctx, 59 | func(ctx context.Context, s table.Session) (err error) { 60 | _, res, err = s.Execute(ctx, readTx, query, table.NewQueryParameters( 61 | table.ValueParam("$timestamp", types.Uint64Value(timestamp)), 62 | table.ValueParam("$prev_timestamp", types.Uint64Value(prevTimestamp)), 63 | table.ValueParam("$prev_doc_id", types.Uint64Value(prevDocID)), 64 | )) 65 | return err 66 | }, 67 | ) 68 | if err != nil { 69 | return nil, err 70 | } 71 | if res.Err() != nil { 72 | return nil, res.Err() 73 | } 74 | return res, nil 75 | } 76 | 77 | func deleteDocumentWithTimestamp(ctx context.Context, c table.Client, prefix string, queue, lastDocID, timestamp uint64) error { 78 | query := fmt.Sprintf(` 79 | PRAGMA TablePathPrefix("%v"); 80 | 81 | DECLARE $doc_id AS Uint64; 82 | DECLARE $timestamp AS Uint64; 83 | 84 | DELETE FROM documents 85 | WHERE doc_id = $doc_id AND ts = $timestamp; 86 | 87 | DELETE FROM expiration_queue_%v 88 | WHERE ts = $timestamp AND doc_id = $doc_id;`, prefix, queue) 89 | 90 | writeTx := table.TxControl(table.BeginTx(table.WithSerializableReadWrite()), table.CommitTx()) 91 | 92 | err := c.Do(ctx, 93 | func(ctx context.Context, s table.Session) (err error) { 94 | _, _, err = s.Execute(ctx, writeTx, query, table.NewQueryParameters( 95 | table.ValueParam("$doc_id", types.Uint64Value(lastDocID)), 96 | table.ValueParam("$timestamp", types.Uint64Value(timestamp)), 97 | )) 98 | return err 99 | }, 100 | ) 101 | return err 102 | } 103 | 104 | func deleteExpired(ctx context.Context, c table.Client, prefix string, queue, timestamp uint64) (err error) { 105 | fmt.Printf("> DeleteExpired from queue #%d:\n", queue) 106 | empty := false 107 | lastTimestamp := uint64(0) 108 | lastDocID := uint64(0) 109 | 110 | for !empty { 111 | err = func() (err error) { // for isolate defer inside lambda 112 | res, err := readExpiredBatchTransaction(ctx, c, prefix, queue, timestamp, lastTimestamp, lastDocID) 113 | if err != nil { 114 | return err 115 | } 116 | defer func() { 117 | _ = res.Close() 118 | }() 119 | 120 | empty = true 121 | res.NextResultSet(ctx) 122 | for res.NextRow() { 123 | empty = false 124 | err = res.ScanNamed( 125 | named.OptionalWithDefault("doc_id", &lastDocID), 126 | named.OptionalWithDefault("ts", &lastTimestamp), 127 | ) 128 | if err != nil { 129 | return err 130 | } 131 | fmt.Printf("\tDocId: %d\n\tTimestamp: %d\n", lastDocID, lastTimestamp) 132 | 133 | err = deleteDocumentWithTimestamp(ctx, c, prefix, queue, lastDocID, lastTimestamp) 134 | if err != nil { 135 | return err 136 | } 137 | } 138 | return res.Err() 139 | }() 140 | if err != nil { 141 | return err 142 | } 143 | } 144 | return nil 145 | } 146 | 147 | func readDocument(ctx context.Context, c table.Client, prefix, url string) error { 148 | fmt.Printf("> ReadDocument \"%v\":\n", url) 149 | 150 | query := fmt.Sprintf(` 151 | PRAGMA TablePathPrefix("%v"); 152 | 153 | DECLARE $url AS Text; 154 | 155 | $doc_id = Digest::CityHash($url); 156 | 157 | SELECT doc_id, url, html, ts 158 | FROM documents 159 | WHERE doc_id = $doc_id;`, prefix) 160 | 161 | readTx := table.TxControl(table.BeginTx(table.WithOnlineReadOnly()), table.CommitTx()) 162 | 163 | err := c.Do(ctx, 164 | func(ctx context.Context, s table.Session) (err error) { 165 | _, res, err := s.Execute(ctx, readTx, query, table.NewQueryParameters( 166 | table.ValueParam("$url", types.TextValue(url)), 167 | )) 168 | if err != nil { 169 | return err 170 | } 171 | defer func() { 172 | _ = res.Close() 173 | }() 174 | var ( 175 | docID *uint64 176 | docURL *string 177 | ts *uint64 178 | html *string 179 | ) 180 | if res.NextResultSet(ctx) && res.NextRow() { 181 | err = res.ScanNamed( 182 | named.Optional("doc_id", &docID), 183 | named.Optional("url", &docURL), 184 | named.Optional("ts", &ts), 185 | named.Optional("html", &html), 186 | ) 187 | if err != nil { 188 | return err 189 | } 190 | fmt.Printf("\tDocId: %v\n", docID) 191 | fmt.Printf("\tUrl: %v\n", docURL) 192 | fmt.Printf("\tTimestamp: %v\n", ts) 193 | fmt.Printf("\tHtml: %v\n", html) 194 | } else { 195 | fmt.Println("\tNot found") 196 | } 197 | return res.Err() 198 | }, 199 | ) 200 | return err 201 | } 202 | 203 | func addDocument(ctx context.Context, c table.Client, prefix, url, html string, timestamp uint64) error { 204 | fmt.Printf("> AddDocument: \n\tUrl: %v\n\tTimestamp: %v\n", url, timestamp) 205 | 206 | queue := rand.Intn(expirationQueueCount) 207 | query := fmt.Sprintf(` 208 | PRAGMA TablePathPrefix("%v"); 209 | 210 | DECLARE $url AS Text; 211 | DECLARE $html AS Text; 212 | DECLARE $timestamp AS Uint64; 213 | 214 | $doc_id = Digest::CityHash($url); 215 | 216 | REPLACE INTO documents 217 | (doc_id, url, html, ts) 218 | VALUES 219 | ($doc_id, $url, $html, $timestamp); 220 | 221 | REPLACE INTO expiration_queue_%v 222 | (ts, doc_id) 223 | VALUES 224 | ($timestamp, $doc_id);`, prefix, queue) 225 | 226 | writeTx := table.TxControl(table.BeginTx(table.WithSerializableReadWrite()), table.CommitTx()) 227 | 228 | err := c.Do(ctx, 229 | func(ctx context.Context, s table.Session) (err error) { 230 | _, _, err = s.Execute(ctx, writeTx, query, table.NewQueryParameters( 231 | table.ValueParam("$url", types.TextValue(url)), 232 | table.ValueParam("$html", types.TextValue(html)), 233 | table.ValueParam("$timestamp", types.Uint64Value(timestamp)), 234 | )) 235 | return err 236 | }, 237 | ) 238 | return err 239 | } 240 | 241 | func createTables(ctx context.Context, c table.Client, prefix string) (err error) { 242 | err = c.Do(ctx, 243 | func(ctx context.Context, s table.Session) error { 244 | return s.CreateTable(ctx, path.Join(prefix, "documents"), 245 | options.WithColumn("doc_id", types.Optional(types.TypeUint64)), 246 | options.WithColumn("url", types.Optional(types.TypeUTF8)), 247 | options.WithColumn("html", types.Optional(types.TypeUTF8)), 248 | options.WithColumn("ts", types.Optional(types.TypeUint64)), 249 | options.WithPrimaryKeyColumn("doc_id"), 250 | options.WithProfile( 251 | options.WithPartitioningPolicy( 252 | options.WithPartitioningPolicyUniformPartitions(uint64(docTablePartitionCount)))), 253 | ) 254 | }, 255 | ) 256 | if err != nil { 257 | return err 258 | } 259 | 260 | for i := 0; i < expirationQueueCount; i++ { 261 | err = c.Do(ctx, 262 | func(ctx context.Context, s table.Session) error { 263 | return s.CreateTable(ctx, path.Join(prefix, fmt.Sprintf("expiration_queue_%v", i)), 264 | options.WithColumn("doc_id", types.Optional(types.TypeUint64)), 265 | options.WithColumn("ts", types.Optional(types.TypeUint64)), 266 | options.WithPrimaryKeyColumn("ts", "doc_id"), 267 | ) 268 | }, 269 | ) 270 | if err != nil { 271 | return err 272 | } 273 | } 274 | 275 | return nil 276 | } 277 | -------------------------------------------------------------------------------- /ttl_readtable/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path" 9 | 10 | environ "github.com/ydb-platform/ydb-go-sdk-auth-environ" 11 | ydb "github.com/ydb-platform/ydb-go-sdk/v3" 12 | "github.com/ydb-platform/ydb-go-sdk/v3/sugar" 13 | ) 14 | 15 | var ( 16 | dsn string 17 | prefix string 18 | ) 19 | 20 | func init() { 21 | required := []string{"ydb"} 22 | flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) 23 | flagSet.Usage = func() { 24 | out := flagSet.Output() 25 | _, _ = fmt.Fprintf(out, "Usage:\n%s [options]\n", os.Args[0]) 26 | _, _ = fmt.Fprintf(out, "\nOptions:\n") 27 | flagSet.PrintDefaults() 28 | } 29 | flagSet.StringVar(&dsn, 30 | "ydb", "", 31 | "YDB connection string", 32 | ) 33 | flagSet.StringVar(&prefix, 34 | "prefix", "", 35 | "tables prefix", 36 | ) 37 | if err := flagSet.Parse(os.Args[1:]); err != nil { 38 | flagSet.Usage() 39 | os.Exit(1) 40 | } 41 | flagSet.Visit(func(f *flag.Flag) { 42 | for i, arg := range required { 43 | if arg == f.Name { 44 | required = append(required[:i], required[i+1:]...) 45 | } 46 | } 47 | }) 48 | if len(required) > 0 { 49 | fmt.Printf("\nSome required options not defined: %v\n\n", required) 50 | flagSet.Usage() 51 | os.Exit(1) 52 | } 53 | } 54 | 55 | func main() { 56 | ctx, cancel := context.WithCancel(context.Background()) 57 | defer cancel() 58 | db, err := ydb.Open(ctx, dsn, 59 | environ.WithEnvironCredentials(ctx), 60 | ) 61 | 62 | if err != nil { 63 | panic(fmt.Errorf("connect error: %w", err)) 64 | } 65 | defer func() { _ = db.Close(ctx) }() 66 | 67 | err = sugar.RemoveRecursive(ctx, db, prefix) 68 | if err != nil { 69 | panic(err) 70 | } 71 | err = sugar.MakeRecursive(ctx, db, prefix) 72 | if err != nil { 73 | panic(err) 74 | } 75 | 76 | prefix = path.Join(db.Name(), prefix) 77 | 78 | err = createTables(ctx, db.Table(), prefix) 79 | if err != nil { 80 | panic(fmt.Errorf("create tables error: %w", err)) 81 | } 82 | 83 | err = addDocument(ctx, db.Table(), prefix, 84 | "https://yandex.ru/", 85 | "

Yandex

", 86 | 1) 87 | if err != nil { 88 | panic(fmt.Errorf("add document failed: %w", err)) 89 | } 90 | 91 | err = addDocument(ctx, db.Table(), prefix, 92 | "https://ya.ru/", 93 | "

Ya

", 94 | 2) 95 | if err != nil { 96 | panic(fmt.Errorf("add document failed: %w", err)) 97 | } 98 | 99 | err = addDocument(ctx, db.Table(), prefix, 100 | "https://mail.yandex.ru/", 101 | "

Mail

", 102 | 3) 103 | if err != nil { 104 | panic(fmt.Errorf("add document failed: %w", err)) 105 | } 106 | 107 | err = addDocument(ctx, db.Table(), prefix, 108 | "https://zen.yandex.ru/", 109 | "

Zen

", 110 | 4) 111 | if err != nil { 112 | panic(fmt.Errorf("add document failed: %w", err)) 113 | } 114 | 115 | err = readDocument(ctx, db.Table(), prefix, "https://yandex.ru/") 116 | if err != nil { 117 | panic(fmt.Errorf("read document failed: %w", err)) 118 | } 119 | 120 | err = readDocument(ctx, db.Table(), prefix, "https://ya.ru/") 121 | if err != nil { 122 | panic(fmt.Errorf("read document failed: %w", err)) 123 | } 124 | 125 | err = readDocument(ctx, db.Table(), prefix, "https://mail.yandex.ru/") 126 | if err != nil { 127 | panic(fmt.Errorf("read document failed: %w", err)) 128 | } 129 | 130 | err = readDocument(ctx, db.Table(), prefix, "https://zen.yandex.ru/") 131 | if err != nil { 132 | panic(fmt.Errorf("read document failed: %w", err)) 133 | } 134 | 135 | err = deleteExpired(ctx, db.Table(), prefix, 2) 136 | if err != nil { 137 | panic(fmt.Errorf("delete expired failed: %w", err)) 138 | } 139 | 140 | err = readDocument(ctx, db.Table(), prefix, "https://yandex.ru/") 141 | if err != nil { 142 | panic(fmt.Errorf("read document failed: %w", err)) 143 | } 144 | 145 | err = readDocument(ctx, db.Table(), prefix, "https://ya.ru/") 146 | if err != nil { 147 | panic(fmt.Errorf("read document failed: %w", err)) 148 | } 149 | 150 | err = readDocument(ctx, db.Table(), prefix, "https://mail.yandex.ru/") 151 | if err != nil { 152 | panic(fmt.Errorf("read document failed: %w", err)) 153 | } 154 | 155 | err = readDocument(ctx, db.Table(), prefix, "https://zen.yandex.ru/") 156 | if err != nil { 157 | panic(fmt.Errorf("read document failed: %w", err)) 158 | } 159 | 160 | err = addDocument(ctx, db.Table(), prefix, 161 | "https://yandex.ru/", 162 | "

Yandex

", 163 | 3) 164 | if err != nil { 165 | panic(fmt.Errorf("add document failed: %w", err)) 166 | } 167 | 168 | err = addDocument(ctx, db.Table(), prefix, 169 | "https://ya.ru/", 170 | "

Ya

", 171 | 4) 172 | if err != nil { 173 | panic(fmt.Errorf("add document failed: %w", err)) 174 | } 175 | 176 | err = deleteExpired(ctx, db.Table(), prefix, 3) 177 | if err != nil { 178 | panic(fmt.Errorf("delete expired failed: %w", err)) 179 | } 180 | 181 | err = readDocument(ctx, db.Table(), prefix, "https://yandex.ru/") 182 | if err != nil { 183 | panic(fmt.Errorf("read document failed: %w", err)) 184 | } 185 | 186 | err = readDocument(ctx, db.Table(), prefix, "https://ya.ru/") 187 | if err != nil { 188 | panic(fmt.Errorf("read document failed: %w", err)) 189 | } 190 | 191 | err = readDocument(ctx, db.Table(), prefix, "https://mail.yandex.ru/") 192 | if err != nil { 193 | panic(fmt.Errorf("read document failed: %w", err)) 194 | } 195 | 196 | err = readDocument(ctx, db.Table(), prefix, "https://zen.yandex.ru/") 197 | if err != nil { 198 | panic(fmt.Errorf("read document failed: %w", err)) 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /ttl_readtable/series.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "path" 7 | 8 | "github.com/ydb-platform/ydb-go-sdk/v3/table" 9 | "github.com/ydb-platform/ydb-go-sdk/v3/table/options" 10 | "github.com/ydb-platform/ydb-go-sdk/v3/table/result" 11 | "github.com/ydb-platform/ydb-go-sdk/v3/table/result/named" 12 | "github.com/ydb-platform/ydb-go-sdk/v3/table/types" 13 | ) 14 | 15 | const ( 16 | docTablePartitionCount = 4 17 | deleteBatchSize = 10 18 | ) 19 | 20 | func deleteExpiredDocuments(ctx context.Context, c table.Client, prefix string, ids []uint64, 21 | timestamp uint64) error { 22 | fmt.Printf("> DeleteExpiredDocuments: %+v\n", ids) 23 | 24 | query := fmt.Sprintf(` 25 | PRAGMA TablePathPrefix("%v"); 26 | 27 | DECLARE $keys AS List>; 30 | 31 | DECLARE $timestamp AS Uint64; 32 | 33 | $expired = ( 34 | SELECT d.doc_id AS doc_id 35 | FROM AS_TABLE($keys) AS k 36 | INNER JOIN documents AS d 37 | ON k.doc_id = d.doc_id 38 | WHERE ts <= $timestamp 39 | ); 40 | 41 | DELETE FROM documents ON 42 | SELECT * FROM $expired;`, prefix) 43 | 44 | keys := types.ListValue(func() []types.Value { 45 | var k = make([]types.Value, len(ids)) 46 | for i := range ids { 47 | k[i] = types.StructValue(types.StructFieldValue("doc_id", types.Uint64Value(ids[i]))) 48 | } 49 | return k 50 | }()...) 51 | 52 | writeTx := table.TxControl(table.BeginTx(table.WithSerializableReadWrite()), table.CommitTx()) 53 | 54 | err := c.Do(ctx, 55 | func(ctx context.Context, s table.Session) (err error) { 56 | _, _, err = s.Execute(ctx, writeTx, query, 57 | table.NewQueryParameters( 58 | table.ValueParam("$keys", keys), 59 | table.ValueParam("$timestamp", types.Uint64Value(timestamp)), 60 | ), 61 | ) 62 | return err 63 | }, 64 | ) 65 | return err 66 | } 67 | 68 | func deleteExpiredRange(ctx context.Context, c table.Client, prefix string, timestamp uint64, 69 | keyRange options.KeyRange) error { 70 | fmt.Printf("> DeleteExpiredRange: %+v\n", keyRange) 71 | 72 | var res result.StreamResult 73 | err := c.Do(ctx, 74 | func(ctx context.Context, s table.Session) (err error) { 75 | res, err = s.StreamReadTable(ctx, path.Join(prefix, "documents"), 76 | options.ReadKeyRange(keyRange), 77 | options.ReadColumn("doc_id"), 78 | options.ReadColumn("ts")) 79 | return err 80 | }, 81 | ) 82 | if err != nil { 83 | return err 84 | } 85 | if err != nil { 86 | return err 87 | } 88 | if err = res.Err(); err != nil { 89 | return err 90 | } 91 | 92 | // As single key range usually represents a single shard, so we batch deletions here 93 | // without introducing distributed transactions. 94 | var ( 95 | docIds []uint64 96 | docID uint64 97 | ts uint64 98 | ) 99 | for res.NextResultSet(ctx) { 100 | for res.NextRow() { 101 | err = res.ScanNamed( 102 | named.OptionalWithDefault("doc_id", &docID), 103 | named.OptionalWithDefault("ts", &ts), 104 | ) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | if ts <= timestamp { 110 | docIds = append(docIds, docID) 111 | } 112 | if len(docIds) >= deleteBatchSize { 113 | if err := deleteExpiredDocuments(ctx, c, prefix, docIds, timestamp); err != nil { 114 | return err 115 | } 116 | docIds = []uint64{} 117 | } 118 | } 119 | if len(docIds) > 0 { 120 | if err := deleteExpiredDocuments(ctx, c, prefix, docIds, timestamp); err != nil { 121 | return err 122 | } 123 | docIds = []uint64{} 124 | } 125 | } 126 | 127 | return nil 128 | } 129 | 130 | func deleteExpired(ctx context.Context, c table.Client, prefix string, timestamp uint64) error { 131 | fmt.Printf("> DeleteExpired: timestamp: %v:\n", timestamp) 132 | 133 | var res options.Description 134 | err := c.Do(ctx, 135 | func(ctx context.Context, s table.Session) (err error) { 136 | res, err = s.DescribeTable(ctx, path.Join(prefix, "documents"), options.WithShardKeyBounds()) 137 | return err 138 | }, 139 | ) 140 | 141 | if err != nil { 142 | return err 143 | } 144 | for _, kr := range res.KeyRanges { 145 | // DeleteExpiredRange can be run in parallel for different ranges. 146 | // Keep in mind that deletion RPS should be somehow limited in this case to avoid 147 | // spikes of cluster load due to TTL. 148 | err = deleteExpiredRange(ctx, c, prefix, timestamp, kr) 149 | if err != nil { 150 | return err 151 | } 152 | } 153 | 154 | return nil 155 | } 156 | 157 | func readDocument(ctx context.Context, c table.Client, prefix, url string) error { 158 | fmt.Printf("> ReadDocument \"%v\":\n", url) 159 | 160 | query := fmt.Sprintf(` 161 | PRAGMA TablePathPrefix("%v"); 162 | 163 | DECLARE $url AS Text; 164 | 165 | $doc_id = Digest::CityHash($url); 166 | 167 | SELECT doc_id, url, html, ts 168 | FROM documents 169 | WHERE doc_id = $doc_id;`, prefix) 170 | 171 | readTx := table.TxControl(table.BeginTx(table.WithOnlineReadOnly()), table.CommitTx()) 172 | 173 | var res result.Result 174 | err := c.Do(ctx, 175 | func(ctx context.Context, s table.Session) (err error) { 176 | _, res, err = s.Execute(ctx, readTx, query, table.NewQueryParameters( 177 | table.ValueParam("$url", types.TextValue(url))), 178 | ) 179 | return err 180 | }, 181 | ) 182 | if err != nil { 183 | return err 184 | } 185 | defer func() { 186 | _ = res.Close() 187 | }() 188 | var ( 189 | docID *uint64 190 | docURL *string 191 | ts *uint64 192 | html *string 193 | ) 194 | if res.NextResultSet(ctx) && res.NextRow() { 195 | err = res.ScanNamed( 196 | named.Optional("doc_id", &docID), 197 | named.Optional("url", &docURL), 198 | named.Optional("ts", &ts), 199 | named.Optional("html", &html), 200 | ) 201 | if err != nil { 202 | return err 203 | } 204 | fmt.Printf("\tDocId: %v\n", docID) 205 | fmt.Printf("\tUrl: %v\n", docURL) 206 | fmt.Printf("\tTimestamp: %v\n", ts) 207 | fmt.Printf("\tHtml: %v\n", html) 208 | } else { 209 | fmt.Println("\tNot found") 210 | } 211 | 212 | return res.Err() 213 | } 214 | 215 | func addDocument(ctx context.Context, c table.Client, prefix, url, html string, timestamp uint64) error { 216 | fmt.Printf("> AddDocument: \n\tUrl: %v\n\tTimestamp: %v\n", url, timestamp) 217 | 218 | query := fmt.Sprintf(` 219 | PRAGMA TablePathPrefix("%v"); 220 | 221 | DECLARE $url AS Text; 222 | DECLARE $html AS Text; 223 | DECLARE $timestamp AS Uint64; 224 | 225 | $doc_id = Digest::CityHash($url); 226 | 227 | REPLACE INTO documents 228 | (doc_id, url, html, ts) 229 | VALUES 230 | ($doc_id, $url, $html, $timestamp);`, prefix) 231 | 232 | writeTx := table.TxControl(table.BeginTx(table.WithSerializableReadWrite()), table.CommitTx()) 233 | 234 | err := c.Do(ctx, 235 | func(ctx context.Context, s table.Session) (err error) { 236 | _, _, err = s.Execute(ctx, writeTx, query, table.NewQueryParameters( 237 | table.ValueParam("$url", types.TextValue(url)), 238 | table.ValueParam("$html", types.TextValue(html)), 239 | table.ValueParam("$timestamp", types.Uint64Value(timestamp))), 240 | ) 241 | return err 242 | }, 243 | ) 244 | return err 245 | } 246 | 247 | func createTables(ctx context.Context, c table.Client, prefix string) (err error) { 248 | err = c.Do(ctx, 249 | func(ctx context.Context, s table.Session) error { 250 | return s.CreateTable(ctx, path.Join(prefix, "documents"), 251 | options.WithColumn("doc_id", types.Optional(types.TypeUint64)), 252 | options.WithColumn("url", types.Optional(types.TypeUTF8)), 253 | options.WithColumn("html", types.Optional(types.TypeUTF8)), 254 | options.WithColumn("ts", types.Optional(types.TypeUint64)), 255 | options.WithPrimaryKeyColumn("doc_id"), 256 | options.WithProfile( 257 | options.WithPartitioningPolicy( 258 | options.WithPartitioningPolicyUniformPartitions(uint64(docTablePartitionCount)))), 259 | ) 260 | }, 261 | ) 262 | if err != nil { 263 | return err 264 | } 265 | 266 | return nil 267 | } 268 | --------------------------------------------------------------------------------