├── .dockerignore
├── .github
    ├── actions
    │   └── spelling
    │   │   ├── README.md
    │   │   ├── advice.md
    │   │   ├── allow.txt
    │   │   ├── block-delimiters.list
    │   │   ├── candidate.patterns
    │   │   ├── excludes.txt
    │   │   ├── expect.txt
    │   │   ├── line_forbidden.patterns
    │   │   ├── patterns.txt
    │   │   └── reject.txt
    └── workflows
    │   ├── go.yml
    │   ├── linters.yaml
    │   ├── regress.yaml
    │   ├── release.yaml
    │   ├── spelling.yaml
    │   └── tests.yaml
├── .gitignore
├── .golangci.yaml
├── Makefile
├── README.md
├── build.Dockerfile
├── cmd
    ├── client
    │   └── main.go
    └── yproxy
    │   └── main.go
├── config
    ├── instance.go
    ├── proxy.go
    ├── storage.go
    └── vacuum.go
├── debian
    ├── changelog
    ├── compat
    ├── control
    ├── install
    ├── rules
    └── source
    │   └── format
├── examples
    └── yproxy.yaml
├── go.mod
├── go.sum
├── pkg
    ├── backups
    │   ├── backups.go
    │   └── s3.go
    ├── client
    │   └── client.go
    ├── clientpool
    │   └── clientpool.go
    ├── core
    │   ├── core.go
    │   ├── parser
    │   │   ├── .gitignore
    │   │   ├── Makefile
    │   │   ├── ast.go
    │   │   ├── gram.go
    │   │   ├── gram.y
    │   │   ├── lex.go
    │   │   ├── lex.rl
    │   │   ├── utils.go
    │   │   └── y_test.go
    │   ├── pg
    │   │   └── pg.go
    │   └── pprof.go
    ├── crypt
    │   └── crypt.go
    ├── database
    │   └── database.go
    ├── message
    │   ├── cat_message.go
    │   ├── cat_message_v2.go
    │   ├── common.go
    │   ├── copy_complete.go
    │   ├── copy_data.go
    │   ├── copy_done.go
    │   ├── copy_message.go
    │   ├── copy_message_v2.go
    │   ├── delete_message.go
    │   ├── error_message.go
    │   ├── gool_message.go
    │   ├── list_message.go
    │   ├── list_message_v2.go
    │   ├── message.go
    │   ├── message_test.go
    │   ├── object_meta_message.go
    │   ├── obsolete_message.go
    │   ├── patch_message.go
    │   ├── put_complete.go
    │   ├── put_message.go
    │   ├── put_message_v2.go
    │   ├── put_message_v3.go
    │   ├── ready_for_query.go
    │   ├── untrashify_message.go
    │   └── utils.go
    ├── mock
    │   ├── backups.go
    │   ├── client
    │   │   └── client.go
    │   ├── database.go
    │   ├── proc
    │   │   └── yio
    │   │   │   └── yrreader.go
    │   └── storage.go
    ├── object
    │   └── objectInfo.go
    ├── proc
    │   ├── delete_handler.go
    │   ├── delete_handler_test.go
    │   ├── interaction.go
    │   ├── proto.go
    │   └── yio
    │   │   ├── yrreader.go
    │   │   ├── yrreader_test.go
    │   │   ├── ywriter.go
    │   │   └── ywriter_test.go
    ├── sdnotifier
    │   └── notifier.go
    ├── settings
    │   └── settings.go
    ├── storage
    │   ├── filestorage.go
    │   ├── s3storage.go
    │   ├── s3storage_test.go
    │   ├── sessionpool.go
    │   ├── sessionpool_test.go
    │   ├── storage.go
    │   ├── utils.go
    │   └── utils_test.go
    ├── tablespace
    │   └── tablespace.go
    ├── testutils
    │   └── testmode.go
    ├── vacuum
    │   └── vacuum.go
    ├── version.go
    └── ylogger
    │   └── logger.go
├── script
    └── gofmtcheck.sh
├── test
    └── regress
    │   ├── .dockerignore
    │   ├── Dockerfile
    │   ├── conf
    │       ├── yproxy.yaml
    │       ├── yproxy_old.yaml
    │       ├── yproxy_prefix.yaml
    │       └── yproxy_second_bucket.yaml
    │   ├── docker-compose.yaml
    │   ├── expected
    │       ├── 01_simple_put.txt
    │       ├── 02_encrypted_put.txt
    │       ├── 03_copy_separate_keys.txt
    │       ├── 04_copy_separate_keys_multiple.txt
    │       ├── 05_copy_identical_keys.txt
    │       ├── 06_copy_identical_keys_prefix.txt
    │       └── 07_copy_identical_keys_multiple.txt
    │   ├── gpg
    │       ├── gpg_1.priv
    │       ├── gpg_1.pub
    │       ├── gpg_2.priv
    │       └── gpg_2.pub
    │   ├── pg-init
    │       └── create-db.sh
    │   ├── run_tests.sh
    │   └── tests
    │       ├── 01_simple_put.sh
    │       ├── 02_encrypted_put.sh
    │       ├── 03_copy_separate_keys.sh
    │       ├── 04_copy_separate_keys_multiple.sh
    │       ├── 05_copy_identical_keys.sh
    │       ├── 06_copy_identical_keys_prefix.sh
    │       └── 07_copy_identical_keys_multiple.sh
└── tests
    └── graceful_restart.sh


/.dockerignore:
--------------------------------------------------------------------------------
1 | Dockerfile
2 | docker-compose.yaml
3 | 


--------------------------------------------------------------------------------
/.github/actions/spelling/README.md:
--------------------------------------------------------------------------------
 1 | # check-spelling/check-spelling configuration
 2 | 
 3 | File | Purpose | Format | Info
 4 | -|-|-|-
 5 | [dictionary.txt](dictionary.txt) | Replacement dictionary (creating this file will override the default dictionary) | one word per line | [dictionary](https://github.com/check-spelling/check-spelling/wiki/Configuration#dictionary)
 6 | [allow.txt](allow.txt) | Add words to the dictionary | one word per line (only letters and `'`s allowed) | [allow](https://github.com/check-spelling/check-spelling/wiki/Configuration#allow)
 7 | [reject.txt](reject.txt) | Remove words from the dictionary (after allow) | grep pattern matching whole dictionary words | [reject](https://github.com/check-spelling/check-spelling/wiki/Configuration-Examples%3A-reject)
 8 | [excludes.txt](excludes.txt) | Files to ignore entirely | perl regular expression | [excludes](https://github.com/check-spelling/check-spelling/wiki/Configuration-Examples%3A-excludes)
 9 | [only.txt](only.txt) | Only check matching files (applied after excludes) | perl regular expression | [only](https://github.com/check-spelling/check-spelling/wiki/Configuration-Examples%3A-only)
10 | [patterns.txt](patterns.txt) | Patterns to ignore from checked lines | perl regular expression (order matters, first match wins) | [patterns](https://github.com/check-spelling/check-spelling/wiki/Configuration-Examples%3A-patterns)
11 | [candidate.patterns](candidate.patterns) | Patterns that might be worth adding to [patterns.txt](patterns.txt) | perl regular expression with optional comment block introductions (all matches will be suggested) | [candidates](https://github.com/check-spelling/check-spelling/wiki/Feature:-Suggest-patterns)
12 | [line_forbidden.patterns](line_forbidden.patterns) | Patterns to flag in checked lines | perl regular expression (order matters, first match wins) | [patterns](https://github.com/check-spelling/check-spelling/wiki/Configuration-Examples%3A-patterns)
13 | [expect.txt](expect.txt) | Expected words that aren't in the dictionary | one word per line (sorted, alphabetically) | [expect](https://github.com/check-spelling/check-spelling/wiki/Configuration#expect)
14 | [advice.md](advice.md) | Supplement for GitHub comment when unrecognized words are found | GitHub Markdown | [advice](https://github.com/check-spelling/check-spelling/wiki/Configuration-Examples%3A-advice)
15 | [block-delimiters.list](block-delimiters.list) | Define block begin/end markers to ignore lines of text | line with _literal string_ for **start** followed by line with _literal string_ for **end** | [block ignore](https://github.com/check-spelling/check-spelling/wiki/Feature%3A-Block-Ignore#status)
16 | 
17 | Note: you can replace any of these files with a directory by the same name (minus the suffix)
18 | and then include multiple files inside that directory (with that suffix) to merge multiple files together.
19 | 


--------------------------------------------------------------------------------
/.github/actions/spelling/advice.md:
--------------------------------------------------------------------------------
 1 | <!-- See https://github.com/check-spelling/check-spelling/wiki/Configuration-Examples%3A-advice --> <!-- markdownlint-disable MD033 MD041 -->
 2 | <details><summary>If the flagged items are :exploding_head: false positives</summary>
 3 | 
 4 | If items relate to a ...
 5 | * binary file (or some other file you wouldn't want to check at all).
 6 | 
 7 |   Please add a file path to the `excludes.txt` file matching the containing file.
 8 | 
 9 |   File paths are Perl 5 Regular Expressions - you can [test](
10 | https://www.regexplanet.com/advanced/perl/) yours before committing to verify it will match your files.
11 | 
12 |   `^` refers to the file's path from the root of the repository, so `^README\.md$` would exclude [README.md](
13 | ../tree/HEAD/README.md) (on whichever branch you're using).
14 | 
15 | * well-formed pattern.
16 | 
17 |   If you can write a [pattern](
18 | https://github.com/check-spelling/check-spelling/wiki/Configuration-Examples:-patterns
19 | ) that would match it,
20 |   try adding it to the `patterns.txt` file.
21 | 
22 |   Patterns are Perl 5 Regular Expressions - you can [test](
23 | https://www.regexplanet.com/advanced/perl/) yours before committing to verify it will match your lines.
24 | 
25 |   Note that patterns can't match multiline strings.
26 | 
27 | </details>
28 | 
29 | <!-- adoption information-->
30 | :steam_locomotive: If you're seeing this message and your PR is from a branch that doesn't have check-spelling,
31 | please merge to your PR's base branch to get the version configured for your repository.
32 | 


--------------------------------------------------------------------------------
/.github/actions/spelling/allow.txt:
--------------------------------------------------------------------------------
 1 | yezzey
 2 | extversion
 3 | dbname
 4 | extname
 5 | github
 6 | https
 7 | ssh
 8 | ubuntu
 9 | workarounds
10 | 


--------------------------------------------------------------------------------
/.github/actions/spelling/block-delimiters.list:
--------------------------------------------------------------------------------
 1 | # Public Keys
 2 | -----BEGIN PUBLIC KEY-----
 3 | -----END PUBLIC KEY-----
 4 | 
 5 | # Private Keys
 6 | -----BEGIN PRIVATE KEY-----
 7 | -----END PRIVATE KEY-----
 8 | 
 9 | # RSA Private Key
10 | -----BEGIN RSA PRIVATE KEY-----
11 | -----END RSA PRIVATE KEY-----
12 | 
13 | # GPG Public Key
14 | -----BEGIN PGP PUBLIC KEY BLOCK-----
15 | -----END PGP PUBLIC KEY BLOCK-----
16 | 
17 | # GPG Signature
18 | -----BEGIN PGP SIGNATURE-----
19 | -----END PGP SIGNATURE-----
20 | 
21 | # Certificates
22 | -----BEGIN CERTIFICATE-----
23 | -----END CERTIFICATE-----
24 | 
25 | # All Contributors
26 | <!-- ALL-CONTRIBUTORS-LIST:START
27 | <!-- ALL-CONTRIBUTORS-LIST:END
28 | 


--------------------------------------------------------------------------------
/.github/actions/spelling/excludes.txt:
--------------------------------------------------------------------------------
 1 | # See https://github.com/check-spelling/check-spelling/wiki/Configuration-Examples:-excludes
 2 | (?:^|/)(?i)COPYRIGHT
 3 | (?:^|/)(?i)LICEN[CS]E
 4 | (?:^|/)(?i)third[-_]?party/
 5 | (?:^|/)3rdparty/
 6 | (?:^|/)generated/
 7 | (?:^|/)go\.sum$
 8 | (?:^|/)package(?:-lock|)\.json$
 9 | (?:^|/)Pipfile$
10 | (?:^|/)pyproject.toml
11 | (?:^|/)vendor/
12 | (?:^|/|\b)requirements(?:-dev|-doc|-test|)\.txt$
13 | \.a$
14 | \.ai$
15 | \.all-contributorsrc$
16 | \.avi$
17 | \.bmp$
18 | \.bz2$
19 | \.cert?$|\.crt$
20 | \.class$
21 | \.coveragerc$
22 | \.crl$
23 | \.csr$
24 | \.dll$
25 | \.docx?$
26 | \.drawio$
27 | \.DS_Store$
28 | \.eot$
29 | \.eps$
30 | \.exe$
31 | \.gif$
32 | \.git-blame-ignore-revs$
33 | \.gitattributes$
34 | \.gitkeep$
35 | \.graffle$
36 | \.gz$
37 | \.icns$
38 | \.ico$
39 | \.ipynb$
40 | \.jar$
41 | \.jks$
42 | \.jpe?g$
43 | \.key$
44 | \.lib$
45 | \.lock$
46 | \.map$
47 | \.min\..
48 | \.mo$
49 | \.mod$
50 | \.mp[34]$
51 | \.o$
52 | \.ocf$
53 | \.otf$
54 | \.p12$
55 | \.parquet$
56 | \.pdf$
57 | \.pem$
58 | \.pfx$
59 | \.png$
60 | \.psd$
61 | \.pyc$
62 | \.pylintrc$
63 | \.qm$
64 | \.s$
65 | \.sig$
66 | \.so$
67 | \.svgz?$
68 | \.sys$
69 | \.tar$
70 | \.tgz$
71 | \.tiff?$
72 | \.ttf$
73 | \.wav$
74 | \.webm$
75 | \.webp$
76 | \.woff2?$
77 | \.xcf$
78 | \.xlsx?$
79 | \.xpm$
80 | \.xz$
81 | \.zip$
82 | ^\.github/actions/spelling/
83 | ^\Q.github/workflows/spelling.yml\E$
84 | ^\Qbuild.Dockerfile\E$
85 | ^test/regress/gpg/
86 | ignore$
87 | pkg/message/message.go
88 | test/regress/pg-init/create-db.sh
89 | 


--------------------------------------------------------------------------------
/.github/actions/spelling/expect.txt:
--------------------------------------------------------------------------------
  1 | ababa
  2 | aboba
  3 | ABRT
  4 | AExpr
  5 | aocs
  6 | Appendf
  7 | Atoi
  8 | aws
  9 | basebackups
 10 | BConst
 11 | buf
 12 | buildpackage
 13 | Buildx
 14 | caio
 15 | cdsdsd
 16 | cfg
 17 | Chk
 18 | Cleartext
 19 | clientpid
 20 | clientpool
 21 | cmdline
 22 | Cmp
 23 | cnf
 24 | compat
 25 | Cred
 26 | crypter
 27 | Cstring
 28 | ctx
 29 | CURDIR
 30 | datallowconn
 31 | datname
 32 | dattablespace
 33 | debhelper
 34 | debian
 35 | decrypter
 36 | DEK
 37 | devbin
 38 | djewikdeowp
 39 | dpkg
 40 | dws
 41 | EADDRINUSE
 42 | Errflag
 43 | Errorf
 44 | Exca
 45 | expvar
 46 | Fatalf
 47 | fbreak
 48 | fileinfo
 49 | filestorage
 50 | fmtcheck
 51 | gengeneric
 52 | Getenv
 53 | gettime
 54 | gofmt
 55 | gofmtcheck
 56 | goimports
 57 | golangci
 58 | gomock
 59 | gopkg
 60 | govet
 61 | goyacc
 62 | gpdata
 63 | gpdb
 64 | gpg
 65 | gpyezzey
 66 | Greenplum
 67 | healthcheck
 68 | hiuefheiufheuif
 69 | horiz
 70 | httpmock
 71 | iclistener
 72 | iconst
 73 | Iface
 74 | initdb
 75 | inp
 76 | Interactor
 77 | interractor
 78 | isgomock
 79 | isready
 80 | jackc
 81 | jarcoal
 82 | jdoiwejoidew
 83 | kek
 84 | KEKDEK
 85 | Kirill
 86 | KKB
 87 | kobain
 88 | kurt
 89 | ldflags
 90 | len
 91 | lexer
 92 | Libpq
 93 | lsn
 94 | lval
 95 | Messsage
 96 | minio
 97 | mockcl
 98 | mockgen
 99 | mockio
100 | myminio
101 | myname
102 | mynextname
103 | myoldcfg
104 | Nerrs
105 | ninteger
106 | nodetree
107 | nsecs
108 | nwr
109 | nyys
110 | objs
111 | oid
112 | OPENGPDB
113 | openpgp
114 | opstart
115 | optyp
116 | optype
117 | persisnt
118 | PGHOST
119 | pgo
120 | PGP
121 | pgproto
122 | pgtype
123 | PGUSER
124 | pgx
125 | pid
126 | PITR
127 | postgres
128 | postgresql
129 | pprof
130 | println
131 | psql
132 | ptr
133 | qidentifier
134 | ragel
135 | rbuf
136 | RDWR
137 | reshke
138 | Retryer
139 | REUSEADDR
140 | REUSEPORT
141 | rfq
142 | rowsdb
143 | sconst
144 | sdnotifier
145 | segnum
146 | sessionpool
147 | Setsockopt
148 | shm
149 | SIGHUP
150 | SIGINT
151 | SIGTERM
152 | SIGUSR
153 | slen
154 | spf
155 | Sprintf
156 | SScopy
157 | ssl
158 | Statenames
159 | Statname
160 | strconv
161 | stretchr
162 | substr
163 | sys
164 | syscall
165 | tablespace
166 | tcase
167 | TCLOSEBR
168 | TCOMMA
169 | tcp
170 | TDB
171 | tdigest
172 | TEQ
173 | testmode
174 | testutils
175 | textoid
176 | timespec
177 | tmp
178 | tok
179 | tokname
180 | TOKSTART
181 | toml
182 | TOPENBR
183 | Trasn
184 | TSEMICOLON
185 | uber
186 | uint
187 | uintptr
188 | unixgram
189 | unk
190 | unmarshal
191 | unssuported
192 | untrash
193 | untrashified
194 | untrashify
195 | USEC
196 | usr
197 | wal
198 | webserver
199 | WORKDIR
200 | workflows
201 | WRONLY
202 | wtf
203 | xcinside
204 | xcstart
205 | xcstop
206 | xlog
207 | yaccgen
208 | yaccpar
209 | yacctab
210 | yandex
211 | yandexcloud
212 | ycl
213 | YClient
214 | yio
215 | ylogger
216 | yml
217 | yproxy
218 | YPROXYHELLO
219 | YRestart
220 | YRetry
221 | yrreader
222 | ywriter
223 | yydefault
224 | yyfmt
225 | yyg
226 | yyj
227 | yylex
228 | yyn
229 | yynewstate
230 | yynt
231 | yyp
232 | yypt
233 | yyrcvr
234 | yys
235 | yystack
236 | yystate
237 | yytoken
238 | zerolog
239 | 


--------------------------------------------------------------------------------
/.github/actions/spelling/patterns.txt:
--------------------------------------------------------------------------------
 1 | # See https://github.com/check-spelling/check-spelling/wiki/Configuration-Examples:-patterns
 2 | 
 3 | # Questionably acceptable forms of `in to`
 4 | # Personally, I prefer `log into`, but people object
 5 | # https://www.tprteaching.com/log-into-log-in-to-login/
 6 | \b(?:(?:[Ll]og(?:g(?=[a-z])|)|[Ss]ign)(?:ed|ing)?) in to\b
 7 | 
 8 | # to opt in
 9 | \bto opt in\b
10 | 
11 | # acceptable duplicates
12 | # ls directory listings
13 | [-bcdlpsw](?:[-r][-w][-SsTtx]){3}[\.+*]?\s+\d+\s+\S+\s+\S+\s+[.\d]+(?:[KMGT]|)\s+
14 | # mount
15 | \bmount\s+-t\s+(\w+)\s+\g{-1}\b
16 | # C types and repeated CSS values
17 | \s(auto|buffalo|center|div|inherit|long|LONG|none|normal|solid|thin|transparent|very)(?: \g{-1})+\s
18 | # C enum and struct
19 | \b(?:enum|struct)\s+(\w+)\s+\g{-1}\b
20 | # go templates
21 | \s(\w+)\s+\g{-1}\s+\`(?:graphql|inject|json|yaml):
22 | # doxygen / javadoc / .net
23 | (?:[\\@](?:brief|defgroup|groupname|link|t?param|return|retval)|(?:public|private|\[Parameter(?:\(.+\)|)\])(?:\s+(?:static|override|readonly|required|virtual))*)(?:\s+\{\w+\}|)\s+(\w+)\s+\g{-1}\s
24 | 
25 | # macOS file path
26 | (?:Contents\W+|(?!iOS)/)MacOS\b
27 | 
28 | # Python package registry has incorrect spelling for macOS / Mac OS X
29 | "Operating System :: MacOS :: MacOS X"
30 | 
31 | # "company" in Germany
32 | \bGmbH\b
33 | 
34 | # IntelliJ
35 | \bIntelliJ\b
36 | 
37 | # Commit message -- Signed-off-by and friends
38 | ^\s*(?:(?:Based-on-patch|Co-authored|Helped|Mentored|Reported|Reviewed|Signed-off)-by|Thanks-to): (?:[^<]*<[^>]*>|[^<]*)\s*$
39 | 
40 | # Autogenerated revert commit message
41 | ^This reverts commit [0-9a-f]{40}\.$
42 | 
43 | # ignore long runs of a single character:
44 | \b([A-Za-z])\g{-1}{3,}\b
45 | 
46 | # hex runs
47 | \b[0-9a-fA-F]{16,}\b
48 | 
49 | # 'uses' in workflows
50 | uses:\s+[-a-zA-Z0-9_]+/[-a-zA-Z0-9_]+@.+
51 | 


--------------------------------------------------------------------------------
/.github/actions/spelling/reject.txt:
--------------------------------------------------------------------------------
 1 | ^attache$
 2 | ^bellows?$
 3 | benefitting
 4 | occurences?
 5 | ^dependan.*
 6 | ^diables?$
 7 | ^oer$
 8 | Sorce
 9 | ^[Ss]pae.*
10 | ^Teh$
11 | ^untill$
12 | ^untilling$
13 | ^venders?$
14 | ^wether.*
15 | 


--------------------------------------------------------------------------------
/.github/workflows/go.yml:
--------------------------------------------------------------------------------
 1 | # This workflow will build a golang project
 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
 3 | 
 4 | name: Go
 5 | 
 6 | on:
 7 |   push:
 8 |     branches: [ "master" ]
 9 |   pull_request:
10 |     branches: [ "master" ]
11 | 
12 | jobs:
13 | 
14 |   build:
15 |     runs-on: ubuntu-latest
16 |     steps:
17 |     - uses: actions/checkout@v3
18 | 
19 |     - name: Set up Go
20 |       uses: actions/setup-go@v4
21 |       with:
22 |         go-version: '1.21'
23 | 
24 |     - name: Build
25 |       run: go build -v ./...
26 | 
27 |     - name: Test
28 |       run: make unittest
29 | 
30 |     - name: Run bash tests
31 |       run: |
32 |         make build
33 |         bash tests/graceful_restart.sh


--------------------------------------------------------------------------------
/.github/workflows/linters.yaml:
--------------------------------------------------------------------------------
 1 | name: linters
 2 | 
 3 | on:
 4 |   push:
 5 |     branches: [ master ]
 6 |   pull_request:
 7 |     branches: [ master ]
 8 | 
 9 | permissions:
10 |   contents: read
11 | 
12 | jobs:
13 |   fmtcheck:
14 |     name: go fmt check
15 |     runs-on: ubuntu-latest
16 |     steps:
17 |       - uses: actions/checkout@v4
18 |       - uses: actions/setup-go@v5
19 |         with:
20 |           go-version: '1.23.0'
21 |           cache: false
22 |       - name: go fmt check
23 |         run: make fmtcheck
24 | 
25 |   golangci:
26 |     name: golangci-lint
27 |     runs-on: ubuntu-latest
28 |     steps:
29 |       - uses: actions/checkout@v4
30 |       - uses: actions/setup-go@v5
31 |         with:
32 |           go-version: '1.23.0'
33 |           cache: false
34 |       - name: golangci-lint
35 |         uses: golangci/golangci-lint-action@v7
36 |         with:
37 |           version: latest
38 |           args: --timeout=10m
39 | 
40 |   goimports:
41 |     runs-on: ubuntu-latest
42 |     name: goimports check
43 |     steps:
44 |       - name: checkout
45 |         uses: actions/checkout@v4
46 |       - name: goimports check
47 |         uses: DarthBenro008/goimports-check-action@v0.2.0
48 |         with:
49 |           root-path: './' # The relative root path to run goimports
50 |           excludes: "./yacc/"
51 | 


--------------------------------------------------------------------------------
/.github/workflows/regress.yaml:
--------------------------------------------------------------------------------
 1 | name: Regression tests
 2 | 
 3 | on:
 4 |   push:
 5 |     branches: [ "master" ]
 6 |   pull_request:
 7 |     branches: [ "*" ]
 8 | 
 9 | jobs:
10 |   build:
11 |     runs-on: ubuntu-latest
12 |     steps:
13 |     - uses: actions/checkout@v3
14 | 
15 |     - name: Regression tests
16 |       run: make regress
17 | 


--------------------------------------------------------------------------------
/.github/workflows/release.yaml:
--------------------------------------------------------------------------------
 1 | name: release
 2 | 
 3 | on:
 4 |   release:
 5 |     type: [created]
 6 | 
 7 | jobs:
 8 |   release:
 9 |     name: Release
10 |     runs-on: ubuntu-latest
11 |     steps:
12 | 
13 |       - name: Set up Go 1.x
14 |         uses: actions/setup-go@v4
15 |         with:
16 |           go-version: 1.21
17 |         id: go
18 | 
19 |       - name: Check out code into the Go module directory
20 |         uses: actions/checkout@v3
21 | 
22 |       - name: Get dependencies
23 |         run:
24 |           sudo apt update && sudo apt install -y debhelper
25 | 
26 |       - name: Set env
27 |         run: echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV
28 | 
29 |       - name: build package
30 |         run: |
31 |           make package
32 |           gh release upload ${TAG} $(echo ../* | tr ' ' '\n' | grep -v '^../yproxy$' | tr '\n' ' ')
33 |         env:
34 |           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}


--------------------------------------------------------------------------------
/.github/workflows/spelling.yaml:
--------------------------------------------------------------------------------
 1 | name: Check Spelling
 2 | 
 3 | on:
 4 |   pull_request:
 5 |     branches:
 6 |     - "**"
 7 |   issue_comment:
 8 |     types:
 9 |     - 'created'
10 | 
11 | jobs:
12 |   spelling:
13 |     name: Check Spelling
14 |     permissions:
15 |       contents: read
16 |       pull-requests: read
17 |       actions: read
18 |       security-events: write
19 |     outputs:
20 |       followup: ${{ steps.spelling.outputs.followup }}
21 |     runs-on: ubuntu-latest
22 |     if: ${{ contains(github.event_name, 'pull_request') || github.event_name == 'push' }}
23 |     concurrency:
24 |       group: spelling-${{ github.event.pull_request.number || github.ref }}
25 |       # note: If you use only_check_changed_files, you do not want cancel-in-progress
26 |       cancel-in-progress: true
27 |     steps:
28 |     - name: check-spelling
29 |       id: spelling
30 |       uses: check-spelling/check-spelling@main
31 |       with:
32 |         suppress_push_for_open_pull_request: ${{ github.actor != 'dependabot[bot]' && 1 }}
33 |         checkout: true
34 |         check_file_names: 1
35 |         spell_check_this: check-spelling/spell-check-this@prerelease
36 |         post_comment: 0
37 |         use_magic_file: 1
38 |         report-timing: 1
39 |         warnings: bad-regex,binary-file,deprecated-feature,large-file,limited-references,no-newline-at-eof,noisy-file,non-alpha-in-dictionary,token-is-substring,unexpected-line-ending,whitespace-in-dictionary,minified-file,unsupported-configuration,no-files-to-check
40 |         experimental_apply_changes_via_bot: 1
41 |         use_sarif: ${{ (!github.event.pull_request || (github.event.pull_request.head.repo.full_name == github.repository)) && 1 }}
42 |         extra_dictionary_limit: 20
43 |         extra_dictionaries:
44 |           cspell:software-terms/dict/softwareTerms.txt
45 | 
46 |   comment-pr:
47 |     name: Report (PR)
48 |     # If you workflow isn't running on pull_request*, you can remove this job
49 |     runs-on: ubuntu-latest
50 |     needs: spelling
51 |     permissions:
52 |       contents: read
53 |       pull-requests: write
54 |     if: (success() || failure()) && needs.spelling.outputs.followup && contains(github.event_name, 'pull_request')
55 |     steps:
56 |     - name: comment
57 |       uses: check-spelling/check-spelling@main
58 |       with:
59 |         checkout: true
60 |         spell_check_this: check-spelling/spell-check-this@prerelease
61 |         task: ${{ needs.spelling.outputs.followup }}
62 |         experimental_apply_changes_via_bot: 1
63 | 


--------------------------------------------------------------------------------
/.github/workflows/tests.yaml:
--------------------------------------------------------------------------------
 1 | name: Run yezzey integration tests
 2 | 
 3 | on:
 4 |   pull_request:
 5 |     branches: "*"
 6 |   push:
 7 |     branches:
 8 |       - "master"
 9 | jobs:
10 |   test:
11 |     name: Run tests
12 |     runs-on: ubuntu-latest
13 |     steps:
14 | 
15 |       - name: Set up QEMU
16 |         uses: docker/setup-qemu-action@v2
17 | 
18 |       - name: Set up Docker Buildx
19 |         uses: docker/setup-buildx-action@v2
20 | 
21 |       - name: Check out code
22 |         uses: actions/checkout@v3
23 |         with:
24 |           repository: open-gpdb/gpdb
25 |           ref: OPENGPDB_STABLE
26 | 
27 |       - name: set PR ref
28 |         if: ${{github.event_name == 'pull_request'}}
29 |         run: echo "GITHUB_REF=${{github.head_ref}}" >> $GITHUB_ENV
30 | 
31 |       - name: set push ref
32 |         if: ${{github.event_name == 'push'}}
33 |         run: echo "GITHUB_REF=${{github.ref_name}}" >> $GITHUB_ENV
34 | 
35 |       - name: Build
36 |         run: docker compose -f docker/yezzey/docker-compose.yaml build
37 | 
38 |       - name: Run Docker
39 |         shell: bash
40 |         run: docker compose -f docker/yezzey/docker-compose.yaml run --remove-orphans yezzey
41 | 
42 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | vendor
2 | devbin/*
3 | yezzey.log
4 | 


--------------------------------------------------------------------------------
/.golangci.yaml:
--------------------------------------------------------------------------------
1 | version: "2"
2 | linters:
3 |   exclusions:
4 |     rules:
5 |       - path: 'pkg/core/parser'
6 |         linters:
7 |           - unused
8 |           - govet


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | 
 2 | GIT_REVISION=`git rev-parse --short HEAD`
 3 | YPROXY_VERSION=`git describe --tags --abbrev=0`
 4 | LDFLAGS=-ldflags "-X github.com/yezzey-gp/yproxy/pkg.GitRevision=${GIT_REVISION} -X github.com/yezzey-gp/yproxy/pkg.YproxyVersion=${YPROXY_VERSION}"
 5 | GOFMT_FILES?=$$(find . -name '*.go' | grep -v .git | grep -v parser | grep -v vendor)
 6 | 
 7 | ####################### BUILD #######################
 8 | 
 9 | build:
10 | 	mkdir -p devbin
11 | 	go build -pgo=auto -o devbin/yproxy $(LDFLAGS) ./cmd/yproxy
12 | 	go build -o devbin/yp-client ./cmd/client
13 | 
14 | ####################### TESTS #######################
15 | 
16 | unittest:
17 | 	go test -race ./pkg/message/... ./pkg/proc/... ./pkg/core/... ./pkg/storage/...
18 | 
19 | regress:
20 | 	docker compose -f test/regress/docker-compose.yaml down
21 | 	docker compose -f test/regress/docker-compose.yaml run --remove-orphans --build yproxy
22 | 
23 | mockgen:
24 | 	mockgen -source=pkg/proc/yio/yrreader.go -destination=pkg/mock/proc/yio/yrreader.go -package=mock
25 | 	mockgen -source=pkg/client/client.go -destination=pkg/mock/client/client.go -package=mock
26 | 	mockgen -source=pkg/database/database.go -destination=pkg/mock/database.go -package=mock
27 | 	mockgen -source=pkg/backups/backups.go -destination=pkg/mock/backups.go -package=mock
28 | 	mockgen -source=pkg/storage/storage.go -destination=pkg/mock/storage.go -package=mock
29 | 
30 | version = $(shell git describe --tags --abbrev=0)
31 | package:
32 | 	sed -i 's/YPROXY_VERSION/${version}/g' debian/changelog
33 | 	dpkg-buildpackage -us -uc
34 | 
35 | 
36 | ####################### LINTERS #######################
37 | 
38 | fmt:
39 | 	gofmt -w $(GOFMT_FILES)
40 | 
41 | fmtcheck:
42 | 	@sh -c "'$(CURDIR)/script/gofmtcheck.sh'"
43 | 
44 | lint:
45 | 	golangci-lint run --timeout=10m
46 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # yproxy
 2 | 
 3 | yproxy - a service for efficient transfer data external storages.
 4 | 
 5 | 
 6 | ### debugging
 7 | 
 8 | 1. set `debug_port` and `debug_minutes` in configuration file
 9 | 2. send SIGHUP to `yproxy`
10 | 3. target golang tools to /debug/pprof to profile application


--------------------------------------------------------------------------------
/build.Dockerfile:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/open-gpdb/yproxy/1add3d98ab307ef2931a54997b92cae5fd7bb573/build.Dockerfile


--------------------------------------------------------------------------------
/cmd/yproxy/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"log"
 5 | 
 6 | 	"github.com/spf13/cobra"
 7 | 
 8 | 	"github.com/yezzey-gp/yproxy/config"
 9 | 	"github.com/yezzey-gp/yproxy/pkg"
10 | 	"github.com/yezzey-gp/yproxy/pkg/core"
11 | 	"github.com/yezzey-gp/yproxy/pkg/testutils"
12 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
13 | )
14 | 
15 | var cfgPath string
16 | 
17 | var logLevel string
18 | 
19 | var rootCmd = &cobra.Command{
20 | 	Use:   "",
21 | 	Short: "",
22 | 	RunE: func(cmd *cobra.Command, args []string) error {
23 | 		ylogger.Zero.Debug().Str("config-path", cfgPath).Msg("using config path")
24 | 		err := config.LoadInstanceConfig(cfgPath)
25 | 		if err != nil {
26 | 			return err
27 | 		}
28 | 
29 | 		instanceCnf := config.InstanceConfig()
30 | 
31 | 		instance := core.NewInstance()
32 | 
33 | 		if instanceCnf.LogPath != "" {
34 | 			ylogger.ReloadLogger(instanceCnf.LogPath)
35 | 		}
36 | 		if logLevel == "" {
37 | 			logLevel = instanceCnf.LogLevel
38 | 		}
39 | 		if err := ylogger.UpdateZeroLogLevel(logLevel); err != nil {
40 | 			log.Printf("failed to update log level: %s\n", err)
41 | 		}
42 | 
43 | 		return instance.Run(instanceCnf)
44 | 	},
45 | 	Version: pkg.YproxyVersionRevision,
46 | }
47 | 
48 | func init() {
49 | 	rootCmd.PersistentFlags().StringVarP(&cfgPath, "config", "c", "/etc/yproxy/yproxy.yaml", "path to yproxy config file")
50 | 	rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", "", "log level")
51 | 	rootCmd.PersistentFlags().BoolVarP(&testutils.TestMode, "test-mode", "", false, "enable test mode. Use only in regression tests")
52 | }
53 | 
54 | func main() {
55 | 	if err := rootCmd.Execute(); err != nil {
56 | 		ylogger.Zero.Fatal().Err(err).Msg("")
57 | 	}
58 | }
59 | 


--------------------------------------------------------------------------------
/config/instance.go:
--------------------------------------------------------------------------------
  1 | package config
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"fmt"
  6 | 	"log"
  7 | 	"os"
  8 | 	"strings"
  9 | 
 10 | 	"github.com/BurntSushi/toml"
 11 | 	"gopkg.in/yaml.v2"
 12 | )
 13 | 
 14 | type Instance struct {
 15 | 	StorageCnf       Storage `json:"storage" toml:"storage" yaml:"storage"`
 16 | 	BackupStorageCnf Storage `json:"backup_storage" toml:"backup_storage" yaml:"backup_storage"`
 17 | 	ProxyCnf         Proxy   `json:"proxy" toml:"proxy" yaml:"proxy"`
 18 | 
 19 | 	CryptoCnf Crypto `json:"crypto" toml:"crypto" yaml:"crypto"`
 20 | 
 21 | 	VacuumCnf Vacuum `json:"vacuum" toml:"vacuum" yaml:"vacuum"`
 22 | 
 23 | 	LogPath                string `json:"log_path" toml:"log_path" yaml:"log_path"`
 24 | 	LogLevel               string `json:"log_level" toml:"log_level" yaml:"log_level"`
 25 | 	SocketPath             string `json:"socket_path" toml:"socket_path" yaml:"socket_path"`
 26 | 	StatPort               int    `json:"stat_port" toml:"stat_port" yaml:"stat_port"`
 27 | 	PsqlPort               int    `json:"psql_port" toml:"psql_port" yaml:"psql_port"`
 28 | 	InterconnectSocketPath string `json:"interconnect_socket_path" toml:"interconnect_socket_path" yaml:"interconnect_socket_path"`
 29 | 	DebugPort              int    `json:"debug_port" toml:"debug_port" yaml:"debug_port"`
 30 | 	DebugMinutes           int    `json:"debug_minutes" toml:"debug_minutes" yaml:"debug_minutes"`
 31 | 
 32 | 	SystemdNotificationsDebug bool `json:"sd_notifications_debug" toml:"sd_notifications_debug" yaml:"sd_notifications_debug"`
 33 | 	systemdSocketPath         string
 34 | }
 35 | 
 36 | func (i *Instance) ReadSystemdSocketPath() {
 37 | 	path := os.Getenv("NOTIFY_SOCKET")
 38 | 	if path != "" {
 39 | 		i.systemdSocketPath = path
 40 | 	}
 41 | }
 42 | 
 43 | func (i *Instance) GetSystemdSocketPath() string {
 44 | 	return i.systemdSocketPath
 45 | }
 46 | 
 47 | var cfgInstance Instance
 48 | 
 49 | func InstanceConfig() *Instance {
 50 | 	return &cfgInstance
 51 | }
 52 | 
 53 | func initInstanceConfig(file *os.File, cfgInstance *Instance) error {
 54 | 	cfgInstance.VacuumCnf.CheckBackup = true
 55 | 	if strings.HasSuffix(file.Name(), ".toml") {
 56 | 		_, err := toml.NewDecoder(file).Decode(cfgInstance)
 57 | 		return err
 58 | 	}
 59 | 	if strings.HasSuffix(file.Name(), ".yaml") {
 60 | 		return yaml.NewDecoder(file).Decode(&cfgInstance)
 61 | 	}
 62 | 	if strings.HasSuffix(file.Name(), ".json") {
 63 | 		return json.NewDecoder(file).Decode(&cfgInstance)
 64 | 	}
 65 | 	return fmt.Errorf("unknown config format type: %s. Use .toml, .yaml or .json suffix in filename", file.Name())
 66 | }
 67 | 
 68 | const (
 69 | 	DefaultStorageConcurrency = 100
 70 | 	DefaultStatPort           = 7432
 71 | 	DefaultPsqlPort           = 8432
 72 | 
 73 | 	DefaultEndpointSourceScheme = "https"
 74 | )
 75 | 
 76 | func EmbedDefaults(cfgInstance *Instance) {
 77 | 	if cfgInstance.StorageCnf.StorageType == "" {
 78 | 		cfgInstance.StorageCnf.StorageType = "s3"
 79 | 	}
 80 | 	if cfgInstance.StorageCnf.StorageConcurrency == 0 {
 81 | 		cfgInstance.StorageCnf.StorageConcurrency = DefaultStorageConcurrency
 82 | 	}
 83 | 	if cfgInstance.BackupStorageCnf.StorageType == "" {
 84 | 		cfgInstance.BackupStorageCnf.StorageType = "s3"
 85 | 	}
 86 | 	if cfgInstance.BackupStorageCnf.StorageConcurrency == 0 {
 87 | 		cfgInstance.BackupStorageCnf.StorageConcurrency = DefaultStorageConcurrency
 88 | 	}
 89 | 	if cfgInstance.StatPort == 0 {
 90 | 		cfgInstance.StatPort = DefaultStatPort
 91 | 	}
 92 | 	if cfgInstance.PsqlPort == 0 {
 93 | 		cfgInstance.PsqlPort = DefaultPsqlPort
 94 | 	}
 95 | 	if cfgInstance.StorageCnf.EndpointSourceScheme == "" {
 96 | 		cfgInstance.StorageCnf.EndpointSourceScheme = DefaultEndpointSourceScheme
 97 | 	}
 98 | }
 99 | 
100 | func LoadInstanceConfig(cfgPath string) (err error) {
101 | 	cfgInstance, err = ReadInstanceConfig(cfgPath)
102 | 	if err != nil {
103 | 		return
104 | 	}
105 | 
106 | 	cfgInstance.ReadSystemdSocketPath()
107 | 	EmbedDefaults(&cfgInstance)
108 | 
109 | 	configBytes, err := json.MarshalIndent(cfgInstance, "", "  ")
110 | 	if err != nil {
111 | 		return
112 | 	}
113 | 
114 | 	log.Println("Running config:", string(configBytes))
115 | 	return
116 | }
117 | 
118 | func ReadInstanceConfig(cfgPath string) (Instance, error) {
119 | 	var cfg Instance
120 | 	file, err := os.Open(cfgPath)
121 | 	if err != nil {
122 | 		return cfg, err
123 | 	}
124 | 	defer func(file *os.File) {
125 | 		err := file.Close()
126 | 		if err != nil {
127 | 			log.Fatalf("failed to close config file: %v", err)
128 | 		}
129 | 	}(file)
130 | 
131 | 	if err := initInstanceConfig(file, &cfg); err != nil {
132 | 		return cfg, err
133 | 	}
134 | 
135 | 	return cfg, nil
136 | }
137 | 


--------------------------------------------------------------------------------
/config/proxy.go:
--------------------------------------------------------------------------------
1 | package config
2 | 
3 | type Proxy struct {
4 | 	ConsolePort string `json:"console_port" toml:"console_port" yaml:"console_port"`
5 | 
6 | 	BucketCachePath string `json:"bucket_cache_path" toml:"bucket_cache_path" yaml:"bucket_cache_path"`
7 | }
8 | 


--------------------------------------------------------------------------------
/config/storage.go:
--------------------------------------------------------------------------------
 1 | package config
 2 | 
 3 | import "net/url"
 4 | 
 5 | type Crypto struct {
 6 | 	GPGKeyId   string `json:"gpg_key_id" toml:"gpg_key_id" yaml:"gpg_key_id"`
 7 | 	GPGKeyPath string `json:"gpg_key_path" toml:"gpg_key_path" yaml:"gpg_key_path"`
 8 | }
 9 | 
10 | type StorageCredentials struct {
11 | 	AccessKeyId     string `json:"access_key_id" toml:"access_key_id" yaml:"access_key_id"`
12 | 	SecretAccessKey string `json:"secret_access_key" toml:"secret_access_key" yaml:"secret_access_key"`
13 | }
14 | 
15 | type Storage struct {
16 | 	StorageEndpoint string `json:"storage_endpoint" toml:"storage_endpoint" yaml:"storage_endpoint"`
17 | 	AccessKeyId     string `json:"access_key_id" toml:"access_key_id" yaml:"access_key_id"`
18 | 	SecretAccessKey string `json:"secret_access_key" toml:"secret_access_key" yaml:"secret_access_key"`
19 | 	StoragePrefix   string `json:"storage_prefix" toml:"storage_prefix" yaml:"storage_prefix"`
20 | 	StorageBucket   string `json:"storage_bucket" toml:"storage_bucket" yaml:"storage_bucket"`
21 | 
22 | 	CredentialMap map[string]StorageCredentials `json:"credential_map" toml:"credential_map" yaml:"credential_map"`
23 | 
24 | 	TablespaceMap map[string]string `json:"tablespace_map" toml:"tablespace_map" yaml:"tablespace_map"`
25 | 
26 | 	// how many concurrent connection acquire allowed
27 | 	StorageConcurrency int64 `json:"storage_concurrency" toml:"storage_concurrency" yaml:"storage_concurrency"`
28 | 
29 | 	StorageRegion string `json:"storage_region" toml:"storage_region" yaml:"storage_region"`
30 | 
31 | 	// File storage default s3. Available: s3, fs
32 | 	StorageType string `json:"storage_type" toml:"storage_type" yaml:"storage_type"`
33 | 
34 | 	EndpointSourceHost   string `json:"storage_endpoint_source_host" toml:"storage_endpoint_source_host" yaml:"storage_endpoint_source_host"`
35 | 	EndpointSourcePort   string `json:"storage_endpoint_source_port" toml:"storage_endpoint_source_port" yaml:"storage_endpoint_source_port"`
36 | 	EndpointSourceScheme string `json:"storage_endpoint_source_scheme" toml:"storage_endpoint_source_scheme" yaml:"storage_endpoint_source_scheme"`
37 | }
38 | 
39 | func (s Storage) ID() string {
40 | 	id, _ := url.JoinPath(s.StorageEndpoint, s.StorageBucket, s.StoragePrefix)
41 | 	return id
42 | }
43 | 


--------------------------------------------------------------------------------
/config/vacuum.go:
--------------------------------------------------------------------------------
1 | package config
2 | 
3 | type Vacuum struct {
4 | 	CheckBackup bool `json:"check_backup" toml:"check_backup" yaml:"check_backup"`
5 | }
6 | 


--------------------------------------------------------------------------------
/debian/changelog:
--------------------------------------------------------------------------------
1 | yproxy (YPROXY_VERSION) stable; urgency=low
2 | 
3 |   * Package build
4 | 
5 |  -- Kirill Reshke <reshke@double.cloud>  Mon, 13 Nov 2023 12:00:00 +0000
6 | 


--------------------------------------------------------------------------------
/debian/compat:
--------------------------------------------------------------------------------
1 | 9
2 | 


--------------------------------------------------------------------------------
/debian/control:
--------------------------------------------------------------------------------
 1 | Source: yproxy
 2 | Section: database
 3 | Priority: optional
 4 | Maintainer: Kirill Reshke <reshke@yandex-team.ru>
 5 | Build-Depends: 
 6 | Standards-Version: 3.8.0
 7 | Homepage: https://github.com/yezzey-gp/yproxy
 8 | 
 9 | Package: yproxy
10 | Conflicts: 
11 | Architecture: any
12 | Depends: 
13 | Description: Local daemon for S3 access, caching & throttling
14 |  This package will install yproxy binaries into /usr/bin
15 | 


--------------------------------------------------------------------------------
/debian/install:
--------------------------------------------------------------------------------
1 | debian/build/* /usr/bin
2 | 


--------------------------------------------------------------------------------
/debian/rules:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/make -f
 2 | %:
 3 | 	dh $@
 4 | 
 5 | override_dh_auto_build:
 6 | 	make build
 7 | 
 8 | override_dh_auto_test:
 9 | 
10 | 
11 | override_dh_auto_install:
12 | 	mkdir -p $(PWD)/debian/build
13 | 	cp $(PWD)/devbin/* $(PWD)/debian/build
14 | 
15 | 


--------------------------------------------------------------------------------
/debian/source/format:
--------------------------------------------------------------------------------
1 | 3.0 (native)
2 | 


--------------------------------------------------------------------------------
/examples/yproxy.yaml:
--------------------------------------------------------------------------------
1 | socket_path: "/tmp/yproxy.sock"
2 | interconnect_socket_path: "/tmp/ic.sock"
3 | 
4 | storage:
5 |   storage_endpoint: "https://storage.yandexcloud.net"
6 |   storage_prefix: "wal-e/6/"
7 |   storage_bucket: "gpyezzey"
8 |   storage_region: "us-west-2"
9 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/yezzey-gp/yproxy
 2 | 
 3 | go 1.22.0
 4 | 
 5 | toolchain go1.23.1
 6 | 
 7 | require (
 8 | 	github.com/BurntSushi/toml v1.3.2
 9 | 	github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c
10 | 	github.com/jackc/pgx/v5 v5.6.0
11 | 	github.com/jarcoal/httpmock v1.3.1
12 | 	github.com/pkg/errors v0.9.1
13 | 	github.com/rs/zerolog v1.31.0
14 | 	github.com/spf13/cobra v1.8.0
15 | 	github.com/stretchr/testify v1.8.4
16 | 	github.com/yezzey-gp/aws-sdk-go v0.1.0
17 | 	go.uber.org/mock v0.4.0
18 | 	golang.org/x/sync v0.10.0
19 | 	gopkg.in/yaml.v2 v2.4.0
20 | )
21 | 
22 | require golang.org/x/text v0.21.0 // indirect
23 | 
24 | require (
25 | 	github.com/cockroachdb/apd v1.1.0 // indirect
26 | 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
27 | 	github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect
28 | 	github.com/kr/text v0.2.0 // indirect
29 | 	github.com/lib/pq v1.10.9 // indirect
30 | 	github.com/rogpeppe/go-internal v1.12.0 // indirect
31 | 	github.com/shopspring/decimal v1.4.0 // indirect
32 | )
33 | 
34 | require (
35 | 	github.com/golang/mock v1.6.0 // indirect
36 | 	github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 // indirect
37 | 	golang.org/x/crypto v0.30.0 // indirect
38 | 	golang.org/x/mod v0.17.0 // indirect
39 | 	golang.org/x/net v0.32.0 // indirect
40 | 	golang.org/x/sys v0.28.0 // indirect
41 | 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
42 | 	gonum.org/v1/gonum v0.15.1 // indirect
43 | )
44 | 
45 | require (
46 | 	github.com/caio/go-tdigest v3.1.0+incompatible
47 | 	github.com/cloudflare/circl v1.3.3 // indirect
48 | 	github.com/davecgh/go-spew v1.1.1 // indirect
49 | 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
50 | 	github.com/jackc/pgx v3.6.2+incompatible
51 | 	github.com/jmespath/go-jmespath v0.4.0 // indirect
52 | 	github.com/mattn/go-colorable v0.1.13 // indirect
53 | 	github.com/mattn/go-isatty v0.0.19 // indirect
54 | 	github.com/pmezard/go-difflib v1.0.0 // indirect
55 | 	github.com/spf13/pflag v1.0.5 // indirect
56 | 	gopkg.in/yaml.v3 v3.0.1 // indirect
57 | )
58 | 


--------------------------------------------------------------------------------
/pkg/backups/backups.go:
--------------------------------------------------------------------------------
 1 | package backups
 2 | 
 3 | type BackupLSN struct {
 4 | 	Lsn uint64 `json:"LSN"`
 5 | }
 6 | 
 7 | //go:generate mockgen -destination=pkg/mock/backups.go -package=mock
 8 | type BackupInterractor interface {
 9 | 	GetFirstLSN(seg uint64) (uint64, error)
10 | }
11 | 


--------------------------------------------------------------------------------
/pkg/backups/s3.go:
--------------------------------------------------------------------------------
 1 | package backups
 2 | 
 3 | import (
 4 | 	"encoding/json"
 5 | 	"fmt"
 6 | 	"io"
 7 | 	"strings"
 8 | 
 9 | 	"github.com/yezzey-gp/yproxy/pkg/storage"
10 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
11 | )
12 | 
13 | type StorageBackupInteractor struct {
14 | 	Storage storage.StorageInteractor
15 | }
16 | 
17 | // get lsn of the oldest backup
18 | func (b *StorageBackupInteractor) GetFirstLSN(seg uint64) (uint64, error) {
19 | 	objects, err := b.Storage.ListPath(fmt.Sprintf("segments_005/seg%d/basebackups_005/", seg), true, nil)
20 | 	if err != nil {
21 | 		ylogger.Zero.Debug().Err(err).Msg("GetFirstLSN: list result")
22 | 		return 0, err
23 | 	}
24 | 	ylogger.Zero.Debug().Int("size", len(objects)).Msg("GetFirstLSN: list result size")
25 | 
26 | 	minLSN := BackupLSN{Lsn: ^uint64(0)}
27 | 	for _, obj := range objects {
28 | 		ylogger.Zero.Debug().Str("path", obj.Path).Msg("GetFirstLSN: checking")
29 | 		if !strings.HasSuffix(obj.Path, "backup_stop_sentinel.json") {
30 | 			continue
31 | 		}
32 | 
33 | 		// cat file
34 | 		reader, err := b.Storage.CatFileFromStorage(obj.Path, 0, nil)
35 | 		if err != nil {
36 | 			continue
37 | 		}
38 | 		content, err := io.ReadAll(reader)
39 | 		if err != nil {
40 | 			ylogger.Zero.Debug().Str("path", obj.Path).Err(err).Msg("GetFirstLSN: read all")
41 | 			continue
42 | 		}
43 | 
44 | 		lsn := BackupLSN{}
45 | 		err = json.Unmarshal(content, &lsn)
46 | 		if err != nil {
47 | 			ylogger.Zero.Debug().Str("path", obj.Path).Err(err).Msg("GetFirstLSN: unmarshal")
48 | 			continue
49 | 		}
50 | 
51 | 		ylogger.Zero.Debug().Str("path", obj.Path).Uint64("lsn", lsn.Lsn).Msg("GetFirstLSN: parsed backup lsn")
52 | 
53 | 		if lsn.Lsn < minLSN.Lsn {
54 | 			minLSN.Lsn = lsn.Lsn
55 | 		}
56 | 	}
57 | 
58 | 	return minLSN.Lsn, err
59 | }
60 | 


--------------------------------------------------------------------------------
/pkg/client/client.go:
--------------------------------------------------------------------------------
  1 | package client
  2 | 
  3 | import (
  4 | 	"io"
  5 | 	"net"
  6 | 	"reflect"
  7 | 	"time"
  8 | 
  9 | 	"github.com/yezzey-gp/yproxy/pkg/message"
 10 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
 11 | )
 12 | 
 13 | type YproxyClient interface {
 14 | 	ID() uint
 15 | 	ReplyError(err error, msg string) error
 16 | 	GetRW() io.ReadWriteCloser
 17 | 
 18 | 	SetOPType(optype message.MessageType)
 19 | 	OPType() message.MessageType
 20 | 	OPStart() time.Time
 21 | 
 22 | 	/* Read/Write progress */
 23 | 	SetByteOffset(int64)
 24 | 	ByteOffset() int64
 25 | 
 26 | 	SetExternalFilePath(path string)
 27 | 	ExternalFilePath() string
 28 | 
 29 | 	Close() error
 30 | }
 31 | 
 32 | type YClient struct {
 33 | 	Conn    net.Conn
 34 | 	op      message.MessageType
 35 | 	opstart time.Time
 36 | 	path    string
 37 | 
 38 | 	progress int64
 39 | }
 40 | 
 41 | // ByteOffset implements YproxyClient.
 42 | func (y *YClient) ByteOffset() int64 {
 43 | 	return y.progress
 44 | }
 45 | 
 46 | func (y *YClient) SetByteOffset(n int64) {
 47 | 	y.progress = n
 48 | }
 49 | 
 50 | // ExternalFilePath implements YproxyClient.
 51 | func (y *YClient) ExternalFilePath() string {
 52 | 	return y.path
 53 | }
 54 | 
 55 | // SetExternalFilePath implements YproxyClient.
 56 | func (y *YClient) SetExternalFilePath(path string) {
 57 | 	y.path = path
 58 | }
 59 | 
 60 | // OPType implements YproxyClient.
 61 | func (y *YClient) OPType() message.MessageType {
 62 | 	return y.op
 63 | }
 64 | 
 65 | func (y *YClient) OPStart() time.Time {
 66 | 	return y.opstart
 67 | }
 68 | 
 69 | // SetOPType implements YproxyClient.
 70 | func (y *YClient) SetOPType(optype message.MessageType) {
 71 | 	y.op = optype
 72 | 	y.opstart = time.Now()
 73 | }
 74 | 
 75 | // Close implements YproxyClient.
 76 | func (y *YClient) Close() error {
 77 | 	return y.Conn.Close()
 78 | }
 79 | 
 80 | // GetPointer do the same thing like fmt.Sprintf("%p", &num) but fast
 81 | // GetPointer returns the memory address of the given value as an unsigned integer.
 82 | func GetPointer(value interface{}) uint {
 83 | 	ptr := reflect.ValueOf(value).Pointer()
 84 | 	uintPtr := uintptr(ptr)
 85 | 	return uint(uintPtr)
 86 | }
 87 | 
 88 | // ID implements YproxyClient.
 89 | func (y *YClient) ID() uint {
 90 | 	return GetPointer(y)
 91 | }
 92 | 
 93 | func NewYClient(c net.Conn) YproxyClient {
 94 | 	return &YClient{
 95 | 		Conn: c,
 96 | 	}
 97 | }
 98 | 
 99 | func (y *YClient) ReplyError(err error, msg string) error {
100 | 	ylogger.Zero.Error().Err(err).Msg(msg)
101 | 
102 | 	_, _ = y.Conn.Write(message.NewErrorMessage(err, msg).Encode())
103 | 	return nil
104 | }
105 | 
106 | func (y *YClient) GetRW() io.ReadWriteCloser {
107 | 	return y.Conn
108 | }
109 | 


--------------------------------------------------------------------------------
/pkg/clientpool/clientpool.go:
--------------------------------------------------------------------------------
  1 | package clientpool
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"sync"
  6 | 	"time"
  7 | 
  8 | 	"github.com/caio/go-tdigest"
  9 | 	"github.com/yezzey-gp/yproxy/pkg/client"
 10 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
 11 | )
 12 | 
 13 | type Pool interface {
 14 | 	ClientPoolForeach(cb func(client client.YproxyClient) error) error
 15 | 
 16 | 	Put(client client.YproxyClient) error
 17 | 	Pop(id uint) (bool, error)
 18 | 
 19 | 	Quantile(ct int, q []float64) []QuantInfo
 20 | 
 21 | 	Shutdown() error
 22 | }
 23 | 
 24 | type PoolImpl struct {
 25 | 	mu   sync.Mutex
 26 | 	pool map[uint]client.YproxyClient
 27 | 	/* size category -> optype -> speed quantiles */
 28 | 	opSpeed map[int]map[string]*tdigest.TDigest
 29 | }
 30 | 
 31 | var _ Pool = &PoolImpl{}
 32 | 
 33 | func SizeToCat(sz int64) int {
 34 | 	if sz < 1024*1024 {
 35 | 		return 0
 36 | 	}
 37 | 	if sz < 16*1024*1024 {
 38 | 		return 1
 39 | 	}
 40 | 	return 2
 41 | }
 42 | 
 43 | func (c *PoolImpl) Put(client client.YproxyClient) error {
 44 | 	c.mu.Lock()
 45 | 	defer c.mu.Unlock()
 46 | 
 47 | 	c.pool[client.ID()] = client
 48 | 
 49 | 	return nil
 50 | }
 51 | 
 52 | type QuantInfo struct {
 53 | 	Op string
 54 | 	Q  []float64
 55 | }
 56 | 
 57 | func (c *PoolImpl) Quantile(ct int, q []float64) []QuantInfo {
 58 | 	c.mu.Lock()
 59 | 	defer c.mu.Unlock()
 60 | 	var ret []QuantInfo
 61 | 	for k, v := range c.opSpeed[ct] {
 62 | 		var Qs []float64
 63 | 		for _, qq := range q {
 64 | 			Qs = append(Qs, v.Quantile(qq))
 65 | 		}
 66 | 		ret = append(ret, QuantInfo{
 67 | 			Op: k,
 68 | 			Q:  Qs,
 69 | 		})
 70 | 	}
 71 | 	return ret
 72 | }
 73 | 
 74 | func (c *PoolImpl) Pop(id uint) (bool, error) {
 75 | 	c.mu.Lock()
 76 | 	defer c.mu.Unlock()
 77 | 
 78 | 	cl, ok := c.pool[id]
 79 | 	if ok {
 80 | 
 81 | 		total := cl.ByteOffset()
 82 | 		if total > 0 {
 83 | 			ct := SizeToCat(total)
 84 | 			timeTotal := time.Since(cl.OPStart()).Nanoseconds()
 85 | 
 86 | 			optyp := cl.OPType().String()
 87 | 			if c.opSpeed[ct][optyp] == nil {
 88 | 				c.opSpeed[ct][optyp], _ = tdigest.New()
 89 | 			}
 90 | 
 91 | 			_ = c.opSpeed[ct][optyp].Add(float64(total) / float64(timeTotal))
 92 | 		}
 93 | 
 94 | 		delete(c.pool, id)
 95 | 		/* be conservative */
 96 | 		_ = cl.Close()
 97 | 		return true, nil
 98 | 	}
 99 | 
100 | 	return ok, fmt.Errorf("failed to find client %d connection", id)
101 | }
102 | 
103 | func (c *PoolImpl) Shutdown() error {
104 | 	c.mu.Lock()
105 | 	defer c.mu.Unlock()
106 | 
107 | 	for _, cl := range c.pool {
108 | 		go func(cl client.YproxyClient) {
109 | 			if err := cl.Close(); err != nil {
110 | 				ylogger.Zero.Error().Err(err).Msg("")
111 | 			}
112 | 		}(cl)
113 | 	}
114 | 
115 | 	return nil
116 | }
117 | func (c *PoolImpl) ClientPoolForeach(cb func(client client.YproxyClient) error) error {
118 | 	c.mu.Lock()
119 | 	defer c.mu.Unlock()
120 | 
121 | 	for _, cl := range c.pool {
122 | 		if err := cb(cl); err != nil {
123 | 			ylogger.Zero.Error().Err(err).Msg("")
124 | 		}
125 | 	}
126 | 
127 | 	return nil
128 | }
129 | 
130 | func NewClientPool() Pool {
131 | 	return &PoolImpl{
132 | 		pool: map[uint]client.YproxyClient{},
133 | 		mu:   sync.Mutex{},
134 | 		opSpeed: map[int]map[string]*tdigest.TDigest{
135 | 			0: {},
136 | 			1: {},
137 | 			2: {},
138 | 		},
139 | 	}
140 | }
141 | 


--------------------------------------------------------------------------------
/pkg/core/core.go:
--------------------------------------------------------------------------------
  1 | package core
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"errors"
  6 | 	"fmt"
  7 | 	"net"
  8 | 	"os"
  9 | 	"os/signal"
 10 | 	"sync"
 11 | 	"syscall"
 12 | 	"time"
 13 | 
 14 | 	"golang.org/x/sys/unix"
 15 | 
 16 | 	"github.com/yezzey-gp/yproxy/config"
 17 | 	"github.com/yezzey-gp/yproxy/pkg/client"
 18 | 	"github.com/yezzey-gp/yproxy/pkg/clientpool"
 19 | 	"github.com/yezzey-gp/yproxy/pkg/core/pg"
 20 | 	"github.com/yezzey-gp/yproxy/pkg/crypt"
 21 | 	"github.com/yezzey-gp/yproxy/pkg/message"
 22 | 	"github.com/yezzey-gp/yproxy/pkg/proc"
 23 | 	"github.com/yezzey-gp/yproxy/pkg/sdnotifier"
 24 | 	"github.com/yezzey-gp/yproxy/pkg/storage"
 25 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
 26 | )
 27 | 
 28 | type Instance struct {
 29 | 	pool clientpool.Pool
 30 | 
 31 | 	startTs time.Time
 32 | }
 33 | 
 34 | func NewInstance() *Instance {
 35 | 	return &Instance{
 36 | 		pool:    clientpool.NewClientPool(),
 37 | 		startTs: time.Now(),
 38 | 	}
 39 | }
 40 | 
 41 | func (i *Instance) DispatchServer(listener net.Listener, server func(net.Conn)) {
 42 | 	go func() {
 43 | 		defer func() { _ = listener.Close() }()
 44 | 		for {
 45 | 			clConn, err := listener.Accept()
 46 | 			if err != nil {
 47 | 				if errors.Is(err, net.ErrClosed) {
 48 | 					break
 49 | 				}
 50 | 				ylogger.Zero.Error().Err(err).Msg("failed to accept connection")
 51 | 				continue
 52 | 			}
 53 | 			ylogger.Zero.Debug().Str("addr", clConn.LocalAddr().String()).Msg("accepted client connection")
 54 | 
 55 | 			go server(clConn)
 56 | 		}
 57 | 	}()
 58 | }
 59 | 
 60 | func (i *Instance) Run(instanceCnf *config.Instance) error {
 61 | 
 62 | 	sigs := make(chan os.Signal, 1)
 63 | 	signal.Notify(sigs, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1, syscall.SIGUSR2)
 64 | 
 65 | 	ctx, cancelCtx := context.WithCancel(context.Background())
 66 | 	defer cancelCtx()
 67 | 
 68 | 	var listener net.Listener
 69 | 	var iclistener net.Listener
 70 | 	var dws *DebugWebServer
 71 | 
 72 | 	go func() {
 73 | 		defer cancelCtx()
 74 | 
 75 | 		for {
 76 | 			s := <-sigs
 77 | 			ylogger.Zero.Info().Str("signal", s.String()).Msg("received signal")
 78 | 
 79 | 			switch s {
 80 | 			case syscall.SIGUSR1:
 81 | 				ylogger.ReloadLogger(instanceCnf.LogPath)
 82 | 			case syscall.SIGHUP:
 83 | 				if dws != nil {
 84 | 					err := dws.ServeFor(time.Duration(instanceCnf.DebugMinutes) * time.Minute)
 85 | 					if err != nil {
 86 | 						ylogger.Zero.Error().Err(err).Msg("Error in debug server")
 87 | 					}
 88 | 				}
 89 | 			case syscall.SIGUSR2:
 90 | 				fallthrough
 91 | 			case syscall.SIGINT, syscall.SIGTERM:
 92 | 				// make better
 93 | 				fallthrough
 94 | 			default:
 95 | 				if listener != nil {
 96 | 					if err := listener.Close(); err != nil {
 97 | 						ylogger.Zero.Error().Err(err).Msg("failed to close socket")
 98 | 					}
 99 | 				}
100 | 				if iclistener != nil {
101 | 					if err := iclistener.Close(); err != nil {
102 | 						ylogger.Zero.Error().Err(err).Msg("failed to close ic socket")
103 | 					}
104 | 				}
105 | 				return
106 | 			}
107 | 		}
108 | 	}()
109 | 
110 | 	/* dispatch statistic server */
111 | 	if instanceCnf.StatPort != 0 {
112 | 		config := &net.ListenConfig{Control: reusePort}
113 | 		statListener, err := config.Listen(context.Background(), "tcp", fmt.Sprintf("localhost:%v", instanceCnf.StatPort))
114 | 		if err != nil {
115 | 			ylogger.Zero.Error().Err(err).Msg("failed to start socket listener")
116 | 			return err
117 | 		}
118 | 
119 | 		i.DispatchServer(statListener, func(clConn net.Conn) {
120 | 			defer func() { _ = clConn.Close() }()
121 | 
122 | 			_, _ = clConn.Write([]byte("Hello from stats server!!\n"))
123 | 			_, _ = clConn.Write([]byte("Client id | Optype | External Path \n"))
124 | 
125 | 			if err := i.pool.ClientPoolForeach(func(cl client.YproxyClient) error {
126 | 				_, err := clConn.Write(fmt.Appendf(nil, "%v | %v | %v\n", cl.ID(), cl.OPType(), cl.ExternalFilePath()))
127 | 				return err
128 | 			}); err != nil {
129 | 				ylogger.Zero.Error().Err(err).Msg("failed to write client stats data")
130 | 			}
131 | 		})
132 | 	}
133 | 
134 | 	/* dispatch /debug/pprof server */
135 | 	if instanceCnf.DebugPort != 0 {
136 | 		debugAddr := fmt.Sprintf("[::1]:%d", instanceCnf.DebugPort)
137 | 		dws = NewDebugWebServer(debugAddr)
138 | 	}
139 | 
140 | 	s, err := storage.NewStorage(
141 | 		&instanceCnf.StorageCnf,
142 | 	)
143 | 	if err != nil {
144 | 		return err
145 | 	}
146 | 
147 | 	bs, err := storage.NewStorage(&instanceCnf.BackupStorageCnf)
148 | 	if err != nil {
149 | 		return err
150 | 	}
151 | 
152 | 	if instanceCnf.PsqlPort != 0 {
153 | 		config := &net.ListenConfig{Control: reusePort}
154 | 		psqlListener, err := config.Listen(context.Background(), "tcp", fmt.Sprintf("localhost:%v", instanceCnf.PsqlPort))
155 | 		if err != nil {
156 | 			ylogger.Zero.Error().Err(err).Msg("failed to start socket listener")
157 | 			return err
158 | 		}
159 | 
160 | 		i.DispatchServer(psqlListener, func(c net.Conn) {
161 | 			pg.PostgresIface(c, i.pool, i.startTs, s)
162 | 		})
163 | 	}
164 | 
165 | 	activeConnections := sync.WaitGroup{}
166 | 	retryTicker := time.NewTicker(time.Second)
167 | 
168 | 	for listener == nil {
169 | 		select {
170 | 		case <-retryTicker.C:
171 | 			listener, err = net.Listen("unix", instanceCnf.SocketPath)
172 | 			if err != nil {
173 | 				if errors.Is(err, syscall.EADDRINUSE) {
174 | 					continue
175 | 				}
176 | 				ylogger.Zero.Error().Err(err).Msg("failed to start socket listener")
177 | 				return err
178 | 			}
179 | 		case <-ctx.Done():
180 | 			return ctx.Err()
181 | 		}
182 | 	}
183 | 	ylogger.Zero.Info().Str("socket", instanceCnf.SocketPath).Msg("yproxy is listening unix socket")
184 | 
185 | 	var cr crypt.Crypter = nil
186 | 	if instanceCnf.CryptoCnf.GPGKeyPath != "" {
187 | 		cr, err = crypt.NewCrypto(&instanceCnf.CryptoCnf)
188 | 	}
189 | 
190 | 	i.DispatchServer(listener, func(clConn net.Conn) {
191 | 		activeConnections.Add(1)
192 | 		defer activeConnections.Done()
193 | 		defer func() { _ = clConn.Close() }()
194 | 		ycl := client.NewYClient(clConn)
195 | 		if err := i.pool.Put(ycl); err != nil {
196 | 			// ?? wtf
197 | 			ylogger.Zero.Debug().Uint("id", ycl.ID()).Err(err).Msg("error puting client to pool")
198 | 		}
199 | 		if err := proc.ProcConn(s, bs, cr, ycl, &instanceCnf.VacuumCnf); err != nil {
200 | 			ylogger.Zero.Debug().Uint("id", ycl.ID()).Err(err).Msg("error serving client")
201 | 		}
202 | 		if _, err := i.pool.Pop(ycl.ID()); err != nil {
203 | 			// ?? wtf
204 | 			ylogger.Zero.Debug().Uint("id", ycl.ID()).Err(err).Msg("error erasing client from pool")
205 | 		}
206 | 	})
207 | 
208 | 	for iclistener == nil {
209 | 		select {
210 | 		case <-retryTicker.C:
211 | 			iclistener, err = net.Listen("unix", instanceCnf.InterconnectSocketPath)
212 | 			if err != nil {
213 | 				if errors.Is(err, syscall.EADDRINUSE) {
214 | 					continue
215 | 				}
216 | 				ylogger.Zero.Error().Err(err).Msg("failed to start socket listener")
217 | 				return err
218 | 			}
219 | 		case <-ctx.Done():
220 | 			return ctx.Err()
221 | 		}
222 | 	}
223 | 
224 | 	ylogger.Zero.Debug().Msg("try to start interconnect socket listener")
225 | 	if err != nil {
226 | 		ylogger.Zero.Error().Err(err).Msg("failed to start interconnect socket listener")
227 | 		return err
228 | 	}
229 | 
230 | 	i.DispatchServer(iclistener, func(clConn net.Conn) {
231 | 		activeConnections.Add(1)
232 | 		defer activeConnections.Done()
233 | 		defer func() { _ = clConn.Close() }()
234 | 		ycl := client.NewYClient(clConn)
235 | 		r := proc.NewProtoReader(ycl)
236 | 
237 | 		mt, _, err := r.ReadPacket()
238 | 
239 | 		if err != nil {
240 | 			ylogger.Zero.Error().Err(err).Msg("failed to accept interconnection")
241 | 		}
242 | 
243 | 		switch mt {
244 | 		case message.MessageTypeGool:
245 | 			msg := message.ReadyForQueryMessage{}
246 | 			_, _ = ycl.GetRW().Write(msg.Encode())
247 | 		default:
248 | 			_ = ycl.ReplyError(fmt.Errorf("wrong message type"), "")
249 | 
250 | 		}
251 | 		ylogger.Zero.Debug().Msg("interconnection closed")
252 | 	})
253 | 
254 | 	notifier, err := sdnotifier.NewNotifier(instanceCnf.GetSystemdSocketPath(), instanceCnf.SystemdNotificationsDebug)
255 | 	if err != nil {
256 | 		ylogger.Zero.Error().Err(err).Msg("failed to initialize systemd notifier")
257 | 		if instanceCnf.SystemdNotificationsDebug {
258 | 			return err
259 | 		}
260 | 	}
261 | 	_ = notifier.Ready()
262 | 
263 | 	go func() {
264 | 		for {
265 | 			_ = notifier.Notify()
266 | 			time.Sleep(sdnotifier.Timeout)
267 | 		}
268 | 	}()
269 | 
270 | 	<-ctx.Done()
271 | 	activeConnections.Wait()
272 | 	return nil
273 | }
274 | 
275 | func reusePort(network, address string, conn syscall.RawConn) error {
276 | 	return conn.Control(func(descriptor uintptr) {
277 | 		_ = syscall.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unix.SO_REUSEADDR|unix.SO_REUSEPORT, 1)
278 | 	})
279 | }
280 | 


--------------------------------------------------------------------------------
/pkg/core/parser/.gitignore:
--------------------------------------------------------------------------------
1 | y.output


--------------------------------------------------------------------------------
/pkg/core/parser/Makefile:
--------------------------------------------------------------------------------
1 | yaccgen:
2 | 	goyacc -o gram.go -p yy gram.y
3 | gen:
4 | 	ragel -Z -G2 -o lex.go lex.rl
5 | 
6 | build: gen yaccgen


--------------------------------------------------------------------------------
/pkg/core/parser/ast.go:
--------------------------------------------------------------------------------
 1 | package parser
 2 | 
 3 | type Node interface {
 4 | 	iNode()
 5 | }
 6 | 
 7 | type SayHelloCommand struct {
 8 | 	Node
 9 | }
10 | 
11 | type ShowCommand struct {
12 | 	Node
13 | 	Type string
14 | }
15 | 
16 | type KKBCommand struct {
17 | 	Node
18 | }
19 | 
20 | type CopyCommand struct {
21 | 	Node
22 | 	Path    string
23 | 	Options []Node
24 | }
25 | 
26 | type Option struct {
27 | 	Node
28 | 	Name string
29 | 	Arg  Node
30 | }
31 | 
32 | type AExprSConst struct {
33 | 	Node
34 | 	Value string
35 | }
36 | 
37 | type AExprIConst struct {
38 | 	Node
39 | 	Value int
40 | }
41 | 
42 | type AExprBConst struct {
43 | 	Node
44 | 	Value bool
45 | }
46 | 


--------------------------------------------------------------------------------
/pkg/core/parser/gram.y:
--------------------------------------------------------------------------------
  1 | %{
  2 | package parser
  3 | 
  4 | 
  5 | type YpParser yyParser
  6 | 
  7 | func NewYpParser() YpParser {
  8 | 	return yyNewParser()
  9 | }
 10 | 
 11 | %}
 12 | 
 13 | // fields inside this union end up as the fields in a structure known
 14 | // as ${PREFIX}SymType, of which a reference is passed to the lexer.
 15 | %union {
 16 | 	str                    string
 17 |     int                    int
 18 |     node				   Node
 19 |     nodeList		       []Node
 20 |     bool			       bool
 21 | }
 22 | 
 23 | // any non-terminal which returns a value needs a type, which is
 24 | // really a field name in the above union struct
 25 | 
 26 | // CMDS
 27 | %type <node> command
 28 | 
 29 | %type<node> say_hello_command show_command copy_command
 30 | %type<node> kurt_kobain_command
 31 | 
 32 | %type<str> reversed_keyword
 33 | 
 34 | 
 35 | /* basic words */
 36 | %token<str> SAY HELLO
 37 | 
 38 | /* stats */
 39 | %token<str> SHOW
 40 | 
 41 | /* copy */
 42 | %token<str> COPY WITH
 43 | %type<node> copy_gengeneric_opt_elem copy_generic_opt_arg
 44 | %type<bool> opt_boolean
 45 | %type<str> ColLabel
 46 | %type<nodeList> copy_options copy_gengeneric_opt_list
 47 | %token<str> TOPENBR TCLOSEBR TCOMMA
 48 | %token<str> FALSE_P TRUE_P 
 49 | 
 50 | /* pseudo-sql */
 51 | %token<str> SELECT FROM WHERE ORDER BY SORT ASC DESC GROUP
 52 | 
 53 | /* misc */
 54 | %token<str> KURT KOBAIN STOP SYSTEM
 55 | 
 56 | // same for terminals
 57 | %token <str> SCONST IDENT
 58 | %token <int> ICONST
 59 | 
 60 | /* '=' */
 61 | %token<str> TEQ
 62 | 
 63 | /* ';' != */
 64 | %token<str> TSEMICOLON 
 65 | 
 66 | %start any_command
 67 | 
 68 | %%
 69 | 
 70 | any_command:
 71 |     command semicolon_opt
 72 | 
 73 | semicolon_opt:
 74 |     TSEMICOLON {}
 75 |     | /*empty*/ {}
 76 |     ;
 77 | 
 78 | reversed_keyword:
 79 |       SAY {$$=$1}
 80 |     | HELLO {$$=$1}
 81 |     ;
 82 | 
 83 | 
 84 | command:
 85 |     say_hello_command
 86 |     {
 87 |         setParseTree(yylex, $1)
 88 |     } |
 89 |     show_command {
 90 |         setParseTree(yylex, $1)
 91 |     } | 
 92 |     kurt_kobain_command {
 93 |         setParseTree(yylex, $1)
 94 |     } | 
 95 |     copy_command {
 96 |         setParseTree(yylex, $1)
 97 |     } | /* nothing */ { $$ = nil }
 98 | 
 99 | say_hello_command:
100 |     SAY HELLO { $$ = &SayHelloCommand{} } 
101 |     ;
102 | 
103 | show_command:
104 |     SHOW IDENT {
105 |         $$ = &ShowCommand{
106 |             Type: $2,
107 |         }
108 |     }
109 |     ;
110 | 
111 | copy_command:
112 |     COPY SCONST WITH copy_options {
113 |         $$ = &CopyCommand{
114 |             Path: $2,
115 |             Options: $4,
116 |         }
117 |     }
118 |     ;
119 | 
120 | copy_options: TOPENBR copy_gengeneric_opt_list TCLOSEBR { $$ = $2; };
121 | 
122 | copy_gengeneric_opt_list:
123 |     copy_gengeneric_opt_elem
124 |     {
125 |         $$ = []Node{$1};
126 |     }
127 |     | copy_gengeneric_opt_list TCOMMA copy_gengeneric_opt_elem
128 |     {
129 |         $$ = append($1, $3);
130 |     }
131 |     ;
132 | 
133 | copy_gengeneric_opt_elem:
134 |     ColLabel copy_generic_opt_arg
135 |     {
136 |         $$ = &Option{
137 |             Name: $1,
138 |             Arg: $2,
139 |         }
140 |     }
141 |     ;
142 | 
143 | copy_generic_opt_arg:
144 |     opt_boolean			            { $$ = &AExprBConst{Value: $1} }
145 |     | ICONST					    { $$ = &AExprIConst{Value: $1} }
146 |     | SCONST                        { $$ = &AExprSConst{Value: $1} }
147 |     | /* EMPTY */					{  }
148 |     ;
149 | 
150 | opt_boolean:
151 |     TRUE_P									{ $$ = true }
152 |     | FALSE_P								{ $$ =  false }
153 |     //| NonReservedWord_or_SCONST				{ $$ = $1; }
154 |     ;
155 | 
156 | ColLabel:	
157 |     IDENT									{ $$ = $1; }
158 |     ;
159 | 
160 | kurt_kobain_command:
161 |     STOP SYSTEM {
162 |         $$ = &KKBCommand{
163 |         }
164 |     } | KURT KOBAIN {
165 |         $$ = &KKBCommand{
166 |         }
167 |     }
168 |     ;
169 | 


--------------------------------------------------------------------------------
/pkg/core/parser/lex.rl:
--------------------------------------------------------------------------------
  1 | package parser
  2 | 
  3 | import (
  4 |     "strconv"
  5 | )
  6 | 
  7 | %%{ 
  8 |     machine lexer;
  9 |     write data;
 10 |     access lex.;
 11 |     variable p lex.p;
 12 |     variable pe lex.pe;
 13 | }%%
 14 | 
 15 | type Lexer struct {
 16 | 	data         []byte
 17 | 	p, pe, cs    int
 18 | 	ts, te, act  int
 19 | 
 20 | 	result []string
 21 | }
 22 | 
 23 | func NewLexer(data []byte) *Lexer {
 24 |     lex := &Lexer{ 
 25 |         data: data,
 26 |         pe: len(data),
 27 |     }
 28 |     %% write init;
 29 |     return lex
 30 | }
 31 | 
 32 | func ResetLexer(lex *Lexer, data []byte) {
 33 |     lex.pe = len(data)
 34 |     lex.data = data
 35 |     %% write init;
 36 | }
 37 | 
 38 | func (l *Lexer) Error(msg string) {
 39 | 	println(msg)
 40 | }
 41 | 
 42 | 
 43 | func (lex *Lexer) Lex(lval *yySymType) int {
 44 |     eof := lex.pe
 45 |     var tok int
 46 | 
 47 |     %%{
 48 |         # /* digit = [0-9] ; already defined */
 49 | 
 50 | 
 51 | #        xcstart		=	\/\*{op_chars}*;
 52 | #        xcstop		=	\*+\/;
 53 | #        xcinside	=	[^*/]+;
 54 | 
 55 |         integer = digit+;
 56 |         ninteger = '-' integer;
 57 |         param = '$' integer;
 58 |         
 59 |         decimal	= ((digit*'.'digit+)|(digit+'.'digit*));
 60 |         real = (decimal)|('-'decimal);
 61 | 
 62 |         ident_start	=	[A-Za-z\200-\377_];
 63 |         ident_cont	=	[A-Za-z\200-\377_0-9$];
 64 | 
 65 |         identifier	=	ident_start ident_cont*;
 66 | 
 67 |         qidentifier	=	'"' ident_start ident_cont* '"' ;
 68 | 
 69 | 
 70 | #        space		=	[ \t\n\r\f];
 71 |         horiz_space	= [ \t\f];
 72 |         newline		=	[\n\r];
 73 |         non_newline	=	[^\n\r];
 74 | 
 75 |         sql_comment = '-''-' non_newline*;
 76 |         c_style_comment = '/''*' (any - '*''/')* '*''/';
 77 |         comment		= sql_comment | c_style_comment;
 78 | 
 79 | 
 80 | #       whitespace	=	({space}+|{comment});
 81 |         whitespace	=	space+;
 82 | 
 83 | 
 84 |         op_chars	=	( '~' | '!' | '@' | '#' | '^' | '&' | '|' | '`' | '?' | '+' | '-' | '*' | '\\' | '%' | '<' | '>' | '=' ) ;
 85 |         operator	=	op_chars+;
 86 | 
 87 |         sconst = '\'' (any-'\'')* '\'';
 88 |         
 89 |         main := |*
 90 |             whitespace => { /* do nothing */ };
 91 |             # integer const is string const 
 92 |             comment => {/* nothing */};
 93 | 
 94 |             integer =>  { lval.int, _ = strconv.Atoi(string(lex.data[lex.ts:lex.te])); tok = ICONST; fbreak;};
 95 |             ninteger => { lval.int, _ = strconv.Atoi(string(lex.data[lex.ts:lex.te])); tok = ICONST; fbreak;};
 96 | 
 97 |             real =>  { lval.str = string(lex.data[lex.ts:lex.te]); tok = SCONST; fbreak;};
 98 | 
 99 |             /SAY/i => { lval.str = string(lex.data[lex.ts:lex.te]); tok = SAY; fbreak;};
100 |             /HELLO/i => { lval.str = string(lex.data[lex.ts:lex.te]); tok = HELLO; fbreak;};
101 |             /SHOW/i => { lval.str = string(lex.data[lex.ts:lex.te]); tok = SHOW; fbreak;};
102 | 
103 |             /COPY/i => { lval.str = string(lex.data[lex.ts:lex.te]); tok = COPY; fbreak;};
104 |             /WITH/i => { lval.str = string(lex.data[lex.ts:lex.te]); tok = WITH; fbreak;};
105 | 
106 |             /KURT/i => { lval.str = string(lex.data[lex.ts:lex.te]); tok = KURT; fbreak;};
107 |             /KOBAIN/i => { lval.str = string(lex.data[lex.ts:lex.te]); tok = KOBAIN; fbreak;};
108 | 
109 |             /STOP/i => { lval.str = string(lex.data[lex.ts:lex.te]); tok = STOP; fbreak;};
110 |             /SYSTEM/i => { lval.str = string(lex.data[lex.ts:lex.te]); tok = SYSTEM; fbreak;};
111 | 
112 |             qidentifier      => { lval.str = string(lex.data[lex.ts + 1:lex.te - 1]); tok = IDENT; fbreak;};
113 |             identifier      => { lval.str = string(lex.data[lex.ts:lex.te]); tok = IDENT; fbreak;};
114 |             sconst      => { lval.str = string(lex.data[lex.ts + 1:lex.te - 1]); tok = SCONST; fbreak;};
115 | 
116 |             '=' => { lval.str = string(lex.data[lex.ts:lex.te]); tok = TEQ; fbreak;};
117 |             '(' => { lval.str = string(lex.data[lex.ts:lex.te]); tok = TOPENBR; fbreak;};
118 |             ')' => { lval.str = string(lex.data[lex.ts:lex.te]); tok = TCLOSEBR; fbreak;};
119 |             ',' => { lval.str = string(lex.data[lex.ts:lex.te]); tok = TCOMMA; fbreak;};
120 | 
121 |             /true/i => { lval.str = string(lex.data[lex.ts:lex.te]); tok = TRUE_P; fbreak;};
122 |             /false/i => { lval.str = string(lex.data[lex.ts:lex.te]); tok = FALSE_P; fbreak;};
123 | 
124 |         *|;
125 | 
126 |         write exec;
127 |     }%%
128 | 
129 |     return int(tok);
130 | }


--------------------------------------------------------------------------------
/pkg/core/parser/utils.go:
--------------------------------------------------------------------------------
 1 | package parser
 2 | 
 3 | import (
 4 | 	"errors"
 5 | 	"fmt"
 6 | )
 7 | 
 8 | // Tokenizer is the struct used to generate SQL
 9 | // tokens for the parser.
10 | type Tokenizer struct {
11 | 	s   string
12 | 	pos int
13 | 
14 | 	ParseTree Node
15 | 	LastError string
16 | 	l         *Lexer
17 | }
18 | 
19 | func (t *Tokenizer) Error(s string) {
20 | 	t.LastError = s
21 | }
22 | 
23 | func NewStringTokenizer(sql string) *Tokenizer {
24 | 	return &Tokenizer{
25 | 		s: sql,
26 | 		l: NewLexer([]byte(sql)),
27 | 	}
28 | }
29 | 
30 | func (t *Tokenizer) Lex(lval *yySymType) int {
31 | 	return t.l.Lex(lval)
32 | }
33 | 
34 | func (t *Tokenizer) LexT() int {
35 | 	return t.l.Lex(new(yySymType))
36 | }
37 | 
38 | func (t *Tokenizer) Reset(sql string) {
39 | 	t.s = sql
40 | 	ResetLexer(t.l, []byte(sql))
41 | }
42 | 
43 | func setParseTree(yylex interface{}, stmt Node) {
44 | 	yylex.(*Tokenizer).ParseTree = stmt
45 | }
46 | 
47 | func Parse(sql string) (Node, error) {
48 | 	tokenizer := NewStringTokenizer(sql)
49 | 	if yyParse(tokenizer) != 0 {
50 | 		return nil, errors.New(tokenizer.LastError + fmt.Sprintf(" on pos %d", tokenizer.l.ts))
51 | 	}
52 | 	ast := tokenizer.ParseTree
53 | 	return ast, nil
54 | }
55 | 
56 | func ParseWithLexerParser(l YpParser, t *Tokenizer, sql string) (Node, error) {
57 | 	t.Reset(sql)
58 | 	if l.Parse(t) != 0 {
59 | 		return nil, errors.New(t.LastError + fmt.Sprintf(" on pos %d", t.l.ts))
60 | 	}
61 | 	ast := t.ParseTree
62 | 	return ast, nil
63 | }
64 | 


--------------------------------------------------------------------------------
/pkg/core/parser/y_test.go:
--------------------------------------------------------------------------------
 1 | package parser_test
 2 | 
 3 | import (
 4 | 	"testing"
 5 | 
 6 | 	"github.com/stretchr/testify/assert"
 7 | 	"github.com/yezzey-gp/yproxy/pkg/core/parser"
 8 | )
 9 | 
10 | func TestParse(t *testing.T) {
11 | 	assert := assert.New(t)
12 | 
13 | 	type tcase struct {
14 | 		query string
15 | 		exp   parser.Node
16 | 		err   error
17 | 	}
18 | 
19 | 	/*  */
20 | 	for _, tt := range []tcase{
21 | 		{
22 | 			query: "say hello",
23 | 			exp:   &parser.SayHelloCommand{},
24 | 			err:   nil,
25 | 		},
26 | 		{
27 | 			query: "show clients",
28 | 			exp: &parser.ShowCommand{
29 | 				Type: "clients",
30 | 			},
31 | 			err: nil,
32 | 		},
33 | 		{
34 | 			query: "show connections",
35 | 			exp: &parser.ShowCommand{
36 | 				Type: "connections",
37 | 			},
38 | 			err: nil,
39 | 		},
40 | 		{
41 | 			query: `STOP SYSTEM`,
42 | 			exp:   &parser.KKBCommand{},
43 | 			err:   nil,
44 | 		},
45 | 		{
46 | 			query: `COPY '/prefix' WITH (CONFIG '/etc/yproxy/old_config.yaml', PORT 6002)`,
47 | 			exp: &parser.CopyCommand{
48 | 				Path: "/prefix",
49 | 				Options: []parser.Node{
50 | 					&parser.Option{Name: "CONFIG", Arg: &parser.AExprSConst{Value: "/etc/yproxy/old_config.yaml"}},
51 | 					&parser.Option{Name: "PORT", Arg: &parser.AExprIConst{Value: 6002}},
52 | 				},
53 | 			},
54 | 			err: nil,
55 | 		},
56 | 	} {
57 | 		tmp, err := parser.Parse(tt.query)
58 | 
59 | 		assert.NoError(err, "query %s", tt.query)
60 | 
61 | 		assert.Equal(tt.exp, tmp, "query %s", tt.query)
62 | 	}
63 | }
64 | 


--------------------------------------------------------------------------------
/pkg/core/pprof.go:
--------------------------------------------------------------------------------
 1 | package core
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"expvar"
 6 | 	"fmt"
 7 | 	"net/http"
 8 | 	"net/http/pprof"
 9 | 	"sync"
10 | 	"time"
11 | 
12 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
13 | )
14 | 
15 | func NewDebugWebServer(httpAddr string) *DebugWebServer {
16 | 	return &DebugWebServer{
17 | 		httpAddr: httpAddr,
18 | 	}
19 | }
20 | 
21 | type DebugWebServer struct {
22 | 	*http.Server
23 | 	*http.ServeMux
24 | 	mu       sync.Mutex
25 | 	httpAddr string
26 | 	timer    *time.Timer
27 | }
28 | 
29 | // ServeFor starts pprof webserver in the background goroutine.
30 | // After Duration `t` server shuts down for security reasons.
31 | func (dws *DebugWebServer) ServeFor(t time.Duration) error {
32 | 	dws.mu.Lock()
33 | 	defer dws.mu.Unlock()
34 | 
35 | 	if dws.timer != nil {
36 | 		postponed := dws.timer.Reset(t)
37 | 		if !postponed {
38 | 			// Reset() started new countdown (because previous were executed), however we just
39 | 			// observed that `dws.timer` is not nil... it means that AfterFunc`s is running
40 | 			// concurrently (waiting for mutex)
41 | 			return fmt.Errorf("previous debug server is being shutdown, retry later")
42 | 		}
43 | 		return nil
44 | 	}
45 | 
46 | 	dws.configureDebugWebServer()
47 | 	dws.timer = time.AfterFunc(t, func() {
48 | 		err := dws.Shutdown(context.Background())
49 | 		if err != nil {
50 | 			ylogger.Zero.Error().Err(err).Msg("Failed to shutdown debug webserver")
51 | 		}
52 | 	})
53 | 	go func() {
54 | 		_ = dws.ListenAndServe()
55 | 	}()
56 | 
57 | 	return nil
58 | }
59 | 
60 | func (dws *DebugWebServer) configureDebugWebServer() {
61 | 	mux := http.NewServeMux()
62 | 	dws.Server = &http.Server{
63 | 		Addr:    dws.httpAddr,
64 | 		Handler: mux,
65 | 	}
66 | 	dws.ServeMux = mux
67 | 
68 | 	dws.enablePprofEndpoints()
69 | 	dws.enableExpVarEndpoints()
70 | }
71 | 
72 | // Shutdown synchronously shuts down debug server (if started).
73 | func (dws *DebugWebServer) Shutdown(ctx context.Context) error {
74 | 	dws.mu.Lock()
75 | 	defer dws.mu.Unlock()
76 | 
77 | 	if dws.timer == nil {
78 | 		return nil
79 | 	}
80 | 	dws.timer = nil
81 | 	return dws.Server.Shutdown(ctx)
82 | }
83 | 
84 | // enablePprofEndpoints exposes pprof http endpoints.
85 | func (dws *DebugWebServer) enablePprofEndpoints() {
86 | 	dws.HandleFunc("/debug/pprof/", pprof.Index)
87 | 	dws.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
88 | 	dws.HandleFunc("/debug/pprof/profile", pprof.Profile)
89 | 	dws.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
90 | 	dws.HandleFunc("/debug/pprof/trace", pprof.Trace)
91 | }
92 | 
93 | // enableExpVarEndpoints exposes expvar http endpoints.
94 | func (dws *DebugWebServer) enableExpVarEndpoints() {
95 | 	dws.HandleFunc("/debug/vars", expvar.Handler().ServeHTTP)
96 | }
97 | 


--------------------------------------------------------------------------------
/pkg/crypt/crypt.go:
--------------------------------------------------------------------------------
  1 | package crypt
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	"io"
  6 | 	"os"
  7 | 
  8 | 	"github.com/pkg/errors"
  9 | 	"github.com/yezzey-gp/yproxy/config"
 10 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
 11 | 
 12 | 	"github.com/ProtonMail/go-crypto/openpgp"
 13 | )
 14 | 
 15 | type KeyVersion byte
 16 | 
 17 | const (
 18 | 	SingleKeyEncryption = KeyVersion(iota + 1)
 19 | 	KEKDEKEncryption
 20 | )
 21 | 
 22 | type Crypter interface {
 23 | 	Decrypt(reader io.ReadCloser) (io.Reader, error)
 24 | 	Encrypt(writer io.WriteCloser) (io.WriteCloser, error)
 25 | 
 26 | 	CmpKey(path string) (bool, error)
 27 | }
 28 | 
 29 | type GPGCrypter struct {
 30 | 	EntityList openpgp.EntityList
 31 | 
 32 | 	cnf *config.Crypto
 33 | }
 34 | 
 35 | func NewCrypto(cnf *config.Crypto) (Crypter, error) {
 36 | 	cr := &GPGCrypter{
 37 | 		cnf: cnf,
 38 | 	}
 39 | 
 40 | 	err := cr.loadSecret()
 41 | 	if err != nil {
 42 | 		return nil, err
 43 | 	}
 44 | 
 45 | 	return cr, nil
 46 | }
 47 | 
 48 | func (g *GPGCrypter) readKey(path string) (io.Reader, error) {
 49 | 	byteData, err := os.ReadFile(path)
 50 | 
 51 | 	if err != nil {
 52 | 		return nil, err
 53 | 	}
 54 | 
 55 | 	return bytes.NewReader(byteData), nil
 56 | }
 57 | 
 58 | func (g *GPGCrypter) readPGPKey() (openpgp.EntityList, error) {
 59 | 	gpgKeyReader, err := g.readKey(g.cnf.GPGKeyPath)
 60 | 
 61 | 	if err != nil {
 62 | 		return nil, err
 63 | 	}
 64 | 
 65 | 	entityList, err := openpgp.ReadArmoredKeyRing(gpgKeyReader)
 66 | 
 67 | 	if err != nil {
 68 | 		return nil, err
 69 | 	}
 70 | 
 71 | 	return entityList, nil
 72 | }
 73 | 
 74 | func (g *GPGCrypter) loadSecret() error {
 75 | 	entityList, err := g.readPGPKey()
 76 | 
 77 | 	if err != nil {
 78 | 		return errors.WithStack(err)
 79 | 	}
 80 | 
 81 | 	g.EntityList = entityList
 82 | 
 83 | 	return nil
 84 | }
 85 | 
 86 | func (g *GPGCrypter) Decrypt(reader io.ReadCloser) (io.Reader, error) {
 87 | 
 88 | 	ylogger.Zero.Debug().Str("gpg path", g.cnf.GPGKeyPath).Msg("loaded gpg key")
 89 | 
 90 | 	md, err := openpgp.ReadMessage(reader, g.EntityList, nil, nil)
 91 | 
 92 | 	if err != nil {
 93 | 		return nil, errors.WithStack(err)
 94 | 	}
 95 | 
 96 | 	return md.UnverifiedBody, nil
 97 | }
 98 | 
 99 | func (g *GPGCrypter) Encrypt(writer io.WriteCloser) (io.WriteCloser, error) {
100 | 	ylogger.Zero.Debug().Str("gpg path", g.cnf.GPGKeyPath).Msg("loaded gpg key")
101 | 
102 | 	encryptedWriter, err := openpgp.Encrypt(writer, g.EntityList, nil, nil, nil)
103 | 
104 | 	if err != nil {
105 | 		return nil, errors.WithStack(err)
106 | 	}
107 | 
108 | 	return encryptedWriter, nil
109 | }
110 | 
111 | func (g *GPGCrypter) CmpKey(path string) (bool, error) {
112 | 	this, err := os.ReadFile(g.cnf.GPGKeyPath)
113 | 	if err != nil {
114 | 		return false, err
115 | 	}
116 | 	other, err := os.ReadFile(path)
117 | 	if err != nil {
118 | 		return false, err
119 | 	}
120 | 	return bytes.Equal(this, other), nil
121 | }
122 | 


--------------------------------------------------------------------------------
/pkg/database/database.go:
--------------------------------------------------------------------------------
  1 | package database
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 
  6 | 	"github.com/jackc/pgx"
  7 | 	"github.com/jackc/pgx/pgtype"
  8 | 	"github.com/pkg/errors"
  9 | 	"github.com/yezzey-gp/yproxy/pkg/testutils"
 10 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
 11 | )
 12 | 
 13 | //go:generate mockgen -destination=../mock/mock_database_interractor.go -package mock
 14 | type DatabaseInterractor interface {
 15 | 	GetVirtualExpireIndexes(port uint64) (map[string]bool, map[string]uint64, error)
 16 | }
 17 | 
 18 | type DatabaseHandler struct {
 19 | }
 20 | 
 21 | type DB struct {
 22 | 	name       string
 23 | 	tablespace pgtype.OID
 24 | 	oid        pgtype.OID
 25 | }
 26 | 
 27 | type ExpireHint struct {
 28 | 	expireLsn string
 29 | 	x_path    string
 30 | }
 31 | type LSN struct {
 32 | 	lsn string
 33 | }
 34 | 
 35 | func checkVersion(c *pgx.Conn, exp string) (bool, error) {
 36 | 	rows, err := c.Query(`SELECT extversion FROM pg_extension WHERE extname = 'yezzey';`)
 37 | 	if err != nil {
 38 | 		return false, fmt.Errorf("unable to get ao/aocs tables %v", err) //fix
 39 | 	}
 40 | 
 41 | 	defer rows.Close()
 42 | 	ylogger.Zero.Debug().Msg("executed select")
 43 | 
 44 | 	if rows.Next() {
 45 | 		var ver string
 46 | 		if err := rows.Scan(&ver); err != nil {
 47 | 			return false, fmt.Errorf("unable to parse query output %v", err)
 48 | 		}
 49 | 		if rows.Next() {
 50 | 			return false, fmt.Errorf("unable to get yezzey extension version: duplicate output")
 51 | 		}
 52 | 
 53 | 		/* we compare versions lexicographically */
 54 | 		return ver >= exp, nil
 55 | 	}
 56 | 
 57 | 	return false, fmt.Errorf("unable to get yezzey extension version")
 58 | }
 59 | 
 60 | func (database *DatabaseHandler) GetVirtualExpireIndex(port uint64, db DB, virtualIndex *map[string]bool, expireIndex *map[string]uint64) error {
 61 | 	ylogger.Zero.Debug().Str("database name", db.name).Msg("received database")
 62 | 	conn, err := connectToDatabase(port, db.name)
 63 | 	if err != nil {
 64 | 		return err
 65 | 	}
 66 | 	defer func() { _ = conn.Close() }() //error
 67 | 	ylogger.Zero.Debug().Str("database name", db.name).Msg("GetVirtualExpireIndex: connected to database")
 68 | 
 69 | 	/* Todo: check that yezzey version >= 1.8.4 */
 70 | 	if ch, err := checkVersion(conn, "1.8.4"); err != nil {
 71 | 		ylogger.Zero.Debug().Err(err).Msg("GetVirtualExpireIndex: failed")
 72 | 		return err
 73 | 	} else if ch {
 74 | 		rows, err := conn.Query(`SELECT x_path, lsn FROM yezzey.yezzey_expire_hint;`)
 75 | 		if err != nil {
 76 | 			return fmt.Errorf("unable to get ao/aocs tables %v", err) //fix
 77 | 		}
 78 | 		defer rows.Close()
 79 | 		ylogger.Zero.Debug().Msg("executed select")
 80 | 
 81 | 		for rows.Next() {
 82 | 			row := ExpireHint{}
 83 | 			if err := rows.Scan(&row.x_path, &row.expireLsn); err != nil {
 84 | 				return fmt.Errorf("unable to parse query output %v", err)
 85 | 			}
 86 | 
 87 | 			lsn, err := pgx.ParseLSN(row.expireLsn)
 88 | 			if err != nil {
 89 | 				return fmt.Errorf("unable to parse query output %v", err)
 90 | 			}
 91 | 
 92 | 			ylogger.Zero.Debug().Str("x_path", row.x_path).Str("lsn", row.expireLsn).Msg("added file to expire hint")
 93 | 			(*expireIndex)[row.x_path] = lsn
 94 | 		}
 95 | 		ylogger.Zero.Debug().Msg("fetched expire hint info")
 96 | 	}
 97 | 
 98 | 	viRows, err := conn.Query(`SELECT x_path FROM yezzey.yezzey_virtual_index;`)
 99 | 	if err != nil {
100 | 		return fmt.Errorf("unable to get ao/aocs tables %v", err) //fix
101 | 	}
102 | 	defer viRows.Close()
103 | 
104 | 	for viRows.Next() {
105 | 		xpath := ""
106 | 		if err := viRows.Scan(&xpath); err != nil {
107 | 			return fmt.Errorf("unable to parse query output %v", err)
108 | 		}
109 | 		(*virtualIndex)[xpath] = true
110 | 		ylogger.Zero.Debug().Str("x_path", xpath).Msg("added")
111 | 	}
112 | 	ylogger.Zero.Debug().Msg("fetched virtual index info")
113 | 
114 | 	return err
115 | }
116 | 
117 | func (database *DatabaseHandler) GetNextLSN(port uint64, dbname string) (uint64, error) {
118 | 	ylogger.Zero.Debug().Str("database name", dbname).Msg("received database")
119 | 	conn, err := connectToDatabase(port, dbname)
120 | 	if err != nil {
121 | 		return 0, err
122 | 	}
123 | 	defer func() { _ = conn.Close() }() //error
124 | 	ylogger.Zero.Debug().Str("database name", dbname).Msg("GetNextLSN: connected to database")
125 | 
126 | 	dbRow := conn.QueryRow(`select pg_current_xlog_location();`)
127 | 	ylogger.Zero.Debug().Msg("executed select")
128 | 
129 | 	row := LSN{}
130 | 	if err := dbRow.Scan(&row.lsn); err != nil {
131 | 		return 0, fmt.Errorf("unable to parse query output %v", err)
132 | 	}
133 | 	lsn, err := pgx.ParseLSN(row.lsn)
134 | 	if err != nil {
135 | 		return 0, fmt.Errorf("unable to parse query output %v", err)
136 | 	}
137 | 
138 | 	ylogger.Zero.Debug().Uint64("lsn", lsn).Msg("received lsn")
139 | 	return lsn, nil
140 | }
141 | 
142 | func (database *DatabaseHandler) GetVirtualExpireIndexes(port uint64) (map[string]bool, map[string]uint64, error) { //TODO несколько баз
143 | 	databases, err := getDatabase(port)
144 | 	if err != nil || databases == nil {
145 | 		return nil, nil, fmt.Errorf("unable to get ao/aocs tables %v", err) //fix
146 | 	}
147 | 
148 | 	expireIndex := make(map[string]uint64, 0)
149 | 	virtualIndex := make(map[string]bool, 0)
150 | 	for _, db := range databases {
151 | 		err = database.GetVirtualExpireIndex(port, db, &virtualIndex, &expireIndex)
152 | 		if err != nil {
153 | 			return nil, nil, err
154 | 		}
155 | 
156 | 	}
157 | 	return virtualIndex, expireIndex, nil
158 | }
159 | 
160 | func (database *DatabaseHandler) GetConnectToDatabase(port uint64, dbname string) (*pgx.Conn, error) {
161 | 	ylogger.Zero.Debug().Str("database name", dbname).Msg("received database")
162 | 	conn, err := connectToDatabase(port, dbname)
163 | 	if err != nil {
164 | 		return nil, err
165 | 	}
166 | 	ylogger.Zero.Debug().Str("database name", dbname).Msg("connected to database")
167 | 	return conn, nil
168 | 
169 | }
170 | func (database *DatabaseHandler) AddToExpireIndex(conn *pgx.Conn, port uint64, dbname string, filename string, lsn uint64) error {
171 | 	rows, err := conn.Query(`INSERT INTO yezzey.yezzey_expire_hint (lsn,x_path) VALUES ($1 , $2);`, pgx.FormatLSN(lsn), filename)
172 | 	if err != nil {
173 | 		return fmt.Errorf("unable to update yezzey_expire_hint %v", err) //fix
174 | 	}
175 | 	defer rows.Close()
176 | 
177 | 	return nil
178 | }
179 | 
180 | func (database *DatabaseHandler) DeleteFromExpireIndex(conn *pgx.Conn, port uint64, dbname string, filename string) error {
181 | 	rows, err := conn.Query(`DELETE FROM yezzey.yezzey_expire_hint WHERE x_path = $1;`, filename)
182 | 	if err != nil {
183 | 		return fmt.Errorf("unable to delete from yezzey_expire_hint %v", err) //fix
184 | 	}
185 | 	defer rows.Close()
186 | 
187 | 	return nil
188 | }
189 | func getDatabase(port uint64) ([]DB, error) {
190 | 	var databases = []DB{}
191 | 	conn, err := connectToDatabase(port, "postgres")
192 | 	if err != nil {
193 | 		return nil, err
194 | 	}
195 | 	defer func() { _ = conn.Close() }() //error
196 | 	ylogger.Zero.Debug().Msg("connected to db")
197 | 	rows, err := conn.Query(`SELECT dattablespace, oid, datname FROM pg_database WHERE datallowconn;`)
198 | 	if err != nil {
199 | 		return nil, err
200 | 	}
201 | 	defer rows.Close()
202 | 	ylogger.Zero.Debug().Msg("received db list")
203 | 
204 | 	for rows.Next() {
205 | 		row := DB{}
206 | 		if err := rows.Scan(&row.tablespace, &row.oid, &row.name); err != nil {
207 | 			return nil, err
208 | 		}
209 | 		ylogger.Zero.Debug().Str("db", row.name).Int("db", int(row.oid)).Int("db", int(row.tablespace)).Msg("database")
210 | 		if row.name == "postgres" {
211 | 			continue
212 | 		}
213 | 
214 | 		ylogger.Zero.Debug().Str("db", row.name).Msg("check database")
215 | 		connDb, err := connectToDatabase(port, row.name)
216 | 		if err != nil {
217 | 			return nil, err
218 | 		}
219 | 		defer func() { _ = connDb.Close() }() //error
220 | 
221 | 		rowsdb, err := connDb.Query(`SELECT exists(SELECT * FROM information_schema.schemata WHERE schema_name='yezzey');`)
222 | 		if err != nil {
223 | 			return nil, err
224 | 		}
225 | 		defer rowsdb.Close()
226 | 		var ans bool
227 | 		rowsdb.Next()
228 | 		err = rowsdb.Scan(&ans)
229 | 		if err != nil {
230 | 			ylogger.Zero.Error().AnErr("error", err).Msg("error during yezzey check")
231 | 			return nil, err
232 | 		}
233 | 		ylogger.Zero.Debug().Bool("result", ans).Msg("find yezzey schema")
234 | 		if ans {
235 | 			ylogger.Zero.Debug().Str("db", row.name).Msg("found yezzey schema in database")
236 | 			ylogger.Zero.Debug().Int("db", int(row.oid)).Int("db", int(row.tablespace)).Msg("found yezzey schema in database")
237 | 			databases = append(databases, row)
238 | 		}
239 | 
240 | 		ylogger.Zero.Debug().Str("db", row.name).Msg("no yezzey schema in database")
241 | 	}
242 | 	if len(databases) == 0 {
243 | 		return nil, fmt.Errorf("no yezzey schema across databases")
244 | 
245 | 	} else {
246 | 		return databases, nil
247 | 	}
248 | }
249 | 
250 | func connectToDatabase(port uint64, database string) (*pgx.Conn, error) {
251 | 	config, err := pgx.ParseEnvLibpq()
252 | 	if err != nil {
253 | 		return nil, errors.Wrap(err, "Connect: unable to read environment variables")
254 | 	}
255 | 
256 | 	config.Port = uint16(port)
257 | 	config.Database = database
258 | 
259 | 	if testutils.TestMode {
260 | 		// Do not set GP-specific params
261 | 		return pgx.Connect(config)
262 | 	}
263 | 	config.RuntimeParams["gp_role"] = "utility"
264 | 	conn, err := pgx.Connect(config)
265 | 	if err != nil {
266 | 		config.RuntimeParams["gp_session_role"] = "utility"
267 | 		conn, err = pgx.Connect(config)
268 | 		if err != nil {
269 | 			fmt.Printf("error in connection %v", err) // delete this
270 | 			return nil, err
271 | 		}
272 | 	}
273 | 	return conn, nil
274 | }
275 | 


--------------------------------------------------------------------------------
/pkg/message/cat_message.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | )
 6 | 
 7 | type CatMessage struct {
 8 | 	Decrypt     bool
 9 | 	Name        string
10 | 	StartOffset uint64
11 | }
12 | 
13 | var _ ProtoMessage = &CatMessage{}
14 | 
15 | func NewCatMessage(name string, decrypt bool, StartOffset uint64) *CatMessage {
16 | 	return &CatMessage{
17 | 		Name:        name,
18 | 		Decrypt:     decrypt,
19 | 		StartOffset: StartOffset,
20 | 	}
21 | }
22 | 
23 | func (c *CatMessage) Encode() []byte {
24 | 	bt := []byte{
25 | 		byte(MessageTypeCat),
26 | 		0,
27 | 		0,
28 | 		0,
29 | 	}
30 | 
31 | 	if c.Decrypt {
32 | 		bt[1] = byte(DecryptMessage)
33 | 	} else {
34 | 		bt[1] = byte(NoDecryptMessage)
35 | 	}
36 | 
37 | 	if c.StartOffset != 0 {
38 | 		bt[2] = byte(ExtendedMesssage)
39 | 	}
40 | 
41 | 	bt = append(bt, []byte(c.Name)...)
42 | 	bt = append(bt, 0)
43 | 	if c.StartOffset != 0 {
44 | 		bt = binary.BigEndian.AppendUint64(bt, c.StartOffset)
45 | 	}
46 | 	ln := len(bt) + 8
47 | 
48 | 	bs := make([]byte, 8)
49 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
50 | 	return append(bs, bt...)
51 | }
52 | 
53 | func (c *CatMessage) Decode(body []byte) {
54 | 	c.Name, _ = GetCstring(body[4:])
55 | 	if body[1] == byte(DecryptMessage) {
56 | 		c.Decrypt = true
57 | 	}
58 | 	if body[2] == byte(ExtendedMesssage) {
59 | 		c.StartOffset = binary.BigEndian.Uint64(body[4+len(c.Name)+1:])
60 | 	}
61 | }
62 | 


--------------------------------------------------------------------------------
/pkg/message/cat_message_v2.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | 
 6 | 	"github.com/yezzey-gp/yproxy/pkg/settings"
 7 | )
 8 | 
 9 | type CatMessageV2 struct {
10 | 	Decrypt     bool
11 | 	KEK         bool // If we need to use key encryption key when decrypting data
12 | 	Name        string
13 | 	StartOffset uint64
14 | 
15 | 	Settings []settings.StorageSettings
16 | }
17 | 
18 | var _ ProtoMessage = &CatMessage{}
19 | 
20 | func NewCatMessageV2(name string, decrypt bool, kek bool, StartOffset uint64, Settings []settings.StorageSettings) *CatMessageV2 {
21 | 	return &CatMessageV2{
22 | 		Name:        name,
23 | 		Decrypt:     decrypt,
24 | 		KEK:         kek,
25 | 		StartOffset: StartOffset,
26 | 		Settings:    Settings,
27 | 	}
28 | }
29 | 
30 | func (c *CatMessageV2) Encode() []byte {
31 | 	bt := []byte{
32 | 		byte(MessageTypeCatV2),
33 | 		0,
34 | 		0,
35 | 		0,
36 | 	}
37 | 
38 | 	if c.Decrypt {
39 | 		bt[1] = byte(DecryptMessage)
40 | 	} else {
41 | 		bt[1] = byte(NoDecryptMessage)
42 | 	}
43 | 	if c.KEK {
44 | 		bt[2] = byte(UseKEK)
45 | 	} else {
46 | 		bt[2] = byte(NoUseKEK)
47 | 	}
48 | 
49 | 	bt = append(bt, []byte(c.Name)...)
50 | 	bt = append(bt, 0)
51 | 	bt = binary.BigEndian.AppendUint64(bt, c.StartOffset)
52 | 
53 | 	bt = binary.BigEndian.AppendUint64(bt, uint64(len(c.Settings)))
54 | 
55 | 	for _, s := range c.Settings {
56 | 
57 | 		bt = append(bt, []byte(s.Name)...)
58 | 		bt = append(bt, 0)
59 | 
60 | 		bt = append(bt, []byte(s.Value)...)
61 | 		bt = append(bt, 0)
62 | 	}
63 | 
64 | 	ln := len(bt) + 8
65 | 
66 | 	bs := make([]byte, 8)
67 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
68 | 	return append(bs, bt...)
69 | }
70 | 
71 | func (c *CatMessageV2) Decode(body []byte) {
72 | 	var off uint64
73 | 	c.Name, off = GetCstring(body[4:])
74 | 	if body[1] == byte(DecryptMessage) {
75 | 		c.Decrypt = true
76 | 	}
77 | 	if body[2] == byte(UseKEK) {
78 | 		c.KEK = true
79 | 	}
80 | 	c.StartOffset = binary.BigEndian.Uint64(body[4+len(c.Name)+1:])
81 | 
82 | 	settLen := binary.BigEndian.Uint64(body[4+8+off : 4+off+8+8])
83 | 
84 | 	totalOff := 4 + off + 8 + 8
85 | 
86 | 	c.Settings = make([]settings.StorageSettings, settLen)
87 | 
88 | 	for i := range int(settLen) {
89 | 
90 | 		var currOff uint64
91 | 
92 | 		c.Settings[i].Name, currOff = GetCstring(body[totalOff:])
93 | 		totalOff += currOff
94 | 
95 | 		c.Settings[i].Value, currOff = GetCstring(body[totalOff:])
96 | 		totalOff += currOff
97 | 	}
98 | }
99 | 


--------------------------------------------------------------------------------
/pkg/message/common.go:
--------------------------------------------------------------------------------
1 | package message
2 | 
3 | const (
4 | 	StorageClassSetting = "StorageClass"
5 | 	TableSpaceSetting   = "TableSpace"
6 | 	MultipartChunkSize  = "MultipartChunkSize"
7 | 	MultipartUpload     = "MultipartUpload"
8 | )
9 | 


--------------------------------------------------------------------------------
/pkg/message/copy_complete.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | )
 6 | 
 7 | type CopyCompleteMessage struct {
 8 | 	KeyVersion byte
 9 | }
10 | 
11 | var _ ProtoMessage = &CopyCompleteMessage{}
12 | 
13 | func NewCopyCompleteMessage(keyVersion byte) *CopyCompleteMessage {
14 | 	return &CopyCompleteMessage{
15 | 		KeyVersion: keyVersion,
16 | 	}
17 | }
18 | 
19 | func (c *CopyCompleteMessage) Encode() []byte {
20 | 	bt := []byte{
21 | 		byte(MessageTypeCopyComplete),
22 | 		c.KeyVersion,
23 | 		0,
24 | 		0,
25 | 	}
26 | 	ln := len(bt) + 8
27 | 
28 | 	bs := make([]byte, 8)
29 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
30 | 	return append(bs, bt...)
31 | }
32 | 
33 | func (c *CopyCompleteMessage) Decode(body []byte) {
34 | 	c.KeyVersion = body[1]
35 | }
36 | 


--------------------------------------------------------------------------------
/pkg/message/copy_data.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | )
 6 | 
 7 | type CopyDataMessage struct {
 8 | 	ProtoMessage
 9 | 	Sz   uint64
10 | 	Data []byte
11 | }
12 | 
13 | func NewCopyDataMessage() *CopyDataMessage {
14 | 	return &CopyDataMessage{}
15 | }
16 | 
17 | func (cc *CopyDataMessage) Encode() []byte {
18 | 	bt := make([]byte, 4+8+cc.Sz)
19 | 
20 | 	bt[0] = byte(MessageTypeCopyData)
21 | 
22 | 	// sizeof(sz) + data
23 | 	ln := len(bt) + 8
24 | 
25 | 	bs := make([]byte, 8)
26 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
27 | 
28 | 	binary.BigEndian.PutUint64(bt[4:], uint64(cc.Sz))
29 | 
30 | 	// check data len more than cc.sz?
31 | 	copy(bt[(4+8):], cc.Data[:cc.Sz])
32 | 
33 | 	return append(bs, bt...)
34 | }
35 | 
36 | func (cc *CopyDataMessage) Decode(data []byte) {
37 | 	msgLenBuf := data[4:12]
38 | 	cc.Sz = binary.BigEndian.Uint64(msgLenBuf)
39 | 	cc.Data = make([]byte, cc.Sz)
40 | 	copy(cc.Data, data[12:12+cc.Sz])
41 | }
42 | 


--------------------------------------------------------------------------------
/pkg/message/copy_done.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import "encoding/binary"
 4 | 
 5 | type CopyDoneMessage struct {
 6 | }
 7 | 
 8 | var _ ProtoMessage = &CopyDoneMessage{}
 9 | 
10 | func NewCopyDoneMessage() *CopyDoneMessage {
11 | 	return &CopyDoneMessage{}
12 | }
13 | 
14 | func (cc *CopyDoneMessage) Encode() []byte {
15 | 	bt := []byte{
16 | 		byte(MessageTypeCopyDone),
17 | 		0,
18 | 		0,
19 | 		0,
20 | 	}
21 | 
22 | 	ln := len(bt) + 8
23 | 
24 | 	bs := make([]byte, 8)
25 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
26 | 	return append(bs, bt...)
27 | }
28 | 
29 | func (c *CopyDoneMessage) Decode(body []byte) {
30 | }
31 | 


--------------------------------------------------------------------------------
/pkg/message/copy_message.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | 
 6 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
 7 | )
 8 | 
 9 | type CopyMessage struct {
10 | 	Decrypt    bool
11 | 	Encrypt    bool
12 | 	Name       string
13 | 	OldCfgPath string
14 | 	Port       uint64
15 | 	Confirm    bool
16 | }
17 | 
18 | var _ ProtoMessage = &CopyMessage{}
19 | 
20 | func NewCopyMessage(name, oldCfgPath string, encrypt, decrypt, confirm bool, port uint64) *CopyMessage {
21 | 	return &CopyMessage{
22 | 		Name:       name,
23 | 		Encrypt:    encrypt,
24 | 		Decrypt:    decrypt,
25 | 		OldCfgPath: oldCfgPath,
26 | 		Port:       port,
27 | 		Confirm:    confirm,
28 | 	}
29 | }
30 | 
31 | func (message *CopyMessage) Encode() []byte {
32 | 	encodedMessage := []byte{
33 | 		byte(MessageTypeCopy),
34 | 		byte(NoDecryptMessage),
35 | 		byte(NoEncryptMessage),
36 | 		0,
37 | 	}
38 | 
39 | 	if message.Decrypt {
40 | 		encodedMessage[1] = byte(DecryptMessage)
41 | 	}
42 | 
43 | 	if message.Encrypt {
44 | 		encodedMessage[2] = byte(EncryptMessage)
45 | 	}
46 | 
47 | 	byteName := []byte(message.Name)
48 | 	byteLen := make([]byte, 8)
49 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(byteName)))
50 | 	encodedMessage = append(encodedMessage, byteLen...)
51 | 	encodedMessage = append(encodedMessage, byteName...)
52 | 
53 | 	byteOldCfg := []byte(message.OldCfgPath)
54 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(byteOldCfg)))
55 | 	encodedMessage = append(encodedMessage, byteLen...)
56 | 	encodedMessage = append(encodedMessage, byteOldCfg...)
57 | 
58 | 	port := make([]byte, 8)
59 | 	binary.BigEndian.PutUint64(port, uint64(message.Port))
60 | 	encodedMessage = append(encodedMessage, port...)
61 | 
62 | 	if message.Confirm {
63 | 		encodedMessage = append(encodedMessage, 1)
64 | 	} else {
65 | 		encodedMessage = append(encodedMessage, 0)
66 | 	}
67 | 
68 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(encodedMessage)+8))
69 | 	ylogger.Zero.Debug().Str("type", MessageType(encodedMessage[0]).String()).Msg("send")
70 | 	ylogger.Zero.Debug().Str("object-path", MessageType(encodedMessage[0]).String()).Msg("decrypt object")
71 | 	return append(byteLen, encodedMessage...)
72 | }
73 | 
74 | func (encodedMessage *CopyMessage) Decode(data []byte) {
75 | 	if data[1] == byte(DecryptMessage) {
76 | 		encodedMessage.Decrypt = true
77 | 	}
78 | 	if data[2] == byte(EncryptMessage) {
79 | 		encodedMessage.Encrypt = true
80 | 	}
81 | 
82 | 	nameLen := binary.BigEndian.Uint64(data[4:12])
83 | 	encodedMessage.Name = string(data[12 : 12+nameLen])
84 | 	oldConfLen := binary.BigEndian.Uint64(data[12+nameLen : 12+nameLen+8])
85 | 	encodedMessage.OldCfgPath = string(data[12+nameLen+8 : 12+nameLen+8+oldConfLen])
86 | 	encodedMessage.Port = binary.BigEndian.Uint64(data[12+nameLen+8+oldConfLen : 12+nameLen+8+oldConfLen+8])
87 | 
88 | 	ind := 12 + nameLen + 8 + oldConfLen + 8
89 | 	if data[ind] == 1 {
90 | 		encodedMessage.Confirm = true
91 | 	}
92 | }
93 | 


--------------------------------------------------------------------------------
/pkg/message/copy_message_v2.go:
--------------------------------------------------------------------------------
  1 | package message
  2 | 
  3 | import (
  4 | 	"encoding/binary"
  5 | 
  6 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
  7 | )
  8 | 
  9 | const (
 10 | 	CopyEncrypt    = 0b1
 11 | 	CopyDecrypt    = 0b10
 12 | 	CopyUseKEK     = 0b100
 13 | 	CopyServerSide = 0b1000
 14 | )
 15 | 
 16 | type CopyMessageV2 struct {
 17 | 	Decrypt        bool
 18 | 	Encrypt        bool
 19 | 	Name           string
 20 | 	OldCfgPath     string
 21 | 	Port           uint64
 22 | 	Confirm        bool
 23 | 	KEKDecrypt     bool
 24 | 	ServerSideCopy bool
 25 | }
 26 | 
 27 | var _ ProtoMessage = &CopyMessageV2{}
 28 | 
 29 | func NewCopyMessageV2(name, oldCfgPath string, encrypt, decrypt, confirm, kEKDecrypt, ssCopy bool, port uint64) *CopyMessageV2 {
 30 | 	return &CopyMessageV2{
 31 | 		Name:           name,
 32 | 		Encrypt:        encrypt,
 33 | 		Decrypt:        decrypt,
 34 | 		OldCfgPath:     oldCfgPath,
 35 | 		Port:           port,
 36 | 		Confirm:        confirm,
 37 | 		KEKDecrypt:     kEKDecrypt,
 38 | 		ServerSideCopy: ssCopy,
 39 | 	}
 40 | }
 41 | 
 42 | func (message *CopyMessageV2) Encode() []byte {
 43 | 	flags := byte(0)
 44 | 	if message.Decrypt {
 45 | 		flags |= CopyDecrypt
 46 | 	}
 47 | 	if message.Encrypt {
 48 | 		flags |= CopyEncrypt
 49 | 	}
 50 | 	if message.KEKDecrypt {
 51 | 		flags |= CopyUseKEK
 52 | 	}
 53 | 	if message.ServerSideCopy {
 54 | 		flags |= CopyServerSide
 55 | 	}
 56 | 
 57 | 	encodedMessage := []byte{
 58 | 		byte(MessageTypeCopyV2),
 59 | 		flags,
 60 | 		0,
 61 | 		0,
 62 | 	}
 63 | 
 64 | 	byteName := []byte(message.Name)
 65 | 	byteLen := make([]byte, 8)
 66 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(byteName)))
 67 | 	encodedMessage = append(encodedMessage, byteLen...)
 68 | 	encodedMessage = append(encodedMessage, byteName...)
 69 | 
 70 | 	byteOldCfg := []byte(message.OldCfgPath)
 71 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(byteOldCfg)))
 72 | 	encodedMessage = append(encodedMessage, byteLen...)
 73 | 	encodedMessage = append(encodedMessage, byteOldCfg...)
 74 | 
 75 | 	port := make([]byte, 8)
 76 | 	binary.BigEndian.PutUint64(port, uint64(message.Port))
 77 | 	encodedMessage = append(encodedMessage, port...)
 78 | 
 79 | 	if message.Confirm {
 80 | 		encodedMessage = append(encodedMessage, 1)
 81 | 	} else {
 82 | 		encodedMessage = append(encodedMessage, 0)
 83 | 	}
 84 | 
 85 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(encodedMessage)+8))
 86 | 	ylogger.Zero.Debug().Str("type", MessageType(encodedMessage[0]).String()).Msg("send")
 87 | 	ylogger.Zero.Debug().Str("object-path", MessageType(encodedMessage[0]).String()).Msg("decrypt object")
 88 | 	return append(byteLen, encodedMessage...)
 89 | }
 90 | 
 91 | func (encodedMessage *CopyMessageV2) Decode(data []byte) {
 92 | 	if data[1]&CopyDecrypt != 0 {
 93 | 		encodedMessage.Decrypt = true
 94 | 	}
 95 | 	if data[1]&CopyEncrypt != 0 {
 96 | 		encodedMessage.Encrypt = true
 97 | 	}
 98 | 	if data[1]&CopyUseKEK != 0 {
 99 | 		encodedMessage.KEKDecrypt = true
100 | 	}
101 | 	if data[1]&CopyServerSide != 0 {
102 | 		encodedMessage.ServerSideCopy = true
103 | 	}
104 | 
105 | 	nameLen := binary.BigEndian.Uint64(data[4:12])
106 | 	encodedMessage.Name = string(data[12 : 12+nameLen])
107 | 	oldConfLen := binary.BigEndian.Uint64(data[12+nameLen : 12+nameLen+8])
108 | 	encodedMessage.OldCfgPath = string(data[12+nameLen+8 : 12+nameLen+8+oldConfLen])
109 | 	encodedMessage.Port = binary.BigEndian.Uint64(data[12+nameLen+8+oldConfLen : 12+nameLen+8+oldConfLen+8])
110 | 
111 | 	ind := 12 + nameLen + 8 + oldConfLen + 8
112 | 	if data[ind] == 1 {
113 | 		encodedMessage.Confirm = true
114 | 	}
115 | }
116 | 


--------------------------------------------------------------------------------
/pkg/message/delete_message.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | )
 6 | 
 7 | type DeleteMessage struct { //seg port
 8 | 	Name      string
 9 | 	Port      uint64
10 | 	Segnum    uint64
11 | 	Confirm   bool
12 | 	Garbage   bool
13 | 	CrazyDrop bool
14 | }
15 | 
16 | var _ ProtoMessage = &DeleteMessage{}
17 | 
18 | func NewDeleteMessage(name string, port uint64, seg uint64, confirm bool, garbage bool) *DeleteMessage {
19 | 	return &DeleteMessage{
20 | 		Name:    name,
21 | 		Port:    port,
22 | 		Segnum:  seg,
23 | 		Confirm: confirm,
24 | 		Garbage: garbage,
25 | 	}
26 | }
27 | 
28 | func (c *DeleteMessage) Encode() []byte {
29 | 	bt := []byte{
30 | 		byte(MessageTypeDelete),
31 | 		0,
32 | 		0,
33 | 		0,
34 | 	}
35 | 
36 | 	if c.Confirm {
37 | 		bt[1] = 1
38 | 	}
39 | 	if c.Garbage {
40 | 		bt[2] = 1
41 | 	}
42 | 	if c.CrazyDrop {
43 | 		bt[3] = 1
44 | 	}
45 | 
46 | 	bt = append(bt, []byte(c.Name)...)
47 | 	bt = append(bt, 0)
48 | 
49 | 	p := make([]byte, 8)
50 | 	binary.BigEndian.PutUint64(p, uint64(c.Port))
51 | 	bt = append(bt, p...)
52 | 
53 | 	p = make([]byte, 8)
54 | 	binary.BigEndian.PutUint64(p, uint64(c.Segnum))
55 | 	bt = append(bt, p...)
56 | 
57 | 	ln := len(bt) + 8
58 | 	bs := make([]byte, 8)
59 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
60 | 	return append(bs, bt...)
61 | }
62 | 
63 | func (c *DeleteMessage) Decode(body []byte) {
64 | 	if body[1] == 1 {
65 | 		c.Confirm = true
66 | 	}
67 | 	if body[2] == 1 {
68 | 		c.Garbage = true
69 | 	}
70 | 	if body[3] == 1 {
71 | 		c.CrazyDrop = true
72 | 	}
73 | 	c.Name, _ = GetCstring(body[4:])
74 | 	c.Port = binary.BigEndian.Uint64(body[len(body)-16 : len(body)-8])
75 | 	c.Segnum = binary.BigEndian.Uint64(body[len(body)-8:])
76 | }
77 | 


--------------------------------------------------------------------------------
/pkg/message/error_message.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | )
 6 | 
 7 | type ErrorMessage struct {
 8 | 	Error   string
 9 | 	Message string
10 | }
11 | 
12 | var _ ProtoMessage = &ListMessage{}
13 | 
14 | func NewErrorMessage(err error, msg string) *ErrorMessage {
15 | 	return &ErrorMessage{
16 | 		Error:   err.Error(),
17 | 		Message: msg,
18 | 	}
19 | }
20 | 
21 | func (c *ErrorMessage) Encode() []byte {
22 | 	encodedMessage := []byte{
23 | 		byte(MessageTypeError),
24 | 		0,
25 | 		0,
26 | 		0,
27 | 	}
28 | 
29 | 	byteError := []byte(c.Error)
30 | 	byteLen := make([]byte, 8)
31 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(byteError)))
32 | 	encodedMessage = append(encodedMessage, byteLen...)
33 | 	encodedMessage = append(encodedMessage, byteError...)
34 | 
35 | 	byteMessage := []byte(c.Message)
36 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(byteMessage)))
37 | 	encodedMessage = append(encodedMessage, byteLen...)
38 | 	encodedMessage = append(encodedMessage, byteMessage...)
39 | 
40 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(encodedMessage)+8))
41 | 	return append(byteLen, encodedMessage...)
42 | }
43 | 
44 | func (c *ErrorMessage) Decode(data []byte) {
45 | 	errorLen := binary.BigEndian.Uint64(data[4:12])
46 | 	c.Error = string(data[12 : 12+errorLen])
47 | 	messageLen := binary.BigEndian.Uint64(data[12+errorLen : 12+errorLen+8])
48 | 	c.Message = string(data[12+errorLen+8 : 12+errorLen+8+messageLen])
49 | }
50 | 


--------------------------------------------------------------------------------
/pkg/message/gool_message.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"bytes"
 5 | 	"encoding/binary"
 6 | )
 7 | 
 8 | type GoolMessage struct {
 9 | 	Name string
10 | }
11 | 
12 | func (c *GoolMessage) Encode() []byte {
13 | 	bt := []byte{
14 | 		byte(MessageTypeGool),
15 | 		0,
16 | 		0,
17 | 		0,
18 | 	}
19 | 
20 | 	bt = append(bt, []byte("GOOL "+c.Name)...)
21 | 	bt = append(bt, 0)
22 | 	ln := len(bt) + 8
23 | 
24 | 	bs := make([]byte, 8)
25 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
26 | 	return append(bs, bt...)
27 | }
28 | 
29 | func (c *GoolMessage) Decode(body []byte) {
30 | 	c.Name = c.GetGoolName(body[4:])
31 | }
32 | 
33 | func (c *GoolMessage) GetGoolName(b []byte) string {
34 | 	buff := bytes.NewBufferString("")
35 | 
36 | 	for i := range b {
37 | 		if b[i] == 0 {
38 | 			break
39 | 		}
40 | 		buff.WriteByte(b[i])
41 | 	}
42 | 
43 | 	return buff.String()
44 | }
45 | 
46 | var _ ProtoMessage = &GoolMessage{}
47 | 
48 | func NewGoolMessage(name string) *GoolMessage {
49 | 	return &GoolMessage{
50 | 		Name: name,
51 | 	}
52 | }
53 | 


--------------------------------------------------------------------------------
/pkg/message/list_message.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | )
 6 | 
 7 | type ListMessage struct {
 8 | 	Prefix string
 9 | }
10 | 
11 | var _ ProtoMessage = &ListMessage{}
12 | 
13 | func NewListMessage(name string) *ListMessage {
14 | 	return &ListMessage{
15 | 		Prefix: name,
16 | 	}
17 | }
18 | 
19 | func (c *ListMessage) Encode() []byte {
20 | 	bt := []byte{
21 | 		byte(MessageTypeList),
22 | 		0,
23 | 		0,
24 | 		0,
25 | 	}
26 | 
27 | 	bt = append(bt, []byte(c.Prefix)...)
28 | 	bt = append(bt, 0)
29 | 	ln := len(bt) + 8
30 | 
31 | 	bs := make([]byte, 8)
32 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
33 | 	return append(bs, bt...)
34 | }
35 | 
36 | func (c *ListMessage) Decode(body []byte) {
37 | 	c.Prefix, _ = GetCstring(body[4:])
38 | }
39 | 


--------------------------------------------------------------------------------
/pkg/message/list_message_v2.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | 
 6 | 	"github.com/yezzey-gp/yproxy/pkg/settings"
 7 | )
 8 | 
 9 | type ListMessageV2 struct {
10 | 	Prefix string
11 | 
12 | 	Settings []settings.StorageSettings
13 | }
14 | 
15 | var _ ProtoMessage = &ListMessage{}
16 | 
17 | func NewListMessageV2(name string, Settings []settings.StorageSettings) *ListMessageV2 {
18 | 	return &ListMessageV2{
19 | 		Prefix:   name,
20 | 		Settings: Settings,
21 | 	}
22 | }
23 | 
24 | func (c *ListMessageV2) Encode() []byte {
25 | 	bt := []byte{
26 | 		byte(MessageTypeListV2),
27 | 		0,
28 | 		0,
29 | 		0,
30 | 	}
31 | 
32 | 	bt = append(bt, []byte(c.Prefix)...)
33 | 	bt = append(bt, 0)
34 | 
35 | 	bt = binary.BigEndian.AppendUint64(bt, uint64(len(c.Settings)))
36 | 
37 | 	for _, s := range c.Settings {
38 | 
39 | 		bt = append(bt, []byte(s.Name)...)
40 | 		bt = append(bt, 0)
41 | 
42 | 		bt = append(bt, []byte(s.Value)...)
43 | 		bt = append(bt, 0)
44 | 	}
45 | 
46 | 	ln := len(bt) + 8
47 | 
48 | 	bs := make([]byte, 8)
49 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
50 | 	return append(bs, bt...)
51 | }
52 | 
53 | func (c *ListMessageV2) Decode(body []byte) {
54 | 	var off uint64
55 | 	c.Prefix, off = GetCstring(body[4:])
56 | 
57 | 	settLen := binary.BigEndian.Uint64(body[4+off : 4+off+8])
58 | 
59 | 	totalOff := 4 + off + 8
60 | 
61 | 	c.Settings = make([]settings.StorageSettings, settLen)
62 | 
63 | 	for i := range int(settLen) {
64 | 
65 | 		var currOff uint64
66 | 
67 | 		c.Settings[i].Name, currOff = GetCstring(body[totalOff:])
68 | 		totalOff += currOff
69 | 
70 | 		c.Settings[i].Value, currOff = GetCstring(body[totalOff:])
71 | 		totalOff += currOff
72 | 	}
73 | }
74 | 


--------------------------------------------------------------------------------
/pkg/message/message.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | type ProtoMessage interface {
 4 | 	Decode([]byte)
 5 | 	Encode() []byte
 6 | }
 7 | 
 8 | type MessageType byte
 9 | 
10 | type RequestEncryption byte
11 | 
12 | type KEKUsage byte
13 | 
14 | const (
15 | 	MessageTypeCat = MessageType(iota + 42)
16 | 	MessageTypePut
17 | 	MessageTypeCopyDone
18 | 	MessageTypeReadyForQuery
19 | 	MessageTypeCopyData
20 | 	MessageTypeDelete
21 | 	MessageTypeList
22 | 	MessageTypeObjectMeta
23 | 	MessageTypePatch
24 | 	MessageTypeCopy
25 | 	MessageTypeGool
26 | 	MessageTypePutV2
27 | 	MessageTypeCatV2
28 | 	MessageTypeError
29 | 	MessageTypePutV3
30 | 	MessageTypePutComplete
31 | 	MessageTypeUntrashify
32 | 	MessageTypeCopyV2
33 | 	MessageTypeCopyComplete
34 | 	MessageTypeListV2
35 | 
36 | 	MessageCollectObsolete = MessageType(64)
37 | 	MessageDeleteObsolete  = MessageType(65)
38 | 
39 | 	DecryptMessage   = RequestEncryption(1)
40 | 	NoDecryptMessage = RequestEncryption(0)
41 | 
42 | 	EncryptMessage   = RequestEncryption(1)
43 | 	NoEncryptMessage = RequestEncryption(0)
44 | 
45 | 	NoUseKEK = KEKUsage(0)
46 | 	UseKEK   = KEKUsage(1)
47 | 
48 | 	ExtendedMesssage = byte(1)
49 | )
50 | 
51 | func (m MessageType) String() string {
52 | 	switch m {
53 | 	case MessageTypeCat:
54 | 		return "CAT"
55 | 	case MessageTypeCatV2:
56 | 		return "CATV2"
57 | 	case MessageTypePut:
58 | 		return "PUT"
59 | 	case MessageTypePutV2:
60 | 		return "PUTV2"
61 | 	case MessageTypePutV3:
62 | 		return "PUTV3"
63 | 	case MessageTypePutComplete:
64 | 		return "PUTCOMPLETE"
65 | 	case MessageTypeCopyDone:
66 | 		return "COMMAND COMPLETE"
67 | 	case MessageTypeReadyForQuery:
68 | 		return "READY FOR QUERY"
69 | 	case MessageTypeCopyData:
70 | 		return "COPY DATA"
71 | 	case MessageTypeDelete:
72 | 		return "DELETE"
73 | 	case MessageTypeList:
74 | 		return "LIST"
75 | 	case MessageTypeListV2:
76 | 		return "LISTV2"
77 | 	case MessageTypeObjectMeta:
78 | 		return "OBJECT META"
79 | 	case MessageTypeCopy:
80 | 		return "COPY"
81 | 	case MessageTypeCopyV2:
82 | 		return "COPYV2"
83 | 	case MessageTypeGool:
84 | 		return "GOOL"
85 | 	case MessageTypeError:
86 | 		return "ERROR"
87 | 	case MessageTypeUntrashify:
88 | 		return "UNTRASHIFY"
89 | 	case MessageCollectObsolete:
90 | 		return "COLLECT OBSOLETE"
91 | 	case MessageDeleteObsolete:
92 | 		return "DELETE OBSOLETE"
93 | 	case MessageTypeCopyComplete:
94 | 		return "COPY COMPLETE"
95 | 	}
96 | 	return "UNKNOWN"
97 | }
98 | 


--------------------------------------------------------------------------------
/pkg/message/message_test.go:
--------------------------------------------------------------------------------
  1 | package message_test
  2 | 
  3 | import (
  4 | 	"encoding/binary"
  5 | 	"fmt"
  6 | 	"testing"
  7 | 
  8 | 	"github.com/stretchr/testify/assert"
  9 | 	"github.com/yezzey-gp/yproxy/pkg/message"
 10 | 	"github.com/yezzey-gp/yproxy/pkg/settings"
 11 | )
 12 | 
 13 | func TestCatMsg(t *testing.T) {
 14 | 	assert := assert.New(t)
 15 | 
 16 | 	type tcase struct {
 17 | 		name    string
 18 | 		decrypt bool
 19 | 		off     uint64
 20 | 		err     error
 21 | 	}
 22 | 
 23 | 	for _, tt := range []tcase{
 24 | 		{
 25 | 			"nam1",
 26 | 			true,
 27 | 			0,
 28 | 			nil,
 29 | 		},
 30 | 		{
 31 | 			"nam1",
 32 | 			true,
 33 | 			10,
 34 | 			nil,
 35 | 		},
 36 | 	} {
 37 | 
 38 | 		msg := message.NewCatMessage(tt.name, tt.decrypt, tt.off)
 39 | 		body := msg.Encode()
 40 | 
 41 | 		msg2 := message.CatMessage{}
 42 | 
 43 | 		msg2.Decode(body[8:])
 44 | 
 45 | 		assert.Equal(msg.Name, msg2.Name)
 46 | 		assert.Equal(msg.Decrypt, msg2.Decrypt)
 47 | 	}
 48 | }
 49 | 
 50 | func TestPutMsg(t *testing.T) {
 51 | 	assert := assert.New(t)
 52 | 
 53 | 	type tcase struct {
 54 | 		name    string
 55 | 		encrypt bool
 56 | 		err     error
 57 | 	}
 58 | 
 59 | 	for _, tt := range []tcase{
 60 | 		{
 61 | 			"nam1",
 62 | 			true,
 63 | 			nil,
 64 | 		},
 65 | 	} {
 66 | 
 67 | 		msg := message.NewPutMessage(tt.name, tt.encrypt)
 68 | 		body := msg.Encode()
 69 | 
 70 | 		msg2 := message.PutMessage{}
 71 | 
 72 | 		msg2.Decode(body[8:])
 73 | 
 74 | 		assert.Equal(msg.Name, msg2.Name)
 75 | 		assert.Equal(msg.Encrypt, msg2.Encrypt)
 76 | 	}
 77 | }
 78 | 
 79 | func TestPutV2Msg(t *testing.T) {
 80 | 	assert := assert.New(t)
 81 | 
 82 | 	type tcase struct {
 83 | 		name     string
 84 | 		encrypt  bool
 85 | 		err      error
 86 | 		settings []settings.StorageSettings
 87 | 	}
 88 | 
 89 | 	for _, tt := range []tcase{
 90 | 		{
 91 | 			"nam1",
 92 | 			true,
 93 | 			nil,
 94 | 			[]settings.StorageSettings{
 95 | 				{
 96 | 					Name:  "a",
 97 | 					Value: "b",
 98 | 				},
 99 | 				{
100 | 					Name:  "cdsdsd",
101 | 					Value: "ds",
102 | 				},
103 | 			},
104 | 		},
105 | 	} {
106 | 
107 | 		msg := message.NewPutMessageV2(tt.name, tt.encrypt, tt.settings)
108 | 		body := msg.Encode()
109 | 
110 | 		msg2 := message.PutMessageV2{}
111 | 
112 | 		msg2.Decode(body[8:])
113 | 
114 | 		assert.Equal(msg.Name, msg2.Name)
115 | 		assert.Equal(msg.Encrypt, msg2.Encrypt)
116 | 		assert.Equal(msg.Settings, msg2.Settings)
117 | 	}
118 | }
119 | 
120 | func TestPutMsgV3(t *testing.T) {
121 | 	assert := assert.New(t)
122 | 
123 | 	type tcase struct {
124 | 		name     string
125 | 		encrypt  bool
126 | 		err      error
127 | 		settings []settings.StorageSettings
128 | 	}
129 | 
130 | 	for _, tt := range []tcase{
131 | 		{
132 | 			"nam1",
133 | 			true,
134 | 			nil,
135 | 			[]settings.StorageSettings{
136 | 				{
137 | 					Name:  "a",
138 | 					Value: "b",
139 | 				},
140 | 				{
141 | 					Name:  "cdsdsd",
142 | 					Value: "ds",
143 | 				},
144 | 			},
145 | 		},
146 | 	} {
147 | 
148 | 		msg := message.NewPutMessageV3(tt.name, tt.encrypt, tt.settings)
149 | 		body := msg.Encode()
150 | 
151 | 		msg2 := message.PutMessageV3{}
152 | 
153 | 		msg2.Decode(body[8:])
154 | 
155 | 		assert.Equal(msg.Name, msg2.Name)
156 | 		assert.Equal(msg.Encrypt, msg2.Encrypt)
157 | 		assert.Equal(msg.Settings, msg2.Settings)
158 | 	}
159 | }
160 | 
161 | func TestPutCompleteMsg(t *testing.T) {
162 | 	assert := assert.New(t)
163 | 
164 | 	type tcase struct {
165 | 		keyVersion uint16
166 | 	}
167 | 
168 | 	for _, tt := range []tcase{
169 | 		{
170 | 			1,
171 | 		},
172 | 	} {
173 | 
174 | 		msg := message.NewPutCompleteMessage(tt.keyVersion)
175 | 		body := msg.Encode()
176 | 
177 | 		msg2 := message.PutCompleteMessage{}
178 | 
179 | 		msg2.Decode(body[8:])
180 | 
181 | 		assert.Equal(msg.KeyVersion, msg2.KeyVersion)
182 | 	}
183 | }
184 | 
185 | func TestCatMsgV2(t *testing.T) {
186 | 	assert := assert.New(t)
187 | 
188 | 	type tcase struct {
189 | 		name    string
190 | 		decrypt bool
191 | 		kek     bool
192 | 		off     uint64
193 | 
194 | 		settings []settings.StorageSettings
195 | 		err      error
196 | 	}
197 | 
198 | 	for _, tt := range []tcase{
199 | 		{
200 | 			"nam1",
201 | 			true,
202 | 			true,
203 | 			0,
204 | 			[]settings.StorageSettings{
205 | 				{
206 | 					Name:  "a",
207 | 					Value: "b",
208 | 				},
209 | 				{
210 | 					Name:  "cdsdsd",
211 | 					Value: "ds",
212 | 				},
213 | 			},
214 | 			nil,
215 | 		},
216 | 		{
217 | 			"nam1",
218 | 			true,
219 | 			true,
220 | 			10,
221 | 			[]settings.StorageSettings{
222 | 				{
223 | 					Name:  "a",
224 | 					Value: "b",
225 | 				},
226 | 				{
227 | 					Name:  "cdsdsd",
228 | 					Value: "ds",
229 | 				},
230 | 			},
231 | 			nil,
232 | 		},
233 | 		{
234 | 			"nam1",
235 | 			true,
236 | 			false,
237 | 			0,
238 | 			[]settings.StorageSettings{
239 | 				{
240 | 					Name:  "a",
241 | 					Value: "b",
242 | 				},
243 | 				{
244 | 					Name:  "cdsdsd",
245 | 					Value: "ds",
246 | 				},
247 | 			},
248 | 			nil,
249 | 		},
250 | 		{
251 | 			"nam1",
252 | 			true,
253 | 			false,
254 | 			10,
255 | 			[]settings.StorageSettings{
256 | 				{
257 | 					Name:  "a",
258 | 					Value: "b",
259 | 				},
260 | 				{
261 | 					Name:  "cdsdsd",
262 | 					Value: "ds",
263 | 				},
264 | 			},
265 | 			nil,
266 | 		},
267 | 	} {
268 | 
269 | 		msg := message.NewCatMessageV2(tt.name, tt.decrypt, tt.kek, tt.off, tt.settings)
270 | 		body := msg.Encode()
271 | 
272 | 		msg2 := message.CatMessageV2{}
273 | 
274 | 		msg2.Decode(body[8:])
275 | 
276 | 		assert.Equal(msg.Name, msg2.Name)
277 | 		assert.Equal(msg.Decrypt, msg2.Decrypt)
278 | 		assert.Equal(msg.KEK, msg2.KEK)
279 | 		assert.Equal(msg.StartOffset, msg2.StartOffset)
280 | 		assert.Equal(msg.Settings, msg2.Settings)
281 | 	}
282 | }
283 | 
284 | func TestPatchMsg(t *testing.T) {
285 | 	assert := assert.New(t)
286 | 
287 | 	type tcase struct {
288 | 		name    string
289 | 		encrypt bool
290 | 		off     uint64
291 | 		err     error
292 | 	}
293 | 
294 | 	for _, tt := range []tcase{
295 | 		{
296 | 			"nam1",
297 | 			true,
298 | 			1235,
299 | 			nil,
300 | 		},
301 | 	} {
302 | 
303 | 		msg := message.NewPatchMessage(tt.name, tt.off, tt.encrypt)
304 | 		body := msg.Encode()
305 | 
306 | 		msg2 := message.PatchMessage{}
307 | 
308 | 		msg2.Decode(body[8:])
309 | 
310 | 		assert.Equal(msg.Name, msg2.Name)
311 | 		assert.Equal(msg.Encrypt, msg2.Encrypt)
312 | 		assert.Equal(msg.Offset, msg2.Offset)
313 | 	}
314 | }
315 | 
316 | func TestCopyDataMsg(t *testing.T) {
317 | 	assert := assert.New(t)
318 | 
319 | 	type tcase struct {
320 | 		body []byte
321 | 		err  error
322 | 	}
323 | 
324 | 	for _, tt := range []tcase{
325 | 		{
326 | 			[]byte(
327 | 				"hiuefheiufheuif",
328 | 			),
329 | 			nil,
330 | 		},
331 | 	} {
332 | 
333 | 		msg := message.NewCopyDataMessage()
334 | 		msg.Data = tt.body
335 | 		msg.Sz = uint64(len(tt.body))
336 | 		body := msg.Encode()
337 | 
338 | 		msg2 := message.CopyDataMessage{}
339 | 
340 | 		msg2.Decode(body[8:])
341 | 
342 | 		sz := binary.BigEndian.Uint64(body[:8])
343 | 
344 | 		assert.Equal(int(sz), len(body))
345 | 
346 | 		assert.Equal(msg.Data, msg2.Data)
347 | 		assert.Equal(msg.Sz, msg2.Sz)
348 | 	}
349 | }
350 | 
351 | func TestListMsg(t *testing.T) {
352 | 	assert := assert.New(t)
353 | 
354 | 	type tcase struct {
355 | 		prefix string
356 | 	}
357 | 
358 | 	for _, tt := range []tcase{
359 | 		{
360 | 			"nam1",
361 | 		},
362 | 	} {
363 | 
364 | 		msg := message.NewListMessage(tt.prefix)
365 | 		body := msg.Encode()
366 | 
367 | 		msg2 := message.ListMessage{}
368 | 
369 | 		msg2.Decode(body[8:])
370 | 
371 | 		assert.Equal(msg.Prefix, msg2.Prefix)
372 | 	}
373 | }
374 | 
375 | func TestListMsgV2(t *testing.T) {
376 | 	assert := assert.New(t)
377 | 
378 | 	type tcase struct {
379 | 		prefix string
380 | 		Set    []settings.StorageSettings
381 | 	}
382 | 
383 | 	for _, tt := range []tcase{
384 | 		{
385 | 			prefix: "nam1",
386 | 			Set:    []settings.StorageSettings{},
387 | 		},
388 | 		{
389 | 			prefix: "nam1",
390 | 			Set: []settings.StorageSettings{
391 | 				{
392 | 					Name:  "completely normal setting name",
393 | 					Value: "some value",
394 | 				},
395 | 				{
396 | 					Name:  "another completely normal setting name",
397 | 					Value: "no meaningful value",
398 | 				},
399 | 			},
400 | 		},
401 | 	} {
402 | 
403 | 		msg := message.NewListMessageV2(tt.prefix, tt.Set)
404 | 		body := msg.Encode()
405 | 
406 | 		msg2 := message.ListMessageV2{}
407 | 
408 | 		msg2.Decode(body[8:])
409 | 
410 | 		assert.Equal(msg.Prefix, msg2.Prefix)
411 | 		assert.Equal(msg.Settings, msg2.Settings)
412 | 	}
413 | }
414 | 
415 | func TestCopyMsg(t *testing.T) {
416 | 	assert := assert.New(t)
417 | 
418 | 	msg := message.NewCopyMessage("myname/mynextname", "myoldcfg/path", true, true, true, 5432)
419 | 	body := msg.Encode()
420 | 
421 | 	assert.Equal(body[8], byte(message.MessageTypeCopy))
422 | 
423 | 	msg2 := message.CopyMessage{}
424 | 	msg2.Decode(body[8:])
425 | 
426 | 	assert.Equal("myname/mynextname", msg2.Name)
427 | 	assert.Equal("myoldcfg/path", msg2.OldCfgPath)
428 | 	assert.True(msg2.Decrypt)
429 | 	assert.True(msg2.Encrypt)
430 | 	assert.Equal(uint64(5432), msg2.Port)
431 | 	assert.True(msg2.Confirm)
432 | }
433 | 
434 | func TestDeleteMsg(t *testing.T) {
435 | 	assert := assert.New(t)
436 | 
437 | 	msg := message.NewDeleteMessage("myname/mynextname", 5432, 42, true, true)
438 | 	body := msg.Encode()
439 | 
440 | 	assert.Equal(body[8], byte(message.MessageTypeDelete))
441 | 
442 | 	msg2 := message.DeleteMessage{}
443 | 	msg2.Decode(body[8:])
444 | 
445 | 	assert.Equal("myname/mynextname", msg2.Name)
446 | 	assert.Equal(uint64(5432), msg2.Port)
447 | 	assert.Equal(uint64(42), msg2.Segnum)
448 | 	assert.True(msg2.Confirm)
449 | 	assert.True(msg2.Garbage)
450 | }
451 | 
452 | func TestUntrashifyMsg(t *testing.T) {
453 | 	assert := assert.New(t)
454 | 
455 | 	msg := message.NewUntrashifyMessage("myname/mynextname", 42, true)
456 | 	body := msg.Encode()
457 | 
458 | 	assert.Equal(body[8], byte(message.MessageTypeUntrashify))
459 | 
460 | 	msg2 := message.UntrashifyMessage{}
461 | 	msg2.Decode(body[8:])
462 | 
463 | 	assert.Equal("myname/mynextname", msg2.Name)
464 | 	assert.Equal(uint64(42), msg2.Segnum)
465 | 	assert.True(msg2.Confirm)
466 | }
467 | 
468 | func TestErrorMsg(t *testing.T) {
469 | 	assert := assert.New(t)
470 | 
471 | 	errString := "could not list objects: NoCredentialProviders: no valid providers in chain."
472 | 	messageString := "Deprecated. For verbose messaging see aws.Config.CredentialsChainVerboseErrors"
473 | 	err := fmt.Errorf("%s", errString)
474 | 
475 | 	msg := message.NewErrorMessage(err, messageString)
476 | 	body := msg.Encode()
477 | 
478 | 	assert.Equal(body[8], byte(message.MessageTypeError))
479 | 
480 | 	msg2 := message.ErrorMessage{}
481 | 	msg2.Decode(body[8:])
482 | 
483 | 	assert.Equal(errString, msg2.Error)
484 | 	assert.Equal(messageString, msg2.Message)
485 | }
486 | 


--------------------------------------------------------------------------------
/pkg/message/object_meta_message.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"bytes"
 5 | 	"encoding/binary"
 6 | 
 7 | 	"github.com/yezzey-gp/yproxy/pkg/object"
 8 | )
 9 | 
10 | type ObjectInfoMessage struct {
11 | 	Content []*object.ObjectInfo
12 | }
13 | 
14 | var _ ProtoMessage = &ObjectInfoMessage{}
15 | 
16 | func NewObjectMetaMessage(content []*object.ObjectInfo) *ObjectInfoMessage {
17 | 	return &ObjectInfoMessage{
18 | 		Content: content,
19 | 	}
20 | }
21 | 
22 | func (c *ObjectInfoMessage) Encode() []byte {
23 | 	bt := []byte{
24 | 		byte(MessageTypeObjectMeta),
25 | 		0,
26 | 		0,
27 | 		0,
28 | 	}
29 | 
30 | 	for _, objMeta := range c.Content {
31 | 		bt = append(bt, []byte(objMeta.Path)...)
32 | 		bt = append(bt, 0)
33 | 
34 | 		bn := make([]byte, 8)
35 | 		binary.BigEndian.PutUint64(bn, uint64(objMeta.Size))
36 | 		bt = append(bt, bn...)
37 | 	}
38 | 
39 | 	ln := len(bt) + 8
40 | 	bs := make([]byte, 8)
41 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
42 | 	return append(bs, bt...)
43 | }
44 | 
45 | func (c *ObjectInfoMessage) Decode(body []byte) {
46 | 	body = body[4:]
47 | 	c.Content = make([]*object.ObjectInfo, 0)
48 | 	for len(body) > 0 {
49 | 		name, index := c.GetString(body)
50 | 		size := int64(binary.BigEndian.Uint64(body[index : index+8]))
51 | 
52 | 		c.Content = append(c.Content, &object.ObjectInfo{
53 | 			Path: name,
54 | 			Size: size,
55 | 		})
56 | 		body = body[index+8:]
57 | 	}
58 | }
59 | 
60 | func (c *ObjectInfoMessage) GetString(b []byte) (string, int) {
61 | 	buff := bytes.NewBufferString("")
62 | 
63 | 	i := 0
64 | 	for ; i < len(b); i++ {
65 | 		if b[i] == 0 {
66 | 			break
67 | 		}
68 | 		buff.WriteByte(b[i])
69 | 	}
70 | 
71 | 	return buff.String(), i + 1
72 | }
73 | 


--------------------------------------------------------------------------------
/pkg/message/obsolete_message.go:
--------------------------------------------------------------------------------
  1 | package message
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	"encoding/binary"
  6 | )
  7 | 
  8 | type CollectObsoleteMessage struct {
  9 | 	Segnum  uint64
 10 | 	Port    uint64
 11 | 	DBName  string
 12 | 	Message string
 13 | }
 14 | 
 15 | type DeleteObsoleteMessage struct {
 16 | 	Segnum  uint64
 17 | 	Port    uint64
 18 | 	DBName  string
 19 | 	Message string
 20 | }
 21 | 
 22 | var _ ProtoMessage = &ListMessage{}
 23 | 
 24 | // TODO
 25 | func NewCollectObsoleteMessage(dbname string, msg string) *CollectObsoleteMessage {
 26 | 	return &CollectObsoleteMessage{
 27 | 		DBName:  dbname,
 28 | 		Message: msg,
 29 | 	}
 30 | }
 31 | 
 32 | func (c *CollectObsoleteMessage) Encode() []byte {
 33 | 	encodedMessage := []byte{
 34 | 		byte(MessageCollectObsolete),
 35 | 		0,
 36 | 		0,
 37 | 		0,
 38 | 	}
 39 | 
 40 | 	byteError := []byte(c.DBName)
 41 | 	byteLen := make([]byte, 8)
 42 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(byteError)))
 43 | 	encodedMessage = append(encodedMessage, byteLen...)
 44 | 	encodedMessage = append(encodedMessage, byteError...)
 45 | 
 46 | 	byteMessage := []byte(c.Message)
 47 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(byteMessage)))
 48 | 	encodedMessage = append(encodedMessage, byteLen...)
 49 | 	encodedMessage = append(encodedMessage, byteMessage...)
 50 | 
 51 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(encodedMessage)+8))
 52 | 	return append(byteLen, encodedMessage...)
 53 | }
 54 | 
 55 | func (c *CollectObsoleteMessage) Decode(data []byte) {
 56 | 	c.Segnum = binary.BigEndian.Uint64(data[4:12])
 57 | 	c.Port = binary.BigEndian.Uint64(data[12:20])
 58 | 	n := bytes.IndexByte(data[20:], 0)
 59 | 	c.DBName = string(data[20 : 20+n])
 60 | 	c.Message = string(data[20+n+1 : len(data)-1])
 61 | }
 62 | 
 63 | func NewDeleteObsoleteMessage(dbname string, msg string) *DeleteObsoleteMessage {
 64 | 	return &DeleteObsoleteMessage{
 65 | 		DBName:  dbname,
 66 | 		Message: msg,
 67 | 	}
 68 | }
 69 | 
 70 | func (c *DeleteObsoleteMessage) Encode() []byte {
 71 | 	encodedMessage := []byte{
 72 | 		byte(MessageDeleteObsolete),
 73 | 		0,
 74 | 		0,
 75 | 		0,
 76 | 	}
 77 | 
 78 | 	byteError := []byte(c.DBName)
 79 | 	byteLen := make([]byte, 8)
 80 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(byteError)))
 81 | 	encodedMessage = append(encodedMessage, byteLen...)
 82 | 	encodedMessage = append(encodedMessage, byteError...)
 83 | 
 84 | 	byteMessage := []byte(c.Message)
 85 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(byteMessage)))
 86 | 	encodedMessage = append(encodedMessage, byteLen...)
 87 | 	encodedMessage = append(encodedMessage, byteMessage...)
 88 | 
 89 | 	binary.BigEndian.PutUint64(byteLen, uint64(len(encodedMessage)+8))
 90 | 	return append(byteLen, encodedMessage...)
 91 | }
 92 | 
 93 | func (c *DeleteObsoleteMessage) Decode(data []byte) {
 94 | 	c.Segnum = binary.BigEndian.Uint64(data[4:12])
 95 | 	c.Port = binary.BigEndian.Uint64(data[12:20])
 96 | 	n := bytes.IndexByte(data[20:], 0)
 97 | 
 98 | 	c.DBName = string(data[20 : 20+n])
 99 | 	c.Message = string(data[20+n+1 : len(data)-1])
100 | }
101 | 


--------------------------------------------------------------------------------
/pkg/message/patch_message.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"bytes"
 5 | 	"encoding/binary"
 6 | )
 7 | 
 8 | type PatchMessage struct {
 9 | 	Encrypt bool
10 | 	Offset  uint64
11 | 	Name    string
12 | }
13 | 
14 | var _ ProtoMessage = &PatchMessage{}
15 | 
16 | func NewPatchMessage(name string, offset uint64, encrypt bool) *PatchMessage {
17 | 	return &PatchMessage{
18 | 		Encrypt: encrypt,
19 | 		Offset:  offset,
20 | 		Name:    name,
21 | 	}
22 | }
23 | 
24 | func (c *PatchMessage) Encode() []byte {
25 | 	bt := []byte{
26 | 		byte(MessageTypePatch),
27 | 		0,
28 | 		0,
29 | 		0,
30 | 	}
31 | 
32 | 	if c.Encrypt {
33 | 		bt[1] = byte(EncryptMessage)
34 | 	} else {
35 | 		bt[1] = byte(NoEncryptMessage)
36 | 	}
37 | 	offset := make([]byte, 8)
38 | 	binary.BigEndian.PutUint64(offset, uint64(c.Offset))
39 | 
40 | 	bt = append(bt, offset...)
41 | 	bt = append(bt, []byte(c.Name)...)
42 | 	bt = append(bt, 0)
43 | 	ln := len(bt) + 8
44 | 
45 | 	bs := make([]byte, 8)
46 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
47 | 	return append(bs, bt...)
48 | }
49 | 
50 | func (c *PatchMessage) GetPatchName(b []byte) string {
51 | 	buff := bytes.NewBufferString("")
52 | 
53 | 	for i := 0; i < len(b); i++ {
54 | 		if b[i] == 0 {
55 | 			break
56 | 		}
57 | 		buff.WriteByte(b[i])
58 | 	}
59 | 
60 | 	return buff.String()
61 | }
62 | 
63 | func (c *PatchMessage) Decode(body []byte) {
64 | 	if body[1] == byte(EncryptMessage) {
65 | 		c.Encrypt = true
66 | 	}
67 | 
68 | 	c.Offset = binary.BigEndian.Uint64(body[4:12])
69 | 	c.Name = c.GetPatchName(body[12:])
70 | }
71 | 


--------------------------------------------------------------------------------
/pkg/message/put_complete.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | )
 6 | 
 7 | type PutCompleteMessage struct {
 8 | 	KeyVersion uint16
 9 | }
10 | 
11 | var _ ProtoMessage = &PutCompleteMessage{}
12 | 
13 | func NewPutCompleteMessage(keyVersion uint16) *PutCompleteMessage {
14 | 	return &PutCompleteMessage{
15 | 		KeyVersion: keyVersion,
16 | 	}
17 | }
18 | 
19 | func (c *PutCompleteMessage) Encode() []byte {
20 | 	bt := []byte{
21 | 		byte(MessageTypePutComplete),
22 | 		0,
23 | 		0,
24 | 		0,
25 | 	}
26 | 
27 | 	kv := make([]byte, 2)
28 | 	binary.BigEndian.PutUint16(kv, uint16(c.KeyVersion))
29 | 	bt = append(bt, kv...)
30 | 
31 | 	ln := len(bt) + 8
32 | 
33 | 	bs := make([]byte, 8)
34 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
35 | 	return append(bs, bt...)
36 | }
37 | 
38 | func (c *PutCompleteMessage) Decode(body []byte) {
39 | 	c.KeyVersion = binary.BigEndian.Uint16(body[4 : 4+2])
40 | }
41 | 


--------------------------------------------------------------------------------
/pkg/message/put_message.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | )
 6 | 
 7 | type PutMessage struct {
 8 | 	Encrypt bool
 9 | 	Name    string
10 | }
11 | 
12 | var _ ProtoMessage = &PutMessage{}
13 | 
14 | func NewPutMessage(name string, encrypt bool) *PutMessage {
15 | 	return &PutMessage{
16 | 		Name:    name,
17 | 		Encrypt: encrypt,
18 | 	}
19 | }
20 | 
21 | func (c *PutMessage) Encode() []byte {
22 | 	bt := []byte{
23 | 		byte(MessageTypePut),
24 | 		0,
25 | 		0,
26 | 		0,
27 | 	}
28 | 
29 | 	if c.Encrypt {
30 | 		bt[1] = byte(EncryptMessage)
31 | 	} else {
32 | 		bt[1] = byte(NoEncryptMessage)
33 | 	}
34 | 
35 | 	bt = append(bt, []byte(c.Name)...)
36 | 	bt = append(bt, 0)
37 | 	ln := len(bt) + 8
38 | 
39 | 	bs := make([]byte, 8)
40 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
41 | 	return append(bs, bt...)
42 | }
43 | 
44 | func (c *PutMessage) Decode(body []byte) {
45 | 	if body[1] == byte(EncryptMessage) {
46 | 		c.Encrypt = true
47 | 	}
48 | 	c.Name, _ = GetCstring(body[4:])
49 | }
50 | 


--------------------------------------------------------------------------------
/pkg/message/put_message_v2.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | 
 6 | 	"github.com/yezzey-gp/yproxy/pkg/settings"
 7 | )
 8 | 
 9 | type PutMessageV2 struct {
10 | 	Encrypt bool
11 | 	Name    string
12 | 
13 | 	Settings []settings.StorageSettings
14 | }
15 | 
16 | var _ ProtoMessage = &PutMessageV2{}
17 | 
18 | func NewPutMessageV2(name string, encrypt bool, settings []settings.StorageSettings) *PutMessageV2 {
19 | 	return &PutMessageV2{
20 | 		Name:     name,
21 | 		Encrypt:  encrypt,
22 | 		Settings: settings,
23 | 	}
24 | }
25 | 
26 | func (c *PutMessageV2) Encode() []byte {
27 | 	bt := []byte{
28 | 		byte(MessageTypePutV2),
29 | 		0,
30 | 		0,
31 | 		0,
32 | 	}
33 | 
34 | 	if c.Encrypt {
35 | 		bt[1] = byte(EncryptMessage)
36 | 	} else {
37 | 		bt[1] = byte(NoEncryptMessage)
38 | 	}
39 | 
40 | 	bt = append(bt, []byte(c.Name)...)
41 | 	bt = append(bt, 0)
42 | 
43 | 	slen := make([]byte, 8)
44 | 	binary.BigEndian.PutUint64(slen, uint64(len(c.Settings)))
45 | 	bt = append(bt, slen...)
46 | 
47 | 	for _, s := range c.Settings {
48 | 
49 | 		bt = append(bt, []byte(s.Name)...)
50 | 		bt = append(bt, 0)
51 | 
52 | 		bt = append(bt, []byte(s.Value)...)
53 | 		bt = append(bt, 0)
54 | 	}
55 | 
56 | 	ln := len(bt) + 8
57 | 
58 | 	bs := make([]byte, 8)
59 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
60 | 	return append(bs, bt...)
61 | }
62 | 
63 | func (c *PutMessageV2) Decode(body []byte) {
64 | 	if body[1] == byte(EncryptMessage) {
65 | 		c.Encrypt = true
66 | 	}
67 | 	var off uint64
68 | 	c.Name, off = GetCstring(body[4:])
69 | 
70 | 	settLen := binary.BigEndian.Uint64(body[4+off : 4+off+8])
71 | 
72 | 	totalOff := 4 + off + 8
73 | 
74 | 	c.Settings = make([]settings.StorageSettings, settLen)
75 | 
76 | 	for i := 0; i < int(settLen); i++ {
77 | 
78 | 		var currOff uint64
79 | 
80 | 		c.Settings[i].Name, currOff = GetCstring(body[totalOff:])
81 | 		totalOff += currOff
82 | 
83 | 		c.Settings[i].Value, currOff = GetCstring(body[totalOff:])
84 | 		totalOff += currOff
85 | 	}
86 | }
87 | 


--------------------------------------------------------------------------------
/pkg/message/put_message_v3.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | 
 6 | 	"github.com/yezzey-gp/yproxy/pkg/settings"
 7 | )
 8 | 
 9 | type PutMessageV3 struct {
10 | 	Encrypt bool
11 | 	Name    string
12 | 
13 | 	Settings []settings.StorageSettings
14 | }
15 | 
16 | var _ ProtoMessage = &PutMessageV3{}
17 | 
18 | func NewPutMessageV3(name string, encrypt bool, settings []settings.StorageSettings) *PutMessageV3 {
19 | 	return &PutMessageV3{
20 | 		Name:     name,
21 | 		Encrypt:  encrypt,
22 | 		Settings: settings,
23 | 	}
24 | }
25 | 
26 | func (c *PutMessageV3) Encode() []byte {
27 | 	bt := []byte{
28 | 		byte(MessageTypePutV3),
29 | 		0,
30 | 		0,
31 | 		0,
32 | 	}
33 | 
34 | 	if c.Encrypt {
35 | 		bt[1] = byte(EncryptMessage)
36 | 	} else {
37 | 		bt[1] = byte(NoEncryptMessage)
38 | 	}
39 | 
40 | 	bt = append(bt, []byte(c.Name)...)
41 | 	bt = append(bt, 0)
42 | 
43 | 	slen := make([]byte, 8)
44 | 	binary.BigEndian.PutUint64(slen, uint64(len(c.Settings)))
45 | 	bt = append(bt, slen...)
46 | 
47 | 	for _, s := range c.Settings {
48 | 
49 | 		bt = append(bt, []byte(s.Name)...)
50 | 		bt = append(bt, 0)
51 | 
52 | 		bt = append(bt, []byte(s.Value)...)
53 | 		bt = append(bt, 0)
54 | 	}
55 | 
56 | 	ln := len(bt) + 8
57 | 
58 | 	bs := make([]byte, 8)
59 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
60 | 	return append(bs, bt...)
61 | }
62 | 
63 | func (c *PutMessageV3) Decode(body []byte) {
64 | 	if body[1] == byte(EncryptMessage) {
65 | 		c.Encrypt = true
66 | 	}
67 | 	var off uint64
68 | 	c.Name, off = GetCstring(body[4:])
69 | 
70 | 	settLen := binary.BigEndian.Uint64(body[4+off : 4+off+8])
71 | 
72 | 	totalOff := 4 + off + 8
73 | 
74 | 	c.Settings = make([]settings.StorageSettings, settLen)
75 | 
76 | 	for i := range int(settLen) {
77 | 
78 | 		var currOff uint64
79 | 
80 | 		c.Settings[i].Name, currOff = GetCstring(body[totalOff:])
81 | 		totalOff += currOff
82 | 
83 | 		c.Settings[i].Value, currOff = GetCstring(body[totalOff:])
84 | 		totalOff += currOff
85 | 	}
86 | }
87 | 


--------------------------------------------------------------------------------
/pkg/message/ready_for_query.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import "encoding/binary"
 4 | 
 5 | type ReadyForQueryMessage struct {
 6 | 	ProtoMessage
 7 | }
 8 | 
 9 | func NewReadyForQueryMessage() *ReadyForQueryMessage {
10 | 	return &ReadyForQueryMessage{}
11 | }
12 | 
13 | func (cc *ReadyForQueryMessage) Encode() []byte {
14 | 	bt := []byte{
15 | 		byte(MessageTypeReadyForQuery),
16 | 		0,
17 | 		0,
18 | 		0,
19 | 	}
20 | 
21 | 	ln := len(bt) + 8
22 | 
23 | 	bs := make([]byte, 8)
24 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
25 | 	return append(bs, bt...)
26 | }
27 | 
28 | func (c *ReadyForQueryMessage) Decode(body []byte) error {
29 | 	return nil
30 | }
31 | 


--------------------------------------------------------------------------------
/pkg/message/untrashify_message.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | )
 6 | 
 7 | type UntrashifyMessage struct { //seg port
 8 | 	Name    string
 9 | 	Segnum  uint64
10 | 	Confirm bool
11 | }
12 | 
13 | var _ ProtoMessage = &UntrashifyMessage{}
14 | 
15 | func NewUntrashifyMessage(name string, seg uint64, confirm bool) *UntrashifyMessage {
16 | 	return &UntrashifyMessage{
17 | 		Name:    name,
18 | 		Segnum:  seg,
19 | 		Confirm: confirm,
20 | 	}
21 | }
22 | 
23 | func (c *UntrashifyMessage) Encode() []byte {
24 | 	bt := []byte{
25 | 		byte(MessageTypeUntrashify),
26 | 		0,
27 | 		0,
28 | 		0,
29 | 	}
30 | 
31 | 	if c.Confirm {
32 | 		bt[1] = 1
33 | 	}
34 | 
35 | 	bt = append(bt, []byte(c.Name)...)
36 | 	bt = append(bt, 0)
37 | 
38 | 	p := make([]byte, 8)
39 | 	binary.BigEndian.PutUint64(p, uint64(c.Segnum))
40 | 	bt = append(bt, p...)
41 | 
42 | 	ln := len(bt) + 8
43 | 	bs := make([]byte, 8)
44 | 	binary.BigEndian.PutUint64(bs, uint64(ln))
45 | 	return append(bs, bt...)
46 | }
47 | 
48 | func (c *UntrashifyMessage) Decode(body []byte) {
49 | 	if body[1] == 1 {
50 | 		c.Confirm = true
51 | 	}
52 | 	c.Name, _ = GetCstring(body[4:])
53 | 	c.Segnum = binary.BigEndian.Uint64(body[len(body)-8:])
54 | }
55 | 


--------------------------------------------------------------------------------
/pkg/message/utils.go:
--------------------------------------------------------------------------------
 1 | package message
 2 | 
 3 | import "bytes"
 4 | 
 5 | func GetCstring(b []byte) (string, uint64) {
 6 | 	offset := uint64(0)
 7 | 	buff := bytes.NewBufferString("")
 8 | 
 9 | 	for i := 0; i < len(b); i++ {
10 | 		offset++
11 | 		if b[i] == 0 {
12 | 			break
13 | 		}
14 | 		buff.WriteByte(b[i])
15 | 	}
16 | 
17 | 	return buff.String(), offset
18 | }
19 | 


--------------------------------------------------------------------------------
/pkg/mock/backups.go:
--------------------------------------------------------------------------------
 1 | // Code generated by MockGen. DO NOT EDIT.
 2 | // Source: pkg/backups/backups.go
 3 | //
 4 | // Generated by this command:
 5 | //
 6 | //	mockgen -source=pkg/backups/backups.go -destination=pkg/mock/backups.go -package=mock
 7 | //
 8 | 
 9 | // Package mock is a generated GoMock package.
10 | package mock
11 | 
12 | import (
13 | 	reflect "reflect"
14 | 
15 | 	gomock "go.uber.org/mock/gomock"
16 | )
17 | 
18 | // MockBackupInterractor is a mock of BackupInterractor interface.
19 | type MockBackupInterractor struct {
20 | 	ctrl     *gomock.Controller
21 | 	recorder *MockBackupInterractorMockRecorder
22 | 	isgomock struct{}
23 | }
24 | 
25 | // MockBackupInterractorMockRecorder is the mock recorder for MockBackupInterractor.
26 | type MockBackupInterractorMockRecorder struct {
27 | 	mock *MockBackupInterractor
28 | }
29 | 
30 | // NewMockBackupInterractor creates a new mock instance.
31 | func NewMockBackupInterractor(ctrl *gomock.Controller) *MockBackupInterractor {
32 | 	mock := &MockBackupInterractor{ctrl: ctrl}
33 | 	mock.recorder = &MockBackupInterractorMockRecorder{mock}
34 | 	return mock
35 | }
36 | 
37 | // EXPECT returns an object that allows the caller to indicate expected use.
38 | func (m *MockBackupInterractor) EXPECT() *MockBackupInterractorMockRecorder {
39 | 	return m.recorder
40 | }
41 | 
42 | // GetFirstLSN mocks base method.
43 | func (m *MockBackupInterractor) GetFirstLSN(seg uint64) (uint64, error) {
44 | 	m.ctrl.T.Helper()
45 | 	ret := m.ctrl.Call(m, "GetFirstLSN", seg)
46 | 	ret0, _ := ret[0].(uint64)
47 | 	ret1, _ := ret[1].(error)
48 | 	return ret0, ret1
49 | }
50 | 
51 | // GetFirstLSN indicates an expected call of GetFirstLSN.
52 | func (mr *MockBackupInterractorMockRecorder) GetFirstLSN(seg any) *gomock.Call {
53 | 	mr.mock.ctrl.T.Helper()
54 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFirstLSN", reflect.TypeOf((*MockBackupInterractor)(nil).GetFirstLSN), seg)
55 | }
56 | 


--------------------------------------------------------------------------------
/pkg/mock/client/client.go:
--------------------------------------------------------------------------------
  1 | // Code generated by MockGen. DO NOT EDIT.
  2 | // Source: pkg/client/client.go
  3 | //
  4 | // Generated by this command:
  5 | //
  6 | //	mockgen -source=pkg/client/client.go -destination=pkg/mock/client/client.go -package=mock
  7 | //
  8 | 
  9 | // Package mock is a generated GoMock package.
 10 | package mock
 11 | 
 12 | import (
 13 | 	io "io"
 14 | 	reflect "reflect"
 15 | 	time "time"
 16 | 
 17 | 	message "github.com/yezzey-gp/yproxy/pkg/message"
 18 | 	gomock "go.uber.org/mock/gomock"
 19 | )
 20 | 
 21 | // MockYproxyClient is a mock of YproxyClient interface.
 22 | type MockYproxyClient struct {
 23 | 	ctrl     *gomock.Controller
 24 | 	recorder *MockYproxyClientMockRecorder
 25 | 	isgomock struct{}
 26 | }
 27 | 
 28 | // MockYproxyClientMockRecorder is the mock recorder for MockYproxyClient.
 29 | type MockYproxyClientMockRecorder struct {
 30 | 	mock *MockYproxyClient
 31 | }
 32 | 
 33 | // NewMockYproxyClient creates a new mock instance.
 34 | func NewMockYproxyClient(ctrl *gomock.Controller) *MockYproxyClient {
 35 | 	mock := &MockYproxyClient{ctrl: ctrl}
 36 | 	mock.recorder = &MockYproxyClientMockRecorder{mock}
 37 | 	return mock
 38 | }
 39 | 
 40 | // EXPECT returns an object that allows the caller to indicate expected use.
 41 | func (m *MockYproxyClient) EXPECT() *MockYproxyClientMockRecorder {
 42 | 	return m.recorder
 43 | }
 44 | 
 45 | // ByteOffset mocks base method.
 46 | func (m *MockYproxyClient) ByteOffset() int64 {
 47 | 	m.ctrl.T.Helper()
 48 | 	ret := m.ctrl.Call(m, "ByteOffset")
 49 | 	ret0, _ := ret[0].(int64)
 50 | 	return ret0
 51 | }
 52 | 
 53 | // ByteOffset indicates an expected call of ByteOffset.
 54 | func (mr *MockYproxyClientMockRecorder) ByteOffset() *gomock.Call {
 55 | 	mr.mock.ctrl.T.Helper()
 56 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ByteOffset", reflect.TypeOf((*MockYproxyClient)(nil).ByteOffset))
 57 | }
 58 | 
 59 | // Close mocks base method.
 60 | func (m *MockYproxyClient) Close() error {
 61 | 	m.ctrl.T.Helper()
 62 | 	ret := m.ctrl.Call(m, "Close")
 63 | 	ret0, _ := ret[0].(error)
 64 | 	return ret0
 65 | }
 66 | 
 67 | // Close indicates an expected call of Close.
 68 | func (mr *MockYproxyClientMockRecorder) Close() *gomock.Call {
 69 | 	mr.mock.ctrl.T.Helper()
 70 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockYproxyClient)(nil).Close))
 71 | }
 72 | 
 73 | // ExternalFilePath mocks base method.
 74 | func (m *MockYproxyClient) ExternalFilePath() string {
 75 | 	m.ctrl.T.Helper()
 76 | 	ret := m.ctrl.Call(m, "ExternalFilePath")
 77 | 	ret0, _ := ret[0].(string)
 78 | 	return ret0
 79 | }
 80 | 
 81 | // ExternalFilePath indicates an expected call of ExternalFilePath.
 82 | func (mr *MockYproxyClientMockRecorder) ExternalFilePath() *gomock.Call {
 83 | 	mr.mock.ctrl.T.Helper()
 84 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExternalFilePath", reflect.TypeOf((*MockYproxyClient)(nil).ExternalFilePath))
 85 | }
 86 | 
 87 | // GetRW mocks base method.
 88 | func (m *MockYproxyClient) GetRW() io.ReadWriteCloser {
 89 | 	m.ctrl.T.Helper()
 90 | 	ret := m.ctrl.Call(m, "GetRW")
 91 | 	ret0, _ := ret[0].(io.ReadWriteCloser)
 92 | 	return ret0
 93 | }
 94 | 
 95 | // GetRW indicates an expected call of GetRW.
 96 | func (mr *MockYproxyClientMockRecorder) GetRW() *gomock.Call {
 97 | 	mr.mock.ctrl.T.Helper()
 98 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRW", reflect.TypeOf((*MockYproxyClient)(nil).GetRW))
 99 | }
100 | 
101 | // ID mocks base method.
102 | func (m *MockYproxyClient) ID() uint {
103 | 	m.ctrl.T.Helper()
104 | 	ret := m.ctrl.Call(m, "ID")
105 | 	ret0, _ := ret[0].(uint)
106 | 	return ret0
107 | }
108 | 
109 | // ID indicates an expected call of ID.
110 | func (mr *MockYproxyClientMockRecorder) ID() *gomock.Call {
111 | 	mr.mock.ctrl.T.Helper()
112 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ID", reflect.TypeOf((*MockYproxyClient)(nil).ID))
113 | }
114 | 
115 | // OPStart mocks base method.
116 | func (m *MockYproxyClient) OPStart() time.Time {
117 | 	m.ctrl.T.Helper()
118 | 	ret := m.ctrl.Call(m, "OPStart")
119 | 	ret0, _ := ret[0].(time.Time)
120 | 	return ret0
121 | }
122 | 
123 | // OPStart indicates an expected call of OPStart.
124 | func (mr *MockYproxyClientMockRecorder) OPStart() *gomock.Call {
125 | 	mr.mock.ctrl.T.Helper()
126 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OPStart", reflect.TypeOf((*MockYproxyClient)(nil).OPStart))
127 | }
128 | 
129 | // OPType mocks base method.
130 | func (m *MockYproxyClient) OPType() message.MessageType {
131 | 	m.ctrl.T.Helper()
132 | 	ret := m.ctrl.Call(m, "OPType")
133 | 	ret0, _ := ret[0].(message.MessageType)
134 | 	return ret0
135 | }
136 | 
137 | // OPType indicates an expected call of OPType.
138 | func (mr *MockYproxyClientMockRecorder) OPType() *gomock.Call {
139 | 	mr.mock.ctrl.T.Helper()
140 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OPType", reflect.TypeOf((*MockYproxyClient)(nil).OPType))
141 | }
142 | 
143 | // ReplyError mocks base method.
144 | func (m *MockYproxyClient) ReplyError(err error, msg string) error {
145 | 	m.ctrl.T.Helper()
146 | 	ret := m.ctrl.Call(m, "ReplyError", err, msg)
147 | 	ret0, _ := ret[0].(error)
148 | 	return ret0
149 | }
150 | 
151 | // ReplyError indicates an expected call of ReplyError.
152 | func (mr *MockYproxyClientMockRecorder) ReplyError(err, msg any) *gomock.Call {
153 | 	mr.mock.ctrl.T.Helper()
154 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReplyError", reflect.TypeOf((*MockYproxyClient)(nil).ReplyError), err, msg)
155 | }
156 | 
157 | // SetByteOffset mocks base method.
158 | func (m *MockYproxyClient) SetByteOffset(arg0 int64) {
159 | 	m.ctrl.T.Helper()
160 | 	m.ctrl.Call(m, "SetByteOffset", arg0)
161 | }
162 | 
163 | // SetByteOffset indicates an expected call of SetByteOffset.
164 | func (mr *MockYproxyClientMockRecorder) SetByteOffset(arg0 any) *gomock.Call {
165 | 	mr.mock.ctrl.T.Helper()
166 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetByteOffset", reflect.TypeOf((*MockYproxyClient)(nil).SetByteOffset), arg0)
167 | }
168 | 
169 | // SetExternalFilePath mocks base method.
170 | func (m *MockYproxyClient) SetExternalFilePath(path string) {
171 | 	m.ctrl.T.Helper()
172 | 	m.ctrl.Call(m, "SetExternalFilePath", path)
173 | }
174 | 
175 | // SetExternalFilePath indicates an expected call of SetExternalFilePath.
176 | func (mr *MockYproxyClientMockRecorder) SetExternalFilePath(path any) *gomock.Call {
177 | 	mr.mock.ctrl.T.Helper()
178 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetExternalFilePath", reflect.TypeOf((*MockYproxyClient)(nil).SetExternalFilePath), path)
179 | }
180 | 
181 | // SetOPType mocks base method.
182 | func (m *MockYproxyClient) SetOPType(optype message.MessageType) {
183 | 	m.ctrl.T.Helper()
184 | 	m.ctrl.Call(m, "SetOPType", optype)
185 | }
186 | 
187 | // SetOPType indicates an expected call of SetOPType.
188 | func (mr *MockYproxyClientMockRecorder) SetOPType(optype any) *gomock.Call {
189 | 	mr.mock.ctrl.T.Helper()
190 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetOPType", reflect.TypeOf((*MockYproxyClient)(nil).SetOPType), optype)
191 | }
192 | 


--------------------------------------------------------------------------------
/pkg/mock/database.go:
--------------------------------------------------------------------------------
 1 | // Code generated by MockGen. DO NOT EDIT.
 2 | // Source: pkg/database/database.go
 3 | //
 4 | // Generated by this command:
 5 | //
 6 | //	mockgen -source=pkg/database/database.go -destination=pkg/mock/database.go -package=mock
 7 | //
 8 | 
 9 | // Package mock is a generated GoMock package.
10 | package mock
11 | 
12 | import (
13 | 	reflect "reflect"
14 | 
15 | 	gomock "go.uber.org/mock/gomock"
16 | )
17 | 
18 | // MockDatabaseInterractor is a mock of DatabaseInterractor interface.
19 | type MockDatabaseInterractor struct {
20 | 	ctrl     *gomock.Controller
21 | 	recorder *MockDatabaseInterractorMockRecorder
22 | 	isgomock struct{}
23 | }
24 | 
25 | // MockDatabaseInterractorMockRecorder is the mock recorder for MockDatabaseInterractor.
26 | type MockDatabaseInterractorMockRecorder struct {
27 | 	mock *MockDatabaseInterractor
28 | }
29 | 
30 | // NewMockDatabaseInterractor creates a new mock instance.
31 | func NewMockDatabaseInterractor(ctrl *gomock.Controller) *MockDatabaseInterractor {
32 | 	mock := &MockDatabaseInterractor{ctrl: ctrl}
33 | 	mock.recorder = &MockDatabaseInterractorMockRecorder{mock}
34 | 	return mock
35 | }
36 | 
37 | // EXPECT returns an object that allows the caller to indicate expected use.
38 | func (m *MockDatabaseInterractor) EXPECT() *MockDatabaseInterractorMockRecorder {
39 | 	return m.recorder
40 | }
41 | 
42 | // GetVirtualExpireIndexes mocks base method.
43 | func (m *MockDatabaseInterractor) GetVirtualExpireIndexes(port uint64) (map[string]bool, map[string]uint64, error) {
44 | 	m.ctrl.T.Helper()
45 | 	ret := m.ctrl.Call(m, "GetVirtualExpireIndexes", port)
46 | 	ret0, _ := ret[0].(map[string]bool)
47 | 	ret1, _ := ret[1].(map[string]uint64)
48 | 	ret2, _ := ret[2].(error)
49 | 	return ret0, ret1, ret2
50 | }
51 | 
52 | // GetVirtualExpireIndexes indicates an expected call of GetVirtualExpireIndexes.
53 | func (mr *MockDatabaseInterractorMockRecorder) GetVirtualExpireIndexes(port any) *gomock.Call {
54 | 	mr.mock.ctrl.T.Helper()
55 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVirtualExpireIndexes", reflect.TypeOf((*MockDatabaseInterractor)(nil).GetVirtualExpireIndexes), port)
56 | }
57 | 


--------------------------------------------------------------------------------
/pkg/mock/proc/yio/yrreader.go:
--------------------------------------------------------------------------------
 1 | // Code generated by MockGen. DO NOT EDIT.
 2 | // Source: pkg/proc/yio/yrreader.go
 3 | //
 4 | // Generated by this command:
 5 | //
 6 | //	mockgen -source=pkg/proc/yio/yrreader.go -destination=pkg/mock/proc/yio/yrreader.go -package=mock
 7 | //
 8 | 
 9 | // Package mock is a generated GoMock package.
10 | package mock
11 | 
12 | import (
13 | 	reflect "reflect"
14 | 
15 | 	gomock "go.uber.org/mock/gomock"
16 | )
17 | 
18 | // MockRestartReader is a mock of RestartReader interface.
19 | type MockRestartReader struct {
20 | 	ctrl     *gomock.Controller
21 | 	recorder *MockRestartReaderMockRecorder
22 | 	isgomock struct{}
23 | }
24 | 
25 | // MockRestartReaderMockRecorder is the mock recorder for MockRestartReader.
26 | type MockRestartReaderMockRecorder struct {
27 | 	mock *MockRestartReader
28 | }
29 | 
30 | // NewMockRestartReader creates a new mock instance.
31 | func NewMockRestartReader(ctrl *gomock.Controller) *MockRestartReader {
32 | 	mock := &MockRestartReader{ctrl: ctrl}
33 | 	mock.recorder = &MockRestartReaderMockRecorder{mock}
34 | 	return mock
35 | }
36 | 
37 | // EXPECT returns an object that allows the caller to indicate expected use.
38 | func (m *MockRestartReader) EXPECT() *MockRestartReaderMockRecorder {
39 | 	return m.recorder
40 | }
41 | 
42 | // Close mocks base method.
43 | func (m *MockRestartReader) Close() error {
44 | 	m.ctrl.T.Helper()
45 | 	ret := m.ctrl.Call(m, "Close")
46 | 	ret0, _ := ret[0].(error)
47 | 	return ret0
48 | }
49 | 
50 | // Close indicates an expected call of Close.
51 | func (mr *MockRestartReaderMockRecorder) Close() *gomock.Call {
52 | 	mr.mock.ctrl.T.Helper()
53 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockRestartReader)(nil).Close))
54 | }
55 | 
56 | // Read mocks base method.
57 | func (m *MockRestartReader) Read(p []byte) (int, error) {
58 | 	m.ctrl.T.Helper()
59 | 	ret := m.ctrl.Call(m, "Read", p)
60 | 	ret0, _ := ret[0].(int)
61 | 	ret1, _ := ret[1].(error)
62 | 	return ret0, ret1
63 | }
64 | 
65 | // Read indicates an expected call of Read.
66 | func (mr *MockRestartReaderMockRecorder) Read(p any) *gomock.Call {
67 | 	mr.mock.ctrl.T.Helper()
68 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Read", reflect.TypeOf((*MockRestartReader)(nil).Read), p)
69 | }
70 | 
71 | // Restart mocks base method.
72 | func (m *MockRestartReader) Restart(offsetStart int64) error {
73 | 	m.ctrl.T.Helper()
74 | 	ret := m.ctrl.Call(m, "Restart", offsetStart)
75 | 	ret0, _ := ret[0].(error)
76 | 	return ret0
77 | }
78 | 
79 | // Restart indicates an expected call of Restart.
80 | func (mr *MockRestartReaderMockRecorder) Restart(offsetStart any) *gomock.Call {
81 | 	mr.mock.ctrl.T.Helper()
82 | 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Restart", reflect.TypeOf((*MockRestartReader)(nil).Restart), offsetStart)
83 | }
84 | 


--------------------------------------------------------------------------------
/pkg/object/objectInfo.go:
--------------------------------------------------------------------------------
1 | package object
2 | 
3 | type ObjectInfo struct {
4 | 	Path string
5 | 	Size int64
6 | }
7 | 


--------------------------------------------------------------------------------
/pkg/proc/delete_handler.go:
--------------------------------------------------------------------------------
  1 | package proc
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"path"
  6 | 	"strings"
  7 | 
  8 | 	"github.com/pkg/errors"
  9 | 	"github.com/yezzey-gp/yproxy/config"
 10 | 	"github.com/yezzey-gp/yproxy/pkg/backups"
 11 | 	"github.com/yezzey-gp/yproxy/pkg/database"
 12 | 	"github.com/yezzey-gp/yproxy/pkg/message"
 13 | 	"github.com/yezzey-gp/yproxy/pkg/storage"
 14 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
 15 | )
 16 | 
 17 | //go:generate mockgen -destination=../../../test/mocks/mock_object.go -package mocks -build_flags -mod=readonly github.com/wal-g/wal-g/pkg/storages/storage Object
 18 | type GarbageMgr interface {
 19 | 	HandleDeleteGarbage(message.DeleteMessage) error
 20 | 	HandleDeleteFile(message.DeleteMessage) error
 21 | 	HandleUntrashifyFile(message.UntrashifyMessage) error
 22 | }
 23 | 
 24 | type BasicGarbageMgr struct {
 25 | 	BackupInterractor  backups.BackupInterractor
 26 | 	DbInterractor      database.DatabaseInterractor
 27 | 	StorageInterractor storage.StorageInteractor
 28 | 
 29 | 	Cnf *config.Vacuum
 30 | }
 31 | 
 32 | var _ GarbageMgr = &BasicGarbageMgr{}
 33 | 
 34 | func TrashPathFromRegPath(p string, segnum int) string {
 35 | 
 36 | 	filePathParts := strings.Split(p, "/")
 37 | 
 38 | 	destPath := path.Join(
 39 | 		"trash",
 40 | 		"segments_005",
 41 | 		fmt.Sprintf("seg%d", segnum),
 42 | 		"basebackups_005",
 43 | 		"yezzey", filePathParts[len(filePathParts)-1])
 44 | 
 45 | 	return destPath
 46 | }
 47 | 
 48 | func RegPathFromTrasnPath(p string, segnum int) string {
 49 | 
 50 | 	filePathParts := strings.Split(p, "/")
 51 | 
 52 | 	destPath := path.Join(
 53 | 		"segments_005",
 54 | 		fmt.Sprintf("seg%d", segnum),
 55 | 		"basebackups_005",
 56 | 		"yezzey", filePathParts[len(filePathParts)-1])
 57 | 
 58 | 	return destPath
 59 | }
 60 | 
 61 | // HandleUntrashifyFile implements GarbageMgr.
 62 | func (dh *BasicGarbageMgr) HandleUntrashifyFile(msg message.UntrashifyMessage) error {
 63 | 
 64 | 	ylogger.Zero.Info().Str("path", msg.Name).Msg("listing prefix")
 65 | 	objectMetas, err := dh.StorageInterractor.ListPath(msg.Name, true, nil)
 66 | 	if err != nil {
 67 | 		return errors.Wrap(err, "could not list objects")
 68 | 	}
 69 | 
 70 | 	for _, file := range objectMetas {
 71 | 		ylogger.Zero.Info().Str("file", file.Path).Str("dest-path", RegPathFromTrasnPath(file.Path, int(msg.Segnum))).Msg("file will be untrashified")
 72 | 	}
 73 | 
 74 | 	if !msg.Confirm { //do not delete files if no confirmation flag provided
 75 | 		return nil
 76 | 	}
 77 | 
 78 | 	for _, file := range objectMetas {
 79 | 		tp := RegPathFromTrasnPath(file.Path, int(msg.Segnum))
 80 | 		/* XXX: fix this */
 81 | 		err = dh.StorageInterractor.MoveObject(dh.StorageInterractor.DefaultBucket(), file.Path, tp)
 82 | 		if err != nil {
 83 | 			return err
 84 | 		}
 85 | 	}
 86 | 
 87 | 	return nil
 88 | }
 89 | 
 90 | func (dh *BasicGarbageMgr) DeleteGarbageInBucket(bucket string, msg message.DeleteMessage) error {
 91 | 	fileList, err := dh.ListGarbageFiles(bucket, msg)
 92 | 	if err != nil {
 93 | 		return errors.Wrap(err, "failed to delete file")
 94 | 	}
 95 | 	uploads, err := dh.StorageInterractor.ListFailedMultipartUploads(bucket)
 96 | 	if err != nil {
 97 | 		return err
 98 | 	}
 99 | 	ylogger.Zero.Info().Str("bucket", bucket).Int("amount", len(uploads)).Msg("multipart uploads will be aborted")
100 | 
101 | 	for _, file := range fileList {
102 | 		ylogger.Zero.Info().Str("bucket", bucket).Bool("crazy mode", msg.CrazyDrop).Str("file", file).Msg("file will be deleted")
103 | 	}
104 | 	for _, upload := range uploads {
105 | 		ylogger.Zero.Info().Str("bucket", bucket).Str("uploadId", upload).Msg("upload will be aborted")
106 | 	}
107 | 
108 | 	if !msg.Confirm { //do not delete files if no confirmation flag provided
109 | 		ylogger.Zero.Info().Msg("do not perform actual delete files as no confirmation flag provided")
110 | 		return nil
111 | 	}
112 | 
113 | 	var failed []string
114 | 	retryCount := 0
115 | 	for len(fileList) > 0 && retryCount < 10 {
116 | 		retryCount++
117 | 		for i := 0; i < len(fileList); i++ {
118 | 
119 | 			if msg.CrazyDrop {
120 | 				ylogger.Zero.Info().Str("bucket", bucket).Str("path", fileList[i]).Msg("simply delete without any 'plan B'")
121 | 				err = dh.StorageInterractor.DeleteObject(bucket, fileList[i])
122 | 			} else {
123 | 				tp := TrashPathFromRegPath(fileList[i], int(msg.Segnum))
124 | 				err = dh.StorageInterractor.MoveObject(bucket, fileList[i], tp)
125 | 			}
126 | 			if err != nil {
127 | 				ylogger.Zero.Warn().AnErr("err", err).Str("bucket", bucket).Str("file", fileList[i]).Msg("failed to obsolete file")
128 | 				failed = append(failed, fileList[i])
129 | 			}
130 | 		}
131 | 		fileList = failed
132 | 		failed = make([]string, 0)
133 | 	}
134 | 
135 | 	if len(fileList) > 0 {
136 | 		ylogger.Zero.Error().Str("bucket", bucket).Int("failed files count", len(fileList)).Msg("some files were not moved")
137 | 		ylogger.Zero.Error().Str("bucket", bucket).Any("failed files", fileList).Msg("failed to move some files")
138 | 		return errors.Wrap(err, "failed to move some files")
139 | 	}
140 | 
141 | 	for key, uploadId := range uploads {
142 | 		if err := dh.StorageInterractor.AbortMultipartUpload(bucket, key, uploadId); err != nil {
143 | 			return err
144 | 		}
145 | 	}
146 | 
147 | 	return nil
148 | }
149 | 
150 | func (dh *BasicGarbageMgr) HandleDeleteGarbage(msg message.DeleteMessage) error {
151 | 	for _, b := range dh.StorageInterractor.ListBuckets() {
152 | 		if err := dh.DeleteGarbageInBucket(b, msg); err != nil {
153 | 			return err
154 | 		}
155 | 	}
156 | 	return nil
157 | }
158 | 
159 | func (dh *BasicGarbageMgr) HandleDeleteFile(msg message.DeleteMessage) error {
160 | 	for _, b := range dh.StorageInterractor.ListBuckets() {
161 | 		err := dh.StorageInterractor.DeleteObject(b, msg.Name)
162 | 		if err != nil {
163 | 			ylogger.Zero.Error().AnErr("err", err).Str("name", msg.Name).Msg("failed to delete file")
164 | 			return errors.Wrap(err, "failed to delete file")
165 | 		}
166 | 	}
167 | 	return nil
168 | }
169 | 
170 | func (dh *BasicGarbageMgr) ListGarbageFiles(bucket string, msg message.DeleteMessage) ([]string, error) {
171 | 	//get first backup lsn
172 | 	var firstBackupLSN uint64
173 | 	var err error
174 | 
175 | 	if dh.Cnf.CheckBackup {
176 | 		firstBackupLSN, err = dh.BackupInterractor.GetFirstLSN(msg.Segnum)
177 | 		if err != nil {
178 | 			ylogger.Zero.Error().AnErr("err", err).Msg("failed to get first lsn") //return or just assume there are no backups?
179 | 			return nil, err
180 | 		}
181 | 		ylogger.Zero.Info().Uint64("lsn", firstBackupLSN).Msg("first backup LSN")
182 | 	} else {
183 | 		firstBackupLSN = ^uint64(0)
184 | 		ylogger.Zero.Info().Uint64("lsn", firstBackupLSN).Msg("omit first backup LSN")
185 | 	}
186 | 
187 | 	//list files in storage
188 | 	ylogger.Zero.Info().Str("path", msg.Name).Msg("listing prefix")
189 | 	objectMetas, err := dh.StorageInterractor.ListBucketPath(bucket, msg.Name, true)
190 | 	if err != nil {
191 | 		return nil, errors.Wrap(err, "could not list objects")
192 | 	}
193 | 	ylogger.Zero.Info().Int("amount", len(objectMetas)).Msg("objects count")
194 | 
195 | 	vi, ei, err := dh.DbInterractor.GetVirtualExpireIndexes(msg.Port)
196 | 	if err != nil {
197 | 		ylogger.Zero.Error().AnErr("err", err).Msg("failed to get indexes")
198 | 		return nil, errors.Wrap(err, "could not get virtual and expire indexes")
199 | 	}
200 | 	ylogger.Zero.Info().Msg("received virtual index and expire index")
201 | 	ylogger.Zero.Debug().Int("virtual", len(vi)).Msg("virtual index match count")
202 | 	ylogger.Zero.Debug().Int("expire", len(ei)).Msg("expire index match count")
203 | 
204 | 	filesToDelete := make([]string, 0)
205 | 	for i := range objectMetas {
206 | 		reworkedName := objectMetas[i].Path
207 | 		ylogger.Zero.Debug().Str("reworked name", reworkedName).Msg("lookup chunk")
208 | 
209 | 		if vi[reworkedName] {
210 | 			continue
211 | 		}
212 | 
213 | 		lsn, ok := ei[reworkedName]
214 | 		ylogger.Zero.Debug().Uint64("lsn", lsn).Uint64("backup lsn", firstBackupLSN).Str("path", objectMetas[i].Path).Msg("comparing lsn")
215 | 		if lsn < firstBackupLSN || !ok {
216 | 			ylogger.Zero.Debug().Str("file", objectMetas[i].Path).
217 | 				Bool("file in expire index", ok).
218 | 				Bool("lsn is less than in first backup", lsn < firstBackupLSN).
219 | 				Msg("file does not persist in virtual index, nor needed for PITR, so will be deleted")
220 | 			filesToDelete = append(filesToDelete, objectMetas[i].Path)
221 | 		}
222 | 	}
223 | 
224 | 	ylogger.Zero.Info().Int("amount", len(filesToDelete)).Msg("files will be deleted")
225 | 
226 | 	return filesToDelete, nil
227 | }
228 | 


--------------------------------------------------------------------------------
/pkg/proc/delete_handler_test.go:
--------------------------------------------------------------------------------
 1 | package proc_test
 2 | 
 3 | import (
 4 | 	"testing"
 5 | 
 6 | 	"github.com/stretchr/testify/assert"
 7 | 	"github.com/yezzey-gp/yproxy/config"
 8 | 	"github.com/yezzey-gp/yproxy/pkg/message"
 9 | 	mock "github.com/yezzey-gp/yproxy/pkg/mock"
10 | 	"github.com/yezzey-gp/yproxy/pkg/object"
11 | 	"github.com/yezzey-gp/yproxy/pkg/proc"
12 | 	"go.uber.org/mock/gomock"
13 | )
14 | 
15 | func TestFilesToDeletion(t *testing.T) {
16 | 	ctrl := gomock.NewController(t)
17 | 
18 | 	msg := message.DeleteMessage{
19 | 		Name:    "path",
20 | 		Port:    6000,
21 | 		Segnum:  0,
22 | 		Confirm: false,
23 | 	}
24 | 
25 | 	filesInStorage := []*object.ObjectInfo{
26 | 		{Path: "1663_16530_not-deleted_18002_"},
27 | 		{Path: "1663_16530_deleted-after-backup_18002_"},
28 | 		{Path: "1663_16530_deleted-when-backup-start_18002_"},
29 | 		{Path: "1663_16530_deleted-before-backup_18002_"},
30 | 		{Path: "some_trash"},
31 | 	}
32 | 	storage := mock.NewMockStorageInteractor(ctrl)
33 | 	storage.EXPECT().ListBucketPath("", msg.Name, true).Return(filesInStorage, nil)
34 | 
35 | 	backup := mock.NewMockBackupInterractor(ctrl)
36 | 	backup.EXPECT().GetFirstLSN(msg.Segnum).Return(uint64(1337), nil)
37 | 
38 | 	vi := map[string]bool{
39 | 		"1663_16530_not-deleted_18002_":               true,
40 | 		"1663_16530_deleted-after-backup_18002_":      true,
41 | 		"1663_16530_deleted-when-backup-start_18002_": true,
42 | 	}
43 | 	ei := map[string]uint64{
44 | 		"1663_16530_deleted-after-backup_18002_":      uint64(1400),
45 | 		"1663_16530_deleted-when-backup-start_18002_": uint64(1337),
46 | 		"1663_16530_deleted-before-backup_18002_":     uint64(1300),
47 | 	}
48 | 	database := mock.NewMockDatabaseInterractor(ctrl)
49 | 	database.EXPECT().GetVirtualExpireIndexes(msg.Port).Return(vi, ei, nil)
50 | 
51 | 	handler := proc.BasicGarbageMgr{
52 | 		StorageInterractor: storage,
53 | 		DbInterractor:      database,
54 | 		BackupInterractor:  backup,
55 | 		Cnf:                &config.Vacuum{CheckBackup: true},
56 | 	}
57 | 
58 | 	list, err := handler.ListGarbageFiles("", msg)
59 | 
60 | 	assert.NoError(t, err)
61 | 	assert.Equal(t, 2, len(list))
62 | 	assert.Equal(t, "1663_16530_deleted-before-backup_18002_", list[0])
63 | 	assert.Equal(t, "some_trash", list[1])
64 | }
65 | 
66 | func TestTrashPathConversion(t *testing.T) {
67 | 
68 | 	type tt struct {
69 | 		Path         string
70 | 		ExpTrashPath string
71 | 	}
72 | 
73 | 	for _, tc := range []tt{
74 | 		{
75 | 			Path:         "segments_005/seg60/basebackups_005/yezzey/1663_16712_001128848b0d46158f270005bc9cd82a_28144635_2305__DY_1_xlog_109463918313472",
76 | 			ExpTrashPath: "trash/segments_005/seg60/basebackups_005/yezzey/1663_16712_001128848b0d46158f270005bc9cd82a_28144635_2305__DY_1_xlog_109463918313472",
77 | 		},
78 | 	} {
79 | 
80 | 		resP := proc.TrashPathFromRegPath(tc.Path, 60)
81 | 
82 | 		assert.Equal(t, tc.ExpTrashPath, resP)
83 | 
84 | 		resP2 := proc.RegPathFromTrasnPath(resP, 60)
85 | 
86 | 		assert.Equal(t, tc.Path, resP2)
87 | 	}
88 | }
89 | 


--------------------------------------------------------------------------------
/pkg/proc/proto.go:
--------------------------------------------------------------------------------
 1 | package proc
 2 | 
 3 | import (
 4 | 	"encoding/binary"
 5 | 	"fmt"
 6 | 	"io"
 7 | 
 8 | 	"github.com/yezzey-gp/yproxy/pkg/client"
 9 | 	"github.com/yezzey-gp/yproxy/pkg/message"
10 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
11 | )
12 | 
13 | type ProtoReader struct {
14 | 	c io.ReadWriteCloser
15 | }
16 | 
17 | func NewProtoReader(ycl client.YproxyClient) *ProtoReader {
18 | 	return &ProtoReader{
19 | 		c: ycl.GetRW(),
20 | 	}
21 | }
22 | 
23 | // 1mb of data + header
24 | const maxMsgLen = 1<<20 | 1<<10
25 | 
26 | func (r *ProtoReader) ReadPacket() (message.MessageType, []byte, error) {
27 | 	msgLenBuf := make([]byte, 8)
28 | 	_, err := io.ReadFull(r.c, msgLenBuf)
29 | 	if err != nil {
30 | 		return 0, nil, fmt.Errorf("failed to read params: %w", err)
31 | 	}
32 | 
33 | 	dataLen := binary.BigEndian.Uint64(msgLenBuf)
34 | 
35 | 	if dataLen > maxMsgLen {
36 | 		return 0, nil, fmt.Errorf("message too big %d", dataLen)
37 | 	}
38 | 
39 | 	if dataLen <= 8 {
40 | 		return 0, nil, fmt.Errorf("message empty")
41 | 	}
42 | 
43 | 	dataLen -= 8
44 | 
45 | 	ylogger.Zero.Debug().Uint64("size", dataLen).Msg("requested packet")
46 | 
47 | 	data := make([]byte, dataLen)
48 | 	_, err = io.ReadFull(r.c, data)
49 | 	if err != nil {
50 | 		return 0, nil, err
51 | 	}
52 | 
53 | 	return message.MessageType(data[0]), data, nil
54 | }
55 | 


--------------------------------------------------------------------------------
/pkg/proc/yio/yrreader.go:
--------------------------------------------------------------------------------
  1 | package yio
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"io"
  6 | 	"time"
  7 | 
  8 | 	"github.com/yezzey-gp/yproxy/pkg/client"
  9 | 	"github.com/yezzey-gp/yproxy/pkg/settings"
 10 | 	"github.com/yezzey-gp/yproxy/pkg/storage"
 11 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
 12 | )
 13 | 
 14 | type RestartReader interface {
 15 | 	io.ReadCloser
 16 | 	Restart(offsetStart int64) error
 17 | }
 18 | 
 19 | type YRestartReader struct {
 20 | 	underlying io.ReadCloser
 21 | 	s          storage.StorageInteractor
 22 | 	name       string
 23 | 	settings   []settings.StorageSettings
 24 | }
 25 | 
 26 | // Close implements RestartReader.
 27 | func (y *YRestartReader) Close() error {
 28 | 	if y.underlying != nil {
 29 | 		return y.underlying.Close()
 30 | 	}
 31 | 	return nil
 32 | }
 33 | 
 34 | // Read implements RestartReader.
 35 | func (y *YRestartReader) Read(p []byte) (n int, err error) {
 36 | 	return y.underlying.Read(p)
 37 | }
 38 | 
 39 | func NewRestartReader(s storage.StorageInteractor,
 40 | 	name string, setts []settings.StorageSettings) RestartReader {
 41 | 
 42 | 	return &YRestartReader{
 43 | 		s:        s,
 44 | 		name:     name,
 45 | 		settings: setts,
 46 | 	}
 47 | }
 48 | 
 49 | func (y *YRestartReader) Restart(offsetStart int64) error {
 50 | 	if y.underlying != nil {
 51 | 		_ = y.underlying.Close()
 52 | 	}
 53 | 	if offsetStart == 0 {
 54 | 		ylogger.Zero.Debug().Str("object-path", y.name).Msg("cat object with offset")
 55 | 	} else {
 56 | 		ylogger.Zero.Error().Str("object-path", y.name).Int64("offset", offsetStart).Msg("cat object with offset after possible error")
 57 | 	}
 58 | 	r, err := y.s.CatFileFromStorage(y.name, offsetStart, y.settings)
 59 | 	if err != nil {
 60 | 		return err
 61 | 	}
 62 | 
 63 | 	y.underlying = r
 64 | 
 65 | 	return nil
 66 | }
 67 | 
 68 | type YproxyRetryReader struct {
 69 | 	io.ReadCloser
 70 | 	underlying RestartReader
 71 | 
 72 | 	selfCl client.YproxyClient
 73 | 
 74 | 	offsetReached int64
 75 | 	retryLimit    int
 76 | 	needReacquire bool
 77 | }
 78 | 
 79 | // Close implements io.ReadCloser.
 80 | func (y *YproxyRetryReader) Close() error {
 81 | 	err := y.underlying.Close()
 82 | 	if err != nil {
 83 | 		ylogger.Zero.Error().Err(err).Msg("encounter close error")
 84 | 	}
 85 | 	return err
 86 | }
 87 | 
 88 | // Read implements io.ReadCloser.
 89 | func (y *YproxyRetryReader) Read(p []byte) (int, error) {
 90 | 
 91 | 	for retry := range y.retryLimit {
 92 | 
 93 | 		if y.needReacquire {
 94 | 
 95 | 			err := y.underlying.Restart(y.offsetReached)
 96 | 
 97 | 			if err != nil {
 98 | 				// log error and continue.
 99 | 				// Try to mitigate overload problems with random sleep
100 | 				ylogger.Zero.Error().Err(err).Int("offset reached", int(y.offsetReached)).Int("retry count", int(retry)).Msg("failed to reacquire external storage connection, wait and retry")
101 | 
102 | 				time.Sleep(time.Second)
103 | 				continue
104 | 			}
105 | 
106 | 			y.needReacquire = false
107 | 		}
108 | 
109 | 		n, err := y.underlying.Read(p)
110 | 		if err == io.EOF {
111 | 			return n, err
112 | 		}
113 | 		if err != nil || n < 0 {
114 | 			ylogger.Zero.Error().Err(err).Int("offset reached", int(y.offsetReached)).Int("retry count", int(retry)).Msg("encounter read error")
115 | 
116 | 			// what if close failed?
117 | 			_ = y.underlying.Close()
118 | 
119 | 			// try to reacquire connection to external storage and continue read
120 | 			// from previously reached point
121 | 
122 | 			y.needReacquire = true
123 | 			continue
124 | 		} else {
125 | 			y.offsetReached += int64(n)
126 | 
127 | 			y.selfCl.SetByteOffset(y.offsetReached)
128 | 
129 | 			return n, err
130 | 		}
131 | 	}
132 | 	return -1, fmt.Errorf("failed to upload within retries")
133 | }
134 | 
135 | const (
136 | 	defaultRetryLimit = 100
137 | )
138 | 
139 | func NewYRetryReader(r RestartReader, selfCl client.YproxyClient) io.ReadCloser {
140 | 	return &YproxyRetryReader{
141 | 		underlying:    r,
142 | 		retryLimit:    defaultRetryLimit,
143 | 		selfCl:        selfCl,
144 | 		offsetReached: 0,
145 | 		needReacquire: true, /* do initial storage request */
146 | 	}
147 | }
148 | 
149 | var _ io.ReadCloser = &YproxyRetryReader{}
150 | 


--------------------------------------------------------------------------------
/pkg/proc/yio/yrreader_test.go:
--------------------------------------------------------------------------------
  1 | package yio_test
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"io"
  6 | 	"testing"
  7 | 
  8 | 	"github.com/stretchr/testify/assert"
  9 | 	mockcl "github.com/yezzey-gp/yproxy/pkg/mock/client"
 10 | 	mockio "github.com/yezzey-gp/yproxy/pkg/mock/proc/yio"
 11 | 	"github.com/yezzey-gp/yproxy/pkg/proc/yio"
 12 | 	"go.uber.org/mock/gomock"
 13 | )
 14 | 
 15 | func TestYproxyRetryReaderEmpty(t *testing.T) {
 16 | 
 17 | 	ctrl := gomock.NewController(t)
 18 | 
 19 | 	rr := mockio.NewMockRestartReader(ctrl)
 20 | 
 21 | 	ycl := mockcl.NewMockYproxyClient(ctrl)
 22 | 
 23 | 	yr := yio.NewYRetryReader(rr, ycl)
 24 | 
 25 | 	buf := []byte{1, 233, 45}
 26 | 
 27 | 	rr.EXPECT().Restart(int64(0)).Return(nil)
 28 | 	rr.EXPECT().Read(gomock.Any()).Return(0, io.EOF)
 29 | 	rr.EXPECT().Close().Times(1)
 30 | 
 31 | 	_, err := yr.Read(buf)
 32 | 
 33 | 	assert.Equal(t, io.EOF, err)
 34 | 
 35 | 	assert.Nil(t, yr.Close())
 36 | }
 37 | 
 38 | func TestYproxyRetryReaderSimpleRead(t *testing.T) {
 39 | 
 40 | 	ctrl := gomock.NewController(t)
 41 | 
 42 | 	rr := mockio.NewMockRestartReader(ctrl)
 43 | 
 44 | 	ycl := mockcl.NewMockYproxyClient(ctrl)
 45 | 	yr := yio.NewYRetryReader(rr, ycl)
 46 | 
 47 | 	ycl.EXPECT().SetByteOffset(int64(3)).Times(1)
 48 | 
 49 | 	buf := []byte{0, 0, 0}
 50 | 
 51 | 	rr.EXPECT().Restart(int64(0)).Return(nil)
 52 | 	rr.EXPECT().Read(gomock.Any()).Do(
 53 | 		func(rbuf []byte) {
 54 | 			rbuf[0] = 1
 55 | 			rbuf[1] = 27
 56 | 			rbuf[2] = 33
 57 | 		},
 58 | 	).Return(3, nil)
 59 | 	rr.EXPECT().Read(gomock.Any()).Return(0, io.EOF)
 60 | 	rr.EXPECT().Close().Times(1)
 61 | 
 62 | 	n, err := yr.Read(buf)
 63 | 	assert.NoError(t, err)
 64 | 	assert.Equal(t, 3, n)
 65 | 
 66 | 	assert.Equal(t, buf[0], uint8(1))
 67 | 	assert.Equal(t, buf[1], uint8(27))
 68 | 	assert.Equal(t, buf[2], uint8(33))
 69 | 
 70 | 	_, err = yr.Read(buf)
 71 | 
 72 | 	assert.Equal(t, io.EOF, err)
 73 | 
 74 | 	assert.Nil(t, yr.Close())
 75 | }
 76 | 
 77 | func TestYproxyRetryReaderSimpleReadRetry(t *testing.T) {
 78 | 
 79 | 	ctrl := gomock.NewController(t)
 80 | 
 81 | 	rr := mockio.NewMockRestartReader(ctrl)
 82 | 
 83 | 	ycl := mockcl.NewMockYproxyClient(ctrl)
 84 | 	yr := yio.NewYRetryReader(rr, ycl)
 85 | 
 86 | 	ycl.EXPECT().SetByteOffset(int64(3)).Times(1)
 87 | 
 88 | 	ycl.EXPECT().SetByteOffset(int64(6)).Times(1)
 89 | 
 90 | 	buf := []byte{0, 0, 0}
 91 | 
 92 | 	rr.EXPECT().Restart(int64(0)).Return(nil).Times(1)
 93 | 	rr.EXPECT().Restart(int64(3)).Return(nil).Times(1)
 94 | 	rr.EXPECT().Read(buf).Do(
 95 | 		func(rbuf []byte) {
 96 | 			rbuf[0] = 1
 97 | 			rbuf[1] = 27
 98 | 			rbuf[2] = 33
 99 | 		},
100 | 	).Return(3, nil)
101 | 	rr.EXPECT().Read(gomock.Any()).Return(0, fmt.Errorf("no"))
102 | 	rr.EXPECT().Read(gomock.Any()).Do(
103 | 		func(rbuf []byte) {
104 | 			rbuf[0] = 1
105 | 			rbuf[1] = 27
106 | 			rbuf[2] = 33
107 | 		},
108 | 	).Return(3, nil)
109 | 	rr.EXPECT().Read(gomock.Any()).Return(0, io.EOF)
110 | 
111 | 	rr.EXPECT().Close().Times(2)
112 | 
113 | 	n, err := yr.Read(buf)
114 | 	assert.NoError(t, err)
115 | 	assert.Equal(t, 3, n)
116 | 
117 | 	assert.Equal(t, buf[0], uint8(1))
118 | 	assert.Equal(t, buf[1], uint8(27))
119 | 	assert.Equal(t, buf[2], uint8(33))
120 | 
121 | 	// yr got error, but retries
122 | 	buf2 := []byte{0, 0, 0}
123 | 
124 | 	n, err = yr.Read(buf2)
125 | 	assert.NoError(t, err)
126 | 	assert.Equal(t, 3, n)
127 | 
128 | 	assert.Equal(t, buf2[0], uint8(1))
129 | 	assert.Equal(t, buf2[1], uint8(27))
130 | 	assert.Equal(t, buf2[2], uint8(33))
131 | 
132 | 	_, err = yr.Read([]byte{0, 0, 0})
133 | 
134 | 	assert.Equal(t, io.EOF, err)
135 | 
136 | 	assert.Nil(t, yr.Close())
137 | }
138 | 


--------------------------------------------------------------------------------
/pkg/proc/yio/ywriter.go:
--------------------------------------------------------------------------------
 1 | package yio
 2 | 
 3 | import (
 4 | 	"io"
 5 | 
 6 | 	"github.com/yezzey-gp/yproxy/pkg/client"
 7 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
 8 | )
 9 | 
10 | /* TBD: support restart */
11 | type YproxyWriter struct {
12 | 	underlying io.WriteCloser
13 | 
14 | 	selfCl client.YproxyClient
15 | 
16 | 	offsetReached int64
17 | }
18 | 
19 | // Close implements io.WriteCloser.
20 | func (y *YproxyWriter) Close() error {
21 | 	return y.underlying.Close()
22 | }
23 | 
24 | func (y *YproxyWriter) Write(p []byte) (n int, err error) {
25 | 	n, err = y.underlying.Write(p)
26 | 	y.offsetReached += int64(n)
27 | 	y.selfCl.SetByteOffset(y.offsetReached)
28 | 
29 | 	if err != nil {
30 | 		ylogger.Zero.Error().Uint("client id", y.selfCl.ID()).Int("bytes write", n).Err(err).Msg("failed to write into underlying connection")
31 | 	}
32 | 
33 | 	return n, err
34 | }
35 | 
36 | func NewYproxyWriter(under io.WriteCloser, selfCl client.YproxyClient) io.WriteCloser {
37 | 	return &YproxyWriter{
38 | 		underlying:    under,
39 | 		selfCl:        selfCl,
40 | 		offsetReached: 0,
41 | 	}
42 | }
43 | 
44 | var _ io.WriteCloser = &YproxyWriter{}
45 | 


--------------------------------------------------------------------------------
/pkg/proc/yio/ywriter_test.go:
--------------------------------------------------------------------------------
1 | package yio_test
2 | 


--------------------------------------------------------------------------------
/pkg/sdnotifier/notifier.go:
--------------------------------------------------------------------------------
 1 | package sdnotifier
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"net"
 6 | 	"time"
 7 | 
 8 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
 9 | )
10 | 
11 | /*
12 | #include <time.h>
13 | static unsigned long long get_nsecs(void)
14 | {
15 |     struct timespec ts;
16 |     clock_gettime(CLOCK_MONOTONIC, &ts);
17 |     return (unsigned long long)ts.tv_sec * 1000000000UL + ts.tv_nsec;
18 | }
19 | */
20 | import "C"
21 | 
22 | const Timeout = 4 * time.Second
23 | 
24 | type Notifier struct {
25 | 	sock  net.Conn
26 | 	debug bool
27 | }
28 | 
29 | func NewNotifier(ns string, debug bool) (*Notifier, error) {
30 | 	sock, err := net.Dial("unixgram", ns)
31 | 	if err != nil {
32 | 		sock = nil
33 | 		if debug {
34 | 			return &Notifier{sock: nil, debug: debug}, err
35 | 		}
36 | 	}
37 | 
38 | 	return &Notifier{sock: sock, debug: debug}, nil
39 | }
40 | 
41 | func (n *Notifier) Reloading() error {
42 | 	return n.send([]byte(fmt.Sprintf("RELOADING=1\nMONOTONIC_USEC=%d\n", uint64(C.get_nsecs()))))
43 | }
44 | 
45 | func (n *Notifier) Ready() error {
46 | 	return n.send([]byte("READY=1\n"))
47 | }
48 | 
49 | func (n *Notifier) Notify() error {
50 | 	return n.send([]byte("WATCHDOG=1\n"))
51 | }
52 | 
53 | func (n *Notifier) send(msg []byte) error {
54 | 	if n.sock == nil {
55 | 		return nil
56 | 	}
57 | 
58 | 	ylogger.Zero.Debug().Msg(fmt.Sprintf("sending systemd notification: %s", msg))
59 | 	_, err := n.sock.Write(msg)
60 | 
61 | 	if n.debug {
62 | 		return err
63 | 	}
64 | 	return nil
65 | }
66 | 


--------------------------------------------------------------------------------
/pkg/settings/settings.go:
--------------------------------------------------------------------------------
1 | package settings
2 | 
3 | type StorageSettings struct {
4 | 	Name  string
5 | 	Value string
6 | }
7 | 


--------------------------------------------------------------------------------
/pkg/storage/filestorage.go:
--------------------------------------------------------------------------------
  1 | package storage
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"io"
  6 | 	"io/fs"
  7 | 	"os"
  8 | 	"path"
  9 | 	"path/filepath"
 10 | 	"strings"
 11 | 
 12 | 	"github.com/yezzey-gp/yproxy/config"
 13 | 	"github.com/yezzey-gp/yproxy/pkg/object"
 14 | 	"github.com/yezzey-gp/yproxy/pkg/settings"
 15 | )
 16 | 
 17 | // Storage prefix uses as path to folder.
 18 | // WARNING "/path/to/folder" dont work
 19 | // "/path/to/folder/" + "path/to/file.txt"
 20 | type FileStorageInteractor struct {
 21 | 	cnf *config.Storage
 22 | }
 23 | 
 24 | // ListBucketPath implements StorageInteractor.
 25 | func (s *FileStorageInteractor) ListBucketPath(bucket string, prefix string, useCache bool) ([]*object.ObjectInfo, error) {
 26 | 	return s.ListPath(prefix, false, nil)
 27 | }
 28 | 
 29 | // ListBuckets implements StorageInteractor.
 30 | func (s *FileStorageInteractor) ListBuckets() []string {
 31 | 	keys := []string{s.cnf.StorageBucket}
 32 | 
 33 | 	for k := range s.cnf.CredentialMap {
 34 | 		keys = append(keys, k)
 35 | 	}
 36 | 	return keys
 37 | }
 38 | 
 39 | // DefaultBucket implements StorageInteractor.
 40 | func (s *FileStorageInteractor) DefaultBucket() string {
 41 | 	return s.cnf.StorageBucket
 42 | }
 43 | 
 44 | var _ StorageInteractor = &FileStorageInteractor{}
 45 | 
 46 | func (s *FileStorageInteractor) CatFileFromStorage(name string, offset int64, _ []settings.StorageSettings) (io.ReadCloser, error) {
 47 | 	file, err := os.Open(path.Join(s.cnf.StoragePrefix, name))
 48 | 	if err != nil {
 49 | 		return nil, err
 50 | 	}
 51 | 	_, err = io.CopyN(io.Discard, file, offset)
 52 | 	return file, err
 53 | }
 54 | func (s *FileStorageInteractor) ListPath(prefix string, _ bool, _ []settings.StorageSettings) ([]*object.ObjectInfo, error) {
 55 | 	var data []*object.ObjectInfo
 56 | 	err := filepath.WalkDir(s.cnf.StoragePrefix, func(path string, d fs.DirEntry, err error) error {
 57 | 		if err != nil {
 58 | 			return err
 59 | 		}
 60 | 		if d.IsDir() {
 61 | 			return nil
 62 | 		}
 63 | 		file, err := os.Open(path)
 64 | 		if err != nil {
 65 | 			return err
 66 | 		}
 67 | 		fileinfo, err := file.Stat()
 68 | 		if err != nil {
 69 | 			return err
 70 | 		}
 71 | 
 72 | 		cuttedPrefix, _ := strings.CutPrefix(prefix, "/")
 73 | 		neededPrefix := s.cnf.StoragePrefix + cuttedPrefix
 74 | 		if !strings.HasPrefix(path, neededPrefix) {
 75 | 			return nil
 76 | 		}
 77 | 		cPath, ok := strings.CutPrefix(path, s.cnf.StoragePrefix)
 78 | 
 79 | 		if !ok {
 80 | 			return err
 81 | 		}
 82 | 		data = append(data, &object.ObjectInfo{Path: "/" + cPath, Size: fileinfo.Size()})
 83 | 		return nil
 84 | 	})
 85 | 	return data, err
 86 | }
 87 | 
 88 | func (s *FileStorageInteractor) PutFileToDest(name string, r io.Reader, _ []settings.StorageSettings) error {
 89 | 	fPath := path.Join(s.cnf.StoragePrefix, name)
 90 | 	fDir := path.Dir(fPath)
 91 | 	if err := os.MkdirAll(fDir, 0700); err != nil {
 92 | 		return err
 93 | 	}
 94 | 	file, err := os.Create(fPath)
 95 | 	if err != nil {
 96 | 		return err
 97 | 	}
 98 | 	_, err = io.Copy(file, r)
 99 | 	return err
100 | }
101 | 
102 | func (s *FileStorageInteractor) PatchFile(name string, r io.ReadSeeker, startOffset int64) error {
103 | 	//UNUSED TODO
104 | 	return fmt.Errorf("TODO")
105 | }
106 | 
107 | func (s *FileStorageInteractor) MoveObject(_ /*bucket*/, from string, to string) error {
108 | 	fromPath := path.Join(s.cnf.StoragePrefix, from)
109 | 	toPath := path.Join(s.cnf.StoragePrefix, to)
110 | 	toDir := path.Dir(toPath)
111 | 	if err := os.MkdirAll(toDir, 0700); err != nil {
112 | 		return err
113 | 	}
114 | 	return os.Rename(fromPath, toPath)
115 | }
116 | 
117 | func (s *FileStorageInteractor) CopyObject(from, to, fromStoragePrefix, _, _ string) error {
118 | 	fromPath := path.Join(fromStoragePrefix, from)
119 | 	toPath := path.Join(s.cnf.StoragePrefix, to)
120 | 	toDir := path.Dir(toPath)
121 | 	if err := os.MkdirAll(toDir, 0700); err != nil {
122 | 		return err
123 | 	}
124 | 	fromFile, err := os.Open(fromPath)
125 | 	if err != nil {
126 | 		return err
127 | 	}
128 | 	toFile, err := os.OpenFile(toPath, os.O_CREATE|os.O_WRONLY, 0644)
129 | 	if err != nil {
130 | 		return err
131 | 	}
132 | 	_, err = io.Copy(fromFile, toFile)
133 | 	return err
134 | }
135 | 
136 | func (s *FileStorageInteractor) DeleteObject(_ /*bucket*/, key string) error {
137 | 	return os.Remove(path.Join(s.cnf.StoragePrefix, key))
138 | }
139 | 
140 | func (s *FileStorageInteractor) AbortMultipartUploads() error {
141 | 	return nil
142 | }
143 | 
144 | func (s *FileStorageInteractor) AbortMultipartUpload(bucket, key, uploadId string) error {
145 | 	return nil
146 | }
147 | 
148 | func (s *FileStorageInteractor) ListFailedMultipartUploads(string) (map[string]string, error) {
149 | 	return nil, nil
150 | }
151 | 


--------------------------------------------------------------------------------
/pkg/storage/s3storage_test.go:
--------------------------------------------------------------------------------
 1 | package storage
 2 | 
 3 | import (
 4 | 	"os"
 5 | 	"testing"
 6 | 
 7 | 	"github.com/stretchr/testify/assert"
 8 | 	"github.com/yezzey-gp/yproxy/config"
 9 | 	"github.com/yezzey-gp/yproxy/pkg/object"
10 | )
11 | 
12 | func TestCache(t *testing.T) {
13 | 	assert := assert.New(t)
14 | 
15 | 	f, err := os.CreateTemp("", "*")
16 | 	assert.NoError(err)
17 | 	defer func() { _ = os.Remove(f.Name()) }()
18 | 	defer func() { _ = f.Close() }()
19 | 
20 | 	config.InstanceConfig().ProxyCnf.BucketCachePath = f.Name()
21 | 
22 | 	s1 := config.Storage{StorageBucket: "some_bucket_1"}
23 | 	s2 := config.Storage{StorageBucket: "some_bucket_2"}
24 | 
25 | 	abcObjects := []*object.ObjectInfo{{Path: "/abc1"}, {Path: "/abc2"}}
26 | 	allObjects := append(abcObjects, &object.ObjectInfo{Path: "/def1"})
27 | 	err = putInCache(s1.ID(), allObjects)
28 | 	assert.NoError(err)
29 | 
30 | 	err = putInCache(s2.ID(), allObjects)
31 | 	assert.NoError(err)
32 | 
33 | 	objects, err := readCache(s1, "")
34 | 	assert.NoError(err)
35 | 	assert.Equal(allObjects, objects)
36 | 
37 | 	objects, err = readCache(s2, "abc")
38 | 	assert.NoError(err)
39 | 	assert.Equal(abcObjects, objects)
40 | }
41 | 


--------------------------------------------------------------------------------
/pkg/storage/sessionpool.go:
--------------------------------------------------------------------------------
  1 | package storage
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"fmt"
  6 | 	"io"
  7 | 	"net"
  8 | 	"net/http"
  9 | 	"strings"
 10 | 	"sync/atomic"
 11 | 	"time"
 12 | 
 13 | 	"github.com/pkg/errors"
 14 | 	"github.com/yezzey-gp/aws-sdk-go/aws"
 15 | 	"github.com/yezzey-gp/aws-sdk-go/aws/client"
 16 | 	"github.com/yezzey-gp/aws-sdk-go/aws/credentials"
 17 | 	"github.com/yezzey-gp/aws-sdk-go/aws/defaults"
 18 | 	"github.com/yezzey-gp/aws-sdk-go/aws/request"
 19 | 	"github.com/yezzey-gp/aws-sdk-go/aws/session"
 20 | 	"github.com/yezzey-gp/aws-sdk-go/service/s3"
 21 | 	"github.com/yezzey-gp/yproxy/config"
 22 | 	"github.com/yezzey-gp/yproxy/pkg/ylogger"
 23 | 
 24 | 	"golang.org/x/sync/semaphore"
 25 | )
 26 | 
 27 | type SessionPool interface {
 28 | 	GetSession(ctx context.Context, cr *config.StorageCredentials) (*s3.S3, error)
 29 | 	StorageUsedConcurrency() int
 30 | }
 31 | 
 32 | type S3SessionPool struct {
 33 | 	cnf *config.Storage
 34 | 
 35 | 	usedConnections atomic.Int32
 36 | 	sem             *semaphore.Weighted
 37 | }
 38 | 
 39 | // StorageUsedConcurrency implements SessionPool.
 40 | func (sp *S3SessionPool) StorageUsedConcurrency() int {
 41 | 	return int(sp.usedConnections.Load())
 42 | }
 43 | 
 44 | func NewSessionPool(cnf *config.Storage) SessionPool {
 45 | 	return &S3SessionPool{
 46 | 		cnf: cnf,
 47 | 		sem: semaphore.NewWeighted(cnf.StorageConcurrency),
 48 | 	}
 49 | }
 50 | 
 51 | // TODO : unit tests
 52 | func (sp *S3SessionPool) createSession(cr *config.StorageCredentials) (*session.Session, error) {
 53 | 	s, err := session.NewSession(&aws.Config{
 54 | 		Retryer: client.DefaultRetryer{
 55 | 			NumMaxRetries: 20,
 56 | 			MinRetryDelay: time.Second,
 57 | 			MaxRetryDelay: time.Second * 20,
 58 | 		},
 59 | 	})
 60 | 	if err != nil {
 61 | 		return nil, err
 62 | 	}
 63 | 
 64 | 	provider := &credentials.StaticProvider{Value: credentials.Value{
 65 | 		AccessKeyID:     cr.AccessKeyId,
 66 | 		SecretAccessKey: cr.SecretAccessKey,
 67 | 	}}
 68 | 
 69 | 	ylogger.Zero.Debug().Str("endpoint", sp.cnf.StorageEndpoint).Msg("acquire external storage session")
 70 | 
 71 | 	providers := make([]credentials.Provider, 0)
 72 | 	providers = append(providers, provider)
 73 | 	providers = append(providers, defaults.CredProviders(s.Config, defaults.Handlers())...)
 74 | 	newCredentials := credentials.NewCredentials(&credentials.ChainProvider{
 75 | 		VerboseErrors: aws.BoolValue(s.Config.CredentialsChainVerboseErrors),
 76 | 		Providers:     providers,
 77 | 	})
 78 | 
 79 | 	s.Config.WithRegion(sp.cnf.StorageRegion)
 80 | 	s.Config.S3ForcePathStyle = aws.Bool(true)
 81 | 
 82 | 	s.Config.WithEndpoint(sp.cnf.StorageEndpoint)
 83 | 
 84 | 	if sp.cnf.EndpointSourceHost != "" {
 85 | 		s.Handlers.Validate.PushBack(func(request *request.Request) {
 86 | 			endpoint, err := requestEndpoint(sp.cnf.EndpointSourceHost, sp.cnf.EndpointSourcePort)
 87 | 			if err == nil {
 88 | 				ylogger.Zero.Debug().Str("endpoint", endpoint).Msg("using requested endpoint")
 89 | 				host := strings.TrimPrefix(*s.Config.Endpoint, "https://")
 90 | 				request.HTTPRequest.Host = host
 91 | 				request.HTTPRequest.Header.Add("Host", host)
 92 | 				request.HTTPRequest.URL.Host = endpoint
 93 | 				request.HTTPRequest.URL.Scheme = sp.cnf.EndpointSourceScheme
 94 | 			} else {
 95 | 				ylogger.Zero.Debug().Str("endpoint", *s.Config.Endpoint).Msg("using default endpoint")
 96 | 			}
 97 | 		})
 98 | 	}
 99 | 
100 | 	s.Config.WithCredentials(newCredentials)
101 | 	return s, err
102 | }
103 | 
104 | func requestEndpoint(endpointSource, port string) (string, error) {
105 | 	ylogger.Zero.Debug().Str("source host", endpointSource).Msg("requesting storage endpoint")
106 | 	t := http.DefaultTransport
107 | 	c := http.DefaultClient
108 | 	if tr, ok := t.(*http.Transport); ok {
109 | 		tr.DisableKeepAlives = true
110 | 		c = &http.Client{Transport: tr}
111 | 	}
112 | 	resp, err := c.Get(endpointSource)
113 | 	if err != nil {
114 | 		ylogger.Zero.Error().Err(err).Msg("failed to get S3 endpoint")
115 | 		return "", err
116 | 	}
117 | 	defer func() { _ = resp.Body.Close() }()
118 | 	if resp.StatusCode != 200 {
119 | 		ylogger.Zero.Error().Int("status code", resp.StatusCode).Msg("endpoint source bad status code")
120 | 		return "", fmt.Errorf("endpoint source bad status code %d", resp.StatusCode)
121 | 	}
122 | 	body, err := io.ReadAll(resp.Body)
123 | 	if err != nil {
124 | 		ylogger.Zero.Error().Err(err).Msg("error reading endpoint source reply")
125 | 		return "", err
126 | 	}
127 | 	if port != "" {
128 | 		return net.JoinHostPort(string(body), port), err
129 | 	}
130 | 	return string(body), err
131 | }
132 | 
133 | func (s *S3SessionPool) GetSession(ctx context.Context, cr *config.StorageCredentials) (*s3.S3, error) {
134 | 	_ = s.sem.Acquire(ctx, 1)
135 | 	s.usedConnections.Add(1)
136 | 	defer func() {
137 | 		s.sem.Release(1)
138 | 		s.usedConnections.Add(-1)
139 | 	}()
140 | 
141 | 	sess, err := s.createSession(cr)
142 | 	if err != nil {
143 | 		return nil, errors.Wrap(err, "failed to create new session")
144 | 	}
145 | 	return s3.New(sess), nil
146 | }
147 | 


--------------------------------------------------------------------------------
/pkg/storage/sessionpool_test.go:
--------------------------------------------------------------------------------
  1 | package storage_test
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"testing"
  6 | 
  7 | 	"github.com/jarcoal/httpmock"
  8 | 	"github.com/stretchr/testify/assert"
  9 | 	"github.com/yezzey-gp/aws-sdk-go/aws"
 10 | 	"github.com/yezzey-gp/aws-sdk-go/service/s3"
 11 | 	"github.com/yezzey-gp/yproxy/config"
 12 | 	"github.com/yezzey-gp/yproxy/pkg/storage"
 13 | )
 14 | 
 15 | func TestEndpointSourceHTTP(t *testing.T) {
 16 | 	assert := assert.New(t)
 17 | 
 18 | 	httpmock.Activate()
 19 | 
 20 | 	httpmock.RegisterResponder("GET", "http://endpoint_source/get_proxy",
 21 | 		httpmock.NewStringResponder(200, "storage.proxy"))
 22 | 
 23 | 	httpmock.RegisterResponder("GET", "http://storage.proxy/bucket/key",
 24 | 		httpmock.NewStringResponder(200, ""))
 25 | 
 26 | 	pool := storage.NewSessionPool(&config.Storage{
 27 | 		StorageEndpoint:      "storage.mock",
 28 | 		EndpointSourceHost:   "http://endpoint_source/get_proxy",
 29 | 		EndpointSourceScheme: "http",
 30 | 		AccessKeyId:          "mock_access_key",
 31 | 		SecretAccessKey:      "mock_secret_key",
 32 | 
 33 | 		StorageRegion: "us-east-1",
 34 | 
 35 | 		StorageConcurrency: 1,
 36 | 	})
 37 | 
 38 | 	sess, err := pool.GetSession(context.TODO(), &config.StorageCredentials{
 39 | 		AccessKeyId:     "mock_access_key",
 40 | 		SecretAccessKey: "mock_secret_key",
 41 | 	})
 42 | 	assert.NoError(err)
 43 | 	_, err = sess.GetObject(&s3.GetObjectInput{
 44 | 		Bucket: aws.String("bucket"),
 45 | 		Key:    aws.String("key"),
 46 | 	})
 47 | 	assert.NoError(err)
 48 | 
 49 | 	info := httpmock.GetCallCountInfo()
 50 | 	assert.Equal(1, info["GET http://endpoint_source/get_proxy"])
 51 | 	assert.Equal(1, info["GET http://storage.proxy/bucket/key"])
 52 | }
 53 | 
 54 | func TestEndpointSourceHTTPS(t *testing.T) {
 55 | 	assert := assert.New(t)
 56 | 
 57 | 	httpmock.Activate()
 58 | 
 59 | 	httpmock.RegisterResponder("GET", "https://endpoint_source/get_proxy",
 60 | 		httpmock.NewStringResponder(200, "storage.proxy"))
 61 | 
 62 | 	httpmock.RegisterResponder("GET", "https://storage.proxy/bucket/key",
 63 | 		httpmock.NewStringResponder(200, ""))
 64 | 
 65 | 	pool := storage.NewSessionPool(&config.Storage{
 66 | 		StorageEndpoint:      "storage.mock",
 67 | 		EndpointSourceHost:   "https://endpoint_source/get_proxy",
 68 | 		EndpointSourceScheme: "https",
 69 | 		AccessKeyId:          "mock_access_key",
 70 | 		SecretAccessKey:      "mock_secret_key",
 71 | 
 72 | 		StorageRegion: "us-east-1",
 73 | 
 74 | 		StorageConcurrency: 1,
 75 | 	})
 76 | 
 77 | 	sess, err := pool.GetSession(context.TODO(), &config.StorageCredentials{
 78 | 		AccessKeyId:     "mock_access_key",
 79 | 		SecretAccessKey: "mock_secret_key",
 80 | 	})
 81 | 	assert.NoError(err)
 82 | 	_, err = sess.GetObject(&s3.GetObjectInput{
 83 | 		Bucket: aws.String("bucket"),
 84 | 		Key:    aws.String("key"),
 85 | 	})
 86 | 	assert.NoError(err)
 87 | 
 88 | 	info := httpmock.GetCallCountInfo()
 89 | 	assert.Equal(1, info["GET https://endpoint_source/get_proxy"])
 90 | 	assert.Equal(1, info["GET https://storage.proxy/bucket/key"])
 91 | }
 92 | 
 93 | func TestEndpointSourceSetPort(t *testing.T) {
 94 | 	assert := assert.New(t)
 95 | 
 96 | 	httpmock.Activate()
 97 | 
 98 | 	httpmock.RegisterResponder("GET", "https://endpoint_source/get_proxy",
 99 | 		httpmock.NewStringResponder(200, "storage.proxy"))
100 | 
101 | 	httpmock.RegisterResponder("GET", "https://storage.proxy:8080/bucket/key",
102 | 		httpmock.NewStringResponder(200, ""))
103 | 
104 | 	pool := storage.NewSessionPool(&config.Storage{
105 | 		StorageEndpoint:      "storage.mock",
106 | 		EndpointSourceHost:   "https://endpoint_source/get_proxy",
107 | 		EndpointSourcePort:   "8080",
108 | 		EndpointSourceScheme: "https",
109 | 		AccessKeyId:          "mock_access_key",
110 | 		SecretAccessKey:      "mock_secret_key",
111 | 
112 | 		StorageRegion: "us-east-1",
113 | 
114 | 		StorageConcurrency: 1,
115 | 	})
116 | 
117 | 	sess, err := pool.GetSession(context.TODO(), &config.StorageCredentials{
118 | 		AccessKeyId:     "mock_access_key",
119 | 		SecretAccessKey: "mock_secret_key",
120 | 	})
121 | 	assert.NoError(err)
122 | 	_, err = sess.GetObject(&s3.GetObjectInput{
123 | 		Bucket: aws.String("bucket"),
124 | 		Key:    aws.String("key"),
125 | 	})
126 | 	assert.NoError(err)
127 | 
128 | 	info := httpmock.GetCallCountInfo()
129 | 	assert.Equal(1, info["GET https://endpoint_source/get_proxy"])
130 | 	assert.Equal(1, info["GET https://storage.proxy:8080/bucket/key"])
131 | }
132 | 


--------------------------------------------------------------------------------
/pkg/storage/storage.go:
--------------------------------------------------------------------------------
 1 | package storage
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"io"
 6 | 
 7 | 	"github.com/yezzey-gp/yproxy/config"
 8 | 	"github.com/yezzey-gp/yproxy/pkg/object"
 9 | 	"github.com/yezzey-gp/yproxy/pkg/settings"
10 | 	"github.com/yezzey-gp/yproxy/pkg/tablespace"
11 | )
12 | 
13 | type StorageReader interface {
14 | 	CatFileFromStorage(name string, offset int64, setts []settings.StorageSettings) (io.ReadCloser, error)
15 | }
16 | 
17 | type StorageWriter interface {
18 | 	PutFileToDest(name string, r io.Reader, settings []settings.StorageSettings) error
19 | 	PatchFile(name string, r io.ReadSeeker, startOffset int64) error
20 | }
21 | 
22 | type StorageLister interface {
23 | 	ListPath(prefix string, useCache bool, settings []settings.StorageSettings) ([]*object.ObjectInfo, error)
24 | 	ListBucketPath(bucket, prefix string, useCache bool) ([]*object.ObjectInfo, error)
25 | 	ListFailedMultipartUploads(bucket string) (map[string]string, error)
26 | }
27 | 
28 | type StorageMover interface {
29 | 	MoveObject(bucket, from string, to string) error
30 | 	DeleteObject(bucket, key string) error
31 | 	AbortMultipartUpload(bucket, key, uploadId string) error
32 | }
33 | 
34 | type StorageCopier interface {
35 | 	CopyObject(from, to, fromStoragePrefix, fromStorageBucket, toStorageBucket string) error
36 | }
37 | 
38 | //go:generate mockgen -destination=pkg/mock/storage.go -package=mock
39 | type StorageInteractor interface {
40 | 	StorageReader
41 | 	StorageWriter
42 | 	StorageLister
43 | 	StorageMover
44 | 	StorageCopier
45 | 
46 | 	ListBuckets() []string
47 | 	DefaultBucket() string
48 | }
49 | 
50 | func NewStorage(cnf *config.Storage) (StorageInteractor, error) {
51 | 	switch cnf.StorageType {
52 | 	case "fs":
53 | 		return &FileStorageInteractor{
54 | 			cnf: cnf,
55 | 		}, nil
56 | 	case "s3":
57 | 		return &S3StorageInteractor{
58 | 			pool:          NewSessionPool(cnf),
59 | 			cnf:           cnf,
60 | 			TSToBucketMap: buildBucketMapFromCnf(cnf),
61 | 			credentialMap: buildCredMapFromCnf(cnf),
62 | 		}, nil
63 | 	default:
64 | 		return nil, fmt.Errorf("wrong storage type %s", cnf.StorageType)
65 | 	}
66 | }
67 | 
68 | func buildBucketMapFromCnf(cnf *config.Storage) map[string]string {
69 | 	mp := cnf.TablespaceMap
70 | 	if mp == nil {
71 | 		/* fallback for backward-compatibility if to TableSpace map configured */
72 | 		mp = map[string]string{}
73 | 	}
74 | 	if _, ok := mp[tablespace.DefaultTableSpace]; !ok {
75 | 		mp[tablespace.DefaultTableSpace] = cnf.StorageBucket
76 | 	}
77 | 	return mp
78 | }
79 | 
80 | func buildCredMapFromCnf(cnf *config.Storage) map[string]config.StorageCredentials {
81 | 	mp := cnf.CredentialMap
82 | 	if mp == nil {
83 | 		/* fallback for backward-compatibility if to TableSpace map configured */
84 | 		mp = map[string]config.StorageCredentials{}
85 | 	}
86 | 	if _, ok := mp[cnf.StorageBucket]; !ok {
87 | 		mp[cnf.StorageBucket] = config.StorageCredentials{
88 | 			AccessKeyId:     cnf.AccessKeyId,
89 | 			SecretAccessKey: cnf.SecretAccessKey,
90 | 		}
91 | 	}
92 | 	return mp
93 | }
94 | 


--------------------------------------------------------------------------------
/pkg/storage/utils.go:
--------------------------------------------------------------------------------
 1 | package storage
 2 | 
 3 | import "github.com/yezzey-gp/yproxy/pkg/settings"
 4 | 
 5 | func ResolveStorageSetting(settings []settings.StorageSettings, name, defaultVal string) string {
 6 | 
 7 | 	for _, s := range settings {
 8 | 		if s.Name == name {
 9 | 			return s.Value
10 | 		}
11 | 	}
12 | 
13 | 	return defaultVal
14 | }
15 | 


--------------------------------------------------------------------------------
/pkg/storage/utils_test.go:
--------------------------------------------------------------------------------
 1 | package storage_test
 2 | 
 3 | import (
 4 | 	"testing"
 5 | 
 6 | 	"github.com/stretchr/testify/assert"
 7 | 
 8 | 	"github.com/yezzey-gp/yproxy/pkg/settings"
 9 | 	"github.com/yezzey-gp/yproxy/pkg/storage"
10 | )
11 | 
12 | func TestResolveSettings(t *testing.T) {
13 | 
14 | 	assert := assert.New(t)
15 | 
16 | 	type tcase struct {
17 | 		name     string
18 | 		defaultV string
19 | 		exp      string
20 | 		settings []settings.StorageSettings
21 | 	}
22 | 
23 | 	for _, tt := range []tcase{
24 | 		{
25 | 			"ababa",
26 | 			"aboba",
27 | 			"aboba",
28 | 			nil,
29 | 		},
30 | 		{
31 | 			"ababa",
32 | 			"aboba",
33 | 			"aboba",
34 | 			[]settings.StorageSettings{
35 | 				{
36 | 					Name:  "djewikdeowp",
37 | 					Value: "jdoiwejoidew",
38 | 				},
39 | 			},
40 | 		},
41 | 
42 | 		{
43 | 			"ababa",
44 | 			"aboba",
45 | 			"valval",
46 | 			[]settings.StorageSettings{
47 | 				{
48 | 					Name:  "ababa",
49 | 					Value: "valval",
50 | 				},
51 | 			},
52 | 		},
53 | 	} {
54 | 
55 | 		assert.Equal(tt.exp, storage.ResolveStorageSetting(tt.settings, tt.name, tt.defaultV))
56 | 	}
57 | }
58 | 


--------------------------------------------------------------------------------
/pkg/tablespace/tablespace.go:
--------------------------------------------------------------------------------
1 | package tablespace
2 | 
3 | const DefaultTableSpace = "pg_default"
4 | 


--------------------------------------------------------------------------------
/pkg/testutils/testmode.go:
--------------------------------------------------------------------------------
1 | package testutils
2 | 
3 | var TestMode bool
4 | 


--------------------------------------------------------------------------------
/pkg/vacuum/vacuum.go:
--------------------------------------------------------------------------------
 1 | package vacuum
 2 | 
 3 | // Greenplum + yezzey specific logic for external storage vacuuming
 4 | // Yezzey stores AO/AOCS relations data in one or several files in external
 5 | // storage (chunks). Chunk considered to be obsolete when
 6 | // table referencing this chunk was deleted and all backups (WAL-G) which
 7 | // contains this table was deleted
 8 | 
 9 | // TODO: implement
10 | 


--------------------------------------------------------------------------------
/pkg/version.go:
--------------------------------------------------------------------------------
 1 | package pkg
 2 | 
 3 | import "fmt"
 4 | 
 5 | var (
 6 | 	// These variables are here only to show current version. They are set in makefile during build process
 7 | 	YproxyVersion         = "devel"
 8 | 	GitRevision           = "devel"
 9 | 	YproxyVersionRevision = fmt.Sprintf("%s-%s", YproxyVersion, GitRevision)
10 | )
11 | 


--------------------------------------------------------------------------------
/pkg/ylogger/logger.go:
--------------------------------------------------------------------------------
 1 | package ylogger
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"io"
 6 | 	"log"
 7 | 	"os"
 8 | 	"strings"
 9 | 
10 | 	"github.com/rs/zerolog"
11 | )
12 | 
13 | var Zero = NewZeroLogger("")
14 | 
15 | func newWriter(filepath string) (*os.File, io.Writer, error) {
16 | 	if filepath == "" {
17 | 		return nil, os.Stdout, nil
18 | 	}
19 | 	f, err := os.OpenFile(filepath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
20 | 	if err != nil {
21 | 		log.Fatal(err)
22 | 	}
23 | 	return f, f, nil
24 | }
25 | 
26 | func NewZeroLogger(filepath string) *zerolog.Logger {
27 | 	_, writer, err := newWriter(filepath)
28 | 	if err != nil {
29 | 		fmt.Printf("FAILED TO INITIALIZED LOGGER: %v", err)
30 | 	}
31 | 	logger := zerolog.New(writer).With().Timestamp().Logger()
32 | 
33 | 	return &logger
34 | }
35 | 
36 | func UpdateZeroLogLevel(logLevel string) error {
37 | 	level := parseLevel(logLevel)
38 | 	zeroLogger := Zero.With().Logger().Level(level)
39 | 	Zero = &zeroLogger
40 | 	return nil
41 | }
42 | 
43 | func ReloadLogger(filepath string) {
44 | 	if filepath == "" { //
45 | 		return // this means os.Stdout, so no need to open new file
46 | 	}
47 | 	newLogger := NewZeroLogger(filepath).Level(Zero.GetLevel())
48 | 	Zero = &newLogger
49 | }
50 | 
51 | func parseLevel(level string) zerolog.Level {
52 | 	switch strings.ToLower(level) {
53 | 	case "disabled":
54 | 		return zerolog.Disabled
55 | 	case "debug":
56 | 		return zerolog.DebugLevel
57 | 	case "info":
58 | 		return zerolog.InfoLevel
59 | 	case "warning":
60 | 		return zerolog.WarnLevel
61 | 	case "error":
62 | 		return zerolog.ErrorLevel
63 | 	case "fatal":
64 | 		return zerolog.FatalLevel
65 | 	default:
66 | 		return zerolog.InfoLevel
67 | 	}
68 | }
69 | 


--------------------------------------------------------------------------------
/script/gofmtcheck.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | # Check gofmt
 4 | echo "==> Checking that code complies with gofmt requirements..."
 5 | gofmt_files=$(gofmt -l `find . -name '*.go' | grep -v parser | grep -v yacc | grep -v .git`)
 6 | if [[ -n ${gofmt_files} ]]; then
 7 |     echo 'gofmt needs running on the following files:'
 8 |     echo "${gofmt_files}"
 9 |     echo "You can use the command: \`make fmt\` to reformat code."
10 |     exit 1
11 | fi
12 | echo "OK"
13 | 
14 | 
15 | exit 0


--------------------------------------------------------------------------------
/test/regress/.dockerignore:
--------------------------------------------------------------------------------
1 | Dockerfile


--------------------------------------------------------------------------------
/test/regress/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM golang:latest
 2 | 
 3 | WORKDIR /root
 4 | COPY . /root
 5 | 
 6 | RUN mkdir /data
 7 | 
 8 | ENV PGHOST=pg
 9 | ENV PGUSER=postgres
10 | 
11 | RUN apt update && apt install -y postgresql-client s3cmd
12 | 
13 | RUN go build -pgo=auto -o /usr/bin/yproxy $(LDFLAGS) ./cmd/yproxy && \
14 |     go build -o /usr/bin/yp-client ./cmd/client
15 | 
16 | ENTRYPOINT [ "test/regress/run_tests.sh" ]
17 | 


--------------------------------------------------------------------------------
/test/regress/conf/yproxy.yaml:
--------------------------------------------------------------------------------
 1 | socket_path: "/tmp/yproxy.sock"
 2 | 
 3 | storage:
 4 |   storage_endpoint: http://minio:9000
 5 |   access_key_id: some_key
 6 |   secret_access_key: some_key
 7 |   storage_prefix: ""
 8 |   storage_bucket: "gpyezzey"
 9 |   storage_region: "us-east-1"
10 | 
11 | crypto:
12 |   gpg_key_id: "1C1179B1315EE84B111976108DCF5883160204C0"
13 |   gpg_key_path: /root/test/regress/gpg/gpg_1.priv
14 | 
15 | log_level: "debug"
16 | 


--------------------------------------------------------------------------------
/test/regress/conf/yproxy_old.yaml:
--------------------------------------------------------------------------------
 1 | socket_path: "/tmp/yproxy_old.sock"
 2 | 
 3 | storage:
 4 |   storage_endpoint: http://minio:9000
 5 |   access_key_id: some_key
 6 |   secret_access_key: some_key
 7 |   storage_prefix: ""
 8 |   storage_bucket: gpyezzey2
 9 |   storage_region: "us-east-1"
10 | 
11 | crypto:
12 |   gpg_key_id: "F368E75DD267978CB47DC5D11F4EB669CD9BE224"
13 |   gpg_key_path: /root/test/regress/gpg/gpg_2.priv
14 | 
15 | log_level: "debug"
16 | 


--------------------------------------------------------------------------------
/test/regress/conf/yproxy_prefix.yaml:
--------------------------------------------------------------------------------
 1 | socket_path: "/tmp/yproxy_prefix.sock"
 2 | 
 3 | storage:
 4 |   storage_endpoint: http://minio:9000
 5 |   access_key_id: some_key
 6 |   secret_access_key: some_key
 7 |   storage_prefix: "gpdata"
 8 |   storage_bucket: "gpyezzey"
 9 |   storage_region: "us-east-1"
10 | 
11 | crypto:
12 |   gpg_key_id: "1C1179B1315EE84B111976108DCF5883160204C0"
13 |   gpg_key_path: /root/test/regress/gpg/gpg_1.priv
14 | 
15 | log_level: "debug"
16 | 


--------------------------------------------------------------------------------
/test/regress/conf/yproxy_second_bucket.yaml:
--------------------------------------------------------------------------------
 1 | socket_path: "/tmp/yproxy_identical.sock"
 2 | 
 3 | storage:
 4 |   storage_endpoint: http://minio:9000
 5 |   access_key_id: some_key
 6 |   secret_access_key: some_key
 7 |   storage_prefix: ""
 8 |   storage_bucket: "gpyezzey2"
 9 |   storage_region: "us-east-1"
10 | 
11 | crypto:
12 |   gpg_key_id: "1C1179B1315EE84B111976108DCF5883160204C0"
13 |   gpg_key_path: /root/test/regress/gpg/gpg_1.priv
14 | 
15 | log_level: "debug"
16 | 


--------------------------------------------------------------------------------
/test/regress/docker-compose.yaml:
--------------------------------------------------------------------------------
 1 | services:
 2 |   minio:
 3 |     image: quay.io/minio/minio
 4 |     command: server --console-address ":9001" /data
 5 |     expose:
 6 |       - "9000"
 7 |       - "9001"
 8 |     environment:
 9 |       MINIO_ROOT_USER: some_key
10 |       MINIO_ROOT_PASSWORD: some_key
11 |     healthcheck:
12 |       test: ["CMD", "mc", "ready", "local"]
13 |       interval: 5s
14 |       timeout: 5s
15 |       retries: 5
16 |     hostname: minio
17 |   
18 |   setup-minio:
19 |     image: quay.io/minio/mc
20 |     depends_on:
21 |       minio:
22 |         condition: service_healthy
23 |     entrypoint: |
24 |       /bin/sh -c "
25 |       /usr/bin/mc alias set myminio http://minio:9000 some_key some_key
26 |       /usr/bin/mc mb myminio/gpyezzey
27 |       /usr/bin/mc mb myminio/gpyezzey2
28 |       "
29 |   pg:
30 |     image: postgres
31 |     hostname: pg
32 |     environment:
33 |       POSTGRES_USER: postgres
34 |       POSTGRES_PASSWORD: "password"
35 |       POSTGRES_DB: postgres
36 |       POSTGRES_HOST_AUTH_METHOD: trust
37 |     expose:
38 |       - "5432"
39 |     shm_size: 128mb
40 |     healthcheck:
41 |       test: ["CMD-SHELL", "pg_isready"]
42 |       interval: 1s  
43 |       timeout: 5s
44 |       retries: 10
45 |     volumes:
46 |       - ./pg-init:/docker-entrypoint-initdb.d
47 |   
48 |   yproxy:
49 |     image: yproxy
50 |     build:
51 |       context: ../..
52 |       dockerfile: test/regress/Dockerfile
53 |     depends_on:
54 |       minio:
55 |         condition: service_healthy
56 |       setup-minio:
57 |         condition: service_completed_successfully
58 |       pg:
59 |         condition: service_healthy
60 | 


--------------------------------------------------------------------------------
/test/regress/expected/01_simple_put.txt:
--------------------------------------------------------------------------------
1 | 1
2 | Object: {Name: "/some_file", size: 12}
3 | Sample data
4 | 


--------------------------------------------------------------------------------
/test/regress/expected/02_encrypted_put.txt:
--------------------------------------------------------------------------------
1 | 1
2 | Object: {Name: "/encrypted_file", size: 472}
3 | Sample encrypted data
4 | 


--------------------------------------------------------------------------------
/test/regress/expected/03_copy_separate_keys.txt:
--------------------------------------------------------------------------------
1 | INSERT 0 1
2 | 1
3 | Object: {Name: "/encrypted_file_old_key", size: 481}
4 | Sample encrypted data, old key
5 | 1
6 | Object: {Name: "/encrypted_file_old_key", size: 481}
7 | Sample encrypted data, old key
8 | 


--------------------------------------------------------------------------------
/test/regress/expected/04_copy_separate_keys_multiple.txt:
--------------------------------------------------------------------------------
1 | INSERT 0 2
2 | 1
3 | 1
4 | 1
5 | Object: {Name: "/encrypted_file_old_key_1", size: 481}
6 | Object: {Name: "/encrypted_file_old_key_2", size: 481}
7 | Sample encrypted data, old key
8 | Sample encrypted data, old key
9 | 


--------------------------------------------------------------------------------
/test/regress/expected/05_copy_identical_keys.txt:
--------------------------------------------------------------------------------
1 | 1
2 | Object: {Name: "/encrypted_file_identical_key", size: 487}
3 | Sample encrypted data, identical key
4 | INSERT 0 1
5 | 1
6 | Object: {Name: "/encrypted_file_identical_key", size: 487}
7 | Sample encrypted data, identical key
8 | 


--------------------------------------------------------------------------------
/test/regress/expected/06_copy_identical_keys_prefix.txt:
--------------------------------------------------------------------------------
1 | 1
2 | Object: {Name: "/encrypted_file_identical_key", size: 487}
3 | Sample encrypted data, identical key
4 | INSERT 0 1
5 | 1
6 | Object: {Name: "//encrypted_file_identical_key", size: 487}
7 | Sample encrypted data, identical key
8 | 


--------------------------------------------------------------------------------
/test/regress/expected/07_copy_identical_keys_multiple.txt:
--------------------------------------------------------------------------------
1 | 1
2 | 1
3 | INSERT 0 2
4 | 1
5 | Object: {Name: "/encrypted_file_identical_key_1", size: 487}
6 | Object: {Name: "/encrypted_file_identical_key_2", size: 487}
7 | Sample encrypted data, identical key
8 | Sample encrypted data, identical key
9 | 


--------------------------------------------------------------------------------
/test/regress/gpg/gpg_1.priv:
--------------------------------------------------------------------------------
 1 | -----BEGIN PGP PRIVATE KEY BLOCK-----
 2 | 
 3 | lQVYBGflJvgBDACYyDwaPqPIdWVS517ajR+DmZDaHtGpcatA4vOYNCZSSugXv+xk
 4 | UoQZBMLtzTlm956jhp4hsvugvRDyoOA5B8s0DSzLAzB4VQZYb8lcnmfdyHSKHQwR
 5 | HV9UpfVSDnGp5l9hHUM+t4YTO2eH4pBPC4j0XJyd8AMUhbMlSuVxp47uOVX2svFG
 6 | gQtZEt2tFK3DLq+SI2mIVMMJMMSCNwxlDOL66dlsnEtEksmYrcmIs2qlBAFmTmgY
 7 | aKis4GRekpKigKMn/3h1Y5+nAm5xTwthc/6zROWG43RGp9BJ1iAOLUowhB4Tx6NM
 8 | DufwE7ldBi4ahhtY8a+zKo93osh3Ytl0lhECiU71bmj3X00Irm+9pKPdzz8ITL2l
 9 | O28VtstiLU6aGcfzOKMogdIWq5uR8fNW6E+Yp4tfcDpFzDerWpv12lLTZF81te24
10 | oajVwvQVFHgLcnD8FXXPmv8kOgscnFDdPwSXc7JjjqVH8atTCZfBDw1gI7RsHHKY
11 | IhjOLwvehDx1gWEAEQEAAQAL/0BfCTezkbVJhS3nbwcdM5xybN4B4yuFyXZWyPSl
12 | cEn25o/vHK3uOFsELw8biyZviSbz9pdTPoKsdkIL9z0AJCUjse49Q0RH6psDSm9c
13 | DH4c0cFJWMazPfG/1LtEe65uclxMBIOdtn9O5aGSm2O6g/MVPCWtNNcGOAMr6b03
14 | sFh9ztLY9SBUCZB3iu+8STP17SJhCjtXHy+1nO4SPIYs5kt6kNGajnnvcMID4uhW
15 | U8YACIju2xwBqKPiDVftYJvYH39vfBpgC2/LUl3pi5n8+2NuKZsfmJ4FHlthMTrc
16 | 7EnKmddREPBi6Pbm3isBcF+XQlCffaYESXS5yV93YbxHnQs/LX8/Dil8CmAWNjaM
17 | R9+yRBW4YHAPEmwoM7TPkZQ7FZXj13v7vMPdDyzthPUfiNOX9dnIhPqhc8+p9qOE
18 | Vh852ycee9VWi3mDWPa4QmjDNI7MjrEiLZgDTSW58g4sMiVmYkD+w+NXaiTjCS8u
19 | MNvmqm8uHUciCPe8puBagZXF0wYAwZmLYieP2AHL7tNE+eEkRfj4jnUeLD2ym4AP
20 | y2bA9O/INk9OTdeV30DLPgGZJvuxD6RDMkScmvlhMB0Jh0NvQUVPnAkfg/1f/yA6
21 | 2YTdvfPwhWGkPlHo0A9+c4TWthoWGbXl8vz7FvIlapy8Tpt+QAV/GYgGhUEhRFX/
22 | AUI5inErmfd3W9sZ+SEZsbZNF6jw0s8zkrpH9Nh6vuXmgasEN30QQfcCCqjUlaQA
23 | wunzHHhNsLxyjEzCom4eULy0v+IfBgDKBrcrVDlhyliUS1G0GNiKg6GQvrz/hEVi
24 | gJNO58pS1lEqV78CXJi/BXnUWgHpECOAiYWKS4+WEs+eTarrcS/FnJq2IBZ6PJZt
25 | +H0KocePym9pBjwFaEQf9d/5CNtj/ra0Dk3AtG+JVCVZ1yICPJ8axxwknsDp2NOm
26 | IjCgcKTgKuKkvroeItl+giaKdldVSUcIQ25d5MPURQyKBMJBrBNibeQ5+a5o//jl
27 | ig/ZgabQd0ClvYOf+p/RihoUGxK5LH8F/RO875NpHnvQTi7C+MYEZ0noVGueoITP
28 | 5z4zlfqVtG9SRnlvQjkT5UqsCJ7fVMCC9xgZKn5NnmWDBjGkWNaJlB6pkKc/YbIF
29 | ezECs8j0Sf5ZKW0cWp259bXTt/fZzVSuhyR/XYQzMdDcJRJcGSAy5I8KiMLjCTD1
30 | CS6uA4Mudy0aqJuxnuYt/zHcKCINht/vQ5ZxVF5dWDBSaIs1l+CY8IpFTubx19jY
31 | AU30A5PoFgo+xxI+0V5qfpfmNYwBXjBCMdF8tBNZcHJveHkgdGVzdCA8YUBiLmM+
32 | iQHUBBMBCgA+FiEEHBF5sTFe6EsRGXYQjc9YgxYCBMAFAmflJvgCGwMFCQPCZwAF
33 | CwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQjc9YgxYCBMACAgv8Chubu1Jtl/oE
34 | T9fFKO7tEB2BV8cwTymMPk1S7QDzVQUlNph0tN7WsQhrfgCqhp+C/R8aJdCzqjAG
35 | d6JP8+oVoU/PloQeyfewHarka3qhKZLqdvrPAtaScnKZ1skhRUAC2CFQDIfjsj7W
36 | u5+8Y/qBChLUNoRqvTT1h32xoNbS21NpTi74vPChmdgfKyIKGHG5t/c7nXXuKDI7
37 | Om/dRlXxcfpnq0l0zNRO5W65J3tgyfM6d0ph6mkU7GFi+WwjvGet91M/qNCRQrUC
38 | G5rXXUVGcCzqCh8i8hTaAQZQpXrLlH4sBJlp2wVn00dQtmgS3DApgga7osHwsfUV
39 | 3X59jcKFo/+L5JVj3gG+uniBG0tRgGcfzv9zVZzYSVMrGx0r0KqHQvf/AkrGby7f
40 | Bu2RNyA0cJx19J3pI7So+YlKftslGMv4Lq+ctw+WUPle/wET9WcdBsq5EVmajBYc
41 | AnUT/0E5ayGm8EOhFsJ7mxYhu0aexYXustq/F9plXT1ZUt6BBDkknQVYBGflJvgB
42 | DADDm7B55WgY7Ev8gf8wjCyFR4/4KZwDpXrCAk4Ab1F2n8VST1HYKbZO1mXdlJEG
43 | 3Jy+/S2tnDDtm1snjncikaLztyBAX5FcxmFgpLI0JPPv+RrTZD4CxAovkf7BPceR
44 | 97DPbTEc+uCX6bCdc+qQWFrql0iBwNK+M1FFK9Dxk6nXRe5/KHxJur8pJYMQpKti
45 | pJh3uDznlFM9xlWW2gI/6kW5VQ0vUHKxhnK5DBJF2YwbMwv4KG04h9rrqY8tEP44
46 | 3IAf08Jalk8yiy+3tmtRlRU0qlE4fUgbhoScj6V/jHhwpDXjrNKtTvEvhdytwxrT
47 | e8wCN1JCRdfu5wxPfqhblAKxwyoSuJ9hRhdCd8CrE/6SbZk2AiJbjzZDfetEgTBR
48 | Gw2xUlo0oGNLyMtR4DfNrxdDfq9j7r8T8di6iHORcOKP8iZd6X8Um7UUfZVcQckm
49 | n2PPt/QMC1YO7OwPTWMLUdadBXgojhlbTac4z8s6LI3x/WOGOy9RfcFOnSDYO6mu
50 | X8sAEQEAAQAL+wY425ePs+H8Z5EJVDlnxrrD7fvHhw5D+JsB68KZHTmYF3S+JSz/
51 | wVbR/GeHTvMC7Xwoct2E4bbigYxz5wI907XLlhu2GS+e99X/0dNVdupOkuCKiGvE
52 | xKDYFLHHlKOQrMUhr2BBSAeiXVsvfBSfWt+qHu+nAeJ0S2+B10xNQCfNh6qaemmg
53 | pm1ZTh6naPQ4UXPtc7KCG1ZptVXuarKRxyoACe1UB8dHLuWyppomM2NiGk1d1oDn
54 | 1ro2/eDscMm/lHLb05T66N6KrJ4DRlurWEWwMV7KrT/saqLdxkhcGv3hLnZLtc7a
55 | VqTAgbzl1ZrSu/QLyTK1hAwbdJxYepMV1zOBspkj8/7QvMIHDy6YM6GW0Hy2WM58
56 | TmeiMieQP3pyJZz/VfVXPBNg6gtszobBqPQWniV9/CvUhSMsOwjcEnXSIHUhFjhJ
57 | kq8jgh3CVMGqvdOqizl2Cn3N5LK5godwfMGoNJzbVqQwzoxBirPh9Uq8cbJpy8Dv
58 | OILVnuuXZj30wQYA2nhSe2JSjxajCO1t/J+Vesa+1tf+0fmz0obqvYJs7h86q5Kd
59 | h6fqxOwf07MUi7dc+2HfoyMoBkyV96nKwdIE4cytcO9Fl6kf2qBCgdi36799uG0U
60 | ra4y4w8k/adwRl/oGSPZ8a8NmvW+nk3w1ERql/KjdVZLzS7OYXyXsyEabrWqbixf
61 | DaN0dWqNw71jym3khdHojjKeJUSFx4wrJTAikJQReBb2S44iLvrRzWtV1h+t+yfh
62 | 3aUjSLm8qJDmDV2xBgDlNfgqDTM6zXqrXCRZQKkbLCPnKnrTStfq+e8jATN2i6cK
63 | pQrVkWRZL2nUTXIE6/ZXA6HMZwkb1OjRhEbaoLfQV/pxAfIMfandbRSDMqbz/YJ0
64 | zUYQzVZl2BUQHRTFhF7C4cFtSRYrA1coyHrEq8t7tQiP28lsf/QzDjUiSiJxZPly
65 | 8lZM6PzUWLVopsyDeDYMiMUqrWqRHJb/W7ZZhaO9PaCvU0I6ofTnG+rsTiIATx/d
66 | qQSVIOSBFTJs8lZASDsF/jqjRDKIM0Z+pbJjqv1XxO3/f8VCIKx7Rfd1nfe3Pris
67 | FgYKLuX7LTQxQK6rmFHJ1fvk2k/DwhGA6c5qqdhPTZju7YASPDJe/s4UJFi0HyV1
68 | L2ri74Qbg8ieURqyCDC4A8PMPgE4eyr61hwrUbaS0tfhoydQZoN3OQEc4NbZ8U2+
69 | KuihR5asIt9uuq3cWQh+DMxzfCHN6KpvsL6CPaDORL34Yd6sG2UvRJZQFOgy+tsr
70 | XBkMZQb9R7Rsgaw/OYUGL/G6iQG8BBgBCgAmFiEEHBF5sTFe6EsRGXYQjc9YgxYC
71 | BMAFAmflJvgCGwwFCQPCZwAACgkQjc9YgxYCBMCpRQv+MN2GNcsJfm2gyn3FLF+r
72 | W6ZlUHdW503VKtnSwvE1QEEGCvIWmahE9xG5hIU1mOsFbeh+s53FywX+NvRLGgb5
73 | VwyZNS1E3VR+RGPoDOGiwxuufn3f27Kxsso8nbFItEzGXWKfj/ELBPnO6APi/o17
74 | ynxFk/OYvL7nOowLogx9yFuFsyW3kGLYPqHL5ZylWN6SULv/vpXWhrSl+B1j6QJn
75 | OiHCjWFLeVKZdzOuEs821MCEIEOCeiQUt7gFEVHMA5qEkbrDG8KhMCi03+BiQoxj
76 | YBZechAWRrMVm///SxF68bT+D9rCGFQaMAWW2DD8lkscWnsjlC2c2366+aCYiZuv
77 | 8lB5vwU/Xx1vcSpUXbKZnnZH7tpqtl+8sk5XK776l1VH0/rY72xvyJYNsh3gxjv6
78 | oY4SKbNSfi/8WmXcwMxxBVZN5y/JElI5Te+apUIHdMpiACyRb32ESxTGNk980L3t
79 | XeQ9Je7HgqGGMH6qMFf+6e7ZyFS2wG5hX93igo6Cvuzh
80 | =FJx7
81 | -----END PGP PRIVATE KEY BLOCK-----


--------------------------------------------------------------------------------
/test/regress/gpg/gpg_1.pub:
--------------------------------------------------------------------------------
 1 | -----BEGIN PGP PUBLIC KEY BLOCK-----
 2 | 
 3 | mQGNBGflJvgBDACYyDwaPqPIdWVS517ajR+DmZDaHtGpcatA4vOYNCZSSugXv+xk
 4 | UoQZBMLtzTlm956jhp4hsvugvRDyoOA5B8s0DSzLAzB4VQZYb8lcnmfdyHSKHQwR
 5 | HV9UpfVSDnGp5l9hHUM+t4YTO2eH4pBPC4j0XJyd8AMUhbMlSuVxp47uOVX2svFG
 6 | gQtZEt2tFK3DLq+SI2mIVMMJMMSCNwxlDOL66dlsnEtEksmYrcmIs2qlBAFmTmgY
 7 | aKis4GRekpKigKMn/3h1Y5+nAm5xTwthc/6zROWG43RGp9BJ1iAOLUowhB4Tx6NM
 8 | DufwE7ldBi4ahhtY8a+zKo93osh3Ytl0lhECiU71bmj3X00Irm+9pKPdzz8ITL2l
 9 | O28VtstiLU6aGcfzOKMogdIWq5uR8fNW6E+Yp4tfcDpFzDerWpv12lLTZF81te24
10 | oajVwvQVFHgLcnD8FXXPmv8kOgscnFDdPwSXc7JjjqVH8atTCZfBDw1gI7RsHHKY
11 | IhjOLwvehDx1gWEAEQEAAbQTWXByb3h5IHRlc3QgPGFAYi5jPokB1AQTAQoAPhYh
12 | BBwRebExXuhLERl2EI3PWIMWAgTABQJn5Sb4AhsDBQkDwmcABQsJCAcCBhUKCQgL
13 | AgQWAgMBAh4BAheAAAoJEI3PWIMWAgTAAgIL/Aobm7tSbZf6BE/XxSju7RAdgVfH
14 | ME8pjD5NUu0A81UFJTaYdLTe1rEIa34Aqoafgv0fGiXQs6owBneiT/PqFaFPz5aE
15 | Hsn3sB2q5Gt6oSmS6nb6zwLWknJymdbJIUVAAtghUAyH47I+1rufvGP6gQoS1DaE
16 | ar009Yd9saDW0ttTaU4u+LzwoZnYHysiChhxubf3O5117igyOzpv3UZV8XH6Z6tJ
17 | dMzUTuVuuSd7YMnzOndKYeppFOxhYvlsI7xnrfdTP6jQkUK1Ahua111FRnAs6gof
18 | IvIU2gEGUKV6y5R+LASZadsFZ9NHULZoEtwwKYIGu6LB8LH1Fd1+fY3ChaP/i+SV
19 | Y94Bvrp4gRtLUYBnH87/c1Wc2ElTKxsdK9Cqh0L3/wJKxm8u3wbtkTcgNHCcdfSd
20 | 6SO0qPmJSn7bJRjL+C6vnLcPllD5Xv8BE/VnHQbKuRFZmowWHAJ1E/9BOWshpvBD
21 | oRbCe5sWIbtGnsWF7rLavxfaZV09WVLegQQ5JLkBjQRn5Sb4AQwAw5uweeVoGOxL
22 | /IH/MIwshUeP+CmcA6V6wgJOAG9Rdp/FUk9R2Cm2TtZl3ZSRBtycvv0trZww7Ztb
23 | J453IpGi87cgQF+RXMZhYKSyNCTz7/ka02Q+AsQKL5H+wT3Hkfewz20xHPrgl+mw
24 | nXPqkFha6pdIgcDSvjNRRSvQ8ZOp10Xufyh8Sbq/KSWDEKSrYqSYd7g855RTPcZV
25 | ltoCP+pFuVUNL1BysYZyuQwSRdmMGzML+ChtOIfa66mPLRD+ONyAH9PCWpZPMosv
26 | t7ZrUZUVNKpROH1IG4aEnI+lf4x4cKQ146zSrU7xL4XcrcMa03vMAjdSQkXX7ucM
27 | T36oW5QCscMqErifYUYXQnfAqxP+km2ZNgIiW482Q33rRIEwURsNsVJaNKBjS8jL
28 | UeA3za8XQ36vY+6/E/HYuohzkXDij/ImXel/FJu1FH2VXEHJJp9jz7f0DAtWDuzs
29 | D01jC1HWnQV4KI4ZW02nOM/LOiyN8f1jhjsvUX3BTp0g2Duprl/LABEBAAGJAbwE
30 | GAEKACYWIQQcEXmxMV7oSxEZdhCNz1iDFgIEwAUCZ+Um+AIbDAUJA8JnAAAKCRCN
31 | z1iDFgIEwKlFC/4w3YY1ywl+baDKfcUsX6tbpmVQd1bnTdUq2dLC8TVAQQYK8haZ
32 | qET3EbmEhTWY6wVt6H6zncXLBf429EsaBvlXDJk1LUTdVH5EY+gM4aLDG65+fd/b
33 | srGyyjydsUi0TMZdYp+P8QsE+c7oA+L+jXvKfEWT85i8vuc6jAuiDH3IW4WzJbeQ
34 | Ytg+ocvlnKVY3pJQu/++ldaGtKX4HWPpAmc6IcKNYUt5Upl3M64SzzbUwIQgQ4J6
35 | JBS3uAURUcwDmoSRusMbwqEwKLTf4GJCjGNgFl5yEBZGsxWb//9LEXrxtP4P2sIY
36 | VBowBZbYMPyWSxxaeyOULZzbfrr5oJiJm6/yUHm/BT9fHW9xKlRdspmedkfu2mq2
37 | X7yyTlcrvvqXVUfT+tjvbG/Ilg2yHeDGO/qhjhIps1J+L/xaZdzAzHEFVk3nL8kS
38 | UjlN75qlQgd0ymIALJFvfYRLFMY2T3zQve1d5D0l7seCoYYwfqowV/7p7tnIVLbA
39 | bmFf3eKCjoK+7OE=
40 | =LaF8
41 | -----END PGP PUBLIC KEY BLOCK-----


--------------------------------------------------------------------------------
/test/regress/gpg/gpg_2.priv:
--------------------------------------------------------------------------------
 1 | -----BEGIN PGP PRIVATE KEY BLOCK-----
 2 | 
 3 | lQVYBGflLP0BDADFEEP8nxoYl2Xby5wXn+HlarkhqITz1ObmuJF7aq0T4bpjzr2d
 4 | mkbzS3gTHpJPZUZodZX05fQN99E/B6xDVmVFICjK//jaMj92FND41DL+K1AK9dnU
 5 | FCC3Vk9exuZ4CwUKOowsMSRh6NcqijOVjoGIzhHtdnXZg27QtDHJ5GL++qRZkJTy
 6 | DbtEiPihP3g/oFGilOl0Er75YpIs9vEYzwow5XNU3MxX19xwaQCDlwuZa7Drm2Ad
 7 | B2bvjaKoYum+SMWC2+o35ZmU9ZOWAjU4DWu+rh2HrUMKa3jSq1X41RyxFrpkbFDz
 8 | i/FtzUBdxWo+paIKY5Aa/9p/ml1hO99VjZ1p4/oAy4nkDj6FzmM1/hxJId5YLyDp
 9 | yi+OOTWG/vwiow4uje4+PVFO+nwaF1RorNy4ziiBKriuLtvO5yszXF3bXyVcXwfF
10 | bvT3HtXAkMHzB+5QklSMxdxsMsGcdRv7LFyP4BG1L7I/8u/n4bE6qb0wg3GLShPO
11 | 4QlIF6yWtUoCzbMAEQEAAQAL/jpju+lN0SMpNOaf2WPMnVfnO+yjrKU815/UX1L0
12 | xEzITPyGxPs+O7iaC32KyOpahTatZJ3YRuyembL12I5pOn91bHDd0IQCcHhR6atY
13 | 4LUb608cEot63UjFe6YHBEHcbgVeXKs5cmDKZ3q00V0GiQ4alQhJ2dh1z69qPPhn
14 | wgdwB6N8Kg3gkLTZauYIUt5zZ8znF5creMeW893722sw6YGccwRZmfiLPsdV7W+Q
15 | IUE3vgfJlXx0dYUUb/Y5vK2DjPCSR6iliL4PkWV9rZowrb7Xhh2lpzTtFGn97X5x
16 | zMfwDj4HfWA8dlTtR7sbB9PHkOegHSr5puIW8W/ifIbnY4He4vI93462Wsf/fzq9
17 | yjHwraBjMDxtifJ/G0xXW/mBHEfpQ3qlOKU3G4qNSLROmxRx1HrYSb3HTNsevJRo
18 | q8s3/2S2zBian9CDrylThe449eSCkYkQy+tc6SHB0M6/1a+u53pUB+dN0mXDEYbk
19 | w3ORJ8KVaOL1eCI2zJxH61rJ4QYAytp4Lbr08qoPU0hKDKFod56cf1XpYAKhbe9u
20 | AsFNaBXiBowkA2DG4THVSq240+K7vbNHsIMr+Ozso4zrBs5s3iCaSIscV7bR/JcI
21 | 71h0Qqo5BxqlxjXdm7f+ye0gLJGM0g82vGng1yhHDSt7FteRNqR8EA22XvShm1ey
22 | 28hBh1ZsxIe7plubrahznhiBfeF/mwLBOHkaPrtaAGMBKsGKqrPr+rJ3JID5p5Go
23 | ehmvajcOcMOKP2YdWthgK2kmiEWTBgD4sXdGQpePvELFlS6frNlhefJT4MSzKoNI
24 | dulVgDg6TKqlWI21DNtdfQ4WcAq1xpaoHVTQTdfkYVAkEpM72YSSqoGKMQuv+iVE
25 | 3Rwj0QlvBOx8ZWzEhH0VJKZjfmSpItZtIbkmT7bp88ZvERM+81q4qDv8Z5KZkUty
26 | O45DluBg7ho9NAmtyC277S6vySXS+HX0Lprobf5qpwJ8IoYQ6Ho4KrBBbUGaJ01d
27 | ilwjGPU3YiRQolkRQi3u4gQI/ucEa2EF/A+aN/fIDtxWtWtSzBUQZE5/sC2qccnz
28 | ewx0gl/WCiwJmtzlRFZo/BfYC4ZY147tBUriFEYCwYLfYkZwMwqFI8MfOp/TUAhH
29 | HI5BsNFORY7LQPtlsnCDSg1fh8JTfFi01C2mFh0iB9JJJKsm5/aKMybvAFyPAsft
30 | B9qE61pjosX7/Mjpj6+qkHfvAfhxTHtFiH+zlVZ8wfy5YhPLF9fd/p5yJMfRmzdw
31 | 85SoltLwmh9zTK6INNu0r//oU+b5IjWpzusWtB5ZcHJveHkgdGVzdCAyIDxyZWFs
32 | QGVtYWlsLmNvbT6JAdQEEwEKAD4WIQTzaOdd0meXjLR9xdEfTrZpzZviJAUCZ+Us
33 | /QIbAwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRAfTrZpzZviJA/7
34 | DACz74NyS5g0GwaQmVCIStHf+HX+5o/XvbVBELRpJy4aVV+trpFsLZxjIRK3VGI/
35 | RAiFf7u7lf8i2J/mSo6xHZA5V1lLq4lZ6M5KkFAtHb5xLePDk5v+I61xoYvCfBcl
36 | KyqM0us1gvB3XZDnakCZVRcWDTzvRhbb5jicFFz+IypUoq7J9uXFkim0MLnRk24b
37 | Pxz1n4Vzl3Dg9lsP4qpfzC9ZDeU6rL3pNMl1zaMJ/qQqj8jVJLIdEW4UM5bLWyG9
38 | fgYdpvnUVcJtZqwnpKeHwmvJAH6T7zH1BBBaz0hdopFYTtwSVHQ04vUexpjVue8N
39 | oxvF53Hjt9cwg5yFXCcWZMWBquD4EygU8idIHJ/UocP+hJF0iJUHWMu1HjkQGGbL
40 | Z4MiGtHc12La7mpHmBgVop4Xc5h8wSVe6TyX0SCodNS64Rt5nM9boiM7gTqAYHa9
41 | RynujfOUDAZ+fatk50zwRbFBw7FX7KRehxuhmpDL/s+iXRGlYN/I8p0zD9Q2Up/S
42 | uUadBVgEZ+Us/QEMAOWwaEDlCypiZmzDMgQzt8B0mMxnZnBBlhcWMz0eO2sgX6fs
43 | khNJuZ+tZew48cTwfWYfNYIfkL7W8RjF4/0KLeCVyZkBM01PtPxmHCntlwZF9W64
44 | D0LZCwJulFo2fjy/Fa5zbf1TD6xPslfe6c/2YwS3LJpRV0HUzVVmREVSk2rElNoM
45 | +gqEPb2tX9oMzpKq5CI/Wtx7bTiTcpo08EGa61Mc+kVrDMTaW0vj/gs47xaqoV5F
46 | rm5SgSkBXYqsDAMZ4UT9ujvL0JsKyynVvLPSF7ozzNSxzGcPZ4PBar/1+/LQK1uR
47 | c8ERfRuFqJm4KAHUvFfuYFA3+fSVJXq0Q4mRFXviK92UQimq6V/eb4tSLZwz3GXq
48 | ky563HBM5DqBPu1S5qn7liSw/o+7E0/QMospbfKg9H3conNR969YhIF6R3Pf54in
49 | 6P9yFF/+r9zHK5OJNq8/ulFCIfEVrnBpDWU/cIkn4GyRvdjSbvgBpSeaWcJoUtPq
50 | RSMyM+ueuFk8tnmHwQARAQABAAv+ONiCyNLXA1ekoVTi71LMbqt8uJP7sJsu1gpm
51 | xoDRtFboDbOZSIBF2kzMLZ0/St45LtnZAJom1PB17YtmfFT1W0C/CujW5VKtiI0N
52 | +duFNeaBEnSnxEPau3CaM0SK1CZ3q3wCdACHX74+tYwkIlXIll1uQsgQWCjeY70i
53 | duSzuugprdip1on5jNTYpOroAjkcqpQy/N/oPWkSozMoZhg9Fw4UK7ONjre7AiqM
54 | UIM3+wxiUSZeQAwXzsn5eG1RsL/Jvpti3mHzuZwsiT8hi4cFzA68lV/KZTgHuwuj
55 | 3eo0WjGLNuaNWnbPB6UZBvtkqBD9jrYboTyG24E4PS8P3NUY0JL+zUEE3m3UcK5B
56 | bKRUzLFYedx3hXnKURY4cRHKDPBzhxZQeIRvf76nhc2amwbKK1snZEG/suRBGQnP
57 | /Zg1r4qYwfvmzu09xd8atHBWLbYwkJyFU8PXu6QvGijMRBlgtuqG+kQzWwQIHGCL
58 | e/eaqinAxVQnwTtTdwT1z9rRywPVBgDsO+xYQwJQih2xOQkdFjunNUdaBhS+mfJU
59 | N4jM+lDnkWf+/iInjLH06vS0wbLA6fs29+CWW/86wZFaf+4g1jf5h5GD7npjgZYD
60 | iGFHgUcS1rJcRbhNyEiEGu3fzbPNmA/+XDurtN+YAhNkyxm7LhOht7SC1nF0gzQu
61 | 3X11YeuFuYL5w2yxsCslelE4xxKt9lEghMXpeDxo0l6nIa+f1mYg8/ZaUtwODflu
62 | srTbjeJSrlOldY3NXWolXbj+hsIe9j0GAPjoSsLu2sJAZ/B7q6a6o6TZtnLnSXWb
63 | P18gh+y5UbJaCrz6qc7U9++6W3Gk0i+UrLfF4JQLTc3p0jv8bb+eA+EkoVSQc4sz
64 | NkGZqFakMQdyKT/LDWCHZe+quSl/V6zG0Im7ATSP5Saevnc2YalrzyQmFBnEh6io
65 | 3ETfaIyymqolvbBQn6kvWk1B9ORvS88KmKYltaCH1wMPy5mpkBTwkfn0AHiOvkK2
66 | vdMx8G1ViW/ap0F50xyRgU/nkIH5MVKz1QX+NFFqCvb9yu6hTRJbDy99MtkrNcRB
67 | LWr336IoFub8xWzlAY3OHLMnjbU36D1tjkFnZmuZ/SuwuFjRxRyqmm1orKGzywPN
68 | h1PYnDr7T0labMZgCc2838VvAt3MAsFQnrYYob7G0hcAOM8vhkixAS1HN9X0lEaj
69 | ms5JYYj2h1/e9Nd7bK7COqOFHAyj7Z/1ovBhkRH69Yd0Bdrh9UN2Sqis0vgJyIfp
70 | maRhWyMW7o605I6zPiNP4l619NB2I8HawNpX9fCJAbwEGAEKACYWIQTzaOdd0meX
71 | jLR9xdEfTrZpzZviJAUCZ+Us/QIbDAUJA8JnAAAKCRAfTrZpzZviJLuMC/sFQmse
72 | +mNURdJtQr5eJLTi0ioVr8q5ASOnwdKfI1wBJ8DKcTyT+tn+YgSuT1WZQGwLu6fA
73 | KH4Ic7iaSLSSF1IVLtqHeDedCXoDExsxzJJZvG+bqpxgJoCrmN+AJksOByX5K6sn
74 | 6CBxXVz3QEzVhiJzx+NzQnqK0ltX6sorIKNsJgUs+/ex7wZh+zmTEzlmrI/QqGWE
75 | TCvgt5yLLUn+fiidgt3lPLF1Tw2yfOMbtG1qIuiAscmE/jiv3dQ3nzvfefKVhEjV
76 | zXN3sLcwfoLLMAkS54lA4JYCTpgbZhKi8HHGVRNoX/AgpMxJmBxEL9/SBr5EyZ8v
77 | Dh2KsY97wyBmsLx92KIL3jEnXuFhi1s4f5ZlUV/atMSIPAMUBbSHDYyy5mq22SyV
78 | U9DF7tNm6jE0C31GmUqfQLj5GAKow7XayWgmd8dz2sy5m5pCc0WStJhBe2Skozwv
79 | erWnLyXQRVhQc/Z/mx3SAkYep5JXho5XOkCjVem56UHUAGNjzdlffljAtsE=
80 | =QZgW
81 | -----END PGP PRIVATE KEY BLOCK-----


--------------------------------------------------------------------------------
/test/regress/gpg/gpg_2.pub:
--------------------------------------------------------------------------------
 1 | -----BEGIN PGP PUBLIC KEY BLOCK-----
 2 | 
 3 | mQGNBGflLP0BDADFEEP8nxoYl2Xby5wXn+HlarkhqITz1ObmuJF7aq0T4bpjzr2d
 4 | mkbzS3gTHpJPZUZodZX05fQN99E/B6xDVmVFICjK//jaMj92FND41DL+K1AK9dnU
 5 | FCC3Vk9exuZ4CwUKOowsMSRh6NcqijOVjoGIzhHtdnXZg27QtDHJ5GL++qRZkJTy
 6 | DbtEiPihP3g/oFGilOl0Er75YpIs9vEYzwow5XNU3MxX19xwaQCDlwuZa7Drm2Ad
 7 | B2bvjaKoYum+SMWC2+o35ZmU9ZOWAjU4DWu+rh2HrUMKa3jSq1X41RyxFrpkbFDz
 8 | i/FtzUBdxWo+paIKY5Aa/9p/ml1hO99VjZ1p4/oAy4nkDj6FzmM1/hxJId5YLyDp
 9 | yi+OOTWG/vwiow4uje4+PVFO+nwaF1RorNy4ziiBKriuLtvO5yszXF3bXyVcXwfF
10 | bvT3HtXAkMHzB+5QklSMxdxsMsGcdRv7LFyP4BG1L7I/8u/n4bE6qb0wg3GLShPO
11 | 4QlIF6yWtUoCzbMAEQEAAbQeWXByb3h5IHRlc3QgMiA8cmVhbEBlbWFpbC5jb20+
12 | iQHUBBMBCgA+FiEE82jnXdJnl4y0fcXRH062ac2b4iQFAmflLP0CGwMFCQPCZwAF
13 | CwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQH062ac2b4iQP+wwAs++DckuYNBsG
14 | kJlQiErR3/h1/uaP1721QRC0aScuGlVfra6RbC2cYyESt1RiP0QIhX+7u5X/Itif
15 | 5kqOsR2QOVdZS6uJWejOSpBQLR2+cS3jw5Ob/iOtcaGLwnwXJSsqjNLrNYLwd12Q
16 | 52pAmVUXFg0870YW2+Y4nBRc/iMqVKKuyfblxZIptDC50ZNuGz8c9Z+Fc5dw4PZb
17 | D+KqX8wvWQ3lOqy96TTJdc2jCf6kKo/I1SSyHRFuFDOWy1shvX4GHab51FXCbWas
18 | J6Snh8JryQB+k+8x9QQQWs9IXaKRWE7cElR0NOL1HsaY1bnvDaMbxedx47fXMIOc
19 | hVwnFmTFgarg+BMoFPInSByf1KHD/oSRdIiVB1jLtR45EBhmy2eDIhrR3Ndi2u5q
20 | R5gYFaKeF3OYfMElXuk8l9EgqHTUuuEbeZzPW6IjO4E6gGB2vUcp7o3zlAwGfn2r
21 | ZOdM8EWxQcOxV+ykXocboZqQy/7Pol0RpWDfyPKdMw/UNlKf0rlGuQGNBGflLP0B
22 | DADlsGhA5QsqYmZswzIEM7fAdJjMZ2ZwQZYXFjM9HjtrIF+n7JITSbmfrWXsOPHE
23 | 8H1mHzWCH5C+1vEYxeP9Ci3glcmZATNNT7T8Zhwp7ZcGRfVuuA9C2QsCbpRaNn48
24 | vxWuc239Uw+sT7JX3unP9mMEtyyaUVdB1M1VZkRFUpNqxJTaDPoKhD29rV/aDM6S
25 | quQiP1rce204k3KaNPBBmutTHPpFawzE2ltL4/4LOO8WqqFeRa5uUoEpAV2KrAwD
26 | GeFE/bo7y9CbCssp1byz0he6M8zUscxnD2eDwWq/9fvy0CtbkXPBEX0bhaiZuCgB
27 | 1LxX7mBQN/n0lSV6tEOJkRV74ivdlEIpqulf3m+LUi2cM9xl6pMuetxwTOQ6gT7t
28 | Uuap+5YksP6PuxNP0DKLKW3yoPR93KJzUfevWISBekdz3+eIp+j/chRf/q/cxyuT
29 | iTavP7pRQiHxFa5waQ1lP3CJJ+Bskb3Y0m74AaUnmlnCaFLT6kUjMjPrnrhZPLZ5
30 | h8EAEQEAAYkBvAQYAQoAJhYhBPNo513SZ5eMtH3F0R9OtmnNm+IkBQJn5Sz9AhsM
31 | BQkDwmcAAAoJEB9OtmnNm+Iku4wL+wVCax76Y1RF0m1Cvl4ktOLSKhWvyrkBI6fB
32 | 0p8jXAEnwMpxPJP62f5iBK5PVZlAbAu7p8AofghzuJpItJIXUhUu2od4N50JegMT
33 | GzHMklm8b5uqnGAmgKuY34AmSw4HJfkrqyfoIHFdXPdATNWGInPH43NCeorSW1fq
34 | yisgo2wmBSz797HvBmH7OZMTOWasj9CoZYRMK+C3nIstSf5+KJ2C3eU8sXVPDbJ8
35 | 4xu0bWoi6ICxyYT+OK/d1DefO9958pWESNXNc3ewtzB+gsswCRLniUDglgJOmBtm
36 | EqLwccZVE2hf8CCkzEmYHEQv39IGvkTJny8OHYqxj3vDIGawvH3YogveMSde4WGL
37 | Wzh/lmVRX9q0xIg8AxQFtIcNjLLmarbZLJVT0MXu02bqMTQLfUaZSp9AuPkYAqjD
38 | tdrJaCZ3x3PazLmbmkJzRZK0mEF7ZKSjPC96tacvJdBFWFBz9n+bHdICRh6nkleG
39 | jlc6QKNV6bnpQdQAY2PN2V9+WMC2wQ==
40 | =Pwwd
41 | -----END PGP PUBLIC KEY BLOCK-----


--------------------------------------------------------------------------------
/test/regress/pg-init/create-db.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | set -ex
 3 | 
 4 | psql -U $POSTGRES_USER -d $POSTGRES_DB > /var/log/postgresql/setup.log <<EOF
 5 | CREATE DATABASE test;
 6 | \c test
 7 | CREATE SCHEMA yezzey;
 8 | INSERT INTO pg_extension (oid, extowner, extnamespace, extrelocatable, extname, extversion) VALUES (8555, 10, 11, false, 'yezzey', '1.8.4');
 9 | CREATE TABLE yezzey.yezzey_expire_hint (x_path text, lsn pg_lsn);
10 | CREATE TABLE yezzey.yezzey_virtual_index(x_path text);
11 | INSERT INTO yezzey.yezzey_virtual_index VALUES ('/encrypted_file_old_key');
12 | EOF
13 | 


--------------------------------------------------------------------------------
/test/regress/run_tests.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | set -ex
 3 | 
 4 | yproxy --config test/regress/conf/yproxy.yaml --test-mode &> /var/log/yproxy.log &
 5 | yproxy --config test/regress/conf/yproxy_prefix.yaml --test-mode &> /var/log/yproxy_prefix.log &
 6 | yproxy --config test/regress/conf/yproxy_old.yaml --test-mode &> /var/log/yproxy_old.log &
 7 | yproxy --config test/regress/conf/yproxy_second_bucket.yaml --test-mode &> /var/log/yproxy_second_bucket.log &
 8 | 
 9 | # Wait for yproxy to become available
10 | i=0
11 | while (! [ -S /tmp/yproxy.sock ] || ! [ -S /tmp/yproxy_old.sock ] || ! [ -S /tmp/yproxy_identical.sock ] || ! [ -S /tmp/yproxy_prefix.sock ] ) && [ $i -lt 20 ]; do sleep 1; i=$(($i+1)) ; done
12 | ( [ -S /tmp/yproxy.sock ] && [ -S /tmp/yproxy_old.sock ] && [ -S /tmp/yproxy_identical.sock ] && [ -S /tmp/yproxy_prefix.sock ] ) || ( echo 'yproxy startup failed'; exit 2 )
13 | 
14 | for test in $(ls test/regress/tests | awk '{print(substr($1, 1, length($1)-3))}' )
15 |     do ./test/regress/tests/${test}.sh > output.txt || true
16 |     diff test/regress/expected/${test}.txt output.txt
17 |     for file in $(s3cmd --access_key some_key --secret_key some_key --host minio:9000 --host-bucket "" --no-ssl la -r | awk '{print $4}' )
18 |         do s3cmd --access_key some_key --secret_key some_key --host minio:9000 --host-bucket "" --no-ssl rm $file 
19 |     done
20 |     [ $(s3cmd --access_key some_key --secret_key some_key --host minio:9000 --host-bucket "" --no-ssl la -r | grep "\S" | wc -l ) -eq 0 ] || ( echo s3 not empty; exit 2 )
21 |     psql -h pg -U postgres -d test -c "DELETE FROM yezzey.yezzey_virtual_index"
22 | done
23 | 


--------------------------------------------------------------------------------
/test/regress/tests/01_simple_put.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -ex
3 | 
4 | echo 'Sample data' | yp-client --config test/regress/conf/yproxy.yaml -l fatal put 'some_file'
5 | yp-client --config test/regress/conf/yproxy.yaml -l fatal list ''
6 | yp-client --config test/regress/conf/yproxy.yaml -l fatal cat 'some_file'
7 | 


--------------------------------------------------------------------------------
/test/regress/tests/02_encrypted_put.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -ex
3 | 
4 | echo 'Sample encrypted data' | yp-client --config test/regress/conf/yproxy.yaml -l fatal put --encrypt 'encrypted_file'
5 | yp-client --config test/regress/conf/yproxy.yaml -l fatal list ''
6 | yp-client --config test/regress/conf/yproxy.yaml -l fatal cat --decrypt 'encrypted_file'
7 | 


--------------------------------------------------------------------------------
/test/regress/tests/03_copy_separate_keys.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | set -ex
 3 | 
 4 | psql -h pg -U postgres -d test -c "INSERT INTO yezzey.yezzey_virtual_index VALUES ('/encrypted_file_old_key')"
 5 | 
 6 | echo 'Sample encrypted data, old key' | yp-client --config test/regress/conf/yproxy_old.yaml -l fatal put --encrypt 'encrypted_file_old_key'
 7 | yp-client --config test/regress/conf/yproxy_old.yaml -l fatal list ''
 8 | yp-client --config test/regress/conf/yproxy_old.yaml -l fatal cat --decrypt 'encrypted_file_old_key'
 9 | 
10 | # Copy file encrypted with separate keys
11 | yp-client copy --config test/regress/conf/yproxy.yaml --old-config test/regress/conf/yproxy_old.yaml --encrypt --decrypt --port 5432 --confirm --log-level error 'encrypted_file_old_key'
12 | yp-client --config test/regress/conf/yproxy.yaml -l fatal list ''
13 | yp-client --config test/regress/conf/yproxy.yaml -l fatal cat --decrypt 'encrypted_file_old_key'
14 | 


--------------------------------------------------------------------------------
/test/regress/tests/04_copy_separate_keys_multiple.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | set -ex
 3 | 
 4 | psql -h pg -U postgres -d test -c "INSERT INTO yezzey.yezzey_virtual_index VALUES ('/encrypted_file_old_key_1'), ('/encrypted_file_old_key_2')"
 5 | 
 6 | echo 'Sample encrypted data, old key' | yp-client --config test/regress/conf/yproxy_old.yaml -l fatal put --encrypt 'encrypted_file_old_key_1'
 7 | echo 'Sample encrypted data, old key' | yp-client --config test/regress/conf/yproxy_old.yaml -l fatal put --encrypt 'encrypted_file_old_key_2'
 8 | 
 9 | # Copy file encrypted with separate keys
10 | yp-client copy --config test/regress/conf/yproxy.yaml --old-config test/regress/conf/yproxy_old.yaml --encrypt --decrypt --port 5432 --confirm --log-level error ''
11 | yp-client --config test/regress/conf/yproxy.yaml -l fatal list ''
12 | yp-client --config test/regress/conf/yproxy.yaml -l fatal cat --decrypt 'encrypted_file_old_key_1'
13 | yp-client --config test/regress/conf/yproxy.yaml -l fatal cat --decrypt 'encrypted_file_old_key_2'
14 | 


--------------------------------------------------------------------------------
/test/regress/tests/05_copy_identical_keys.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | set -ex
 3 | 
 4 | echo 'Sample encrypted data, identical key' | yp-client --config test/regress/conf/yproxy_second_bucket.yaml -l fatal put --encrypt 'encrypted_file_identical_key'
 5 | yp-client --config test/regress/conf/yproxy_second_bucket.yaml -l fatal list ''
 6 | yp-client --config test/regress/conf/yproxy_second_bucket.yaml -l fatal cat --decrypt 'encrypted_file_identical_key'
 7 | 
 8 | psql -h pg -U postgres -d test -c "INSERT INTO yezzey.yezzey_virtual_index VALUES ('/encrypted_file_identical_key')"
 9 | 
10 | yp-client copy --config test/regress/conf/yproxy.yaml --old-config test/regress/conf/yproxy_second_bucket.yaml --encrypt --decrypt --port 5432 --confirm --log-level error 'encrypted_file_identical_key'
11 | yp-client --config test/regress/conf/yproxy.yaml -l fatal list ''
12 | yp-client --config test/regress/conf/yproxy.yaml -l fatal cat --decrypt 'encrypted_file_identical_key'
13 | 


--------------------------------------------------------------------------------
/test/regress/tests/06_copy_identical_keys_prefix.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | set -ex
 3 | 
 4 | echo 'Sample encrypted data, identical key' | yp-client --config test/regress/conf/yproxy_second_bucket.yaml -l fatal put --encrypt 'encrypted_file_identical_key'
 5 | yp-client --config test/regress/conf/yproxy_second_bucket.yaml -l fatal list ''
 6 | yp-client --config test/regress/conf/yproxy_second_bucket.yaml -l fatal cat --decrypt 'encrypted_file_identical_key'
 7 | 
 8 | psql -h pg -U postgres -d test -c "INSERT INTO yezzey.yezzey_virtual_index VALUES ('/encrypted_file_identical_key')"
 9 | 
10 | yp-client copy --config test/regress/conf/yproxy_prefix.yaml --old-config test/regress/conf/yproxy_second_bucket.yaml --encrypt --decrypt --port 5432 --confirm --log-level error 'encrypted_file_identical_key'
11 | yp-client --config test/regress/conf/yproxy_prefix.yaml -l fatal list ''
12 | yp-client --config test/regress/conf/yproxy_prefix.yaml -l fatal cat --decrypt 'encrypted_file_identical_key'
13 | 


--------------------------------------------------------------------------------
/test/regress/tests/07_copy_identical_keys_multiple.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | set -ex
 3 | 
 4 | echo 'Sample encrypted data, identical key' | yp-client --config test/regress/conf/yproxy_second_bucket.yaml -l fatal put --encrypt 'encrypted_file_identical_key_1'
 5 | echo 'Sample encrypted data, identical key' | yp-client --config test/regress/conf/yproxy_second_bucket.yaml -l fatal put --encrypt 'encrypted_file_identical_key_2'
 6 | 
 7 | psql -h pg -U postgres -d test -c "INSERT INTO yezzey.yezzey_virtual_index VALUES ('/encrypted_file_identical_key_1'), ('/encrypted_file_identical_key_2')"
 8 | 
 9 | yp-client copy --config test/regress/conf/yproxy.yaml --old-config test/regress/conf/yproxy_second_bucket.yaml --encrypt --decrypt --port 5432 --confirm --log-level error ''
10 | yp-client --config test/regress/conf/yproxy.yaml -l fatal list ''
11 | yp-client --config test/regress/conf/yproxy.yaml -l fatal cat --decrypt 'encrypted_file_identical_key_1'
12 | yp-client --config test/regress/conf/yproxy.yaml -l fatal cat --decrypt 'encrypted_file_identical_key_2'
13 | 


--------------------------------------------------------------------------------
/tests/graceful_restart.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | function cleanup() {
 4 |     kill -USR2 $pid1
 5 |     kill -USR2 $pid2
 6 |     kill -ABRT $clientpid1
 7 |     kill -ABRT $clientpid2
 8 |     wait $pid1
 9 |     wait $pid2
10 | }
11 | 
12 | devbin/yproxy --config examples/yproxy.yaml 2>&1 & pid1=$!
13 | sleep 5s
14 | if ! kill -0 $pid1 2>&1; then
15 |     cleanup
16 |     exit 1
17 | fi
18 | echo 'Instance 1 is running with pid' $pid1
19 | 
20 | devbin/yproxy --config examples/yproxy.yaml 2>&1 & pid2=$!
21 | sleep 5s
22 | if ! kill -0 $pid2 2>&1; then
23 |     cleanup
24 |     exit 1
25 | fi
26 | echo 'Instance 2 is running with pid' $pid2
27 | 
28 | devbin/yp-client --config examples/yproxy.yaml gool asdf 2>&1 & clientpid1=$!
29 | if ! kill -0 $clientpid1 2>&1; then
30 |     cleanup
31 |     exit 1
32 | fi
33 | echo 'yp-client1 received connection with pid' $clientpid1
34 | 
35 | echo 'Killing Instance 1'
36 | kill -USR2 $pid1
37 | # old instance should be still running, serving active connections
38 | if ! kill -0 $pid1 2>&1; then
39 |     cleanup
40 |     exit 1
41 | fi
42 | # client should still have a connection to old instance
43 | if ! kill -0 $clientpid1 2>&1; then
44 |     cleanup
45 |     exit 1
46 | fi
47 | 
48 | # new clients must be served
49 | devbin/yp-client --config examples/yproxy.yaml gool asdf 2>&1 & clientpid2=$!
50 | if ! kill -0 $clientpid2 2>&1; then
51 |     cleanup
52 |     exit 1
53 | fi
54 | echo 'yp-client2 received connection with pid' $clientpid2
55 | 
56 | echo 'Killing Instance 2'
57 | kill -USR2 $pid2
58 | 
59 | cleanup
60 | 
61 | exit 0
62 | 


--------------------------------------------------------------------------------