├── .editorconfig ├── .github ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── lint.yml │ └── test-release.yml ├── .gitignore ├── .golangci.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── api ├── benchmarks_test.go ├── routes.go └── routes_test.go ├── app ├── arweave │ ├── arweave.go │ ├── arweave_test.go │ ├── resolve.go │ └── testdata │ │ ├── claim_search.json │ │ └── resolve.json ├── asynquery │ ├── asynquery.go │ ├── asynquery_test.go │ ├── http_handlers.go │ ├── http_handlers_test.go │ ├── metrics.go │ └── routes.go ├── auth │ ├── auth.go │ ├── middleware.go │ ├── middleware_test.go │ └── usergetter.go ├── geopublish │ ├── db.go │ ├── errors.go │ ├── forklift │ │ ├── carriage.go │ │ ├── carriage_test.go │ │ ├── forklift.go │ │ └── testing.go │ ├── geopublish.go │ ├── http.go │ ├── metrics │ │ └── metrics.go │ ├── middleware.go │ ├── routes.go │ ├── testdata │ │ └── wallet.template │ ├── testing.go │ └── tus_test.go ├── proxy │ ├── accounts_test.go │ ├── proxy.go │ └── proxy_test.go ├── publish │ ├── errors.go │ ├── handler_test.go │ ├── http.go │ ├── middleware.go │ ├── publish.go │ ├── publish_test.go │ ├── stream.go │ ├── testing.go │ ├── tus.go │ └── tus_test.go ├── query │ ├── cache.go │ ├── cache_test.go │ ├── caller.go │ ├── caller_test.go │ ├── const.go │ ├── context.go │ ├── errors.go │ ├── metrics.go │ ├── middleware.go │ ├── paid.go │ ├── paid_test.go │ ├── processors.go │ ├── query.go │ ├── query_test.go │ ├── response_strings_test.go │ ├── testing.go │ └── validation.go ├── sdkrouter │ ├── concurrency_test.go │ ├── middleware.go │ ├── sdkrouter.go │ └── sdkrouter_test.go └── wallet │ ├── cache.go │ ├── cache_test.go │ ├── oauth.go │ ├── oauth_test.go │ ├── remote.go │ ├── testing.go │ ├── tracker │ ├── tracker.go │ └── tracker_test.go │ ├── wallet.go │ └── wallet_test.go ├── apps ├── forklift │ ├── cmd │ │ └── main.go │ ├── config │ │ └── forklift.yml │ ├── forklift.go │ ├── forklift_test.go │ ├── metrics.go │ ├── retriever.go │ └── testing.go ├── lbrytv │ └── config │ │ ├── config.go │ │ └── config_test.go ├── uploads │ ├── cmd │ │ └── main.go │ ├── config │ │ └── uploads.yml │ ├── database │ │ ├── db.go │ │ ├── migrations.go │ │ ├── migrations │ │ │ ├── 0001_init.sql │ │ │ └── 0002_urls.sql │ │ ├── models.go │ │ ├── queries.sql │ │ └── queries.sql.go │ ├── json_logger.go │ ├── metrics.go │ ├── payload.go │ ├── readme.md │ ├── testing.go │ ├── uploads.go │ └── uploads_test.go └── watchman │ ├── Dockerfile │ ├── cmd │ ├── watchman-cli │ │ ├── http.go │ │ └── main.go │ └── watchman │ │ ├── http.go │ │ └── main.go │ ├── config │ └── config.go │ ├── design │ └── design.go │ ├── docker-compose.yml │ ├── gen │ ├── http │ │ ├── cli │ │ │ └── watchman │ │ │ │ └── cli.go │ │ ├── openapi.json │ │ ├── openapi.yaml │ │ ├── openapi3.json │ │ ├── openapi3.yaml │ │ └── reporter │ │ │ ├── client │ │ │ ├── cli.go │ │ │ ├── client.go │ │ │ ├── encode_decode.go │ │ │ ├── paths.go │ │ │ └── types.go │ │ │ └── server │ │ │ ├── encode_decode.go │ │ │ ├── paths.go │ │ │ ├── server.go │ │ │ └── types.go │ └── reporter │ │ ├── client.go │ │ ├── endpoints.go │ │ └── service.go │ ├── log │ └── log.go │ ├── metrics.go │ ├── middleware.go │ ├── olapdb │ ├── batch.go │ ├── batch_test.go │ ├── generate.go │ ├── geoip.go │ ├── geoip_test.go │ ├── olapdb.go │ ├── olapdb_test.go │ ├── schema.go │ └── testdata │ │ └── GeoIP2-City-Test.mmdb │ ├── readme.md │ ├── reporter.go │ ├── reporter_test.go │ └── watchman.yaml ├── build ├── forklift │ └── Dockerfile └── uploads │ └── Dockerfile ├── cmd ├── db_migrate_down.go ├── db_migrate_up.go ├── serve.go └── unload_wallets.go ├── config ├── config.go └── config_test.go ├── dev.sh ├── docker-compose.app.yml ├── docker-compose.yml ├── docker ├── daemon_settings.yml ├── launcher.sh └── oapi.yml ├── docs └── design │ └── publish │ ├── publish_v4_client.mmd │ └── publish_v4_urls_client.mmd ├── go.mod ├── go.sum ├── internal ├── audit │ ├── audit.go │ └── audit_test.go ├── e2etest │ ├── e2etest.go │ ├── publish_v3_test.go │ ├── publish_v4_test.go │ └── wait.go ├── errors │ ├── errors.go │ ├── errors_test.go │ └── stack.go ├── handler │ └── handler.go ├── ip │ ├── ip.go │ ├── ip_test.go │ ├── middleware.go │ └── middleware_test.go ├── lbrynet │ ├── errors.go │ └── errors_test.go ├── metrics │ ├── handlers.go │ ├── handlers_test.go │ ├── metrics.go │ ├── middleware.go │ ├── middleware_test.go │ ├── operations.go │ ├── operations_test.go │ └── timer.go ├── middleware │ └── middleware.go ├── monitor │ ├── middleware.go │ ├── middleware_test.go │ ├── module_logger.go │ ├── monitor.go │ ├── monitor_test.go │ └── sentry.go ├── responses │ └── responses.go ├── status │ ├── status.go │ └── status_test.go ├── storage │ ├── migrations │ │ ├── 0001_initial.sql │ │ ├── 0002_wallet_id.sql │ │ ├── 0003_sdk_router.sql │ │ ├── 0004_drop_wallet_id.sql │ │ ├── 0005_wallet_access.sql │ │ ├── 0006_audit_initial.sql │ │ ├── 0007_private_lbrynet.sql │ │ ├── 0008_oauth_integration.sql │ │ ├── 0009_uploads.sql │ │ ├── 0010_asynqueries.sql │ │ └── 0011_drop_unique_upload.sql │ └── storage.go ├── tasks │ └── tasks.go ├── test │ ├── auth.go │ ├── auth_test.go │ ├── http.go │ ├── json.go │ ├── lbrynet.go │ ├── lbrynet_test.go │ ├── static_assets.go │ ├── templates.go │ ├── test.go │ └── test_test.go └── testdeps │ └── testdeps.go ├── main.go ├── models ├── asynqueries.go ├── boil_queries.go ├── boil_table_names.go ├── boil_types.go ├── gorp_migrations.go ├── lbrynet_servers.go ├── psql_upsert.go ├── publish_queries.go ├── query_log.go ├── uploads.go └── users.go ├── oapi.yml ├── pkg ├── app │ └── app.go ├── blobs │ ├── blobs.go │ ├── blobs_test.go │ └── testdata │ │ └── config.yml ├── chainquery │ ├── chainquery.go │ └── chainquery_test.go ├── configng │ ├── client_s3.go │ ├── client_s3v2.go │ └── configng.go ├── fileanalyzer │ ├── fileanalyzer.go │ ├── fileanalyzer_test.go │ └── mime.go ├── iapi │ ├── iapi.go │ ├── iapi_test.go │ └── responses.go ├── keybox │ ├── keybox.go │ └── keybox_test.go ├── logging │ ├── logging.go │ ├── tracing.go │ └── zapadapter │ │ ├── zapadapter.go │ │ └── zapadapter_test.go ├── migrator │ ├── cli.go │ ├── db.go │ ├── migrator.go │ └── testing.go ├── queue │ ├── metrics.go │ ├── queue.go │ └── queue_test.go ├── redislocker │ ├── metrics.go │ ├── redislocker.go │ └── redislocker_test.go ├── rpcerrors │ ├── rpcerrors.go │ └── rpcerrors_test.go ├── sturdycache │ ├── sturdycache.go │ ├── sturdycache_test.go │ └── testing.go └── testservices │ ├── testservices.go │ └── testservices_test.go ├── readme.md ├── scripts ├── init_test_daemon_settings.sh └── wait_for_wallet.sh ├── server ├── server.go └── server_test.go ├── sqlboiler.toml ├── sqlc.yaml ├── tools ├── empty.go ├── go.mod ├── go.sum └── tools.go └── version └── version.go /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | end_of_line = lf 11 | charset = utf-8 12 | 13 | # Use 2 spaces for the HTML files 14 | [*.html] 15 | indent_size = 2 16 | 17 | # The JSON files contain newlines inconsistently 18 | [*.json] 19 | indent_size = 2 20 | insert_final_newline = ignore 21 | 22 | [*.yml] 23 | indent_size = 2 24 | 25 | [*.yaml] 26 | indent_size = 2 27 | 28 | [**/admin/js/vendor/**] 29 | indent_style = ignore 30 | indent_size = ignore 31 | 32 | # Minified JavaScript files shouldn't be changed 33 | [**.min.js] 34 | indent_style = ignore 35 | insert_final_newline = ignore 36 | 37 | # Makefiles always use tabs for indentation 38 | [Makefile] 39 | indent_style = tab 40 | 41 | [docs/**.txt] 42 | max_line_length = 79 43 | 44 | # Go lang uses tabs by default 45 | [*.go] 46 | indent_style = tab 47 | 48 | [*.mmd] 49 | indent_style = space 50 | indent_size = 2 51 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OdyseeTeam/odysee-api/15e122c36a6dc05f55ba456b23614797b4c4dc11/.github/ISSUE_TEMPLATE.md -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OdyseeTeam/odysee-api/15e122c36a6dc05f55ba456b23614797b4c4dc11/.github/PULL_REQUEST_TEMPLATE.md -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | on: 3 | push: 4 | branches: 5 | - master 6 | - dev 7 | pull_request: 8 | 9 | permissions: 10 | contents: read 11 | # Optional: allow read access to pull request. Use with `only-new-issues` option. 12 | pull-requests: read 13 | 14 | jobs: 15 | golangci: 16 | name: lint 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v4 20 | with: 21 | fetch-depth: 0 22 | - uses: actions/setup-go@v5 23 | with: 24 | go-version: '1.24' 25 | cache: false 26 | - name: golangci-lint 27 | uses: golangci/golangci-lint-action@v7 28 | with: 29 | only-new-issues: true 30 | - id: govulncheck 31 | uses: golang/govulncheck-action@v1 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | /assets/bindata.go 3 | /assets/static/app 4 | /dist/ 5 | coverage.* 6 | secrets.env 7 | /token_privkey.rsa 8 | **/dist/ 9 | **/rundata/ 10 | token_privkey.* 11 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | default: none 4 | enable: 5 | - dupl 6 | - goconst 7 | - gocritic 8 | - gocyclo 9 | - gosec 10 | - govet 11 | - ineffassign 12 | - misspell 13 | - sqlclosecheck 14 | - staticcheck 15 | - unused 16 | settings: 17 | errcheck: 18 | check-type-assertions: true 19 | goconst: 20 | min-len: 2 21 | min-occurrences: 3 22 | exclusions: 23 | generated: lax 24 | presets: 25 | - comments 26 | - common-false-positives 27 | - legacy 28 | - std-error-handling 29 | paths: 30 | - third_party$ 31 | - builtin$ 32 | - examples$ 33 | issues: 34 | new-from-rev: d108866 35 | formatters: 36 | enable: 37 | - gci 38 | - gofmt 39 | - goimports 40 | settings: 41 | gci: 42 | sections: 43 | - standard 44 | - prefix(github.com/lbryio) 45 | - prefix(github.com/OdyseeTeam) 46 | - default 47 | custom-order: true 48 | exclusions: 49 | generated: lax 50 | paths: 51 | - third_party$ 52 | - builtin$ 53 | - examples$ 54 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM odyseeteam/transcoder-ffmpeg:5.1.1 AS ffmpeg 3 | FROM alpine:3.21 4 | EXPOSE 8080 5 | 6 | RUN apk update && \ 7 | apk add --no-cache \ 8 | openssh-keygen 9 | 10 | COPY --from=ffmpeg /build/ffprobe /usr/local/bin/ 11 | 12 | WORKDIR /app 13 | COPY dist/linux_amd64/oapi /app 14 | RUN chmod a+x /app/oapi 15 | COPY ./docker/oapi.yml ./config/oapi.yml 16 | COPY ./docker/launcher.sh ./ 17 | 18 | CMD ["./launcher.sh"] 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 LBRY Inc 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 6 | "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the 8 | following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 11 | 12 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 13 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 14 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 15 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | -------------------------------------------------------------------------------- /app/arweave/arweave.go: -------------------------------------------------------------------------------- 1 | package arweave 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/tidwall/gjson" 8 | "github.com/tidwall/sjson" 9 | ) 10 | 11 | func ReplaceAssetUrls(baseUrl string, structure any, collPath, itemPath string) (any, error) { 12 | var origUrls []string 13 | urlPaths := map[string][]string{} 14 | 15 | jsonData, err := json.Marshal(structure) 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | items := gjson.GetBytes(jsonData, collPath) 21 | items.ForEach(func(key, value gjson.Result) bool { 22 | urlPath := fmt.Sprintf("%s.%s.%s", collPath, key.String(), itemPath) 23 | url := gjson.GetBytes(jsonData, urlPath).String() 24 | origUrls = append(origUrls, url) 25 | if slice, exists := urlPaths[url]; exists { 26 | urlPaths[url] = append(slice, urlPath) 27 | } else { 28 | urlPaths[url] = []string{urlPath} 29 | } 30 | return true 31 | }) 32 | 33 | resolver := NewArfleetResolver(baseUrl) 34 | subsUrls, err := resolver.ResolveUrls(origUrls) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | for oldURL, newURL := range subsUrls { 40 | for _, path := range urlPaths[oldURL] { 41 | jsonData, _ = sjson.SetBytes(jsonData, path, newURL) 42 | } 43 | } 44 | 45 | var d any 46 | return d, json.Unmarshal(jsonData, &d) 47 | } 48 | 49 | func ReplaceAssetUrl(baseUrl string, structure any, path string) (any, error) { 50 | jsonData, err := json.Marshal(structure) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | origUrl := gjson.GetBytes(jsonData, path).String() 56 | 57 | resolver := NewArfleetResolver(baseUrl) 58 | subsUrls, err := resolver.ResolveUrls([]string{origUrl}) 59 | 60 | if err != nil { 61 | return nil, err 62 | } 63 | if newUrl, ok := subsUrls[origUrl]; ok { 64 | jsonData, err = sjson.SetBytes(jsonData, path, newUrl) 65 | if err != nil { 66 | return nil, err 67 | } 68 | } 69 | 70 | var d any 71 | return d, json.Unmarshal(jsonData, &d) 72 | } 73 | 74 | func GetClaimUrl(baseUrl, claim_id string) (string, error) { 75 | resolver := NewArfleetResolver(baseUrl) 76 | r, err := resolver.ResolveClaims([]string{claim_id}) 77 | if err != nil { 78 | return "", err 79 | } 80 | return r[claim_id], nil 81 | } 82 | -------------------------------------------------------------------------------- /app/arweave/arweave_test.go: -------------------------------------------------------------------------------- 1 | package arweave 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "path/filepath" 7 | "regexp" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | "github.com/ybbus/jsonrpc/v2" 13 | ) 14 | 15 | func TestReplaceAssetUrls(t *testing.T) { 16 | t.Skip("skipping this in automated mode as it requires extra setup on arfleet") 17 | 18 | require := require.New(t) 19 | assert := assert.New(t) 20 | 21 | absPath, _ := filepath.Abs("./testdata/claim_search.json") 22 | f, err := os.ReadFile(absPath) 23 | require.NoError(err) 24 | var resp jsonrpc.RPCResponse 25 | require.NoError(json.Unmarshal(f, &resp)) 26 | result, err := ReplaceAssetUrls("http://odycdn.com", resp.Result, "items", "value.thumbnail.url") 27 | require.NoError(err) 28 | 29 | out, err := json.MarshalIndent(result, "", " ") 30 | require.NoError(err) 31 | re := regexp.MustCompile(`http://odycdn.com/explore/\w{64}\?filename=\w{64}\.webp`) 32 | matches := re.FindAllString(string(out), -1) 33 | assert.Equal(2, len(matches)) 34 | } 35 | 36 | func TestReplaceAssetUrl(t *testing.T) { 37 | t.Skip("skipping this in automated mode as it requires extra setup on arfleet") 38 | 39 | require := require.New(t) 40 | assert := assert.New(t) 41 | 42 | absPath, _ := filepath.Abs("./testdata/resolve.json") 43 | f, err := os.ReadFile(absPath) 44 | require.NoError(err) 45 | var resp jsonrpc.RPCResponse 46 | require.NoError(json.Unmarshal(f, &resp)) 47 | result, err := ReplaceAssetUrl("http://odycdn.com", resp.Result.(map[string]any)["lbry://@MySillyReactions#d1ae6a9097b44691d318a5bfc6dc1240311c75e2"], "value.thumbnail.url") 48 | require.NoError(err) 49 | 50 | out, err := json.MarshalIndent(result, "", " ") 51 | require.NoError(err) 52 | assert.Regexp(`http://odycdn.com/explore/\w{64}\?filename=\w{64}\.jpg`, string(out)) 53 | } 54 | 55 | func TestGetClaimUrl(t *testing.T) { 56 | t.Skip("skipping this in automated mode as it requires extra setup on arfleet") 57 | 58 | require := require.New(t) 59 | assert := assert.New(t) 60 | url, err := GetClaimUrl("https://cdnhost.com", "91e8caf6d1e740aaa6235d4eb81b21ec21cb2652") 61 | require.NoError(err) 62 | assert.Regexp(`^https://cdnhost.com/explore/\w+\?data_item_id=\w+&filename=\w+.mp4$`, url) 63 | } 64 | -------------------------------------------------------------------------------- /app/arweave/testdata/resolve.json: -------------------------------------------------------------------------------- 1 | { 2 | "jsonrpc": "2.0", 3 | "result": { 4 | "lbry://@MySillyReactions#d1ae6a9097b44691d318a5bfc6dc1240311c75e2": { 5 | "address": "bN7RZQAUbQPJb3gtDYAGe96j9ifqydR56S", 6 | "amount": "0.005", 7 | "canonical_url": "lbry://@MySillyReactions#d", 8 | "claim_id": "d1ae6a9097b44691d318a5bfc6dc1240311c75e2", 9 | "claim_op": "update", 10 | "confirmations": 82454, 11 | "has_signing_key": false, 12 | "height": 1540061, 13 | "meta": { 14 | "activation_height": 1540061, 15 | "claims_in_channel": 162, 16 | "creation_height": 1539258, 17 | "creation_timestamp": 1712496693, 18 | "effective_amount": "0.005", 19 | "expiration_height": 3642461, 20 | "is_controlling": true, 21 | "reposted": 0, 22 | "support_amount": "0.0", 23 | "take_over_height": 1539258 24 | }, 25 | "name": "@MySillyReactions", 26 | "normalized_name": "@mysillyreactions", 27 | "nout": 0, 28 | "permanent_url": "lbry://@MySillyReactions#d1ae6a9097b44691d318a5bfc6dc1240311c75e2", 29 | "short_url": "lbry://@MySillyReactions#d", 30 | "timestamp": 1712594133, 31 | "txid": "d3b28cf6710788bb73c2165c19328c7762fb5e273241089f0db7974dc28861c6", 32 | "type": "claim", 33 | "value": { 34 | "cover": { 35 | "url": "https://thumbnails.lbry.com/banner-UCl7NKDdDxrxdWPEb_ezVYJg" 36 | }, 37 | "description": "As the channel name suggest I offer my silly reactions with a twist of fun.\n\nSilly Reactions on:\nMovie Trailers\nFilm Songs, Reviews\nRumors Talks,\nFunny Videos\n\n\u0026 More important heart-to-heart live talks with our dear members. \n\nFollow-on Instagram: @my_sillyreactions\n\nFor any feedback mysillyreactions@gmail.com\n", 38 | "languages": [ 39 | "en" 40 | ], 41 | "public_key": "03d29c87bb2ae3f83be70003c9ed7fd23598c9c2ea56fc7423fcd23251de1bf29d", 42 | "public_key_id": "bJiBUzH3JiTiXy7jMEgvPRB9HduxHmFHZB", 43 | "tags": [ 44 | "movie trailers", 45 | "trailer reactions", 46 | "funny reactions", 47 | "silly reactions", 48 | "telugu movie reactions" 49 | ], 50 | "thumbnail": { 51 | "url": "https://thumbnails.lbry.com/UCl7NKDdDxrxdWPEb_ezVYJg" 52 | }, 53 | "title": "My Silly Reactions" 54 | }, 55 | "value_type": "channel" 56 | } 57 | }, 58 | "id": 0 59 | } -------------------------------------------------------------------------------- /app/asynquery/metrics.go: -------------------------------------------------------------------------------- 1 | package asynquery 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | const ( 8 | ns = "asynquery" 9 | labelAreaDB = "db" 10 | ) 11 | const LabelFatal = "fatal" 12 | const LabelCommon = "common" 13 | const LabelProcessingTotal = "total" 14 | const LabelProcessingAnalyze = "analyze" 15 | const LabelProcessingBlobSplit = "blob_split" 16 | const LabelProcessingReflection = "reflection" 17 | const LabelProcessingQuery = "query" 18 | 19 | var ( 20 | InternalErrors = prometheus.NewCounterVec(prometheus.CounterOpts{ 21 | Namespace: ns, 22 | Name: "errors", 23 | }, []string{"area"}) 24 | QueriesSent = prometheus.NewCounter(prometheus.CounterOpts{ 25 | Namespace: ns, 26 | Name: "queries_sent", 27 | }) 28 | QueriesCompleted = prometheus.NewCounter(prometheus.CounterOpts{ 29 | Namespace: ns, 30 | Name: "queries_completed", 31 | }) 32 | QueriesFailed = prometheus.NewCounter(prometheus.CounterOpts{ 33 | Namespace: ns, 34 | Name: "queries_failed", 35 | }) 36 | QueriesErrored = prometheus.NewCounter(prometheus.CounterOpts{ 37 | Namespace: ns, 38 | Name: "queries_errored", 39 | }) 40 | ) 41 | 42 | func registerMetrics() { 43 | prometheus.MustRegister( 44 | InternalErrors, QueriesSent, QueriesCompleted, QueriesFailed, QueriesErrored, 45 | ) 46 | } 47 | -------------------------------------------------------------------------------- /app/auth/auth.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/OdyseeTeam/odysee-api/app/sdkrouter" 9 | "github.com/OdyseeTeam/odysee-api/app/wallet" 10 | "github.com/OdyseeTeam/odysee-api/internal/errors" 11 | "github.com/OdyseeTeam/odysee-api/internal/monitor" 12 | "github.com/OdyseeTeam/odysee-api/models" 13 | "github.com/OdyseeTeam/odysee-api/pkg/iapi" 14 | ) 15 | 16 | var ( 17 | logger = monitor.NewModuleLogger("auth") 18 | nilProvider = func(token, ip string) (*models.User, error) { return nil, nil } 19 | ) 20 | 21 | type ctxKey int 22 | 23 | const userContextKey ctxKey = iota 24 | 25 | type CurrentUser struct { 26 | ipAddr string 27 | iac *iapi.Client 28 | 29 | user *models.User 30 | err error 31 | } 32 | 33 | type IAPIUserClient interface { 34 | } 35 | 36 | // Authenticator authenticates 37 | type Authenticator interface { 38 | Authenticate(token, metaRemoteIP string) (*models.User, error) 39 | GetTokenFromRequest(r *http.Request) (string, error) 40 | } 41 | 42 | // Provider tries to authenticate using the provided auth token 43 | type Provider func(token, metaRemoteIP string) (*models.User, error) 44 | 45 | // FromRequest retrieves user from http.Request that went through our Middleware 46 | func FromRequest(r *http.Request) (*models.User, error) { 47 | cu, err := GetCurrentUserData(r.Context()) 48 | if err != nil { 49 | return nil, err 50 | } 51 | return cu.user, cu.err 52 | } 53 | 54 | func AttachCurrentUser(ctx context.Context, cu *CurrentUser) context.Context { 55 | return context.WithValue(ctx, userContextKey, cu) 56 | } 57 | 58 | // GetCurrentUserData retrieves user from http.Request that went through our Middleware 59 | func GetCurrentUserData(ctx context.Context) (*CurrentUser, error) { 60 | v := ctx.Value(userContextKey) 61 | if v == nil { 62 | return nil, errors.Err("auth middleware is required") 63 | } 64 | res := v.(*CurrentUser) 65 | if res == nil { 66 | return nil, fmt.Errorf("%v is not CurrentUser", v) 67 | } 68 | return res, nil 69 | } 70 | 71 | func NewCurrentUser(u *models.User, ipAddr string, iac *iapi.Client, e error) *CurrentUser { 72 | return &CurrentUser{user: u, ipAddr: ipAddr, iac: iac, err: e} 73 | } 74 | 75 | func (cu CurrentUser) User() *models.User { 76 | return cu.user 77 | } 78 | 79 | func (cu CurrentUser) Err() error { 80 | return cu.err 81 | } 82 | 83 | func (cu CurrentUser) IP() string { 84 | return cu.ipAddr 85 | } 86 | 87 | func (cu CurrentUser) IAPIClient() *iapi.Client { 88 | return cu.iac 89 | } 90 | 91 | // NewIAPIProvider authenticates a user by hitting internal-api with the auth token 92 | // and matching the response to a local user. If auth is successful, the user will have a 93 | // lbrynet server assigned and a wallet that's created and ready to use. 94 | func NewIAPIProvider(router *sdkrouter.Router, internalAPIHost string) Provider { 95 | return func(token, metaRemoteIP string) (*models.User, error) { 96 | return wallet.GetUserWithSDKServer(router, internalAPIHost, token, metaRemoteIP) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /app/auth/usergetter.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/OdyseeTeam/odysee-api/app/wallet" 7 | "github.com/OdyseeTeam/odysee-api/internal/errors" 8 | "github.com/OdyseeTeam/odysee-api/internal/ip" 9 | "github.com/OdyseeTeam/odysee-api/models" 10 | "github.com/OdyseeTeam/odysee-api/pkg/logging" 11 | ) 12 | 13 | type universalUserGetter struct { 14 | logger logging.KVLogger 15 | auther Authenticator 16 | provider Provider 17 | } 18 | 19 | func NewUniversalUserGetter(auther Authenticator, provider Provider, logger logging.KVLogger) *universalUserGetter { 20 | return &universalUserGetter{ 21 | auther: auther, 22 | provider: provider, 23 | logger: logger, 24 | } 25 | } 26 | 27 | func (g *universalUserGetter) FromRequest(r *http.Request) (*models.User, error) { 28 | log := g.logger 29 | token, err := g.auther.GetTokenFromRequest(r) 30 | // No oauth token present in request, try legacy method 31 | if errors.Is(err, wallet.ErrNoAuthInfo) { 32 | // TODO: Remove this pathway after legacy tokens go away. 33 | if token, ok := r.Header[wallet.LegacyTokenHeader]; ok { 34 | addr := ip.ForRequest(r) 35 | user, err := g.provider(token[0], addr) 36 | if err != nil { 37 | log.Info("user authentication failed", "err", err, "method", "token") 38 | return nil, err 39 | } 40 | if user == nil { 41 | err := wallet.ErrNoAuthInfo 42 | log.Info("unauthorized user", "err", err, "method", "token") 43 | return nil, err 44 | } 45 | log.Debug("user authenticated", "user", user.ID, "method", "token") 46 | return user, nil 47 | } 48 | return nil, errors.Err(wallet.ErrNoAuthInfo) 49 | } else if err != nil { 50 | return nil, err 51 | } 52 | 53 | user, err := g.auther.Authenticate(token, ip.ForRequest(r)) 54 | if err != nil { 55 | log.Info("user authentication failed", "err", err, "method", "oauth") 56 | return nil, err 57 | } 58 | if user == nil { 59 | err := wallet.ErrNoAuthInfo 60 | log.Info("unauthorized user", "err", err, "method", "oauth") 61 | return nil, err 62 | } 63 | log.Debug("user authenticated", "user", user.ID, "method", "oauth") 64 | return user, nil 65 | } 66 | -------------------------------------------------------------------------------- /app/geopublish/errors.go: -------------------------------------------------------------------------------- 1 | package geopublish 2 | 3 | import ( 4 | "fmt" 5 | 6 | werrors "github.com/pkg/errors" 7 | ) 8 | 9 | var ErrEmptyRemoteURL = &FetchError{Err: werrors.New("empty remote url")} 10 | 11 | // FetchError reports an error and the remote URL that caused it. 12 | type FetchError struct { 13 | URL string 14 | Err error 15 | } 16 | 17 | func (e *FetchError) Unwrap() error { return e.Err } 18 | func (e *FetchError) Error() string { return fmt.Sprintf("fetch error on %q: %s", e.URL, e.Err) } 19 | 20 | // RequestError reports an error that was due invalid request from client. 21 | type RequestError struct { 22 | Err error 23 | Msg string 24 | } 25 | 26 | func (e *RequestError) Unwrap() error { return e.Err } 27 | func (e *RequestError) Error() string { return fmt.Sprintf("request error: %q %s", e.Msg, e.Err) } 28 | -------------------------------------------------------------------------------- /app/geopublish/forklift/testing.go: -------------------------------------------------------------------------------- 1 | package forklift 2 | 3 | type StreamCreateResponse struct { 4 | Height int `json:"height"` 5 | Hex string `json:"hex"` 6 | Inputs []struct { 7 | Address string `json:"address"` 8 | Amount string `json:"amount"` 9 | Confirmations int `json:"confirmations"` 10 | Height int `json:"height"` 11 | Nout int `json:"nout"` 12 | Timestamp int `json:"timestamp"` 13 | Txid string `json:"txid"` 14 | Type string `json:"type"` 15 | } `json:"inputs"` 16 | Outputs []struct { 17 | Address string `json:"address"` 18 | Amount string `json:"amount"` 19 | ClaimID string `json:"claim_id,omitempty"` 20 | ClaimOp string `json:"claim_op,omitempty"` 21 | Confirmations int `json:"confirmations"` 22 | Height int `json:"height"` 23 | Meta struct { 24 | } `json:"meta,omitempty"` 25 | Name string `json:"name,omitempty"` 26 | NormalizedName string `json:"normalized_name,omitempty"` 27 | Nout int `json:"nout"` 28 | PermanentURL string `json:"permanent_url,omitempty"` 29 | Timestamp interface{} `json:"timestamp"` 30 | Txid string `json:"txid"` 31 | Type string `json:"type"` 32 | Value struct { 33 | Source struct { 34 | Hash string `json:"hash"` 35 | MediaType string `json:"media_type"` 36 | Name string `json:"name"` 37 | SdHash string `json:"sd_hash"` 38 | Size string `json:"size"` 39 | } `json:"source"` 40 | StreamType string `json:"stream_type"` 41 | } `json:"value,omitempty"` 42 | ValueType string `json:"value_type,omitempty"` 43 | } `json:"outputs"` 44 | TotalFee string `json:"total_fee"` 45 | TotalInput string `json:"total_input"` 46 | TotalOutput string `json:"total_output"` 47 | Txid string `json:"txid"` 48 | } 49 | -------------------------------------------------------------------------------- /app/geopublish/middleware.go: -------------------------------------------------------------------------------- 1 | package geopublish 2 | 3 | import ( 4 | "net/http" 5 | "regexp" 6 | 7 | "github.com/getsentry/sentry-go" 8 | "github.com/gorilla/mux" 9 | ) 10 | 11 | var reExtractFileID = regexp.MustCompile(`([^/]+)\/?$`) 12 | 13 | func TracingMiddleware() mux.MiddlewareFunc { 14 | return func(next http.Handler) http.Handler { 15 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 16 | if hub := sentry.GetHubFromContext(r.Context()); hub != nil { 17 | uploadID := extractID(r) 18 | if uploadID != "" { 19 | hub.Scope().AddBreadcrumb(&sentry.Breadcrumb{ 20 | Category: "upload", 21 | Message: "ID", 22 | Level: sentry.LevelInfo, 23 | }, 999) 24 | } 25 | } 26 | 27 | next.ServeHTTP(w, r) 28 | }) 29 | } 30 | } 31 | 32 | func extractID(r *http.Request) string { 33 | params := mux.Vars(r) 34 | return params["id"] 35 | } 36 | -------------------------------------------------------------------------------- /app/geopublish/routes.go: -------------------------------------------------------------------------------- 1 | package geopublish 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "path" 7 | 8 | "github.com/OdyseeTeam/odysee-api/app/geopublish/forklift" 9 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 10 | "github.com/OdyseeTeam/odysee-api/internal/storage" 11 | "github.com/OdyseeTeam/odysee-api/pkg/logging" 12 | "github.com/OdyseeTeam/odysee-api/pkg/redislocker" 13 | "github.com/gorilla/mux" 14 | 15 | "github.com/tus/tusd/pkg/filestore" 16 | tushandler "github.com/tus/tusd/pkg/handler" 17 | ) 18 | 19 | func InstallRoutes(router *mux.Router, userGetter UserGetter, uploadPath, urlPrefix string, logger logging.KVLogger) (*Handler, error) { 20 | redisOpts, err := config.GetRedisLockerOpts() 21 | if err != nil { 22 | return nil, fmt.Errorf("cannot get redis config: %w", err) 23 | } 24 | asynqRedisOpts, err := config.GetRedisBusOpts() 25 | if err != nil { 26 | return nil, fmt.Errorf("cannot get redis config: %w", err) 27 | } 28 | 29 | composer := tushandler.NewStoreComposer() 30 | store := filestore.New(uploadPath) 31 | store.UseIn(composer) 32 | 33 | fl, err := forklift.NewForklift( 34 | path.Join(uploadPath, "blobs"), 35 | config.GetReflectorUpstream(), 36 | asynqRedisOpts, 37 | forklift.WithConcurrency(config.GetGeoPublishConcurrency()), 38 | forklift.WithLogger(logger), 39 | ) 40 | if err != nil { 41 | return nil, fmt.Errorf("cannot initialize forklift: %w", err) 42 | } 43 | err = fl.Start() 44 | if err != nil { 45 | return nil, fmt.Errorf("cannot start forklift: %w", err) 46 | } 47 | 48 | locker, err := redislocker.New(redisOpts) 49 | if err != nil { 50 | return nil, fmt.Errorf("cannot start redislocker: %w", err) 51 | } 52 | locker.UseIn(composer) 53 | 54 | tusCfg := tushandler.Config{ 55 | BasePath: urlPrefix, 56 | StoreComposer: composer, 57 | } 58 | 59 | tusHandler, err := NewHandler( 60 | WithUserGetter(userGetter), 61 | WithTusConfig(tusCfg), 62 | WithUploadPath(uploadPath), 63 | WithDB(storage.DB), 64 | WithQueue(fl), 65 | ) 66 | if err != nil { 67 | return nil, fmt.Errorf("cannot initialize tus handler: %w", err) 68 | } 69 | 70 | r := router 71 | r.Use(tusHandler.Middleware) 72 | r.HandleFunc("/", tusHandler.PostFile).Methods(http.MethodPost).Name("geopublish") 73 | r.HandleFunc("/{id}", tusHandler.HeadFile).Methods(http.MethodHead) 74 | r.HandleFunc("/{id}", tusHandler.PatchFile).Methods(http.MethodPatch) 75 | r.HandleFunc("/{id}", tusHandler.DelFile).Methods(http.MethodDelete) 76 | r.HandleFunc("/{id}/notify", tusHandler.Notify).Methods(http.MethodPost) 77 | r.HandleFunc("/{id}/status", tusHandler.Status).Methods(http.MethodGet) 78 | r.PathPrefix("/").HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {}).Methods(http.MethodOptions) 79 | 80 | return tusHandler, nil 81 | } 82 | -------------------------------------------------------------------------------- /app/geopublish/testdata/wallet.template: -------------------------------------------------------------------------------- 1 | { 2 | "accounts": [ 3 | { 4 | "address_generator": { 5 | "name": "single-address" 6 | }, 7 | "certificates": {}, 8 | "encrypted": false, 9 | "ledger": "lbc_mainnet", 10 | "modified_on": 1657898418, 11 | "name": "Account #bP2uhrhHgHLR6WNHCjQpWFDtb3V8aPpo6Q", 12 | "private_key": "{{.PrivateKey}}", 13 | "public_key": "{{.PublicKey}}" 14 | } 15 | ], 16 | "name": "My Wallet", 17 | "preferences": {}, 18 | "version": 1 19 | } 20 | -------------------------------------------------------------------------------- /app/publish/errors.go: -------------------------------------------------------------------------------- 1 | package publish 2 | 3 | import ( 4 | "fmt" 5 | 6 | werrors "github.com/pkg/errors" 7 | ) 8 | 9 | var ErrEmptyRemoteURL = &FetchError{Err: werrors.New("empty remote url")} 10 | 11 | // FetchError reports an error and the remote URL that caused it. 12 | type FetchError struct { 13 | URL string 14 | Err error 15 | } 16 | 17 | func (e *FetchError) Unwrap() error { return e.Err } 18 | func (e *FetchError) Error() string { return fmt.Sprintf("fetch error on %q: %s", e.URL, e.Err) } 19 | 20 | // RequestError reports an error that was due invalid request from client. 21 | type RequestError struct { 22 | Err error 23 | Msg string 24 | } 25 | 26 | func (e *RequestError) Unwrap() error { return e.Err } 27 | func (e *RequestError) Error() string { return fmt.Sprintf("request error: %q %s", e.Msg, e.Err) } 28 | -------------------------------------------------------------------------------- /app/publish/middleware.go: -------------------------------------------------------------------------------- 1 | package publish 2 | 3 | import ( 4 | "net/http" 5 | "regexp" 6 | 7 | "github.com/getsentry/sentry-go" 8 | "github.com/gorilla/mux" 9 | ) 10 | 11 | var reExtractFileID = regexp.MustCompile(`([^/]+)\/?$`) 12 | 13 | func TracingMiddleware() mux.MiddlewareFunc { 14 | return func(next http.Handler) http.Handler { 15 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 16 | if hub := sentry.GetHubFromContext(r.Context()); hub != nil { 17 | uploadID := extractID(r) 18 | if uploadID != "" { 19 | hub.Scope().AddBreadcrumb(&sentry.Breadcrumb{ 20 | Category: "upload", 21 | Message: "ID", 22 | Level: sentry.LevelInfo, 23 | }, 999) 24 | } 25 | } 26 | 27 | next.ServeHTTP(w, r) 28 | }) 29 | } 30 | } 31 | 32 | func extractID(r *http.Request) string { 33 | params := mux.Vars(r) 34 | return params["id"] 35 | } 36 | -------------------------------------------------------------------------------- /app/publish/publish_test.go: -------------------------------------------------------------------------------- 1 | package publish 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "os/exec" 9 | "path" 10 | "testing" 11 | 12 | "github.com/OdyseeTeam/odysee-api/app/wallet" 13 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 14 | "github.com/OdyseeTeam/odysee-api/internal/storage" 15 | "github.com/OdyseeTeam/odysee-api/internal/test" 16 | "github.com/OdyseeTeam/odysee-api/pkg/migrator" 17 | 18 | "github.com/stretchr/testify/assert" 19 | "github.com/stretchr/testify/require" 20 | ) 21 | 22 | type WalletKeys struct{ PrivateKey, PublicKey string } 23 | 24 | const ( 25 | envPublicKey = "REAL_WALLET_PUBLIC_KEY" 26 | envPrivateKey = "REAL_WALLET_PRIVATE_KEY" 27 | ) 28 | 29 | func copyToContainer(t *testing.T, srcPath, dstPath string) error { 30 | t.Helper() 31 | os.Setenv("PATH", os.Getenv("PATH")+":/usr/local/bin") 32 | if out, err := exec.Command("docker", "cp", srcPath, dstPath).CombinedOutput(); err != nil { 33 | fmt.Println(os.Getenv("PATH")) 34 | return fmt.Errorf("cannot copy %s to %s: %w (%s)", srcPath, dstPath, err, string(out)) 35 | } 36 | return nil 37 | } 38 | 39 | func TestLbrynetPublisher(t *testing.T) { 40 | db, dbCleanup, err := migrator.CreateTestDB(migrator.DBConfigFromApp(config.GetDatabase()), storage.MigrationsFS) 41 | if err != nil { 42 | panic(err) 43 | } 44 | storage.SetDB(db) 45 | dbCleanup() 46 | 47 | data := []byte("test file") 48 | f, err := ioutil.TempFile(os.TempDir(), "*") 49 | require.NoError(t, err) 50 | _, err = f.Write(data) 51 | require.NoError(t, err) 52 | err = f.Close() 53 | require.NoError(t, err) 54 | 55 | err = copyToContainer(t, f.Name(), "lbrynet:/storage") 56 | if err != nil { 57 | t.Skipf("skipping (%s)", err) 58 | } 59 | 60 | req := test.StrToReq(t, `{ 61 | "jsonrpc": "2.0", 62 | "method": "stream_create", 63 | "params": { 64 | "name": "test", 65 | "title": "test", 66 | "description": "test description", 67 | "bid": "0.000001", 68 | "languages": [ 69 | "en" 70 | ], 71 | "tags": [], 72 | "thumbnail_url": "http://smallmedia.com/thumbnail.jpg", 73 | "license": "None", 74 | "release_time": 1567580184, 75 | "file_path": "__POST_FILE__" 76 | }, 77 | "id": 1567580184168 78 | }`) 79 | 80 | userID := 751365 81 | server := test.RandServerAddress(t) 82 | err = wallet.Create(server, userID) 83 | require.NoError(t, err) 84 | 85 | res, err := getCaller(server, path.Join("/storage", path.Base(f.Name())), userID, nil).Call( 86 | context.Background(), req) 87 | require.NoError(t, err) 88 | 89 | // This is all we can check for now without running on testnet or crediting some funds to the test account 90 | assert.Regexp(t, "Not enough funds to cover this transaction", test.ResToStr(t, res)) 91 | } 92 | -------------------------------------------------------------------------------- /app/publish/stream.go: -------------------------------------------------------------------------------- 1 | package publish 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path" 7 | "path/filepath" 8 | 9 | "github.com/lbryio/lbry.go/v3/stream" 10 | pb "github.com/lbryio/types/v2/go" 11 | ) 12 | 13 | const ( 14 | // MaxChunkSize is the max size of decrypted blob. 15 | MaxChunkSize = stream.MaxBlobSize - 1 16 | 17 | // DefaultPrefetchLen is how many blobs we should prefetch ahead. 18 | // 3 should be enough to deliver 2 x 4 = 8MB/s streams. 19 | // however since we can't keep up, let's see if 2 works 20 | DefaultPrefetchLen = 2 21 | ) 22 | 23 | func Streamize(p string) (stream.Stream, *pb.Stream, error) { 24 | file, err := os.Open(p) 25 | if err != nil { 26 | return nil, nil, err 27 | } 28 | defer file.Close() 29 | 30 | enc := stream.NewEncoder(file) 31 | 32 | s, err := enc.Stream() 33 | if err != nil { 34 | return nil, nil, err 35 | } 36 | streamProto := &pb.Stream{ 37 | Source: &pb.Source{ 38 | SdHash: enc.SDBlob().Hash(), 39 | Name: filepath.Base(file.Name()), 40 | Size: uint64(enc.SourceLen()), 41 | Hash: enc.SourceHash(), 42 | }, 43 | } 44 | 45 | err = os.Mkdir(enc.SDBlob().HashHex(), os.ModePerm) 46 | if err != nil { 47 | return nil, nil, err 48 | } 49 | 50 | for _, b := range s { 51 | ioutil.WriteFile(path.Join(enc.SDBlob().HashHex(), b.HashHex()), b, os.ModePerm) 52 | } 53 | 54 | return s, streamProto, nil 55 | } 56 | -------------------------------------------------------------------------------- /app/query/cache_test.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | "github.com/ybbus/jsonrpc/v2" 10 | ) 11 | 12 | func TestGetCacheKey(t *testing.T) { 13 | assert := assert.New(t) 14 | require := require.New(t) 15 | seen := map[string]bool{} 16 | params := []map[string]any{ 17 | {}, 18 | {"uri": "what"}, 19 | {"uri": "odysee"}, 20 | nil, 21 | } 22 | genCacheKey := func(params map[string]any, metaKey string) string { 23 | req := jsonrpc.NewRequest(MethodResolve, params) 24 | query, err := NewQuery(req, "") 25 | require.NoError(err) 26 | cacheReq := NewCacheRequest(query.Method(), query.Params(), metaKey) 27 | return cacheReq.GetCacheKey() 28 | } 29 | for _, p := range params { 30 | t.Run(fmt.Sprintf("%+v", p), func(t *testing.T) { 31 | cacheKey := genCacheKey(p, "") 32 | assert.Len(cacheKey, 32) 33 | assert.NotContains(seen, cacheKey) 34 | seen[cacheKey] = true 35 | }) 36 | t.Run(fmt.Sprintf("%+v", p), func(t *testing.T) { 37 | cacheKey := genCacheKey(p, "user@endpoint") 38 | assert.Len(cacheKey, 32) 39 | assert.NotContains(seen, cacheKey) 40 | seen[cacheKey] = true 41 | }) 42 | } 43 | assert.Contains(seen, genCacheKey(params[1], "user@endpoint")) 44 | } 45 | 46 | func TestCachedResponseMarshal(t *testing.T) { 47 | assert := assert.New(t) 48 | require := require.New(t) 49 | jr, err := decodeResponse(resolveResponseWithPurchase) 50 | require.NoError(err) 51 | require.NotNil(jr.Result) 52 | r := &CachedResponse{ 53 | Result: jr.Result, 54 | Error: jr.Error, 55 | } 56 | mr, err := r.MarshalBinary() 57 | require.NoError(err) 58 | require.NotEmpty(mr) 59 | assert.Less(len(mr), len(resolveResponseWithPurchase)) 60 | r2 := &CachedResponse{} 61 | err = r2.UnmarshalBinary(mr) 62 | require.NoError(err) 63 | assert.Equal(r, r2) 64 | } 65 | -------------------------------------------------------------------------------- /app/query/context.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/sirupsen/logrus" 7 | "github.com/ybbus/jsonrpc/v2" 8 | ) 9 | 10 | type contextKey string 11 | 12 | var ( 13 | contextKeyQuery = contextKey("query") 14 | contextKeyResponse = contextKey("response") 15 | contextKeyLogEntry = contextKey("log-entry") 16 | contextKeyOrigin = contextKey("origin") 17 | ) 18 | 19 | func AttachQuery(ctx context.Context, query *Query) context.Context { 20 | return context.WithValue(ctx, contextKeyQuery, query) 21 | } 22 | 23 | func QueryFromContext(ctx context.Context) *Query { 24 | return ctx.Value(contextKeyQuery).(*Query) 25 | } 26 | 27 | func AttachResponse(ctx context.Context, response *jsonrpc.RPCResponse) context.Context { 28 | return context.WithValue(ctx, contextKeyResponse, response) 29 | } 30 | 31 | func ResponseFromContext(ctx context.Context) *jsonrpc.RPCResponse { 32 | return ctx.Value(contextKeyResponse).(*jsonrpc.RPCResponse) 33 | } 34 | 35 | func AttachLogEntry(ctx context.Context, entry *logrus.Entry) context.Context { 36 | return context.WithValue(ctx, contextKeyLogEntry, entry) 37 | } 38 | 39 | // WithLogField injects additional data into default post-query log entry 40 | func WithLogField(ctx context.Context, key string, value interface{}) { 41 | e := ctx.Value(contextKeyLogEntry).(*logrus.Entry) 42 | e.Data[key] = value 43 | } 44 | 45 | func AttachOrigin(ctx context.Context, origin string) context.Context { 46 | return context.WithValue(ctx, contextKeyOrigin, origin) 47 | } 48 | 49 | func OriginFromContext(ctx context.Context) string { 50 | if origin, ok := ctx.Value(contextKeyOrigin).(string); ok { 51 | return origin 52 | } 53 | return "" 54 | } 55 | -------------------------------------------------------------------------------- /app/query/errors.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/ybbus/jsonrpc/v2" 7 | ) 8 | 9 | var userInputSDKErrors = []string{ 10 | "expected string or bytes-like object", 11 | } 12 | 13 | // isUserInputError checks if error is of the kind where retrying a request with the same input will result in the same error. 14 | func isUserInputError(resp *jsonrpc.RPCResponse) bool { 15 | for _, m := range userInputSDKErrors { 16 | if strings.Contains(resp.Error.Message, m) { 17 | return true 18 | } 19 | 20 | } 21 | return false 22 | } 23 | -------------------------------------------------------------------------------- /app/query/metrics.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/prometheus/client_golang/prometheus/promauto" 8 | ) 9 | 10 | const ( 11 | CacheOperationGet = "get" 12 | CacheOperationSet = "set" 13 | 14 | CacheResultHit = "hit" 15 | CacheResultMiss = "miss" 16 | CacheResultSuccess = "success" 17 | CacheResultError = "error" 18 | 19 | CacheAreaChainquery = "chainquery" 20 | CacheAreaInvalidateCall = "invalidate_call" 21 | 22 | CacheRetrieverErrorNet = "net" 23 | CacheRetrieverErrorSdk = "sdk" 24 | CacheRetrieverErrorInput = "input" 25 | ) 26 | 27 | var ( 28 | QueryCacheRetrievalDurationBuckets = []float64{0.025, 0.05, 0.1, 0.25, 0.4, 1, 2.5, 5, 10, 25, 50, 100, 300} 29 | cacheDurationBuckets = []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0} 30 | 31 | QueryCacheOperationDuration = promauto.NewHistogramVec( 32 | prometheus.HistogramOpts{ 33 | Namespace: "query_cache", 34 | Name: "operation_duration_seconds", 35 | Help: "Cache operation latency", 36 | Buckets: cacheDurationBuckets, 37 | }, 38 | []string{"operation", "result", "method"}, 39 | ) 40 | QueryCacheRetrievalDuration = promauto.NewHistogramVec( 41 | prometheus.HistogramOpts{ 42 | Namespace: "query_cache", 43 | Name: "retrieval_duration_seconds", 44 | Help: "Latency for cold cache retrieval", 45 | Buckets: QueryCacheRetrievalDurationBuckets, 46 | }, 47 | []string{"result", "method"}, 48 | ) 49 | QueryCacheRetrievalFailures = promauto.NewCounterVec( 50 | prometheus.CounterOpts{ 51 | Namespace: "query_cache", 52 | Name: "retrieval_retries", 53 | Help: "Retries for cold cache retrieval", 54 | }, 55 | []string{"kind", "method"}, 56 | ) 57 | QueryCacheRetrySuccesses = promauto.NewSummary( 58 | prometheus.SummaryOpts{ 59 | Namespace: "query_cache", 60 | Name: "retry_successes", 61 | Help: "Successful counts of cache retrieval retries", 62 | }, 63 | ) 64 | QueryCacheErrorCount = promauto.NewCounterVec( 65 | prometheus.CounterOpts{ 66 | Namespace: "query_cache", 67 | Name: "error_count", 68 | Help: "Errors unrelated to cache setting/retrieval", 69 | }, 70 | []string{"area"}, 71 | ) 72 | ) 73 | 74 | func ObserveQueryCacheOperation(operation, result, method string, start time.Time) { 75 | QueryCacheOperationDuration.WithLabelValues(operation, result, method).Observe(float64(time.Since(start).Seconds())) 76 | } 77 | 78 | func ObserveQueryCacheRetrievalDuration(result, method string, start time.Time) { 79 | QueryCacheRetrievalDuration.WithLabelValues(result, method).Observe(float64(time.Since(start).Seconds())) 80 | } 81 | -------------------------------------------------------------------------------- /app/query/middleware.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/gorilla/mux" 8 | ) 9 | 10 | type cacheKey struct{} 11 | 12 | func HasCache(r *http.Request) bool { 13 | return r.Context().Value(cacheKey{}) != nil 14 | } 15 | 16 | func CacheFromRequest(r *http.Request) *QueryCache { 17 | v := r.Context().Value(cacheKey{}) 18 | if v == nil { 19 | panic("query.CacheMiddleware is required") 20 | } 21 | return v.(*QueryCache) 22 | } 23 | 24 | func AddCacheToRequest(cache *QueryCache, fn http.HandlerFunc) http.HandlerFunc { 25 | return func(w http.ResponseWriter, r *http.Request) { 26 | fn(w, r.Clone(context.WithValue(r.Context(), cacheKey{}, cache))) 27 | } 28 | } 29 | 30 | func CacheMiddleware(cache *QueryCache) mux.MiddlewareFunc { 31 | return func(next http.Handler) http.Handler { 32 | return AddCacheToRequest(cache, next.ServeHTTP) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /app/query/paid.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/base64" 6 | "fmt" 7 | "path" 8 | "strings" 9 | ) 10 | 11 | func signStreamURL77(host, filePath, secureToken string, expiryTimestamp int64) (string, error) { 12 | strippedPath := path.Dir(filePath) 13 | 14 | hash := strippedPath + secureToken 15 | if expiryTimestamp > 0 { 16 | hash = fmt.Sprintf("%d%s", expiryTimestamp, hash) 17 | } 18 | 19 | finalHash := md5.Sum([]byte(hash)) 20 | encodedFinalHash := base64.StdEncoding.EncodeToString(finalHash[:]) 21 | encodedFinalHash = strings.NewReplacer("+", "-", "/", "_").Replace(encodedFinalHash) 22 | 23 | // signedURL := "https://" + fmt.Sprintf("%s/%s", host, encodedFinalHash) 24 | // if expiryTimestamp > 0 { 25 | // signedURL += fmt.Sprintf(",%d", expiryTimestamp) 26 | // } 27 | // signedURL += filePath 28 | 29 | if expiryTimestamp > 0 { 30 | return fmt.Sprintf("%s,%d", encodedFinalHash, expiryTimestamp), nil 31 | } 32 | 33 | return encodedFinalHash, nil 34 | } 35 | -------------------------------------------------------------------------------- /app/query/query_test.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/OdyseeTeam/odysee-api/internal/errors" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | "github.com/ybbus/jsonrpc/v2" 11 | ) 12 | 13 | func TestQueryParamsAsMap(t *testing.T) { 14 | q, err := NewQuery(jsonrpc.NewRequest("version"), "") 15 | require.NoError(t, err) 16 | assert.Nil(t, q.ParamsAsMap()) 17 | 18 | q, err = NewQuery(jsonrpc.NewRequest("resolve", map[string]interface{}{"urls": "what"}), "") 19 | require.NoError(t, err) 20 | assert.Equal(t, map[string]interface{}{"urls": "what"}, q.ParamsAsMap()) 21 | 22 | q, err = NewQuery(jsonrpc.NewRequest("account_balance"), "123") 23 | require.NoError(t, err, errors.Unwrap(err)) 24 | assert.Equal(t, map[string]interface{}{"wallet_id": "123"}, q.ParamsAsMap()) 25 | 26 | searchParams := map[string]interface{}{ 27 | "any_tags": []interface{}{ 28 | "art", "automotive", "blockchain", "comedy", "economics", "education", 29 | "gaming", "music", "news", "science", "sports", "technology", 30 | }, 31 | } 32 | q, err = NewQuery(jsonrpc.NewRequest("claim_search", searchParams), "") 33 | require.NoError(t, err) 34 | assert.Equal(t, searchParams, q.ParamsAsMap()) 35 | 36 | q, err = NewQuery(jsonrpc.NewRequest("account_balance"), "123") 37 | require.NoError(t, err, errors.Unwrap(err)) 38 | params := q.ParamsAsMap() 39 | params["new_param"] = "new_param_value" 40 | assert.Equal(t, params, q.ParamsAsMap()) 41 | } 42 | 43 | func TestQueryCopyParamsAsMap(t *testing.T) { 44 | q, err := NewQuery(jsonrpc.NewRequest("account_balance"), "123") 45 | require.NoError(t, err, errors.Unwrap(err)) 46 | params := q.CopyParamsAsMap() 47 | params["new_param"] = "new_param_value" 48 | assert.NotEqual(t, params, q.ParamsAsMap()) 49 | } 50 | 51 | func TestQueryIsAuthenticated(t *testing.T) { 52 | q, err := NewQuery(jsonrpc.NewRequest("resolve"), "12345") 53 | require.NoError(t, err) 54 | assert.True(t, q.IsAuthenticated()) 55 | 56 | q, err = NewQuery(jsonrpc.NewRequest("resolve"), "") 57 | require.NoError(t, err) 58 | assert.False(t, q.IsAuthenticated()) 59 | } 60 | 61 | func TestMethodRequiresWallet(t *testing.T) { 62 | for _, m := range walletSpecificMethods { 63 | if methodInList(m, relaxedMethods) { 64 | assert.False(t, MethodRequiresWallet(m, nil), m) 65 | } else { 66 | assert.True(t, MethodRequiresWallet(m, nil), m) 67 | } 68 | } 69 | } 70 | 71 | func TestMethodAcceptsWallet(t *testing.T) { 72 | for _, m := range walletSpecificMethods { 73 | assert.True(t, MethodAcceptsWallet(m), m) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /app/query/testing.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | 7 | "github.com/ybbus/jsonrpc/v2" 8 | ) 9 | 10 | func decodeResponse(r string) (*jsonrpc.RPCResponse, error) { 11 | decoder := json.NewDecoder(strings.NewReader(r)) 12 | decoder.DisallowUnknownFields() 13 | decoder.UseNumber() 14 | response := &jsonrpc.RPCResponse{} 15 | return response, decoder.Decode(response) 16 | } 17 | -------------------------------------------------------------------------------- /app/query/validation.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | // this should really be in lbry.go 4 | import ( 5 | "crypto/ecdsa" 6 | "crypto/sha256" 7 | "encoding/hex" 8 | "math/big" 9 | 10 | "github.com/OdyseeTeam/odysee-api/internal/errors" 11 | 12 | ljsonrpc "github.com/lbryio/lbry.go/v2/extras/jsonrpc" 13 | "github.com/lbryio/lbry.go/v2/schema/keys" 14 | 15 | "github.com/btcsuite/btcd/btcec" 16 | ) 17 | 18 | func ValidateSignatureFromClaim(channel *ljsonrpc.Claim, signature, signingTS, data string) error { 19 | if channel == nil { 20 | return errors.Err("no channel to validate") 21 | } 22 | if channel.SigningChannel.Value.GetChannel() == nil { 23 | return errors.Err("no channel for public key") 24 | } 25 | pubKey, err := keys.GetPublicKeyFromBytes(channel.SigningChannel.Value.GetChannel().GetPublicKey()) 26 | if err != nil { 27 | return errors.Err(err) 28 | } 29 | 30 | return validateSignature(channel.SigningChannel.ClaimID, signature, signingTS, data, pubKey) 31 | 32 | } 33 | 34 | func validateSignature(channelClaimID, signature, signingTS, data string, publicKey *btcec.PublicKey) error { 35 | injest := sha256.Sum256( 36 | createDigest( 37 | []byte(signingTS), 38 | unhelixifyAndReverse(channelClaimID), 39 | []byte(data), 40 | )) 41 | sig, err := hex.DecodeString(signature) 42 | if err != nil { 43 | return errors.Err(err) 44 | } 45 | signatureBytes := [64]byte{} 46 | copy(signatureBytes[:], sig) 47 | sigValid := isSignatureValid(signatureBytes, publicKey, injest[:]) 48 | if !sigValid { 49 | return errors.Err("could not validate the signature") 50 | } 51 | return nil 52 | } 53 | 54 | func isSignatureValid(signature [64]byte, publicKey *btcec.PublicKey, injest []byte) bool { 55 | R := &big.Int{} 56 | S := &big.Int{} 57 | R.SetBytes(signature[:32]) 58 | S.SetBytes(signature[32:]) 59 | return ecdsa.Verify(publicKey.ToECDSA(), injest, R, S) 60 | } 61 | 62 | func createDigest(pieces ...[]byte) []byte { 63 | var digest []byte 64 | for _, p := range pieces { 65 | digest = append(digest, p...) 66 | } 67 | return digest 68 | } 69 | 70 | // reverseBytes reverses a byte slice. useful for switching endian-ness 71 | func reverseBytes(b []byte) []byte { 72 | r := make([]byte, len(b)) 73 | for left, right := 0, len(b)-1; left < right; left, right = left+1, right-1 { 74 | r[left], r[right] = b[right], b[left] 75 | } 76 | return r 77 | } 78 | 79 | func unhelixifyAndReverse(claimID string) []byte { 80 | b, err := hex.DecodeString(claimID) 81 | if err != nil { 82 | return nil 83 | } 84 | return reverseBytes(b) 85 | } 86 | -------------------------------------------------------------------------------- /app/sdkrouter/concurrency_test.go: -------------------------------------------------------------------------------- 1 | package sdkrouter 2 | 3 | import ( 4 | "math/rand" 5 | "sync" 6 | "testing" 7 | "time" 8 | 9 | "github.com/OdyseeTeam/odysee-api/internal/test" 10 | "github.com/OdyseeTeam/odysee-api/models" 11 | 12 | "github.com/sirupsen/logrus" 13 | "github.com/volatiletech/sqlboiler/boil" 14 | ) 15 | 16 | func TestRouterConcurrency(t *testing.T) { 17 | rpcServer := test.MockHTTPServer(nil) 18 | defer rpcServer.Close() 19 | go func() { 20 | for { 21 | rpcServer.NextResponse <- `{"result": {"items": [], "page": 1, "page_size": 1, "total_pages": 10}}` // mock WalletList response 22 | } 23 | }() 24 | 25 | r := New(map[string]string{"srv": rpcServer.URL}) 26 | servers := r.servers 27 | servers2 := []*models.LbrynetServer{ 28 | {Name: "one", Address: rpcServer.URL}, 29 | {Name: "two", Address: rpcServer.URL}, 30 | } 31 | wg := sync.WaitGroup{} 32 | 33 | db := boil.GetDB() 34 | boil.SetDB(nil) // so we don't get errors about too many Postgres connections 35 | defer func() { boil.SetDB(db) }() 36 | 37 | logLvl := logrus.GetLevel() 38 | logrus.SetLevel(logrus.InfoLevel) // silence the jsonrpc debug messages 39 | defer func() { logrus.SetLevel(logLvl) }() 40 | 41 | for i := 0; i < 1000; i++ { 42 | wg.Add(1) 43 | go func() { 44 | time.Sleep(time.Duration(rand.Intn(150)) * time.Millisecond) 45 | //t.Log("reads") 46 | switch rand.Intn(3) { // do reads in different orders 47 | case 0: 48 | r.RandomServer() 49 | r.GetAll() 50 | r.LeastLoaded() 51 | case 1: 52 | r.GetAll() 53 | r.LeastLoaded() 54 | r.RandomServer() 55 | case 2: 56 | r.LeastLoaded() 57 | r.RandomServer() 58 | r.GetAll() 59 | } 60 | wg.Done() 61 | //t.Log("reads done") 62 | }() 63 | } 64 | for i := 0; i < 500; i++ { 65 | wg.Add(1) 66 | go func(i int) { 67 | time.Sleep(time.Duration(rand.Intn(75)) * time.Millisecond) 68 | //t.Log("WRITE WRITE WRITE") 69 | if i%2 == 0 { 70 | r.setServers(servers) 71 | } else { 72 | r.setServers(servers2) 73 | } 74 | wg.Done() 75 | }(i) 76 | } 77 | for i := 0; i < 500; i++ { 78 | wg.Add(1) 79 | go func() { 80 | time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond) 81 | //t.Log("update metrics") 82 | r.updateLoadAndMetrics() 83 | wg.Done() 84 | }() 85 | } 86 | 87 | wg.Wait() 88 | } 89 | -------------------------------------------------------------------------------- /app/sdkrouter/middleware.go: -------------------------------------------------------------------------------- 1 | package sdkrouter 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/gorilla/mux" 8 | ) 9 | 10 | type ctxKey int 11 | 12 | const contextKey ctxKey = iota 13 | 14 | func FromRequest(r *http.Request) *Router { 15 | v := r.Context().Value(contextKey) 16 | if v == nil { 17 | panic("sdkrouter.Middleware is required") 18 | } 19 | return v.(*Router) 20 | } 21 | 22 | func AddToRequest(rt *Router, fn http.HandlerFunc) http.HandlerFunc { 23 | return func(w http.ResponseWriter, r *http.Request) { 24 | fn(w, r.Clone(context.WithValue(r.Context(), contextKey, rt))) 25 | } 26 | } 27 | 28 | func Middleware(rt *Router) mux.MiddlewareFunc { 29 | return func(next http.Handler) http.Handler { 30 | return AddToRequest(rt, next.ServeHTTP) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /app/wallet/cache.go: -------------------------------------------------------------------------------- 1 | package wallet 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | 7 | "github.com/OdyseeTeam/odysee-api/internal/metrics" 8 | "github.com/OdyseeTeam/odysee-api/internal/monitor" 9 | "github.com/OdyseeTeam/odysee-api/models" 10 | 11 | "github.com/dgraph-io/ristretto" 12 | "golang.org/x/sync/singleflight" 13 | ) 14 | 15 | const ( 16 | ttlUnconfirmed = 15 * time.Second 17 | ttlConfirmed = 15 * time.Minute 18 | ) 19 | 20 | var ( 21 | cacheLogger = monitor.NewModuleLogger("cache") 22 | currentCache *tokenCache 23 | ) 24 | 25 | // tokenCache stores the cache in memory 26 | type tokenCache struct { 27 | cache *ristretto.Cache 28 | sf *singleflight.Group 29 | } 30 | 31 | func init() { 32 | SetTokenCache(NewTokenCache()) 33 | } 34 | 35 | func NewTokenCache() *tokenCache { 36 | rc, _ := ristretto.NewCache(&ristretto.Config{ 37 | MaxCost: 1 << 30, 38 | Metrics: true, 39 | NumCounters: 1e7, 40 | BufferItems: 64, 41 | }) 42 | return &tokenCache{ 43 | cache: rc, 44 | sf: &singleflight.Group{}, 45 | } 46 | } 47 | 48 | func SetTokenCache(c *tokenCache) { 49 | currentCache = c 50 | } 51 | 52 | func (c *tokenCache) get(token string, retriever func() (interface{}, error)) (*models.User, error) { 53 | var err error 54 | cachedUser, ok := c.cache.Get(token) 55 | if !ok { 56 | metrics.AuthTokenCacheMisses.Inc() 57 | cachedUser, err, _ = c.sf.Do(token, retriever) 58 | if err != nil { 59 | return nil, err 60 | } 61 | var baseTTL time.Duration 62 | if cachedUser == nil { 63 | baseTTL = ttlUnconfirmed 64 | } else { 65 | baseTTL = ttlConfirmed 66 | } 67 | c.cache.SetWithTTL(token, cachedUser, 1, baseTTL+time.Duration(rand.Int63n(baseTTL.Nanoseconds()))) 68 | } else { 69 | metrics.AuthTokenCacheHits.Inc() 70 | } 71 | 72 | if cachedUser == nil { 73 | return nil, nil 74 | } 75 | user := cachedUser.(*models.User) 76 | return user, nil 77 | } 78 | 79 | func (c *tokenCache) flush() { 80 | c.cache.Clear() 81 | } 82 | -------------------------------------------------------------------------------- /app/wallet/cache_test.go: -------------------------------------------------------------------------------- 1 | package wallet 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/OdyseeTeam/odysee-api/app/sdkrouter" 8 | "github.com/OdyseeTeam/odysee-api/internal/metrics" 9 | "github.com/OdyseeTeam/odysee-api/internal/test" 10 | "github.com/OdyseeTeam/odysee-api/models" 11 | 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | "github.com/volatiletech/sqlboiler/boil" 15 | ) 16 | 17 | func TestCache(t *testing.T) { 18 | setupTest() 19 | user, err := getOrCreateLocalUser(boil.GetDB(), models.User{ID: dummyUserID}, logger.Log()) 20 | require.NoError(t, err) 21 | srv := test.RandServerAddress(t) 22 | rt := sdkrouter.New(map[string]string{"a": srv}) 23 | err = assignSDKServerToUser(boil.GetDB(), user, rt.LeastLoaded(), logger.Log()) 24 | require.NoError(t, err) 25 | 26 | cases := []struct { 27 | name, token string 28 | retrievedUser *models.User 29 | err error 30 | hitsInc int 31 | }{ 32 | {"UserNotFound", "nonexistingtoken", nil, errors.New("not found"), 0}, 33 | {"ConfirmedUserFound", "confirmedtoken", user, nil, 1}, 34 | {"UnconfirmedUserFound", "unconfirmedtoken", nil, nil, 1}, 35 | } 36 | 37 | for _, c := range cases { 38 | t.Run(c.name, func(t *testing.T) { 39 | hits := metrics.GetCounterValue(metrics.AuthTokenCacheHits) 40 | cachedUser, err := currentCache.get(c.token, func() (interface{}, error) { 41 | return c.retrievedUser, c.err 42 | }) 43 | assert.Equal(t, c.retrievedUser, cachedUser) 44 | assert.Equal(t, c.err, err) 45 | currentCache.cache.Wait() 46 | cachedUser, err = currentCache.get(c.token, func() (interface{}, error) { 47 | return c.retrievedUser, c.err 48 | }) 49 | assert.Equal(t, hits+float64(c.hitsInc), metrics.GetCounterValue(metrics.AuthTokenCacheHits)) 50 | assert.Equal(t, c.retrievedUser, cachedUser) 51 | assert.Equal(t, c.err, err) 52 | }) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /app/wallet/oauth_test.go: -------------------------------------------------------------------------------- 1 | package wallet 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/OdyseeTeam/odysee-api/app/sdkrouter" 8 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 9 | "github.com/OdyseeTeam/odysee-api/internal/test" 10 | "github.com/OdyseeTeam/odysee-api/models" 11 | 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func TestOauthAuthenticatorAuthenticate(t *testing.T) { 17 | setupTest() 18 | srv := test.RandServerAddress(t) 19 | rt := sdkrouter.New(map[string]string{"a": srv}) 20 | _, cleanup := dummyAPI(srv) 21 | defer cleanup() 22 | 23 | auther, err := NewOauthAuthenticator(config.GetOauthProviderURL(), config.GetOauthClientID(), config.GetInternalAPIHost(), rt) 24 | require.NoError(t, err, errors.Unwrap(err)) 25 | 26 | token, err := test.GetTestToken() 27 | require.NoError(t, err, errors.Unwrap(err)) 28 | 29 | u, err := auther.Authenticate("Bearer "+token.AccessToken, "") 30 | require.NoError(t, err, errors.Unwrap(err)) 31 | 32 | count, err := models.Users(models.UserWhere.ID.EQ(u.ID)).CountG() 33 | require.NoError(t, err) 34 | assert.EqualValues(t, 1, count) 35 | assert.True(t, u.LbrynetServerID.IsZero()) // because the server came from a config, it should not have an id set 36 | 37 | // now assign the user a new server thats set in the db 38 | sdk := &models.LbrynetServer{ 39 | Name: "testing", 40 | Address: "test.test.test.test", 41 | } 42 | err = u.SetLbrynetServerG(true, sdk) 43 | require.NoError(t, err) 44 | require.NotEqual(t, 0, sdk.ID) 45 | require.Equal(t, u.LbrynetServerID.Int, sdk.ID) 46 | 47 | // now fetch it all back from the db 48 | u2, err := auther.Authenticate("Bearer "+token.AccessToken, "") 49 | require.NoError(t, err, errors.Unwrap(err)) 50 | require.NotNil(t, u2) 51 | 52 | sdk2, err := u.LbrynetServer().OneG() 53 | require.NoError(t, err) 54 | require.Equal(t, sdk.ID, sdk2.ID) 55 | require.Equal(t, sdk.Address, sdk2.Address) 56 | require.Equal(t, u.LbrynetServerID.Int, sdk2.ID) 57 | } 58 | -------------------------------------------------------------------------------- /app/wallet/remote.go: -------------------------------------------------------------------------------- 1 | package wallet 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "strings" 7 | "time" 8 | 9 | "github.com/OdyseeTeam/odysee-api/internal/metrics" 10 | "github.com/OdyseeTeam/odysee-api/pkg/iapi" 11 | 12 | "golang.org/x/oauth2" 13 | ) 14 | 15 | // remoteUser encapsulates internal-apis user data 16 | type remoteUser struct { 17 | ID int `json:"user_id"` 18 | HasVerifiedEmail bool `json:"has_verified_email"` 19 | Cached bool 20 | } 21 | 22 | func GetRemoteUserLegacy(server, token string, remoteIP string) (remoteUser, error) { 23 | ruser := remoteUser{} 24 | op := metrics.StartOperation(opName, "get_remote_user") 25 | defer op.End() 26 | 27 | iac, err := iapi.NewClient( 28 | iapi.WithLegacyToken(token), 29 | iapi.WithRemoteIP(remoteIP), 30 | iapi.WithServer(server), 31 | ) 32 | if err != nil { 33 | return ruser, err 34 | } 35 | 36 | start := time.Now() 37 | resp := &iapi.UserHasVerifiedEmailResponse{} 38 | err = iac.Call(context.Background(), "user/has_verified_email", map[string]string{}, resp) 39 | duration := time.Since(start).Seconds() 40 | 41 | if err != nil { 42 | if errors.Is(err, iapi.APIError) { 43 | metrics.IAPIAuthFailedDurations.Observe(duration) 44 | } else { 45 | metrics.IAPIAuthErrorDurations.Observe(duration) 46 | } 47 | return remoteUser{}, err 48 | } 49 | 50 | metrics.IAPIAuthSuccessDurations.Observe(duration) 51 | 52 | ruser.ID = resp.Data.UserID 53 | ruser.HasVerifiedEmail = resp.Data.HasVerifiedEmail 54 | 55 | return ruser, nil 56 | } 57 | 58 | func GetRemoteUser(token oauth2.TokenSource, remoteIP string) (remoteUser, error) { 59 | ruser := remoteUser{} 60 | op := metrics.StartOperation(opName, "get_remote_user") 61 | defer op.End() 62 | 63 | t, err := token.Token() 64 | if err != nil { 65 | return ruser, err 66 | } 67 | if t.Type() != "Bearer" { 68 | return ruser, errors.New("internal-apis requires an oAuth token of type 'Bearer'") 69 | } 70 | 71 | iac, err := iapi.NewClient( 72 | iapi.WithOAuthToken(strings.TrimPrefix(t.AccessToken, TokenPrefix)), 73 | iapi.WithRemoteIP(remoteIP), 74 | ) 75 | if err != nil { 76 | return ruser, err 77 | } 78 | 79 | start := time.Now() 80 | resp := &iapi.UserHasVerifiedEmailResponse{} 81 | err = iac.Call(context.Background(), "user/has_verified_email", map[string]string{}, resp) 82 | duration := time.Since(start).Seconds() 83 | 84 | if err != nil { 85 | if errors.Is(err, iapi.APIError) { 86 | metrics.IAPIAuthFailedDurations.Observe(duration) 87 | } else { 88 | metrics.IAPIAuthErrorDurations.Observe(duration) 89 | } 90 | return remoteUser{}, err 91 | } 92 | 93 | metrics.IAPIAuthSuccessDurations.Observe(duration) 94 | 95 | ruser.ID = resp.Data.UserID 96 | ruser.HasVerifiedEmail = resp.Data.HasVerifiedEmail 97 | 98 | return ruser, nil 99 | } 100 | -------------------------------------------------------------------------------- /app/wallet/testing.go: -------------------------------------------------------------------------------- 1 | package wallet 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "net/http" 7 | 8 | "github.com/OdyseeTeam/odysee-api/app/sdkrouter" 9 | "github.com/OdyseeTeam/odysee-api/models" 10 | "github.com/sirupsen/logrus" 11 | 12 | "github.com/volatiletech/null" 13 | "github.com/volatiletech/sqlboiler/boil" 14 | ) 15 | 16 | type Authenticator interface { 17 | Authenticate(token, metaRemoteIP string) (*models.User, error) 18 | GetTokenFromRequest(r *http.Request) (string, error) 19 | } 20 | 21 | // TestAnyAuthenticator will authenticate any token and return a dummy user. 22 | type TestAnyAuthenticator struct{} 23 | 24 | func (a *TestAnyAuthenticator) Authenticate(token, ip string) (*models.User, error) { 25 | return &models.User{ID: 994, IdpID: null.StringFrom("my-idp-id")}, nil 26 | } 27 | 28 | func (a *TestAnyAuthenticator) GetTokenFromRequest(r *http.Request) (string, error) { 29 | return "", nil 30 | } 31 | 32 | // TestMissingTokenAuthenticator will throw a missing token error. 33 | type TestMissingTokenAuthenticator struct{} 34 | 35 | func (a *TestMissingTokenAuthenticator) Authenticate(token, ip string) (*models.User, error) { 36 | return nil, nil 37 | } 38 | 39 | func (a *TestMissingTokenAuthenticator) GetTokenFromRequest(r *http.Request) (string, error) { 40 | return "", ErrNoAuthInfo 41 | } 42 | 43 | // PostProcessAuthenticator allows to manipulate or additionally validate authenticated user before returning it. 44 | type PostProcessAuthenticator struct { 45 | auther Authenticator 46 | postFunc func(*models.User) (*models.User, error) 47 | } 48 | 49 | func NewPostProcessAuthenticator(auther Authenticator, postFunc func(*models.User) (*models.User, error)) *PostProcessAuthenticator { 50 | return &PostProcessAuthenticator{auther, postFunc} 51 | } 52 | 53 | func CreateTestUser(rt *sdkrouter.Router, exec boil.ContextBeginner, id int) (*models.User, error) { 54 | var localUser *models.User 55 | 56 | err := inTx(context.Background(), exec, func(tx *sql.Tx) error { 57 | l := logrus.WithFields(logrus.Fields{}) 58 | localUser, err := getOrCreateLocalUser(tx, models.User{ID: id}, l) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | if localUser.LbrynetServerID.IsZero() { 64 | err := assignSDKServerToUser(tx, localUser, rt.LeastLoaded(), l) 65 | if err != nil { 66 | return err 67 | } 68 | } 69 | return nil 70 | }) 71 | if err != nil { 72 | return nil, err 73 | } 74 | 75 | return localUser, err 76 | } 77 | 78 | func (a *PostProcessAuthenticator) Authenticate(token, ip string) (*models.User, error) { 79 | user, err := a.auther.Authenticate(token, ip) 80 | if err != nil { 81 | return nil, err 82 | } 83 | return a.postFunc(user) 84 | } 85 | 86 | func (a *PostProcessAuthenticator) GetTokenFromRequest(r *http.Request) (string, error) { 87 | return a.auther.GetTokenFromRequest(r) 88 | } 89 | -------------------------------------------------------------------------------- /apps/forklift/config/forklift.yml: -------------------------------------------------------------------------------- 1 | Database: 2 | DSN: postgres://postgres:odyseeteam@localhost/uploads?sslmode=disable 3 | DBName: uploads 4 | AutoMigrations: true 5 | 6 | # ForkliftRequestsConnURL is Redis database where forklift will be listening for complete uploads requests. 7 | # It corresponds to ForkliftRequestsConnURL in uploads.yml config. 8 | ForkliftRequestsConnURL: redis://:odyredis@host.docker.internal:6379/4 9 | # AsynqueryRequestsConnURL is Redis database where asynquery will be listening for finalized uploads requests. 10 | AsynqueryRequestsConnURL: redis://:odyredis@host.docker.internal:6379/3 11 | 12 | IncomingStorage: 13 | Endpoint: http://localhost:9002 14 | Region: us-east-1 15 | Bucket: uploads 16 | Key: minio 17 | Secret: minio123 18 | Flavor: minio 19 | 20 | ReflectorStorage: 21 | DatabaseDSN: 'user:password@tcp(host.com)/blobs' 22 | Destinations: 23 | - Name: wasabi 24 | Endpoint: https://s3.wasabisys.com 25 | Region: us-east-1 26 | Bucket: blobs 27 | AWS_ID: key1 28 | AWS_Secret: secret1 29 | - Name: another 30 | Endpoint: https://s3.wasabisys.com 31 | Region: us-east-2 32 | Bucket: blobs 33 | AWS_ID: key2 34 | AWS_Secret: secret2 35 | Concurrency: 10 36 | BlobPath: /tmp/blobs 37 | UploadPath: /tmp/uploads 38 | 39 | ReflectorWorkers: 5 40 | -------------------------------------------------------------------------------- /apps/forklift/metrics.go: -------------------------------------------------------------------------------- 1 | package forklift 2 | 3 | import ( 4 | "net/http" 5 | "sync" 6 | "time" 7 | 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/prometheus/client_golang/prometheus/promhttp" 10 | ) 11 | 12 | const ns = "forklift" 13 | const LabelFatal = "fatal" 14 | const LabelCommon = "common" 15 | const LabelRetrieve = "retrieve" 16 | const LabelAnalyze = "analyze" 17 | const LabelStreamCreate = "stream_create" 18 | const LabelUpstream = "upstream" 19 | 20 | var onceMetrics sync.Once 21 | 22 | var ( 23 | waitTimeMinutes = prometheus.NewHistogram(prometheus.HistogramOpts{ 24 | Namespace: ns, 25 | Name: "wait_time_minutes", 26 | Buckets: []float64{1, 5, 10, 15, 20, 30, 45, 60, 120}, 27 | }) 28 | processingDurationSeconds = prometheus.NewCounterVec(prometheus.CounterOpts{ 29 | Namespace: ns, 30 | Name: "processing_duration_seconds", 31 | }, []string{"stage"}) 32 | processingErrors = prometheus.NewCounterVec(prometheus.CounterOpts{ 33 | Namespace: ns, 34 | Name: "processing_errors", 35 | }, []string{"stage"}) 36 | 37 | egressVolumeMB = prometheus.NewCounter(prometheus.CounterOpts{ 38 | Namespace: ns, 39 | Name: "egress_volume_mb", 40 | }) 41 | egressDurationSeconds = prometheus.NewCounter(prometheus.CounterOpts{ 42 | Namespace: ns, 43 | Name: "egress_duration_seconds", 44 | }) 45 | ) 46 | 47 | func registerMetrics(registry prometheus.Registerer) { 48 | if registry == nil { 49 | registry = prometheus.DefaultRegisterer 50 | } 51 | registry.MustRegister( 52 | waitTimeMinutes, processingDurationSeconds, processingErrors, egressVolumeMB, egressDurationSeconds, 53 | ) 54 | } 55 | 56 | func BuildMetricsHandler() http.Handler { 57 | registry := prometheus.NewRegistry() 58 | onceMetrics.Do(func() { 59 | registerMetrics(registry) 60 | }) 61 | return promhttp.InstrumentMetricHandler( 62 | registry, promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), 63 | ) 64 | } 65 | 66 | func observeDuration(stage string, start time.Time) { 67 | processingDurationSeconds.WithLabelValues(stage).Add(float64(time.Since(start))) 68 | } 69 | 70 | func observeError(stage string) { 71 | processingErrors.WithLabelValues(stage).Inc() 72 | } 73 | -------------------------------------------------------------------------------- /apps/forklift/testing.go: -------------------------------------------------------------------------------- 1 | package forklift 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "errors" 7 | "os" 8 | "testing" 9 | 10 | "github.com/spf13/viper" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | const EnvTestReflectorConfig = "REFLECTOR_CONFIG" 15 | 16 | var ErrMissingEnv = errors.New("REFLECTOR_CONFIG env var is not set") 17 | 18 | type TestHelper struct { 19 | ReflectorConfig *viper.Viper 20 | } 21 | 22 | func NewTestHelper(t *testing.T) (*TestHelper, error) { 23 | th := &TestHelper{} 24 | os.Setenv("PATH", os.Getenv("PATH")+":/opt/homebrew/bin") 25 | envCfg := os.Getenv(EnvTestReflectorConfig) 26 | 27 | if envCfg == "" { 28 | return nil, ErrMissingEnv 29 | } 30 | 31 | th.ReflectorConfig = DecodeSecretViperConfig(t, EnvTestReflectorConfig) 32 | return th, nil 33 | } 34 | 35 | func DecodeSecretViperConfig(t *testing.T, secretEnvName string) *viper.Viper { 36 | require := require.New(t) 37 | secretValueEncoded := os.Getenv(secretEnvName) 38 | require.NotEmpty(secretValueEncoded) 39 | secretValue, err := base64.StdEncoding.DecodeString(secretValueEncoded) 40 | require.NoError(err) 41 | v := viper.New() 42 | v.SetConfigType("yaml") 43 | err = v.ReadConfig(bytes.NewBuffer(secretValue)) 44 | require.NoError(err) 45 | require.NotNil(v) 46 | return v 47 | } 48 | -------------------------------------------------------------------------------- /apps/lbrytv/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestGetLbrynetServers(t *testing.T) { 11 | Config.Override("LbrynetServers", map[string]string{ 12 | "sdk1": "http://lbrynet1:5279/", 13 | "sdk2": "http://lbrynet2:5279/", 14 | "sdk3": "http://lbrynet3:5279/", 15 | }) 16 | defer Config.RestoreOverridden() 17 | assert.Equal(t, map[string]string{ 18 | "sdk1": "http://lbrynet1:5279/", 19 | "sdk2": "http://lbrynet2:5279/", 20 | "sdk3": "http://lbrynet3:5279/", 21 | }, GetLbrynetServers()) 22 | } 23 | 24 | func TestGetLbrynetServersNoDB(t *testing.T) { 25 | if Config.Viper.GetString(deprecatedLbrynetSetting) != "" && 26 | len(Config.Viper.GetStringMapString(lbrynetServers)) > 0 { 27 | t.Fatalf("Both %s and %s are set. This is a highlander situation...there can be only one.", deprecatedLbrynetSetting, lbrynetServers) 28 | } 29 | } 30 | 31 | func TestGetTokenCacheTimeout(t *testing.T) { 32 | Config.Override("TokenCacheTimeout", "325s") 33 | defer Config.RestoreOverridden() 34 | assert.Equal(t, 325*time.Second, GetTokenCacheTimeout()) 35 | } 36 | 37 | func TestGetRPCTimeout(t *testing.T) { 38 | Config.Override("RPCTimeouts", map[string]string{ 39 | "txo_list": "12s", 40 | "resolve": "200ms", 41 | }) 42 | defer Config.RestoreOverridden() 43 | 44 | assert.Equal(t, 12*time.Second, *GetRPCTimeout("txo_list")) 45 | assert.Equal(t, 200*time.Millisecond, *GetRPCTimeout("resolve")) 46 | assert.Nil(t, GetRPCTimeout("random_method")) 47 | } 48 | -------------------------------------------------------------------------------- /apps/uploads/config/uploads.yml: -------------------------------------------------------------------------------- 1 | Database: 2 | DSN: postgres://postgres:odyseeteam@host.docker.internal/uploads?sslmode=disable 3 | DBName: uploads 4 | AutoMigrations: true 5 | 6 | RedisLocker: redis://:odyredis@host.docker.internal:6379/1 7 | 8 | # ForkliftRequestsConnURL is Redis database where forklift will be listening for complete uploads requests. 9 | ForkliftRequestsConnURL: redis://:odyredis@host.docker.internal:6379/4 10 | 11 | PublicKeyURL: https://api.na-backend.dev.odysee.com/api/v1/asynqueries/auth/pubkey 12 | 13 | Storage: 14 | Endpoint: http://host.docker.internal:9002 15 | Region: us-east-1 16 | Bucket: uploads 17 | Key: minio 18 | Secret: minio123 19 | Flavor: minio 20 | 21 | CORSDomains: 22 | - http://* 23 | - https://* 24 | 25 | GracefulShutdown: 3s 26 | -------------------------------------------------------------------------------- /apps/uploads/database/db.go: -------------------------------------------------------------------------------- 1 | // Code generated by sqlc. DO NOT EDIT. 2 | // versions: 3 | // sqlc v1.19.1 4 | 5 | package database 6 | 7 | import ( 8 | "context" 9 | "database/sql" 10 | ) 11 | 12 | type DBTX interface { 13 | ExecContext(context.Context, string, ...interface{}) (sql.Result, error) 14 | PrepareContext(context.Context, string) (*sql.Stmt, error) 15 | QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) 16 | QueryRowContext(context.Context, string, ...interface{}) *sql.Row 17 | } 18 | 19 | func New(db DBTX) *Queries { 20 | return &Queries{db: db} 21 | } 22 | 23 | type Queries struct { 24 | db DBTX 25 | } 26 | 27 | func (q *Queries) WithTx(tx *sql.Tx) *Queries { 28 | return &Queries{ 29 | db: tx, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /apps/uploads/database/migrations.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import "embed" 4 | 5 | //go:embed migrations/*.sql 6 | var MigrationsFS embed.FS 7 | -------------------------------------------------------------------------------- /apps/uploads/database/migrations/0001_init.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | -- +migrate StatementBegin 3 | CREATE TYPE upload_status AS ENUM ( 4 | 'created', 5 | 'receiving', 6 | 'completed', 7 | 'terminated', 8 | 'processed' 9 | ); 10 | 11 | CREATE TABLE uploads ( 12 | id text NOT NULL UNIQUE PRIMARY KEY CHECK (id <> ''), 13 | user_id int NOT NULL, 14 | filename text NOT NULL, 15 | key text NOT NULL, 16 | 17 | created_at timestamp NOT NULL DEFAULT NOW(), 18 | updated_at timestamp, 19 | 20 | status upload_status NOT NULL, 21 | 22 | size bigint NOT NULL CHECK (size > 0), 23 | received bigint NOT NULL DEFAULT 0, 24 | 25 | sd_hash text NOT NULL, 26 | meta jsonb 27 | ); 28 | 29 | CREATE INDEX uploads_id_user_id ON uploads(id, user_id); 30 | CREATE INDEX uploads_id_user_id_status ON uploads(id, user_id, status); 31 | -- +migrate StatementEnd 32 | 33 | -- +migrate Down 34 | -- +migrate StatementBegin 35 | DROP TABLE uploads; 36 | DROP TYPE upload_status; 37 | -- +migrate StatementEnd 38 | -------------------------------------------------------------------------------- /apps/uploads/database/migrations/0002_urls.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | -- +migrate StatementBegin 3 | CREATE TYPE url_status AS ENUM ( 4 | 'created', 5 | 'downloaded', 6 | 'processed' 7 | ); 8 | 9 | CREATE TABLE urls ( 10 | id text NOT NULL UNIQUE PRIMARY KEY CHECK (id <> ''), 11 | user_id int NOT NULL, 12 | url text NOT NULL CHECK (url <> ''), 13 | filename text NOT NULL CHECK (filename <> ''), 14 | 15 | created_at timestamp NOT NULL DEFAULT NOW(), 16 | updated_at timestamp, 17 | 18 | status url_status NOT NULL, 19 | 20 | size bigint NOT NULL, 21 | sd_hash text NOT NULL, 22 | meta jsonb 23 | ); 24 | 25 | -- CREATE INDEX uploads_id_user_id ON uploads(id, user_id); 26 | -- CREATE INDEX uploads_id_user_id_status ON uploads(id, user_id, status); 27 | -- +migrate StatementEnd 28 | 29 | -- +migrate Down 30 | -- +migrate StatementBegin 31 | DROP TABLE urls; 32 | DROP TYPE url_status; 33 | -- +migrate StatementEnd 34 | -------------------------------------------------------------------------------- /apps/uploads/database/queries.sql: -------------------------------------------------------------------------------- 1 | -- name: CreateUpload :one 2 | INSERT INTO uploads ( 3 | id, user_id, size, status, filename, key, sd_hash 4 | ) VALUES ( 5 | $1, $2, $3, 'created', '', '', '' 6 | ) 7 | RETURNING *; 8 | 9 | -- name: GetUpload :one 10 | SELECT * FROM uploads 11 | WHERE user_id = $1 AND id = $2; 12 | 13 | -- name: RecordUploadProgress :exec 14 | UPDATE uploads SET 15 | updated_at = NOW(), 16 | received = $3 17 | WHERE user_id = $1 AND id = $2 AND status IN ('receiving', 'created'); 18 | 19 | -- name: MarkUploadTerminated :exec 20 | UPDATE uploads SET 21 | updated_at = NOW(), 22 | status = 'completed' 23 | WHERE user_id = $1 AND id = $2; 24 | 25 | -- name: MarkUploadCompleted :exec 26 | UPDATE uploads SET 27 | updated_at = NOW(), 28 | status = 'completed', 29 | filename = $3, 30 | key = $4 31 | WHERE user_id = $1 AND id = $2; 32 | 33 | -- name: MarkUploadProcessed :exec 34 | UPDATE uploads SET 35 | updated_at = NOW(), 36 | status = 'processed', 37 | sd_hash = $2, 38 | meta = $3 39 | WHERE id = $1; 40 | 41 | -- name: CreateURL :one 42 | INSERT INTO urls ( 43 | id, user_id, url, filename, size, sd_hash, status 44 | ) VALUES ( 45 | $1, $2, $3, $4, 0, '', 'created' 46 | ) 47 | RETURNING *; 48 | -------------------------------------------------------------------------------- /apps/uploads/json_logger.go: -------------------------------------------------------------------------------- 1 | package uploads 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/OdyseeTeam/odysee-api/pkg/logging" 8 | 9 | "github.com/go-chi/chi/v5/middleware" 10 | ) 11 | 12 | type JSONLogger struct { 13 | logger logging.KVLogger 14 | } 15 | 16 | func (jl *JSONLogger) Middleware(next http.Handler) http.Handler { 17 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 18 | start := time.Now() 19 | 20 | ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor) 21 | 22 | next.ServeHTTP(ww, r) 23 | 24 | latency := time.Since(start) 25 | 26 | entry := []any{ 27 | "status", ww.Status(), 28 | "method", r.Method, 29 | // "requestURI", r.RequestURI, 30 | "remote_addr", r.RemoteAddr, 31 | "latency", latency.String(), 32 | "bytes_out", ww.BytesWritten(), 33 | } 34 | 35 | jl.logger.Debug(r.URL.Path, entry...) 36 | }) 37 | } 38 | -------------------------------------------------------------------------------- /apps/uploads/metrics.go: -------------------------------------------------------------------------------- 1 | package uploads 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | const ns = "uploads_v4" 8 | 9 | var ( 10 | userAuthErrors = prometheus.NewCounter(prometheus.CounterOpts{ 11 | Namespace: ns, 12 | Name: "user_auth_errors", 13 | }) 14 | sqlErrors = prometheus.NewCounter(prometheus.CounterOpts{ 15 | Namespace: ns, 16 | Name: "sql_errors", 17 | }) 18 | redisErrors = prometheus.NewCounter(prometheus.CounterOpts{ 19 | Namespace: ns, 20 | Name: "redis_errors", 21 | }) 22 | ) 23 | 24 | func registerMetrics(registry prometheus.Registerer) { 25 | if registry == nil { 26 | registry = prometheus.DefaultRegisterer 27 | } 28 | registry.MustRegister( 29 | userAuthErrors, sqlErrors, redisErrors, 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /apps/uploads/readme.md: -------------------------------------------------------------------------------- 1 | # Uploads, a publishing service metrics collector for Odysee 2 | 3 | ## Required tools 4 | 5 | ### sqlc 6 | 7 | ``` 8 | go install github.com/kyleconroy/sqlc/cmd/sqlc@latest 9 | ``` 10 | 11 | ## Schema changes 12 | 13 | Run `sqlc generate` 14 | 15 | -------------------------------------------------------------------------------- /apps/watchman/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | EXPOSE 8080 3 | 4 | RUN apk add --no-cache libc6-compat 5 | 6 | WORKDIR /app 7 | COPY ./dist/linux_amd64/watchman /app 8 | 9 | CMD ["/app/watchman", "serve"] 10 | -------------------------------------------------------------------------------- /apps/watchman/cmd/watchman-cli/http.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | cli "github.com/OdyseeTeam/odysee-api/apps/watchman/gen/http/cli/watchman" 8 | goahttp "goa.design/goa/v3/http" 9 | goa "goa.design/goa/v3/pkg" 10 | ) 11 | 12 | func doHTTP(scheme, host string, timeout int, debug bool) (goa.Endpoint, interface{}, error) { 13 | var ( 14 | doer goahttp.Doer 15 | ) 16 | { 17 | doer = &http.Client{Timeout: time.Duration(timeout) * time.Second} 18 | if debug { 19 | doer = goahttp.NewDebugDoer(doer) 20 | } 21 | } 22 | 23 | return cli.ParseEndpoint( 24 | scheme, 25 | host, 26 | doer, 27 | goahttp.RequestEncoder, 28 | goahttp.ResponseDecoder, 29 | debug, 30 | ) 31 | } 32 | 33 | func httpUsageCommands() string { 34 | return cli.UsageCommands() 35 | } 36 | 37 | func httpUsageExamples() string { 38 | return cli.UsageExamples() 39 | } 40 | -------------------------------------------------------------------------------- /apps/watchman/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | 7 | "github.com/spf13/viper" 8 | ) 9 | 10 | const configName = "watchman" 11 | 12 | func Read() (*viper.Viper, error) { 13 | cfg := viper.New() 14 | cfg.SetConfigName(configName) 15 | cfg.AddConfigPath(ProjectRoot()) 16 | cfg.AddConfigPath(".") 17 | cfg.AddConfigPath("../") 18 | cfg.AddConfigPath("./config") 19 | 20 | return cfg, cfg.ReadInConfig() 21 | } 22 | 23 | func ProjectRoot() string { 24 | ex, err := os.Executable() 25 | if err != nil { 26 | panic(err) 27 | } 28 | return filepath.Dir(ex) 29 | } 30 | -------------------------------------------------------------------------------- /apps/watchman/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | watchman: 5 | image: odyseeteam/watchman:latest 6 | container_name: watchman 7 | restart: on-failure 8 | ports: 9 | - "8080:8080" 10 | volumes: 11 | - "./rundata/geoip:/app/geoip:ro" 12 | clickhouse: 13 | image: yandex/clickhouse-server:21.3.20.1-alpine 14 | container_name: clickhouse 15 | restart: on-failure 16 | ports: 17 | - "8123:8123" 18 | - "9000:9000" 19 | volumes: 20 | - "clickhouse:/var/lib/clickhouse" 21 | grafana: 22 | image: grafana/grafana:7.5.4 23 | container_name: grafana 24 | restart: on-failure 25 | ports: 26 | - "3000:3000" 27 | environment: 28 | - GF_INSTALL_PLUGINS=vertamedia-clickhouse-datasource 29 | volumes: 30 | - "grafana:/var/lib/grafana" 31 | geoipupdate: 32 | image: "maxmindinc/geoipupdate:v4.7.1" 33 | entrypoint: 34 | ["/usr/bin/geoipupdate", "-d", "/geoip-data", "-f", "/conf/GeoIP.conf"] 35 | volumes: 36 | - "./rundata/geoip-conf:/conf" 37 | - "./rundata/geoip:/geoip-data" 38 | 39 | volumes: 40 | grafana: {} 41 | clickhouse: {} 42 | -------------------------------------------------------------------------------- /apps/watchman/gen/http/reporter/client/paths.go: -------------------------------------------------------------------------------- 1 | // Code generated by goa v3.5.2, DO NOT EDIT. 2 | // 3 | // HTTP request path constructors for the reporter service. 4 | // 5 | // Command: 6 | // $ goa gen github.com/OdyseeTeam/odysee-api/apps/watchman/design -o apps/watchman 7 | 8 | package client 9 | 10 | // AddReporterPath returns the URL path to the reporter service add HTTP endpoint. 11 | func AddReporterPath() string { 12 | return "/reports/playback" 13 | } 14 | 15 | // HealthzReporterPath returns the URL path to the reporter service healthz HTTP endpoint. 16 | func HealthzReporterPath() string { 17 | return "/healthz" 18 | } 19 | -------------------------------------------------------------------------------- /apps/watchman/gen/http/reporter/server/paths.go: -------------------------------------------------------------------------------- 1 | // Code generated by goa v3.5.2, DO NOT EDIT. 2 | // 3 | // HTTP request path constructors for the reporter service. 4 | // 5 | // Command: 6 | // $ goa gen github.com/OdyseeTeam/odysee-api/apps/watchman/design -o apps/watchman 7 | 8 | package server 9 | 10 | // AddReporterPath returns the URL path to the reporter service add HTTP endpoint. 11 | func AddReporterPath() string { 12 | return "/reports/playback" 13 | } 14 | 15 | // HealthzReporterPath returns the URL path to the reporter service healthz HTTP endpoint. 16 | func HealthzReporterPath() string { 17 | return "/healthz" 18 | } 19 | -------------------------------------------------------------------------------- /apps/watchman/gen/reporter/client.go: -------------------------------------------------------------------------------- 1 | // Code generated by goa v3.5.2, DO NOT EDIT. 2 | // 3 | // reporter client 4 | // 5 | // Command: 6 | // $ goa gen github.com/OdyseeTeam/odysee-api/apps/watchman/design -o apps/watchman 7 | 8 | package reporter 9 | 10 | import ( 11 | "context" 12 | 13 | goa "goa.design/goa/v3/pkg" 14 | ) 15 | 16 | // Client is the "reporter" service client. 17 | type Client struct { 18 | AddEndpoint goa.Endpoint 19 | HealthzEndpoint goa.Endpoint 20 | } 21 | 22 | // NewClient initializes a "reporter" service client given the endpoints. 23 | func NewClient(add, healthz goa.Endpoint) *Client { 24 | return &Client{ 25 | AddEndpoint: add, 26 | HealthzEndpoint: healthz, 27 | } 28 | } 29 | 30 | // Add calls the "add" endpoint of the "reporter" service. 31 | func (c *Client) Add(ctx context.Context, p *PlaybackReport) (err error) { 32 | _, err = c.AddEndpoint(ctx, p) 33 | return 34 | } 35 | 36 | // Healthz calls the "healthz" endpoint of the "reporter" service. 37 | func (c *Client) Healthz(ctx context.Context) (res string, err error) { 38 | var ires interface{} 39 | ires, err = c.HealthzEndpoint(ctx, nil) 40 | if err != nil { 41 | return 42 | } 43 | return ires.(string), nil 44 | } 45 | -------------------------------------------------------------------------------- /apps/watchman/gen/reporter/endpoints.go: -------------------------------------------------------------------------------- 1 | // Code generated by goa v3.5.2, DO NOT EDIT. 2 | // 3 | // reporter endpoints 4 | // 5 | // Command: 6 | // $ goa gen github.com/OdyseeTeam/odysee-api/apps/watchman/design -o apps/watchman 7 | 8 | package reporter 9 | 10 | import ( 11 | "context" 12 | 13 | goa "goa.design/goa/v3/pkg" 14 | ) 15 | 16 | // Endpoints wraps the "reporter" service endpoints. 17 | type Endpoints struct { 18 | Add goa.Endpoint 19 | Healthz goa.Endpoint 20 | } 21 | 22 | // NewEndpoints wraps the methods of the "reporter" service with endpoints. 23 | func NewEndpoints(s Service) *Endpoints { 24 | return &Endpoints{ 25 | Add: NewAddEndpoint(s), 26 | Healthz: NewHealthzEndpoint(s), 27 | } 28 | } 29 | 30 | // Use applies the given middleware to all the "reporter" service endpoints. 31 | func (e *Endpoints) Use(m func(goa.Endpoint) goa.Endpoint) { 32 | e.Add = m(e.Add) 33 | e.Healthz = m(e.Healthz) 34 | } 35 | 36 | // NewAddEndpoint returns an endpoint function that calls the method "add" of 37 | // service "reporter". 38 | func NewAddEndpoint(s Service) goa.Endpoint { 39 | return func(ctx context.Context, req interface{}) (interface{}, error) { 40 | p := req.(*PlaybackReport) 41 | return nil, s.Add(ctx, p) 42 | } 43 | } 44 | 45 | // NewHealthzEndpoint returns an endpoint function that calls the method 46 | // "healthz" of service "reporter". 47 | func NewHealthzEndpoint(s Service) goa.Endpoint { 48 | return func(ctx context.Context, req interface{}) (interface{}, error) { 49 | return s.Healthz(ctx) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /apps/watchman/gen/reporter/service.go: -------------------------------------------------------------------------------- 1 | // Code generated by goa v3.5.2, DO NOT EDIT. 2 | // 3 | // reporter service 4 | // 5 | // Command: 6 | // $ goa gen github.com/OdyseeTeam/odysee-api/apps/watchman/design -o apps/watchman 7 | 8 | package reporter 9 | 10 | import ( 11 | "context" 12 | ) 13 | 14 | // Media playback reports 15 | type Service interface { 16 | // Add implements add. 17 | Add(context.Context, *PlaybackReport) (err error) 18 | // Healthz implements healthz. 19 | Healthz(context.Context) (res string, err error) 20 | } 21 | 22 | // ServiceName is the name of the service as defined in the design. This is the 23 | // same value that is set in the endpoint request contexts under the ServiceKey 24 | // key. 25 | const ServiceName = "reporter" 26 | 27 | // MethodNames lists the service method names as defined in the design. These 28 | // are the same values that are set in the endpoint request contexts under the 29 | // MethodKey key. 30 | var MethodNames = [2]string{"add", "healthz"} 31 | 32 | // PlaybackReport is the payload type of the reporter service add method. 33 | type PlaybackReport struct { 34 | // LBRY URL (lbry://... without the protocol part) 35 | URL string 36 | // Duration of time between event calls in ms (aiming for between 5s and 30s so 37 | // generally 5000–30000) 38 | Duration int32 39 | // Current playback report stream position, ms 40 | Position int32 41 | // Relative stream position, pct, 0—100 42 | RelPosition int32 43 | // Rebuffering events count during the interval 44 | RebufCount int32 45 | // Sum of total rebuffering events duration in the interval, ms 46 | RebufDuration int32 47 | // Standard binary stream (`stb`), HLS (`hls`) or live stream (`lvs`) 48 | Protocol string 49 | // Cache status of video 50 | Cache *string 51 | // Player server name 52 | Player string 53 | // User ID 54 | UserID string 55 | // Client bandwidth, bit/s 56 | Bandwidth *int32 57 | // Media bitrate, bit/s 58 | Bitrate *int32 59 | // Client device 60 | Device string 61 | } 62 | 63 | // MultiFieldError is the error returned when several fields failed a 64 | // validation rule. 65 | type MultiFieldError struct { 66 | Message string 67 | } 68 | 69 | // Error returns an error description. 70 | func (e *MultiFieldError) Error() string { 71 | return "MultiFieldError is the error returned when several fields failed a validation rule." 72 | } 73 | 74 | // ErrorName returns "MultiFieldError". 75 | func (e *MultiFieldError) ErrorName() string { 76 | return "multi_field_error" 77 | } 78 | -------------------------------------------------------------------------------- /apps/watchman/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "go.uber.org/zap/zapcore" 6 | ) 7 | 8 | var Log = zap.NewNop().Sugar().Named("watchman") 9 | 10 | const () 11 | 12 | const ( 13 | LevelDebug = "debug" 14 | LevelInfo = "info" 15 | EncodingJSON = "json" 16 | EncodingConsole = "console" 17 | ) 18 | 19 | var ( 20 | levels = map[string]zapcore.Level{LevelDebug: zapcore.DebugLevel, LevelInfo: zapcore.InfoLevel} 21 | // encodings = map[int]string{"json": "json", EncodingText: "text"} 22 | ) 23 | 24 | func Configure(level string, encoding string) { 25 | // cfg := zap.NewProductionConfig() 26 | cfg := zap.NewDevelopmentConfig() 27 | cfg.Level = zap.NewAtomicLevelAt(levels[level]) 28 | cfg.Encoding = encoding 29 | l, err := cfg.Build() 30 | if err != nil { 31 | panic(err) 32 | } 33 | Log = l.Sugar().Named("watchman") 34 | Log.Infow("logger configured", "level", level, "encoding", encoding) 35 | } 36 | -------------------------------------------------------------------------------- /apps/watchman/metrics.go: -------------------------------------------------------------------------------- 1 | package watchman 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | var httpResponses = prometheus.NewHistogramVec( 8 | prometheus.HistogramOpts{ 9 | Name: "watchman_http_responses", 10 | Help: "Method call latency distributions", 11 | Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.4, 1, 2, 5, 10}, 12 | }, 13 | []string{"endpoint", "status_code"}, 14 | ) 15 | 16 | func RegisterMetrics() { 17 | prometheus.MustRegister(httpResponses) 18 | } 19 | -------------------------------------------------------------------------------- /apps/watchman/middleware.go: -------------------------------------------------------------------------------- 1 | package watchman 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "net/http" 7 | "strconv" 8 | "time" 9 | 10 | hmw "goa.design/goa/v3/http/middleware" 11 | ) 12 | 13 | type ctxKey int 14 | 15 | const RemoteAddressKey ctxKey = iota + 1 16 | 17 | // ObserveResponse returns a middleware that observes HTTP request processing times and response codes. 18 | func ObserveResponse() func(h http.Handler) http.Handler { 19 | return func(h http.Handler) http.Handler { 20 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 21 | started := time.Now() 22 | rw := hmw.CaptureResponse(w) 23 | h.ServeHTTP(rw, r) 24 | httpResponses.WithLabelValues("playback", strconv.Itoa(rw.StatusCode)).Observe(time.Since(started).Seconds()) 25 | }) 26 | } 27 | } 28 | func RemoteAddressMiddleware() func(http.Handler) http.Handler { 29 | return func(h http.Handler) http.Handler { 30 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 31 | ctx := context.WithValue(r.Context(), RemoteAddressKey, from(r)) 32 | h.ServeHTTP(w, r.WithContext(ctx)) 33 | }) 34 | } 35 | } 36 | 37 | // from makes a best effort to compute the request client IP. 38 | func from(req *http.Request) string { 39 | if f := req.Header.Get("X-Forwarded-For"); f != "" { 40 | return f 41 | } 42 | f := req.RemoteAddr 43 | ip, _, err := net.SplitHostPort(f) 44 | if err != nil { 45 | return f 46 | } 47 | return ip 48 | } 49 | -------------------------------------------------------------------------------- /apps/watchman/olapdb/batch_test.go: -------------------------------------------------------------------------------- 1 | package olapdb 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "testing" 7 | "time" 8 | 9 | "github.com/OdyseeTeam/odysee-api/apps/watchman/gen/reporter" 10 | "github.com/OdyseeTeam/odysee-api/apps/watchman/log" 11 | "github.com/Pallinder/go-randomdata" 12 | "github.com/stretchr/testify/suite" 13 | ) 14 | 15 | type batchWriterSuite struct { 16 | BaseOlapdbSuite 17 | } 18 | 19 | func TestBatchWriterSuite(t *testing.T) { 20 | log.Configure(log.LevelInfo, log.EncodingConsole) 21 | suite.Run(t, new(batchWriterSuite)) 22 | } 23 | 24 | func (s *batchWriterSuite) TestBatch() { 25 | days := 14 * 24 * time.Hour 26 | number := 1000 27 | reports := []*reporter.PlaybackReport{} 28 | bw := NewBatchWriter(100*time.Millisecond, 16) 29 | go bw.Start() 30 | 31 | for t := range timeSeries(number, time.Now().Add(-days)) { 32 | r := PlaybackReportFactory.MustCreate().(*reporter.PlaybackReport) 33 | ts := t.Format(time.RFC1123Z) 34 | err := bw.Write(r, randomdata.StringSample(randomdata.IpV4Address(), randomdata.IpV6Address()), ts) 35 | s.Require().NoError(err) 36 | reports = append(reports, r) 37 | } 38 | 39 | time.Sleep(3 * time.Second) 40 | bw.Stop() 41 | 42 | rows, err := conn.Query(fmt.Sprintf("SELECT URL, Duration from %s.playback ORDER BY Timestamp DESC", database)) 43 | s.Require().NoError(err) 44 | defer rows.Close() 45 | type duration struct { 46 | URL string 47 | Duration int32 48 | } 49 | retDurations := []duration{} 50 | for i, r := range reports { 51 | n := rows.Next() 52 | s.Require().True(n, "only %v rows in db, expected %v", i, len(reports)) 53 | d := duration{} 54 | err = rows.Scan(&d.URL, &d.Duration) 55 | s.Require().NoError(err) 56 | retDurations = append(retDurations, d) 57 | s.Equal(r.URL, d.URL) 58 | s.Equal(r.Duration, d.Duration) 59 | } 60 | s.False(rows.Next()) 61 | 62 | sort.Slice(retDurations, func(i, j int) bool { 63 | return retDurations[i].Duration < retDurations[j].Duration 64 | }) 65 | sort.Slice(reports, func(i, j int) bool { 66 | return reports[i].Duration < reports[j].Duration 67 | }) 68 | 69 | s.Equal(len(reports), len(retDurations)) 70 | for i := range reports { 71 | s.Equal(reports[i].URL, retDurations[i].URL) 72 | s.Equal(reports[i].Duration, retDurations[i].Duration) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /apps/watchman/olapdb/geoip.go: -------------------------------------------------------------------------------- 1 | package olapdb 2 | 3 | import ( 4 | "net" 5 | "strings" 6 | 7 | "github.com/oschwald/geoip2-golang" 8 | ) 9 | 10 | var geodb *geoip2.Reader 11 | 12 | func OpenGeoDB(file string) error { 13 | var err error 14 | geodb, err = geoip2.Open(file) 15 | if err != nil { 16 | return err 17 | } 18 | return nil 19 | } 20 | 21 | func getArea(ip string) (string, string) { 22 | var area, subarea string 23 | 24 | record, err := geodb.City(net.ParseIP(ip)) 25 | if err != nil { 26 | return "", "" 27 | } 28 | 29 | area = record.Country.IsoCode 30 | if len(record.Subdivisions) >= 2 { 31 | subarea = record.Subdivisions[1].IsoCode 32 | } 33 | return strings.ToLower(area), strings.ToLower(subarea) 34 | } 35 | -------------------------------------------------------------------------------- /apps/watchman/olapdb/geoip_test.go: -------------------------------------------------------------------------------- 1 | package olapdb 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func Test_getArea(t *testing.T) { 12 | p, _ := filepath.Abs(filepath.Join("./testdata", "GeoIP2-City-Test.mmdb")) 13 | err := OpenGeoDB(p) 14 | require.NoError(t, err) 15 | a, s := getArea("81.2.69.142") 16 | assert.Equal(t, "gb", a) 17 | assert.Equal(t, "", s) 18 | a, s = getArea("2001:41d0:303:df3e::") 19 | assert.Equal(t, "", a) 20 | assert.Equal(t, "", s) 21 | } 22 | -------------------------------------------------------------------------------- /apps/watchman/olapdb/olapdb_test.go: -------------------------------------------------------------------------------- 1 | package olapdb 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "strings" 7 | "testing" 8 | "time" 9 | 10 | "github.com/OdyseeTeam/odysee-api/apps/watchman/config" 11 | "github.com/OdyseeTeam/odysee-api/apps/watchman/gen/reporter" 12 | "github.com/Pallinder/go-randomdata" 13 | "github.com/stretchr/testify/suite" 14 | ) 15 | 16 | type BaseOlapdbSuite struct { 17 | suite.Suite 18 | cleanup func() 19 | } 20 | 21 | type olapdbSuite struct { 22 | BaseOlapdbSuite 23 | } 24 | 25 | func TestOlapdbSuite(t *testing.T) { 26 | suite.Run(t, new(olapdbSuite)) 27 | } 28 | 29 | func (s *BaseOlapdbSuite) SetupSuite() { 30 | cfg, err := config.Read() 31 | s.Require().NoError(err) 32 | 33 | dbCfg := cfg.GetStringMapString("clickhouse") 34 | dbName := randomdata.Alphanumeric(32) 35 | err = Connect(dbCfg["url"], dbName) 36 | s.cleanup = func() { 37 | MigrateDown(dbName) 38 | 39 | } 40 | s.Require().NoError(err) 41 | 42 | p, _ := filepath.Abs(filepath.Join("./testdata", "GeoIP2-City-Test.mmdb")) 43 | err = OpenGeoDB(p) 44 | s.Require().NoError(err) 45 | } 46 | 47 | func (s *olapdbSuite) TestWriteOne() { 48 | r := PlaybackReportFactory.MustCreate().(*reporter.PlaybackReport) 49 | ts := time.Now().Format(time.RFC1123Z) 50 | err := WriteOne(r, randomdata.StringSample(randomdata.IpV4Address(), randomdata.IpV6Address()), ts) 51 | s.Require().NoError(err) 52 | 53 | var ( 54 | url string 55 | duration int32 56 | ) 57 | rows, err := conn.Query(fmt.Sprintf("select URL, Duration from %s.playback where URL = ?", database), r.URL) 58 | s.Require().NoError(err) 59 | defer rows.Close() 60 | rows.Next() 61 | err = rows.Scan(&url, &duration) 62 | s.Require().NoError(err) 63 | s.Equal(r.Duration, duration) 64 | } 65 | 66 | func (s *olapdbSuite) TestWriteGarbled() { 67 | r := PlaybackReportFactory.MustCreate().(*reporter.PlaybackReport) 68 | cleanPlayer := r.Player 69 | r.Player += ", some-nonsense-at-the-end-abcbcbdagadwsedaddff" 70 | ts := time.Now().Format(time.RFC1123Z) 71 | err := WriteOne(r, randomdata.StringSample(randomdata.IpV4Address(), randomdata.IpV6Address()), ts) 72 | s.Require().NoError(err) 73 | 74 | var ( 75 | url string 76 | duration int32 77 | player string 78 | ) 79 | rows, err := conn.Query(fmt.Sprintf("select URL, Duration, Player from %s.playback where URL = ?", database), r.URL) 80 | s.Require().NoError(err) 81 | defer rows.Close() 82 | rows.Next() 83 | err = rows.Scan(&url, &duration, &player) 84 | s.Require().NoError(err) 85 | s.Equal(r.Duration, duration) 86 | s.Equal(r.URL, url) 87 | s.Equal(r.URL, url) 88 | s.Equal(cleanPlayer, strings.Trim(player, "\x00")) 89 | } 90 | -------------------------------------------------------------------------------- /apps/watchman/olapdb/schema.go: -------------------------------------------------------------------------------- 1 | package olapdb 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | func MigrateUp(dbName string) error { 10 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 11 | defer cancel() 12 | _, err := conn.ExecContext(ctx, fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %v`, dbName)) 13 | if err != nil { 14 | return err 15 | } 16 | _, err = conn.Exec(fmt.Sprintf(` 17 | CREATE TABLE IF NOT EXISTS %v.playback 18 | ( 19 | "URL" String, 20 | "Duration" UInt32, 21 | "Timestamp" Timestamp, 22 | "Position" UInt32, 23 | "RelPosition" UInt8, 24 | "RebufCount" UInt8, 25 | "RebufDuration" UInt32, 26 | "Protocol" FixedString(3), 27 | "Cache" String, 28 | "Player" FixedString(16), 29 | "UserID" String, 30 | "Bandwidth" UInt32, 31 | "Bitrate" UInt32, 32 | "Device" FixedString(3), 33 | "Area" FixedString(2), 34 | "SubArea" FixedString(3), 35 | "IP" IPv6 36 | ) 37 | ENGINE = MergeTree 38 | ORDER BY (Timestamp, UserID, URL) 39 | TTL Timestamp + INTERVAL 15 DAY`, dbName)) 40 | if err != nil { 41 | return err 42 | } 43 | return nil 44 | } 45 | func MigrateDown(dbName string) error { 46 | _, err := conn.Exec(fmt.Sprintf(`DROP DATABASE %v`, dbName)) 47 | if err != nil { 48 | return err 49 | } 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /apps/watchman/olapdb/testdata/GeoIP2-City-Test.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OdyseeTeam/odysee-api/15e122c36a6dc05f55ba456b23614797b4c4dc11/apps/watchman/olapdb/testdata/GeoIP2-City-Test.mmdb -------------------------------------------------------------------------------- /apps/watchman/readme.md: -------------------------------------------------------------------------------- 1 | # Watchman — a playback metrics collector for Odysee 2 | 3 | ## Required tools 4 | 5 | ### Goa 6 | 7 | ``` 8 | go get goa.design/goa/v3/...@v3 9 | ``` 10 | -------------------------------------------------------------------------------- /apps/watchman/reporter.go: -------------------------------------------------------------------------------- 1 | package watchman 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | 7 | reporter "github.com/OdyseeTeam/odysee-api/apps/watchman/gen/reporter" 8 | "github.com/OdyseeTeam/odysee-api/apps/watchman/olapdb" 9 | 10 | "go.uber.org/zap" 11 | ) 12 | 13 | // reporter service example implementation. 14 | // The example methods log the requests and return zero values. 15 | type reportersrvc struct { 16 | db *sql.DB 17 | logger *zap.SugaredLogger 18 | } 19 | 20 | // NewReporter returns the reporter service implementation. 21 | func NewReporter(db *sql.DB, logger *zap.SugaredLogger) reporter.Service { 22 | svc := &reportersrvc{ 23 | db: db, 24 | logger: logger, 25 | } 26 | return svc 27 | } 28 | 29 | // Add implements add. 30 | func (s *reportersrvc) Add(ctx context.Context, p *reporter.PlaybackReport) error { 31 | s.logger.Debug("reporter.add") 32 | 33 | if p.RebufDuration > p.Duration { 34 | return &reporter.MultiFieldError{Message: "rebufferung duration cannot be larger than duration"} 35 | } 36 | addr := ctx.Value(RemoteAddressKey).(string) 37 | err := olapdb.BatchWrite(p, addr, "") 38 | if err != nil { 39 | return err 40 | } 41 | return nil 42 | } 43 | 44 | func (s *reportersrvc) Healthz(ctx context.Context) (string, error) { 45 | return "OK", nil 46 | } 47 | -------------------------------------------------------------------------------- /apps/watchman/watchman.yaml: -------------------------------------------------------------------------------- 1 | Clickhouse: 2 | URL: tcp://127.0.0.1:9000 3 | 4 | GeoIPDB: ./rundata/geoip/GeoLite2-City.mmdb 5 | 6 | Log: 7 | Encoding: console 8 | Level: debug 9 | -------------------------------------------------------------------------------- /build/forklift/Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM odyseeteam/transcoder-ffmpeg:5.1.1 AS ffmpeg 3 | FROM alpine:3.21 4 | EXPOSE 8080 5 | 6 | COPY --from=ffmpeg /build/ffprobe /usr/local/bin/ 7 | 8 | WORKDIR /app 9 | COPY ./dist/linux_amd64/forklift /app 10 | COPY ./apps/forklift/config/forklift.yml /app/config/forklift.yml 11 | 12 | CMD ["/app/forklift", "serve"] 13 | -------------------------------------------------------------------------------- /build/uploads/Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM alpine:3.21 3 | EXPOSE 8080 4 | 5 | # RUN apk add --no-cache libc6-compat 6 | 7 | WORKDIR /app 8 | COPY ./dist/linux_amd64/uploads /app 9 | COPY ./apps/uploads/config/uploads.yml /app/config/uploads.yml 10 | 11 | CMD ["/app/uploads", "serve"] 12 | -------------------------------------------------------------------------------- /cmd/db_migrate_down.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 7 | "github.com/OdyseeTeam/odysee-api/internal/storage" 8 | "github.com/OdyseeTeam/odysee-api/pkg/migrator" 9 | "github.com/sirupsen/logrus" 10 | 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | func init() { 15 | rootCmd.AddCommand(dbMigrateDown) 16 | } 17 | 18 | var dbMigrateDown = &cobra.Command{ 19 | Use: "db_migrate_down", 20 | Short: "Unapply database migrations", 21 | Run: func(_ *cobra.Command, args []string) { 22 | var max int 23 | if len(args) > 0 { 24 | var err error 25 | max, err = strconv.Atoi(args[0]) 26 | if err != nil { 27 | logrus.Error("non integer passed as argument to migration") 28 | } 29 | } 30 | 31 | dbConfig := config.GetDatabase() 32 | db, err := migrator.ConnectDB(migrator.DefaultDBConfig().DSN(dbConfig.Connection).Name(dbConfig.DBName)) 33 | if err != nil { 34 | panic(err) 35 | } 36 | defer db.Close() 37 | m := migrator.New(db, storage.MigrationsFS) 38 | _, err = m.MigrateDown(max) 39 | if err != nil { 40 | panic(err) 41 | } 42 | }, 43 | } 44 | -------------------------------------------------------------------------------- /cmd/db_migrate_up.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 7 | "github.com/OdyseeTeam/odysee-api/internal/storage" 8 | "github.com/OdyseeTeam/odysee-api/pkg/migrator" 9 | 10 | "github.com/sirupsen/logrus" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | func init() { 15 | rootCmd.AddCommand(dbMigrateUp) 16 | } 17 | 18 | var dbMigrateUp = &cobra.Command{ 19 | Use: "db_migrate_up", 20 | Short: "Apply database migrations", 21 | Run: func(cmd *cobra.Command, args []string) { 22 | var max int 23 | if len(args) > 0 { 24 | var err error 25 | max, err = strconv.Atoi(args[0]) 26 | if err != nil { 27 | logrus.Error("non integer passed as argument to migration") 28 | } 29 | } 30 | 31 | dbConfig := config.GetDatabase() 32 | db, err := migrator.ConnectDB(migrator.DefaultDBConfig().DSN(dbConfig.Connection).Name(dbConfig.DBName)) 33 | if err != nil { 34 | panic(err) 35 | } 36 | defer db.Close() 37 | m := migrator.New(db, storage.MigrationsFS) 38 | _, err = m.MigrateUp(max) 39 | if err != nil { 40 | panic(err) 41 | } 42 | }, 43 | } 44 | -------------------------------------------------------------------------------- /cmd/serve.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | 9 | "github.com/OdyseeTeam/odysee-api/api" 10 | "github.com/OdyseeTeam/odysee-api/app/sdkrouter" 11 | "github.com/OdyseeTeam/odysee-api/app/wallet" 12 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 13 | "github.com/OdyseeTeam/odysee-api/server" 14 | "github.com/OdyseeTeam/player-server/pkg/paid" 15 | 16 | "github.com/spf13/cobra" 17 | ) 18 | 19 | var rootCmd = &cobra.Command{ 20 | Use: "oapi", 21 | Short: "backend server for Odysee frontend", 22 | Run: func(_ *cobra.Command, _ []string) { 23 | sdkRouter := sdkrouter.New(config.GetLbrynetServers()) 24 | go sdkRouter.WatchLoad() 25 | 26 | s := server.NewServer(config.GetAddress(), sdkRouter, &api.RoutesOptions{ 27 | EnableProfiling: config.GetProfiling(), 28 | EnableV3Publish: false, 29 | }) 30 | err := s.Start() 31 | if err != nil { 32 | log.Fatal(err) 33 | } 34 | 35 | key, err := ioutil.ReadFile(config.GetPaidTokenPrivKey()) 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | err = paid.InitPrivateKey(key) 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | c := wallet.NewTokenCache() 44 | wallet.SetTokenCache(c) 45 | 46 | // ServeUntilShutdown is blocking, should be last 47 | s.ServeUntilShutdown() 48 | }, 49 | } 50 | 51 | func Execute() { 52 | if err := rootCmd.Execute(); err != nil { 53 | fmt.Println(err) 54 | os.Exit(1) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /cmd/unload_wallets.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/OdyseeTeam/odysee-api/app/wallet/tracker" 9 | "github.com/OdyseeTeam/odysee-api/internal/monitor" 10 | 11 | log "github.com/sirupsen/logrus" 12 | "github.com/spf13/cobra" 13 | "github.com/volatiletech/sqlboiler/boil" 14 | ) 15 | 16 | func init() { 17 | rootCmd.AddCommand(unloadWallets) 18 | } 19 | 20 | var unloadWallets = &cobra.Command{ 21 | Use: "unload_wallets MIN", 22 | Short: "Unload wallets that have not been used in the last MIN minutes", 23 | Args: cobra.ExactArgs(1), 24 | Run: func(cmd *cobra.Command, args []string) { 25 | min, err := strconv.Atoi(args[0]) 26 | if err != nil { 27 | log.Error(args[0] + " is not an integer") 28 | os.Exit(1) 29 | } 30 | 31 | unloadOlderThan := time.Duration(min) * time.Minute 32 | _, err = tracker.Unload(boil.GetDB(), unloadOlderThan) 33 | if err != nil { 34 | log.Error(err) 35 | monitor.ErrorToSentry(err) 36 | os.Exit(1) 37 | } 38 | }, 39 | } 40 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/spf13/viper" 5 | ) 6 | 7 | type ConfigWrapper struct { 8 | Viper *viper.Viper 9 | configName string 10 | overridden map[string]interface{} 11 | } 12 | 13 | type DBConfig struct { 14 | Connection string 15 | DBName string 16 | Options string 17 | } 18 | 19 | func NewConfig() *ConfigWrapper { 20 | return &ConfigWrapper{ 21 | overridden: map[string]interface{}{}, 22 | Viper: viper.New(), 23 | } 24 | } 25 | 26 | // ReadConfig initializes a ConfigWrapper and reads `configName` 27 | func ReadConfig(configName string) *ConfigWrapper { 28 | c := NewConfig() 29 | c.configName = configName 30 | c.initPaths() 31 | c.read() 32 | return c 33 | } 34 | 35 | func (c DBConfig) GetFullDSN() string { 36 | return c.Connection 37 | } 38 | 39 | func (c DBConfig) GetDBName() string { 40 | return c.DBName 41 | } 42 | 43 | func (c *ConfigWrapper) initPaths() { 44 | c.Viper.SetConfigName(c.configName) 45 | c.Viper.AddConfigPath("./config/") 46 | c.Viper.AddConfigPath(".") 47 | c.Viper.AddConfigPath("..") 48 | c.Viper.AddConfigPath("../../") 49 | c.Viper.AddConfigPath("../../../") 50 | } 51 | 52 | func (c *ConfigWrapper) read() { 53 | err := c.Viper.ReadInConfig() 54 | if err != nil { 55 | panic(err) 56 | } 57 | } 58 | 59 | // IsProduction is true if we are running in a production environment 60 | func (c *ConfigWrapper) IsProduction() bool { 61 | return !c.Viper.GetBool("Debug") 62 | } 63 | 64 | // GetDatabase returns postgresql database server connection config 65 | func (c *ConfigWrapper) GetDatabase() DBConfig { 66 | var dbc DBConfig 67 | c.Viper.UnmarshalKey("Database", &dbc) 68 | dbc.Connection = c.Viper.GetString("DatabaseDSN") 69 | return dbc 70 | } 71 | 72 | // Override sets a setting key value to whatever you supply. 73 | // Useful in tests: 74 | // 75 | // config.Override("Lbrynet", "http://www.google.com:8080/api/proxy") 76 | // defer config.RestoreOverridden() 77 | // ... 78 | func (c *ConfigWrapper) Override(key string, value interface{}) { 79 | c.overridden[key] = c.Viper.Get(key) 80 | c.Viper.Set(key, value) 81 | } 82 | 83 | // RestoreOverridden restores original v values overridden by Override 84 | func (c *ConfigWrapper) RestoreOverridden() { 85 | v := c.Viper 86 | if len(c.overridden) == 0 { 87 | return 88 | } 89 | for k, val := range c.overridden { 90 | v.Set(k, val) 91 | } 92 | c.overridden = make(map[string]interface{}) 93 | } 94 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestOverride(t *testing.T) { 12 | c := NewConfig() 13 | err := c.Viper.ReadConfig(strings.NewReader("Lbrynet: http://localhost:5279")) 14 | require.Nil(t, err) 15 | originalSetting := c.Viper.Get("Lbrynet") 16 | c.Override("Lbrynet", "http://www.google.com:8080/api/proxy") 17 | assert.Equal(t, "http://www.google.com:8080/api/proxy", c.Viper.Get("Lbrynet")) 18 | c.RestoreOverridden() 19 | assert.Equal(t, originalSetting, c.Viper.Get("Lbrynet")) 20 | assert.Empty(t, c.overridden) 21 | } 22 | 23 | func TestIsProduction(t *testing.T) { 24 | c := NewConfig() 25 | c.Override("Debug", false) 26 | assert.True(t, c.IsProduction()) 27 | c.Override("Debug", true) 28 | assert.False(t, c.IsProduction()) 29 | } 30 | -------------------------------------------------------------------------------- /dev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 4 | 5 | ( 6 | cd "$DIR" 7 | 8 | export LW_DEBUG=1 9 | 10 | hash reflex 2>/dev/null || go get github.com/cespare/reflex 11 | hash reflex 2>/dev/null || { echo >&2 'Make sure '"$(go env GOPATH)"'/bin is in your $PATH'; exit 1; } 12 | 13 | reflex --decoration=none --start-service=true --regex='\.(go|css|js|html)$' --inverse-regex='assets/bindata\.go' --inverse-regex='vendor/' -- sh -c "go run ." 14 | ) 15 | -------------------------------------------------------------------------------- /docker-compose.app.yml: -------------------------------------------------------------------------------- 1 | services: 2 | oapi: 3 | image: odyseeteam/odysee-api:latest 4 | container_name: oapi 5 | ports: 6 | - 8080:8080 7 | volumes: 8 | - storage:/storage 9 | environment: 10 | LW_DEBUG: 1 11 | depends_on: 12 | - lbrynet 13 | - postgres 14 | labels: 15 | com.centurylinklabs.watchtower.enable: true 16 | 17 | volumes: 18 | storage: {} 19 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | lbrynet: 3 | image: odyseeteam/lbrynet-tv:0.110.0 4 | platform: linux/amd64 5 | container_name: lbrynet 6 | ports: 7 | - "15279:5279" 8 | volumes: 9 | - lbrynet:/storage 10 | - ./docker/daemon_settings.yml:/daemon/daemon_settings.yml 11 | labels: 12 | com.centurylinklabs.watchtower.enable: true 13 | redis: 14 | image: redis:7 15 | container_name: redis 16 | ports: 17 | - '6379:6379' 18 | command: > 19 | --requirepass odyredis --appendonly yes 20 | labels: 21 | com.centurylinklabs.watchtower.enable: false 22 | postgres: 23 | image: postgres:12.5-alpine 24 | container_name: postgres 25 | command: ["postgres", "-c", "log_statement=all", "-c", "log_destination=stderr"] 26 | ports: 27 | - "5432:5432" 28 | volumes: 29 | - pgdata:/pgdata 30 | environment: 31 | POSTGRES_USER: postgres 32 | POSTGRES_PASSWORD: odyseeteam 33 | POSTGRES_DB: oapi 34 | PGDATA: /pgdata 35 | TZ: "UTC" 36 | PGTZ: "UTC" 37 | minio: 38 | image: minio/minio:latest 39 | container_name: minio 40 | ports: 41 | - "9002:9002" 42 | environment: 43 | MINIO_ROOT_USER: minio 44 | MINIO_ROOT_PASSWORD: minio123 45 | command: server /data --address :9002 46 | volumes: 47 | - minio-data:/data 48 | 49 | volumes: 50 | pgdata: {} 51 | lbrynet: {} 52 | minio-data: {} 53 | -------------------------------------------------------------------------------- /docker/daemon_settings.yml: -------------------------------------------------------------------------------- 1 | api: 0.0.0.0:5279 2 | streaming_server: 0.0.0.0:5280 3 | 4 | components_to_skip: 5 | - hash_announcer 6 | - blob_server 7 | - dht 8 | track_bandwidth: false 9 | 10 | data_dir: /storage/lbrynet 11 | download_dir: /storage/download 12 | wallet_dir: /storage/lbryum 13 | 14 | lbryum_servers: ["a-hub1.odysee.com:50001"] 15 | 16 | save_blobs: false 17 | save_files: false 18 | share_usage_data: false 19 | use_upnp: false 20 | save_resolved_claims: false 21 | 22 | reflect_streams: false 23 | -------------------------------------------------------------------------------- /docker/launcher.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | file="token_privkey.rsa" 4 | if [ ! -f "$file" ]; then 5 | ssh-keygen -t rsa -f "$file" -m pem 6 | fi 7 | 8 | ./oapi db_migrate_up 9 | ./oapi 10 | -------------------------------------------------------------------------------- /docker/oapi.yml: -------------------------------------------------------------------------------- 1 | LbrynetServers: 2 | default: http://lbrynet:5279/ 3 | lbrynet1: http://lbrynet:5279/ 4 | lbrynet2: http://lbrynet:5279/ 5 | 6 | Debug: 1 7 | 8 | BaseContentURL: https://player.odycdn.com/api 9 | FreeContentURL: https://player.odycdn.com/api/v4/streams/free/ 10 | PaidContentURL: https://player.odycdn.com/api/v3/streams/paid/ 11 | 12 | StreamsV5: 13 | Host: https://player.odycdn.com 14 | PaidHost: https://secure.odycdn.com 15 | StartPath: /v5/streams/start 16 | HLSPath: /v5/streams/hls 17 | PaidPass: paid-pass 18 | 19 | StreamsV6: 20 | Host: player.odycdn.com 21 | PaidHost: secure.odycdn.com 22 | StartPath: /v6/streams/%s/%s.mp4 23 | Token: cdn-paid-token 24 | 25 | InternalAPIHost: https://api.lbry.com 26 | ProjectURL: https://lbry.tv 27 | 28 | DatabaseDSN: postgres://postgres:odyseeteam@postgres 29 | Database: 30 | DBName: postgres 31 | Options: sslmode=disable 32 | 33 | OAuth: 34 | ClientID: odysee-apis 35 | ProviderURL: https://sso.odysee.com/auth/realms/Users 36 | TokenPath: /protocol/openid-connect/token 37 | 38 | PublishSourceDir: /storage/published 39 | GeoPublishSourceDir: /storage/geopublish 40 | 41 | PaidTokenPrivKey: token_privkey.rsa 42 | 43 | # Change this key for production! 44 | # You can re-generate the key by running: 45 | # openssl ecparam -genkey -name prime256v1 -noout -out private_key.pem && base64 -i private_key.pem 46 | UploadTokenPrivateKey: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUZZYWxQZkhySzNSQ1F2YmhRQ1h6cDZiWG9uODZWOGI5L3B0bjB3QTZxNkxvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFZjhyN3RlQWJwUlVldXZhVWRsNDQzVS9VZkpYZURDd051QkRrbmp5ZnRZaXZ2Tnl6cGt6ZgpYdDl3RE9rc1VZSmEzNVhvSndabjNHMmw2L2EvdVUvWmh3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= 47 | UploadServiceURL: http://uploads-v4/v1/ 48 | 49 | CORSDomains: 50 | - http://localhost:1337 51 | - http://localhost:9090 52 | 53 | RPCTimeouts: 54 | txo_spend: 4m 55 | txo_list: 4m 56 | transaction_list: 4m 57 | publish: 4m 58 | 59 | RedisLocker: redis://:odyredis@redis:6379/1 60 | RedisBus: redis://:odyredis@redis:6379/2 61 | 62 | # AsynqueryRequestsConnURL is Redis database where asynquery will be listening for finalized uploads requests. 63 | # This corresponds to AsynqueryRequestsConnURL in forklift.yml config. 64 | AsynqueryRequestsConnURL: redis://:odyredis@redis:6379/3 65 | 66 | SturdyCache: 67 | Master: redis:6379 68 | Replicas: 69 | - redis:6379 70 | Password: odyredis 71 | 72 | ReflectorUpstream: 73 | DatabaseDSN: 'user:password@tcp(localhost:3306)/blobs' 74 | Endpoint: http://localhost:1337 75 | Region: us-east-1 76 | Bucket: blobs' 77 | Key: key 78 | Secret: secret 79 | 80 | Logging: 81 | Level: debug 82 | Format: console 83 | -------------------------------------------------------------------------------- /docs/design/publish/publish_v4_client.mmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: Publish v4 Flow 3 | --- 4 | sequenceDiagram 5 | actor Client 6 | participant OAPI as Odysee API 7 | participant Uploads as Uploads Service 8 | 9 | Client->>OAPI: POST /api/v1/asynqueries/uploads/ 10 | activate OAPI 11 | OAPI->>OAPI: token auth 12 | OAPI-->>Client: 201 Created (<>, <>) 13 | deactivate OAPI 14 | 15 | Client->>Uploads: POST /:location, <> 16 | activate Uploads 17 | Uploads->>Uploads: validate upload token 18 | Uploads-->>Client: 201 Created (Location: <>) 19 | 20 | loop TUS Uploads 21 | Client->>Uploads: PATCH /:upload-id, <> 22 | Uploads-->>Client: 204 No Content 23 | end 24 | deactivate Uploads 25 | 26 | Client->>OAPI: POST /api/v1/asynqueries/ 27 | activate OAPI 28 | note Left of OAPI: stream_create {file_path: <>} 29 | OAPI-->>Client: 201 Created (<>) 30 | deactivate OAPI 31 | 32 | loop Status Check 33 | Client->>OAPI: GET /api/v1/asynqueries/:query-id 34 | activate OAPI 35 | alt pending 36 | OAPI-->>Client: 204 No Content 37 | else completed 38 | OAPI-->>Client: 200 OK (JSON-RPC response) 39 | else not found 40 | OAPI-->>Client: 404 Not Found 41 | end 42 | deactivate OAPI 43 | end 44 | -------------------------------------------------------------------------------- /docs/design/publish/publish_v4_urls_client.mmd: -------------------------------------------------------------------------------- 1 | --- 2 | title: Publish v4 Flow 3 | --- 4 | sequenceDiagram 5 | actor Client 6 | participant OAPI as Odysee API 7 | participant Uploads as Uploads Service 8 | 9 | Client->>OAPI: POST /api/v1/asynqueries/urls/, 10 | activate OAPI 11 | OAPI->>OAPI: token auth 12 | OAPI-->>Client: 201 Created (<>, <>) 13 | deactivate OAPI 14 | 15 | Client->>Uploads: POST /:location, <>, {url: <>} 16 | activate Uploads 17 | Uploads->>Uploads: validate upload token 18 | Uploads-->>Client: 201 Created (Location: <>) 19 | 20 | Client->>OAPI: POST /api/v1/asynqueries/ 21 | activate OAPI 22 | note Left of OAPI: stream_create {file_path: <>} 23 | OAPI-->>Client: 201 Created (<>) 24 | deactivate OAPI 25 | 26 | loop Status Check 27 | Client->>OAPI: GET /api/v1/asynqueries/:query-id 28 | activate OAPI 29 | alt pending 30 | OAPI-->>Client: 204 No Content 31 | else completed 32 | OAPI-->>Client: 200 OK (JSON-RPC response) 33 | else not found 34 | OAPI-->>Client: 404 Not Found 35 | end 36 | deactivate OAPI 37 | end 38 | -------------------------------------------------------------------------------- /internal/audit/audit.go: -------------------------------------------------------------------------------- 1 | package audit 2 | 3 | import ( 4 | "github.com/OdyseeTeam/odysee-api/internal/monitor" 5 | "github.com/OdyseeTeam/odysee-api/models" 6 | "github.com/volatiletech/null" 7 | "github.com/volatiletech/sqlboiler/boil" 8 | ) 9 | 10 | var logger = monitor.NewModuleLogger("audit") 11 | 12 | func LogQuery(userID int, remoteIP string, method string, body []byte) *models.QueryLog { 13 | qLog := models.QueryLog{Method: method, UserID: null.IntFrom(userID), RemoteIP: remoteIP, Body: null.JSONFrom(body)} 14 | err := qLog.InsertG(boil.Infer()) 15 | if err != nil { 16 | logger.Log().Error("cannot insert query log:", err) 17 | } 18 | return &qLog 19 | } 20 | -------------------------------------------------------------------------------- /internal/audit/audit_test.go: -------------------------------------------------------------------------------- 1 | package audit 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | 8 | "github.com/OdyseeTeam/odysee-api/app/query" 9 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 10 | "github.com/OdyseeTeam/odysee-api/internal/storage" 11 | "github.com/OdyseeTeam/odysee-api/internal/test" 12 | "github.com/OdyseeTeam/odysee-api/models" 13 | "github.com/OdyseeTeam/odysee-api/pkg/migrator" 14 | "github.com/lbryio/lbry.go/v2/extras/null" 15 | 16 | "github.com/stretchr/testify/assert" 17 | "github.com/stretchr/testify/require" 18 | "github.com/ybbus/jsonrpc/v2" 19 | ) 20 | 21 | func TestMain(m *testing.M) { 22 | db, dbCleanup, err := migrator.CreateTestDB(migrator.DBConfigFromApp(config.GetDatabase()), storage.MigrationsFS) 23 | if err != nil { 24 | panic(err) 25 | } 26 | storage.SetDB(db) 27 | code := m.Run() 28 | dbCleanup() 29 | os.Exit(code) 30 | } 31 | 32 | func TestLogQuery(t *testing.T) { 33 | dummyUserID := 1234 34 | jReq := jsonrpc.NewRequest( 35 | query.MethodWalletSend, 36 | map[string]interface{}{"addresses": []string{"dgjkldfjgldkfjgkldfjg"}, "amount": "6.49999000"}) 37 | q := test.ReqToStr(t, jReq) 38 | ql := LogQuery(dummyUserID, "8.8.8.8", query.MethodWalletSend, []byte(q)) 39 | ql, err := models.QueryLogs(models.QueryLogWhere.ID.EQ(ql.ID)).OneG() 40 | require.NoError(t, err) 41 | assert.Equal(t, "8.8.8.8", ql.RemoteIP) 42 | assert.EqualValues(t, null.IntFrom(dummyUserID), ql.UserID) 43 | 44 | loggedReq := &jsonrpc.RPCRequest{} 45 | expReq := &jsonrpc.RPCRequest{} 46 | 47 | err = ql.Body.Unmarshal(&loggedReq) 48 | require.NoError(t, err) 49 | err = json.Unmarshal([]byte(q), expReq) 50 | require.NoError(t, err) 51 | 52 | assert.Equal(t, expReq, loggedReq) 53 | } 54 | 55 | func TestLogQueryNoUserNoRemoteIP(t *testing.T) { 56 | var dummyUserID int 57 | jReq := jsonrpc.NewRequest( 58 | query.MethodWalletSend, 59 | map[string]interface{}{"addresses": []string{"dgjkldfjgldkfjgkldfjg"}, "amount": "6.49999000"}) 60 | q := test.ReqToStr(t, jReq) 61 | ql := LogQuery(dummyUserID, "", query.MethodWalletSend, []byte(q)) 62 | ql, err := models.QueryLogs(models.QueryLogWhere.ID.EQ(ql.ID)).OneG() 63 | require.NoError(t, err) 64 | assert.Equal(t, "", ql.RemoteIP) 65 | assert.EqualValues(t, null.IntFrom(dummyUserID), ql.UserID) 66 | 67 | loggedReq := &jsonrpc.RPCRequest{} 68 | expReq := &jsonrpc.RPCRequest{} 69 | 70 | err = ql.Body.Unmarshal(&loggedReq) 71 | require.NoError(t, err) 72 | err = json.Unmarshal([]byte(q), expReq) 73 | require.NoError(t, err) 74 | 75 | assert.Equal(t, expReq, loggedReq) 76 | } 77 | -------------------------------------------------------------------------------- /internal/e2etest/e2etest.go: -------------------------------------------------------------------------------- 1 | package e2etest 2 | 3 | import ( 4 | "database/sql" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/OdyseeTeam/odysee-api/app/auth" 9 | "github.com/OdyseeTeam/odysee-api/app/sdkrouter" 10 | "github.com/OdyseeTeam/odysee-api/app/wallet" 11 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 12 | "github.com/OdyseeTeam/odysee-api/internal/storage" 13 | "github.com/OdyseeTeam/odysee-api/internal/test" 14 | "github.com/OdyseeTeam/odysee-api/models" 15 | "github.com/OdyseeTeam/odysee-api/pkg/iapi" 16 | "github.com/OdyseeTeam/odysee-api/pkg/migrator" 17 | "github.com/stretchr/testify/require" 18 | ) 19 | 20 | type TestUser struct { 21 | User *models.User 22 | SDKAddress string 23 | CurrentUser *auth.CurrentUser 24 | } 25 | 26 | type UserTestHelper struct { 27 | t *testing.T 28 | TokenHeader string 29 | DB *sql.DB 30 | SDKRouter *sdkrouter.Router 31 | Auther auth.Authenticator 32 | TestUser *TestUser 33 | } 34 | 35 | func (s *UserTestHelper) Setup(t *testing.T) error { 36 | t.Helper() 37 | require := require.New(t) 38 | s.t = t 39 | config.Override("LbrynetServers", "") 40 | 41 | db, dbCleanup, err := migrator.CreateTestDB(migrator.DBConfigFromApp(config.GetDatabase()), storage.MigrationsFS) 42 | require.NoError(err) 43 | storage.SetDB(db) 44 | 45 | sdkr := sdkrouter.New(config.GetLbrynetServers()) 46 | 47 | th, err := test.GetTestTokenHeader() 48 | require.NoError(err) 49 | 50 | auther, err := wallet.NewOauthAuthenticator( 51 | config.GetOauthProviderURL(), config.GetOauthClientID(), 52 | config.GetInternalAPIHost(), sdkr) 53 | require.NoError(err) 54 | 55 | w, err := test.InjectTestingWallet(test.TestUserID) 56 | require.NoError(err) 57 | t.Logf("set up wallet userid=%v", w.UserID) 58 | 59 | u, err := auther.Authenticate(th, "127.0.0.1") 60 | require.NoError(err) 61 | 62 | iac, err := iapi.NewClient( 63 | iapi.WithOAuthToken(strings.TrimPrefix(th, wallet.TokenPrefix)), 64 | iapi.WithRemoteIP("8.8.8.8"), 65 | ) 66 | require.NoError(err) 67 | 68 | cu := auth.NewCurrentUser(u, "8.8.8.8", iac, nil) 69 | 70 | t.Cleanup(func() { 71 | dbCleanup() 72 | w.Unload() 73 | w.RemoveFile() 74 | }) 75 | t.Cleanup(config.RestoreOverridden) 76 | 77 | s.Auther = auther 78 | s.SDKRouter = sdkr 79 | s.TokenHeader = th 80 | s.DB = db 81 | s.TestUser = &TestUser{ 82 | User: u, 83 | SDKAddress: sdkrouter.GetSDKAddress(u), 84 | CurrentUser: cu, 85 | } 86 | return nil 87 | } 88 | 89 | func (s *UserTestHelper) UserID() int { 90 | return s.TestUser.User.ID 91 | } 92 | -------------------------------------------------------------------------------- /internal/e2etest/wait.go: -------------------------------------------------------------------------------- 1 | package e2etest 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | var ErrWaitContinue = errors.New("keep waiting") 11 | 12 | func Wait(t *testing.T, description string, duration, interval time.Duration, run func() error) { 13 | t.Helper() 14 | ctx, cancel := context.WithTimeout(context.Background(), duration) 15 | defer cancel() 16 | wait := time.NewTicker(interval) 17 | Wait: 18 | for { 19 | select { 20 | case <-ctx.Done(): 21 | t.Logf("%s is taking too long", description) 22 | t.FailNow() 23 | case <-wait.C: 24 | err := run() 25 | if err != nil { 26 | if !errors.Is(err, ErrWaitContinue) { 27 | t.Logf("%s failed: %v", description, err) 28 | t.FailNow() 29 | } 30 | } else { 31 | break Wait 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /internal/errors/errors_test.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | base "errors" 5 | "fmt" 6 | "testing" 7 | 8 | pkg "github.com/pkg/errors" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestErr_MultipleLayersOfWrapping(t *testing.T) { 14 | orig := base.New("the base error") 15 | pkg1 := pkg.Wrap(orig, "wrapped pkg 1") 16 | our1 := Err(pkg1) 17 | pkg2 := pkg.Wrap(our1, "wrapped pkg 2") 18 | our2 := Err(pkg2) 19 | assert.True(t, base.Is(our1, orig)) 20 | assert.True(t, base.Is(our2, orig)) 21 | assert.True(t, base.Is(pkg2, orig)) 22 | assert.True(t, base.Is(our2, pkg1)) 23 | assert.True(t, base.Is(our2, our1)) 24 | } 25 | 26 | func TestRecover(t *testing.T) { 27 | var err error 28 | require.NotPanics(t, func() { 29 | err = func() (e error) { 30 | defer Recover(&e) 31 | doPanic() 32 | return nil 33 | }() 34 | }) 35 | 36 | assert.Error(t, err) 37 | assert.Contains(t, err.Error(), "who shall dwell in these worlds") 38 | 39 | withTrace, ok := err.(*traced) 40 | assert.True(t, ok) 41 | 42 | stackFrames := withTrace.StackFrames() 43 | fmt.Println(stackFrames) 44 | assert.Equal(t, "doYetDeeperPanic", stackFrames[0].Name) 45 | assert.Equal(t, "doDeeperPanic", stackFrames[1].Name) 46 | assert.Equal(t, "doPanic", stackFrames[2].Name) 47 | 48 | traceStr := Trace(err) 49 | assert.Contains(t, traceStr, "who shall dwell in these worlds") 50 | } 51 | 52 | func doPanic() { 53 | doDeeperPanic() 54 | } 55 | 56 | func doDeeperPanic() { 57 | doYetDeeperPanic() 58 | } 59 | 60 | func doYetDeeperPanic() { 61 | panic("But who shall dwell in these worlds if they be inhabited?… Are we or they Lords of the World?… And how are all things made for man?") 62 | } 63 | 64 | func TestErr_NilPointer(t *testing.T) { 65 | e := Err(nil) 66 | assert.NotPanics(t, func() { 67 | Unwrap(e) 68 | }) 69 | } 70 | -------------------------------------------------------------------------------- /internal/ip/ip_test.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | var expectedIPs = map[string]string{ 11 | "127.0.0.1, 203.0.113.195": "203.0.113.195", 12 | "127.0.0.1": "", 13 | "2001:db8:85a3:8d3:1319:8a2e:370:7348": "2001:db8:85a3:8d3:1319:8a2e:370:7348", 14 | "127.0.0.1, 2001:db8:85a3:8d3:1319:8a2e:370:7348": "2001:db8:85a3:8d3:1319:8a2e:370:7348", 15 | "127.0.0.1, 127.0.0.1, 127.0.0.1, 150.172.238.178": "150.172.238.178", 16 | "150.172.238.178, 127.0.0.1, 127.0.0.1, 127.0.0.1": "150.172.238.178", 17 | "150.172.238.178, 70.41.3.18, 127.0.0.1": "70.41.3.18", 18 | "127.0.0.1, 192.168.0.1, 70.41.3.18, 127.0.0.1": "70.41.3.18", 19 | } 20 | 21 | func TestForRequest(t *testing.T) { 22 | for val, exp := range expectedIPs { 23 | t.Run(val, func(t *testing.T) { 24 | r, _ := http.NewRequest(http.MethodGet, "", nil) 25 | r.Header.Add("X-Forwarded-For", val) 26 | assert.Equal(t, exp, ForRequest(r)) 27 | }) 28 | } 29 | t.Run("cf header", func(t *testing.T) { 30 | r, _ := http.NewRequest(http.MethodGet, "", nil) 31 | r.Header.Add("X-Forwarded-For", "70.70.70.70") 32 | r.Header.Add(CloudflareIPHeader, "80.80.80.80") 33 | assert.Equal(t, "80.80.80.80", ForRequest(r)) 34 | }) 35 | } 36 | -------------------------------------------------------------------------------- /internal/ip/middleware.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | ) 7 | 8 | type ctxKey int 9 | 10 | const contextKey ctxKey = iota 11 | 12 | // FromRequest retrieves remote user IP from http.Request that went through our middleware 13 | func FromRequest(r *http.Request) string { 14 | v := r.Context().Value(contextKey) 15 | if v == nil { 16 | return "" 17 | } 18 | return v.(string) 19 | } 20 | 21 | // Middleware will attach remote user IP to every request 22 | func Middleware(next http.Handler) http.Handler { 23 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 24 | remoteIP := ForRequest(r) 25 | next.ServeHTTP(w, r.Clone(context.WithValue(r.Context(), contextKey, remoteIP))) 26 | }) 27 | } 28 | -------------------------------------------------------------------------------- /internal/ip/middleware_test.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | 8 | "github.com/OdyseeTeam/odysee-api/internal/middleware" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestMiddleware(t *testing.T) { 14 | for val, exp := range expectedIPs { 15 | t.Run(val, func(t *testing.T) { 16 | r, _ := http.NewRequest(http.MethodGet, "", nil) 17 | r.Header.Add("X-Forwarded-For", val) 18 | 19 | rr := httptest.NewRecorder() 20 | mw := middleware.Apply(Middleware, func(w http.ResponseWriter, r *http.Request) { 21 | assert.Equal(t, exp, FromRequest(r)) 22 | }) 23 | mw.ServeHTTP(rr, r) 24 | }) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /internal/lbrynet/errors.go: -------------------------------------------------------------------------------- 1 | package lbrynet 2 | 3 | import ( 4 | "github.com/OdyseeTeam/odysee-api/internal/errors" 5 | 6 | ljsonrpc "github.com/lbryio/lbry.go/v2/extras/jsonrpc" 7 | ) 8 | 9 | type WalletError struct { 10 | // UserID int 11 | Err error 12 | } 13 | 14 | func (e WalletError) Error() string { return e.Err.Error() } 15 | func (e WalletError) Unwrap() error { return e.Err } 16 | 17 | var ( 18 | ErrWalletNotFound = errors.Base("wallet not found") 19 | ErrWalletExists = errors.Base("wallet exists and is loaded") 20 | ErrWalletNeedsLoading = errors.Base("wallet exists and needs to be loaded") 21 | ErrWalletNotLoaded = errors.Base("wallet is not loaded") 22 | ErrWalletAlreadyLoaded = errors.Base("wallet is already loaded") 23 | ) 24 | 25 | // NewWalletError converts plain SDK error to the typed one 26 | func NewWalletError(err error) error { 27 | var derr ljsonrpc.Error 28 | var ok bool 29 | if derr, ok = err.(ljsonrpc.Error); !ok { 30 | return WalletError{Err: err} 31 | } 32 | switch derr.Name { 33 | case ljsonrpc.ErrorWalletNotFound: 34 | return WalletError{Err: ErrWalletNotFound} 35 | case ljsonrpc.ErrorWalletAlreadyExists: 36 | return WalletError{Err: ErrWalletExists} 37 | case ljsonrpc.ErrorWalletNotLoaded: 38 | return WalletError{Err: ErrWalletNotLoaded} 39 | case ljsonrpc.ErrorWalletAlreadyLoaded: 40 | return WalletError{Err: ErrWalletAlreadyLoaded} 41 | default: 42 | return WalletError{Err: err} 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /internal/lbrynet/errors_test.go: -------------------------------------------------------------------------------- 1 | package lbrynet 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/OdyseeTeam/odysee-api/internal/errors" 7 | ljsonrpc "github.com/lbryio/lbry.go/v2/extras/jsonrpc" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestWalletAlreadyLoaded(t *testing.T) { 13 | err := ljsonrpc.Error{ 14 | Code: 123, 15 | Name: ljsonrpc.ErrorWalletAlreadyLoaded, 16 | Message: "Wallet 123.wallet is already loaded", 17 | } 18 | assert.True(t, errors.Is(NewWalletError(err), ErrWalletAlreadyLoaded)) 19 | } 20 | -------------------------------------------------------------------------------- /internal/metrics/handlers.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | 7 | "github.com/OdyseeTeam/odysee-api/internal/monitor" 8 | "github.com/OdyseeTeam/odysee-api/internal/responses" 9 | 10 | "github.com/spf13/cast" 11 | ) 12 | 13 | var Logger = monitor.NewModuleLogger("metrics") 14 | 15 | func TrackUIMetric(w http.ResponseWriter, req *http.Request) { 16 | responses.AddJSONContentType(w) 17 | resp := make(map[string]string) 18 | code := http.StatusOK 19 | 20 | metricName := req.FormValue("name") 21 | resp["name"] = metricName 22 | 23 | switch metricName { 24 | case "time_to_start": 25 | player := req.FormValue("player") 26 | if len(player) > 64 { 27 | code = http.StatusBadRequest 28 | resp["error"] = "invalid player value" 29 | } 30 | UITimeToStart.WithLabelValues(player).Observe(cast.ToFloat64(req.FormValue("value"))) 31 | default: 32 | code = http.StatusBadRequest 33 | resp["error"] = "invalid metric name" 34 | } 35 | 36 | w.WriteHeader(code) 37 | respByte, _ := json.Marshal(&resp) 38 | w.Write(respByte) 39 | } 40 | -------------------------------------------------------------------------------- /internal/metrics/handlers_test.go: -------------------------------------------------------------------------------- 1 | package metrics_test 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "os" 7 | "testing" 8 | 9 | "github.com/OdyseeTeam/odysee-api/api" 10 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 11 | "github.com/Pallinder/go-randomdata" 12 | 13 | "github.com/gorilla/mux" 14 | "github.com/stretchr/testify/assert" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | func TestInvalidEvent(t *testing.T) { 19 | name := "win-the-lottery" 20 | rr := testMetricUIEvent(t, http.MethodPost, name, map[string]string{}) 21 | assert.Equal(t, http.StatusBadRequest, rr.Code) 22 | assert.Equal(t, `{"error":"invalid metric name","name":"`+name+`"}`, rr.Body.String()) 23 | } 24 | 25 | func TestInvalidMethod(t *testing.T) { 26 | rr := testMetricUIEvent(t, http.MethodGet, "", map[string]string{}) 27 | assert.Equal(t, http.StatusNotFound, rr.Code) 28 | } 29 | 30 | func TestTimeToStartEvent(t *testing.T) { 31 | name := "time_to_start" 32 | rr := testMetricUIEvent(t, http.MethodPost, name, map[string]string{"value": "0.3"}) 33 | assert.Equal(t, http.StatusOK, rr.Code) 34 | assert.Equal(t, `{"name":"`+name+`"}`, rr.Body.String()) 35 | 36 | rr = testMetricUIEvent(t, http.MethodPost, name, map[string]string{"value": "0.3", "player": "sg-p1"}) 37 | assert.Equal(t, http.StatusOK, rr.Code) 38 | assert.Equal(t, `{"name":"`+name+`"}`, rr.Body.String()) 39 | 40 | rr = testMetricUIEvent(t, http.MethodPost, name, map[string]string{"value": "0.3", "player": randomdata.Alphanumeric(96)}) 41 | assert.Equal(t, http.StatusBadRequest, rr.Code) 42 | assert.Equal(t, `{"error":"invalid player value","name":"`+name+`"}`, rr.Body.String()) 43 | } 44 | 45 | func testMetricUIEvent(t *testing.T, method, name string, params map[string]string) *httptest.ResponseRecorder { 46 | req, err := http.NewRequest(method, "/api/v1/metric/ui", nil) 47 | require.NoError(t, err) 48 | 49 | q := req.URL.Query() 50 | q.Add("name", name) 51 | for k, v := range params { 52 | q.Add(k, v) 53 | } 54 | req.URL.RawQuery = q.Encode() 55 | 56 | // override this to temp to avoid permission error when running tests on 57 | // restricted environment. 58 | config.Config.Override("PublishSourceDir", os.TempDir()) 59 | 60 | r := mux.NewRouter() 61 | api.InstallRoutes(r, nil, nil) 62 | rr := httptest.NewRecorder() 63 | r.ServeHTTP(rr, req) 64 | return rr 65 | } 66 | -------------------------------------------------------------------------------- /internal/metrics/middleware.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/OdyseeTeam/odysee-api/internal/errors" 8 | "github.com/gorilla/mux" 9 | "github.com/prometheus/client_golang/prometheus" 10 | ) 11 | 12 | type key int 13 | 14 | const timerContextKey key = iota 15 | 16 | // MeasureMiddleware starts a timer whenever a request is performed. 17 | // It should be added as first in the chain of middlewares. 18 | // Note that it doesn't catch any metrics by itself, 19 | // HTTP handlers are expected to add their own by calling AddObserver. 20 | func MeasureMiddleware() mux.MiddlewareFunc { 21 | return func(next http.Handler) http.Handler { 22 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 23 | t := StartTimer() 24 | Logger.Log().Debugf("timer %s started", t.String()) 25 | rc := r.Clone(context.WithValue(r.Context(), timerContextKey, t)) 26 | defer func() { 27 | t.Stop() 28 | Logger.Log().Debugf("timer %s stopped (%.6fs)", t.String(), t.Duration()) 29 | }() 30 | 31 | next.ServeHTTP(w, rc) 32 | }) 33 | } 34 | } 35 | 36 | // AddObserver adds Prometheus metric to a chain of observers for a given HTTP request. 37 | func AddObserver(r *http.Request, o prometheus.Observer) error { 38 | v := r.Context().Value(timerContextKey) 39 | if v == nil { 40 | return errors.Err("metrics.MeasureMiddleware middleware is required") 41 | } 42 | t := v.(*Timer) 43 | t.AddObserver(o) 44 | return nil 45 | } 46 | 47 | // GetDuration returns current duration of the request in seconds. 48 | // Returns a negative value when MeasureMiddleware middleware is not present. 49 | func GetDuration(r *http.Request) float64 { 50 | v := r.Context().Value(timerContextKey) 51 | if v == nil { 52 | return -1 53 | } 54 | t := v.(*Timer) 55 | return t.Duration() 56 | } 57 | -------------------------------------------------------------------------------- /internal/metrics/operations.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | ) 8 | 9 | type Operation struct { 10 | started time.Time 11 | duration float64 12 | name string 13 | labels prometheus.Labels 14 | } 15 | 16 | func StartOperation(name, tag string) Operation { 17 | return Operation{started: time.Now(), name: name, labels: prometheus.Labels{"name": name, "tag": tag}} 18 | } 19 | 20 | func (o Operation) DurationSeconds() float64 { 21 | return time.Since(o.started).Seconds() 22 | } 23 | 24 | func (o Operation) End() { 25 | o.duration = time.Since(o.started).Seconds() 26 | operations.With(o.labels).Observe(o.duration) 27 | } 28 | -------------------------------------------------------------------------------- /internal/metrics/operations_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "math" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestOperation(t *testing.T) { 12 | op := StartOperation("db", "wallet") 13 | 14 | time.Sleep(20 * time.Millisecond) 15 | 16 | op.End() 17 | 18 | m := GetMetric(operations) 19 | assert.Equal(t, float64(0.02), math.Floor(m.Summary.GetSampleSum()*100)/100) 20 | assert.Equal(t, "name", *m.Label[0].Name) 21 | assert.Equal(t, "db", *m.Label[0].Value) 22 | assert.Equal(t, "tag", *m.Label[1].Name) 23 | assert.Equal(t, "wallet", *m.Label[1].Value) 24 | } 25 | -------------------------------------------------------------------------------- /internal/metrics/timer.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/prometheus/client_golang/prometheus" 8 | ) 9 | 10 | type Timer struct { 11 | Started time.Time 12 | duration float64 13 | observers []prometheus.Observer 14 | } 15 | 16 | func StartTimer() *Timer { 17 | return &Timer{Started: time.Now()} 18 | } 19 | 20 | func (t *Timer) AddObserver(o prometheus.Observer) { 21 | t.observers = append(t.observers, o) 22 | } 23 | 24 | func (t *Timer) Stop() float64 { 25 | if t.duration == 0 { 26 | t.duration = time.Since(t.Started).Seconds() 27 | for _, o := range t.observers { 28 | o.Observe(t.duration) 29 | } 30 | } 31 | return t.duration 32 | } 33 | 34 | func (t *Timer) Duration() float64 { 35 | if t.duration == 0 { 36 | return time.Since(t.Started).Seconds() 37 | } 38 | return t.duration 39 | } 40 | 41 | func (t *Timer) String() string { 42 | return fmt.Sprintf("%.2f", t.Duration()) 43 | } 44 | -------------------------------------------------------------------------------- /internal/middleware/middleware.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/gorilla/mux" 7 | ) 8 | 9 | // Chain chains multiple middleware together 10 | func Chain(mws ...mux.MiddlewareFunc) mux.MiddlewareFunc { 11 | return func(next http.Handler) http.Handler { 12 | for i := len(mws) - 1; i >= 0; i-- { 13 | next = mws[i](next) // apply in reverse to get the intuitive LIFO order 14 | } 15 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 16 | next.ServeHTTP(w, r) 17 | }) 18 | } 19 | } 20 | 21 | // Apply applies middlewares to HandlerFunc 22 | func Apply(mw mux.MiddlewareFunc, handler http.HandlerFunc) http.Handler { 23 | return mw(handler) 24 | } 25 | -------------------------------------------------------------------------------- /internal/monitor/module_logger.go: -------------------------------------------------------------------------------- 1 | package monitor 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | 7 | "github.com/sirupsen/logrus" 8 | ) 9 | 10 | // ModuleLogger contains module-specific logger details. 11 | type ModuleLogger struct { 12 | Entry *logrus.Entry 13 | } 14 | 15 | // NewModuleLogger creates a new ModuleLogger instance carrying module name 16 | // for later `Log()` calls. 17 | func NewModuleLogger(moduleName string) ModuleLogger { 18 | l := logrus.New() 19 | configureLogLevelAndFormat(l) 20 | fields := logrus.Fields{ 21 | "module": moduleName, 22 | } 23 | hostname := os.Getenv("HOSTNAME") 24 | if hostname != "" { 25 | fields["host"] = hostname 26 | } 27 | return ModuleLogger{ 28 | Entry: l.WithFields(fields), 29 | } 30 | } 31 | 32 | // WithFields returns a new log entry containing additional info provided by fields, 33 | // which can be called upon with a corresponding logLevel. 34 | // Example: 35 | // 36 | // logger.WithFields(F{"query": "..."}).Info("query error") 37 | func (m ModuleLogger) WithFields(fields logrus.Fields) *logrus.Entry { 38 | if v, ok := fields[TokenF]; ok && v != "" && isProduction() { 39 | fields[TokenF] = valueMask 40 | } 41 | return m.Entry.WithFields(fields) 42 | } 43 | 44 | // Log returns a new log entry for the module 45 | // which can be called upon with a corresponding logLevel. 46 | // Example: 47 | // 48 | // Log().Info("query error") 49 | func (m ModuleLogger) Log() *logrus.Entry { 50 | return m.Entry.WithFields(nil) 51 | } 52 | 53 | // Disable turns off logging output for this module logger 54 | func (m ModuleLogger) Disable() { 55 | m.Entry.Logger.SetLevel(logrus.PanicLevel) 56 | m.Entry.Logger.SetOutput(ioutil.Discard) 57 | } 58 | -------------------------------------------------------------------------------- /internal/monitor/monitor.go: -------------------------------------------------------------------------------- 1 | package monitor 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 7 | "github.com/OdyseeTeam/odysee-api/version" 8 | 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | var logger = NewModuleLogger("monitor") 13 | var IsProduction = false 14 | 15 | const ( 16 | // TokenF is a token field name that will be stripped from logs in production mode. 17 | TokenF = "token" 18 | // valueMask is what replaces sensitive fields contents in logs. 19 | valueMask = "****" 20 | ) 21 | 22 | var jsonFormatter = logrus.JSONFormatter{DisableTimestamp: true} 23 | var textFormatter = logrus.TextFormatter{FullTimestamp: true, TimestampFormat: "15:04:05"} 24 | 25 | // init magic is needed so logging is set up without calling it in every package explicitly 26 | func init() { 27 | l := logrus.StandardLogger() 28 | configureLogLevelAndFormat(l) 29 | 30 | l.WithFields( 31 | version.BuildInfo(), 32 | ).WithFields(logrus.Fields{ 33 | "mode": LogMode(), 34 | "logLevel": l.Level, 35 | }).Infof("standard logger configured") 36 | } 37 | 38 | func isProduction() bool { 39 | return config.IsProduction() 40 | } 41 | 42 | func LogMode() string { 43 | if isProduction() { 44 | return "production" 45 | } 46 | return "develop" 47 | } 48 | 49 | func configureLogLevelAndFormat(l *logrus.Logger) { 50 | if isProduction() { 51 | l.SetLevel(logrus.InfoLevel) 52 | l.SetFormatter(&jsonFormatter) 53 | } else { 54 | l.SetLevel(logrus.TraceLevel) 55 | l.SetFormatter(&textFormatter) 56 | } 57 | } 58 | 59 | // LogSuccessfulQuery takes a remote method name, execution time and params and logs it 60 | func LogSuccessfulQuery(method string, time float64, params interface{}, response interface{}) { 61 | fields := logrus.Fields{ 62 | "method": method, 63 | "duration": fmt.Sprintf("%.3f", time), 64 | "params": params, 65 | } 66 | // if config.ShouldLogResponses() { 67 | // fields["response"] = response 68 | // } 69 | logger.WithFields(fields).Info("call processed") 70 | } 71 | -------------------------------------------------------------------------------- /internal/monitor/sentry.go: -------------------------------------------------------------------------------- 1 | package monitor 2 | 3 | import ( 4 | "github.com/OdyseeTeam/odysee-api/internal/responses" 5 | 6 | "github.com/getsentry/sentry-go" 7 | ) 8 | 9 | var ignored = []string{ 10 | responses.AuthRequiredErrorMessage, 11 | } 12 | 13 | func ConfigureSentry(dsn, release, env string) { 14 | if dsn == "" { 15 | logger.Log().Info("sentry disabled (no DSN configured)") 16 | return 17 | } 18 | 19 | if err := sentry.Init(sentry.ClientOptions{ 20 | Dsn: dsn, 21 | Release: release, 22 | Environment: env, 23 | AttachStacktrace: true, 24 | IgnoreErrors: ignored, 25 | // TracesSampleRate: .5, 26 | }); err != nil { 27 | logger.Log().Warnf("sentry initialization failed: %v", err) 28 | return 29 | } 30 | logger.Log().Info("sentry initialized") 31 | } 32 | 33 | // ErrorToSentry sends to Sentry general exception info with some optional extra detail (like user email, claim url etc) 34 | func ErrorToSentry(err error, params ...map[string]string) *sentry.EventID { 35 | var extra map[string]string 36 | var eventID *sentry.EventID 37 | if len(params) > 0 { 38 | extra = params[0] 39 | } else { 40 | extra = map[string]string{} 41 | } 42 | 43 | sentry.WithScope(func(scope *sentry.Scope) { 44 | for k, v := range extra { 45 | scope.SetExtra(k, v) 46 | } 47 | sentry.CaptureException(err) 48 | }) 49 | return eventID 50 | } 51 | 52 | func MessageToSentry(msg string, level sentry.Level, params map[string]string) *sentry.EventID { 53 | var eventID *sentry.EventID 54 | sentry.WithScope(func(scope *sentry.Scope) { 55 | for k, v := range params { 56 | scope.SetExtra(k, v) 57 | } 58 | event := sentry.NewEvent() 59 | event.Level = level 60 | event.Message = msg 61 | eventID = sentry.CaptureEvent(event) 62 | }) 63 | return eventID 64 | } 65 | -------------------------------------------------------------------------------- /internal/responses/responses.go: -------------------------------------------------------------------------------- 1 | package responses 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | 7 | "github.com/OdyseeTeam/odysee-api/internal/errors" 8 | 9 | "github.com/ybbus/jsonrpc/v2" 10 | ) 11 | 12 | // this is the message to show when authentication info is required but was not provided in the request 13 | // this is NOT the message for when auth info is provided but is not correct 14 | const AuthRequiredErrorMessage = "authentication required" 15 | 16 | // AddJSONContentType prepares HTTP response writer for JSON content-type. 17 | func AddJSONContentType(w http.ResponseWriter) { 18 | w.Header().Add("content-type", "application/json; charset=utf-8") 19 | } 20 | 21 | func WriteJSON(w http.ResponseWriter, d any) { 22 | rb, err := json.MarshalIndent(d, "", " ") 23 | if err != nil { 24 | w.Write([]byte("error marshaling object: " + err.Error())) 25 | return 26 | } 27 | AddJSONContentType(w) 28 | w.Write(rb) 29 | } 30 | 31 | func JSONRPCSerialize(r *jsonrpc.RPCResponse) ([]byte, error) { 32 | var ( 33 | b []byte 34 | e error 35 | ) 36 | defer errors.Recover(&e) 37 | b, err := json.MarshalIndent(r, "", " ") 38 | if e != nil { 39 | return b, e 40 | } else if err != nil { 41 | return b, err 42 | } 43 | return b, nil 44 | } 45 | -------------------------------------------------------------------------------- /internal/storage/migrations/0001_initial.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | 3 | -- +migrate StatementBegin 4 | CREATE DOMAIN uinteger AS integer 5 | CHECK(VALUE >= 0); 6 | -- +migrate StatementEnd 7 | 8 | -- +migrate StatementBegin 9 | CREATE TABLE "users" ( 10 | "id" uinteger NOT NULL PRIMARY KEY, 11 | 12 | "created_at" timestamp NOT NULL DEFAULT now(), 13 | "updated_at" timestamp NOT NULL DEFAULT now(), 14 | 15 | "sdk_account_id" varchar NOT NULL, 16 | "private_key" varchar NOT NULL, 17 | "public_key" varchar NOT NULL, 18 | "seed" varchar NOT NULL, 19 | 20 | UNIQUE ("sdk_account_id") 21 | ); 22 | -- +migrate StatementEnd 23 | 24 | -- +migrate Down 25 | 26 | -- +migrate StatementBegin 27 | DROP TABLE "users"; 28 | -- +migrate StatementEnd 29 | 30 | -- +migrate StatementBegin 31 | DROP DOMAIN uinteger; 32 | -- +migrate StatementEnd 33 | -------------------------------------------------------------------------------- /internal/storage/migrations/0002_wallet_id.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | 3 | -- +migrate StatementBegin 4 | ALTER TABLE "users" 5 | DROP COLUMN "private_key", 6 | DROP COLUMN "public_key", 7 | DROP COLUMN "seed", 8 | 9 | ADD COLUMN "wallet_id" varchar NOT NULL DEFAULT '', 10 | 11 | ALTER COLUMN "sdk_account_id" DROP NOT NULL 12 | ; 13 | -- +migrate StatementEnd 14 | 15 | -- +migrate Down 16 | 17 | -- +migrate StatementBegin 18 | ALTER TABLE "users" 19 | ADD COLUMN "private_key" varchar NOT NULL DEFAULT '', 20 | ADD COLUMN "public_key" varchar NOT NULL DEFAULT '', 21 | ADD COLUMN "seed" varchar NOT NULL DEFAULT '', 22 | 23 | DROP COLUMN "wallet_id" 24 | ; 25 | -- +migrate StatementEnd 26 | -------------------------------------------------------------------------------- /internal/storage/migrations/0003_sdk_router.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | 3 | -- +migrate StatementBegin 4 | CREATE TABLE lbrynet_servers ( 5 | id SERIAL NOT NULL PRIMARY KEY, 6 | name VARCHAR NOT NULL, 7 | address VARCHAR NOT NULL, 8 | weight INTEGER NOT NULL DEFAULT 0, 9 | created_at TIMESTAMP NOT NULL DEFAULT now(), 10 | updated_at TIMESTAMP NOT NULL DEFAULT now(), 11 | UNIQUE (name) 12 | ); 13 | -- +migrate StatementEnd 14 | 15 | -- +migrate StatementBegin 16 | ALTER TABLE users 17 | ADD COLUMN lbrynet_server_id INTEGER DEFAULT NULL REFERENCES lbrynet_servers(id) ON DELETE SET NULL; 18 | -- +migrate StatementEnd 19 | 20 | -- +migrate StatementBegin 21 | -- This is for *local testing only*, you want to replace 22 | -- these records with actual SDK addresses in a server environment. 23 | INSERT INTO lbrynet_servers(name, address) 24 | VALUES 25 | ('default', 'http://localhost:15279/'), 26 | ('lbrynet2', 'http://localhost:15279/'), 27 | ('lbrynet3', 'http://localhost:15279/'), 28 | ('lbrynet4', 'http://localhost:15279/'), 29 | ('lbrynet5', 'http://localhost:15279/'); 30 | -- +migrate StatementEnd 31 | 32 | -- +migrate StatementBegin 33 | UPDATE users SET lbrynet_server_id = 1; 34 | -- +migrate StatementEnd 35 | 36 | -- +migrate Down 37 | 38 | -- +migrate StatementBegin 39 | ALTER TABLE users 40 | DROP COLUMN lbrynet_server_id; 41 | -- +migrate StatementEnd 42 | 43 | -- +migrate StatementBegin 44 | DROP TABLE lbrynet_servers; 45 | -- +migrate StatementEnd 46 | -------------------------------------------------------------------------------- /internal/storage/migrations/0004_drop_wallet_id.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | 3 | UPDATE users SET lbrynet_server_id = (id % (SELECT COUNT(id) from lbrynet_servers))+1 WHERE lbrynet_server_id IS NULL; 4 | ALTER TABLE users DROP COLUMN wallet_id; 5 | 6 | -- +migrate Down 7 | 8 | ALTER TABLE "users" ADD COLUMN "wallet_id" varchar NOT NULL DEFAULT ''; 9 | -------------------------------------------------------------------------------- /internal/storage/migrations/0005_wallet_access.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | 3 | ALTER TABLE users ADD COLUMN "last_seen_at" timestamp DEFAULT NULL; 4 | CREATE INDEX users_last_seen_at_idx ON users(last_seen_at); 5 | 6 | 7 | -- +migrate Down 8 | 9 | ALTER TABLE users DROP COLUMN "last_seen_at"; 10 | -------------------------------------------------------------------------------- /internal/storage/migrations/0006_audit_initial.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | 3 | -- +migrate StatementBegin 4 | CREATE TABLE "query_log" ( 5 | "id" SERIAL PRIMARY KEY, 6 | "method" varchar NOT NULL, 7 | "timestamp" timestamp NOT NULL DEFAULT now(), 8 | "user_id" uinteger, 9 | 10 | "remote_ip" varchar NOT NULL, 11 | "body" jsonb 12 | ); 13 | CREATE INDEX queries_method_idx ON query_log(method); 14 | CREATE INDEX queries_timestamp_idx ON query_log(timestamp); 15 | CREATE INDEX queries_user_id_idx ON query_log(user_id); 16 | CREATE INDEX queries_remote_ip_idx ON query_log(remote_ip); 17 | -- +migrate StatementEnd 18 | 19 | -- +migrate Down 20 | 21 | -- +migrate StatementBegin 22 | DROP TABLE "query_log"; 23 | -- +migrate StatementEnd 24 | -------------------------------------------------------------------------------- /internal/storage/migrations/0007_private_lbrynet.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | 3 | -- +migrate StatementBegin 4 | ALTER TABLE lbrynet_servers 5 | ADD COLUMN "private" BOOLEAN NOT NULL DEFAULT false; 6 | -- +migrate StatementEnd 7 | 8 | -- +migrate Down 9 | 10 | -- +migrate StatementBegin 11 | ALTER TABLE lbrynet_servers 12 | DROP COLUMN "private"; 13 | -- +migrate StatementEnd 14 | -------------------------------------------------------------------------------- /internal/storage/migrations/0008_oauth_integration.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | 3 | -- +migrate StatementBegin 4 | ALTER TABLE users ADD COLUMN idp_id varchar DEFAULT NULL; 5 | -- +migrate StatementEnd 6 | 7 | -- +migrate StatementBegin 8 | CREATE INDEX users_idp_id_idx ON users(idp_id); 9 | -- +migrate StatementEnd 10 | 11 | -- +migrate Down 12 | 13 | -- +migrate StatementBegin 14 | ALTER TABLE users DROP COLUMN idp_id; 15 | -- +migrate StatementEnd 16 | -------------------------------------------------------------------------------- /internal/storage/migrations/0009_uploads.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | 3 | CREATE TYPE upload_status AS ENUM ( 4 | 'created', 5 | 'uploading', 6 | 'received', 7 | -- 'analyzed', 8 | -- 'split', 9 | -- 'reflected', 10 | -- 'posted', -- query sent 11 | 'terminated', 12 | 'abandoned', 13 | 'failed', 14 | 'finished' 15 | ); 16 | 17 | CREATE TYPE publish_query_status AS ENUM ( 18 | 'received', 19 | 'forwarded', 20 | 'failed', 21 | 'succeeded' 22 | ); 23 | 24 | CREATE TABLE uploads ( 25 | id text NOT NULL UNIQUE PRIMARY KEY CHECK (id <> ''), 26 | user_id int REFERENCES users(id) ON DELETE SET NULL, 27 | path text NOT NULL, 28 | 29 | created_at timestamp NOT NULL DEFAULT NOW(), 30 | updated_at timestamp, 31 | 32 | status upload_status NOT NULL, 33 | error text NOT NULL, 34 | 35 | size bigint NOT NULL CHECK (size > 0), 36 | received bigint NOT NULL DEFAULT 0 37 | ); 38 | 39 | CREATE TABLE publish_queries ( 40 | upload_id text NOT NULL PRIMARY KEY references uploads(id), 41 | 42 | created_at timestamp NOT NULL DEFAULT NOW(), 43 | updated_at timestamp, 44 | 45 | status publish_query_status NOT NULL, 46 | error text NOT NULL, 47 | 48 | query jsonb, 49 | response jsonb 50 | ); 51 | 52 | -- +migrate Down 53 | DROP TABLE publish_queries; 54 | DROP TABLE uploads; 55 | DROP TYPE upload_status; 56 | DROP TYPE publish_query_status; 57 | -------------------------------------------------------------------------------- /internal/storage/migrations/0010_asynqueries.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | -- +migrate StatementBegin 3 | CREATE TYPE asynquery_status AS ENUM ( 4 | 'received', 5 | 'preparing', 6 | 'forwarded', 7 | 'failed', 8 | 'succeeded' 9 | ); 10 | 11 | CREATE TABLE asynqueries ( 12 | id text NOT NULL UNIQUE PRIMARY KEY CHECK (id <> ''), 13 | user_id int REFERENCES users (id) NOT NULL, 14 | created_at timestamp NOT NULL DEFAULT NOW(), 15 | updated_at timestamp, 16 | 17 | status asynquery_status NOT NULL, 18 | error text NOT NULL, 19 | upload_id text NOT NULL UNIQUE, 20 | 21 | body jsonb, 22 | response jsonb 23 | ); 24 | 25 | CREATE INDEX asynqueries_id_user_id ON asynqueries(id, user_id); 26 | CREATE INDEX asynqueries_user_id_upload_id ON asynqueries(user_id, upload_id); 27 | -- +migrate StatementEnd 28 | 29 | -- +migrate Down 30 | -- +migrate StatementBegin 31 | DROP TABLE asynqueries; 32 | DROP TYPE asynquery_status; 33 | -- +migrate StatementEnd 34 | -------------------------------------------------------------------------------- /internal/storage/migrations/0011_drop_unique_upload.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | -- +migrate StatementBegin 3 | ALTER TABLE asynqueries 4 | DROP CONSTRAINT asynqueries_upload_id_key; 5 | -- +migrate StatementEnd 6 | 7 | -- +migrate Down 8 | -- +migrate StatementBegin 9 | ALTER TABLE asynqueries 10 | ADD CONSTRAINT asynqueries_upload_id_key UNIQUE (upload_id); 11 | -- +migrate StatementEnd 12 | -------------------------------------------------------------------------------- /internal/storage/storage.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "database/sql" 5 | "embed" 6 | "time" 7 | 8 | "github.com/OdyseeTeam/odysee-api/internal/metrics" 9 | "github.com/OdyseeTeam/odysee-api/pkg/migrator" 10 | "github.com/volatiletech/sqlboiler/boil" 11 | ) 12 | 13 | //go:embed migrations/*.sql 14 | var MigrationsFS embed.FS 15 | 16 | var DB *sql.DB 17 | var Migrator migrator.Migrator 18 | 19 | func SetDB(db *sql.DB) { 20 | boil.SetDB(db) 21 | DB = db 22 | Migrator = migrator.New(db, MigrationsFS) 23 | } 24 | 25 | func WatchMetrics(interval time.Duration) { 26 | t := time.NewTicker(interval) 27 | for { 28 | <-t.C 29 | stats := DB.Stats() 30 | metrics.LbrytvDBOpenConnections.Set(float64(stats.OpenConnections)) 31 | metrics.LbrytvDBInUseConnections.Set(float64(stats.InUse)) 32 | metrics.LbrytvDBIdleConnections.Set(float64(stats.Idle)) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /internal/tasks/tasks.go: -------------------------------------------------------------------------------- 1 | package tasks 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | const ( 8 | AsynqueryIncomingQuery = "asynquery:incoming" 9 | ForkliftUploadIncoming = "forklift:upload:incoming" 10 | ForkliftURLIncoming = "forklift:url:incoming" 11 | ForkliftUploadDone = "forklift:upload:done" 12 | ) 13 | 14 | type AsynqueryIncomingQueryPayload struct { 15 | QueryID string `json:"query_id"` 16 | UserID int `json:"user_id"` 17 | } 18 | 19 | type ForkliftUploadDonePayload struct { 20 | UploadID string `json:"upload_id"` 21 | UserID int32 `json:"user_id"` 22 | Meta UploadMeta 23 | } 24 | 25 | type ForkliftUploadIncomingPayload struct { 26 | UserID int32 `json:"user_id"` 27 | UploadID string `json:"upload_id"` 28 | FileName string `json:"file_name"` 29 | FileLocation FileLocationS3 `json:"file_location"` 30 | } 31 | 32 | type ForkliftURLIncomingPayload struct { 33 | UserID int32 `json:"user_id"` 34 | UploadID string `json:"upload_id"` 35 | FileName string `json:"file_name"` 36 | FileLocation FileLocationHTTP `json:"file_location"` 37 | } 38 | 39 | type FileLocationS3 struct { 40 | Bucket string 41 | Key string 42 | } 43 | 44 | type FileLocationHTTP struct { 45 | URL string 46 | } 47 | 48 | type UploadMeta struct { 49 | Size uint64 50 | FileName string `json:"file_name"` 51 | SDHash string `json:"sd_hash"` 52 | MIME string 53 | Extension string 54 | Hash string 55 | Duration int `json:",omitempty"` 56 | Width int `json:",omitempty"` 57 | Height int `json:",omitempty"` 58 | } 59 | 60 | func (p AsynqueryIncomingQueryPayload) GetTraceData() map[string]string { 61 | return map[string]string{ 62 | "user_id": strconv.Itoa(int(p.UserID)), 63 | "query_id": p.QueryID, 64 | } 65 | } 66 | 67 | func (p ForkliftUploadDonePayload) GetTraceData() map[string]string { 68 | return map[string]string{ 69 | "user_id": strconv.Itoa(int(p.UserID)), 70 | "upload_id": p.UploadID, 71 | } 72 | } 73 | 74 | func (p ForkliftUploadIncomingPayload) GetTraceData() map[string]string { 75 | return map[string]string{ 76 | "user_id": strconv.Itoa(int(p.UserID)), 77 | "upload_id": p.UploadID, 78 | } 79 | } 80 | 81 | func (p ForkliftURLIncomingPayload) GetTraceData() map[string]string { 82 | return map[string]string{ 83 | "user_id": strconv.Itoa(int(p.UserID)), 84 | "url": p.FileLocation.URL, 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /internal/test/auth.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 9 | 10 | "golang.org/x/oauth2" 11 | ) 12 | 13 | const ( 14 | TestUserID = 418533549 15 | 16 | testClientID = "ci-tester" 17 | 18 | envClientSecret = "OAUTH_TEST_CLIENT_SECRET" 19 | envUsername = "OAUTH_TEST_USERNAME" 20 | envPassword = "OAUTH_TEST_PASSWORD" 21 | 22 | msgMissingEnv = "test oauth client env var %s is not set" 23 | ) 24 | 25 | // GetTestToken is for easily retrieving tokens that can be used in tests utilizing authentication subsystem. 26 | func GetTestToken() (*oauth2.Token, error) { 27 | clientSecret := os.Getenv(envClientSecret) 28 | username := os.Getenv(envUsername) 29 | password := os.Getenv(envPassword) 30 | if clientSecret == "" { 31 | return nil, fmt.Errorf(msgMissingEnv, envClientSecret) 32 | } 33 | if username == "" { 34 | return nil, fmt.Errorf(msgMissingEnv, envUsername) 35 | } 36 | if password == "" { 37 | return nil, fmt.Errorf(msgMissingEnv, envPassword) 38 | } 39 | 40 | ctx := context.Background() 41 | conf := &oauth2.Config{ 42 | // ClientID: config.GetOauthClientID(), 43 | ClientID: testClientID, 44 | ClientSecret: clientSecret, 45 | Endpoint: oauth2.Endpoint{TokenURL: config.GetOauthTokenURL()}, 46 | } 47 | return conf.PasswordCredentialsToken(ctx, username, password) 48 | } 49 | 50 | func GetTestTokenHeader() (string, error) { 51 | t, err := GetTestToken() 52 | if err != nil { 53 | return "", err 54 | } 55 | return "Bearer " + t.AccessToken, nil 56 | } 57 | -------------------------------------------------------------------------------- /internal/test/auth_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/OdyseeTeam/odysee-api/app/wallet" 7 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 8 | 9 | "github.com/stretchr/testify/require" 10 | "golang.org/x/oauth2" 11 | ) 12 | 13 | func TestGetTestToken(t *testing.T) { 14 | token, err := GetTestToken() 15 | require.NoError(t, err) 16 | 17 | _, err = wallet.NewOauthAuthenticator(config.GetOauthProviderURL(), config.GetOauthClientID(), config.GetInternalAPIHost(), nil) 18 | require.NoError(t, err) 19 | 20 | remoteUser, err := wallet.GetRemoteUser(oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token.AccessToken}), "") 21 | require.NoError(t, err) 22 | require.EqualValues(t, TestUserID, remoteUser.ID) 23 | } 24 | -------------------------------------------------------------------------------- /internal/test/json.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "regexp" 8 | "testing" 9 | 10 | "github.com/nsf/jsondiff" 11 | ) 12 | 13 | var testConsoleOptions = jsondiff.Options{ 14 | Added: jsondiff.Tag{Begin: "++", End: "++"}, 15 | Removed: jsondiff.Tag{Begin: "--", End: "--"}, 16 | Changed: jsondiff.Tag{Begin: "[[", End: "]]"}, 17 | Indent: " ", 18 | } 19 | 20 | // JSONCompact removes insignificant space characters from a JSON string. 21 | // It helps compare JSON strings without worrying about whitespace differences. 22 | func JSONCompact(jsonStr string) string { 23 | dst := &bytes.Buffer{} 24 | err := json.Compact(dst, []byte(jsonStr)) 25 | if err != nil { 26 | panic(err) 27 | } 28 | return dst.String() 29 | } 30 | 31 | // GetJSONDiffLog compares two JSON strings or bytes and returns `false` if they match 32 | // or `true` if they don't, plus difference log in a text format suitable for console output. 33 | func GetJSONDiffLog(expected, actual interface{}) (bool, string) { 34 | result, diff := jsondiff.Compare(toBytes(expected), toBytes(actual), &testConsoleOptions) 35 | differs := result != jsondiff.FullMatch 36 | 37 | if !differs { 38 | return false, "" 39 | } 40 | 41 | return differs, diff 42 | } 43 | 44 | // AssertEqualJSON is assert.Equal equivalent for JSON with better comparison and diff output. 45 | func AssertEqualJSON(t *testing.T, expected, actual interface{}, msgAndArgs ...interface{}) bool { 46 | t.Helper() 47 | differs, diff := GetJSONDiffLog(expected, actual) 48 | if !differs { 49 | return true 50 | } 51 | indent := "\t\t" 52 | diffIndented := regexp.MustCompile("(?m)^").ReplaceAll([]byte(diff), []byte("\t"+indent))[len(indent)+1:] 53 | diffLog := fmt.Sprintf("\n\tError:"+indent+"JSON not equal\n\tDiff:"+indent+"%s", diffIndented) 54 | msg := messageFromMsgAndArgs(msgAndArgs...) 55 | if len(msg) > 0 { 56 | t.Errorf(diffLog+"\n\tMessages:"+indent+"%s", msg) 57 | } else { 58 | t.Error(diffLog) 59 | } 60 | return false 61 | } 62 | 63 | func toBytes(v interface{}) []byte { 64 | switch s := v.(type) { 65 | case string: 66 | return []byte(s) 67 | case []byte: 68 | return s 69 | default: 70 | panic(fmt.Sprintf("cannot convert %T to byte slice", v)) 71 | } 72 | } 73 | 74 | // copied from assert.Fail() 75 | func messageFromMsgAndArgs(msgAndArgs ...interface{}) string { 76 | if len(msgAndArgs) == 0 || msgAndArgs == nil { 77 | return "" 78 | } 79 | if len(msgAndArgs) == 1 { 80 | msg := msgAndArgs[0] 81 | if msgAsStr, ok := msg.(string); ok { 82 | return msgAsStr 83 | } 84 | return fmt.Sprintf("%+v", msg) 85 | } 86 | if len(msgAndArgs) > 1 { 87 | return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) 88 | } 89 | return "" 90 | } 91 | -------------------------------------------------------------------------------- /internal/test/lbrynet_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/OdyseeTeam/odysee-api/app/query" 8 | ljsonrpc "github.com/lbryio/lbry.go/v2/extras/jsonrpc" 9 | 10 | "github.com/Pallinder/go-randomdata" 11 | "github.com/shopspring/decimal" 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | "github.com/ybbus/jsonrpc/v2" 15 | ) 16 | 17 | func TestInjectTestingWallet(t *testing.T) { 18 | userID := randomdata.Number(10000, 90000) 19 | w, err := InjectTestingWallet(userID) 20 | require.NoError(t, err) 21 | 22 | c := query.NewCaller(SDKAddress, userID) 23 | res, err := c.Call(context.Background(), jsonrpc.NewRequest("account_balance")) 24 | require.NoError(t, err) 25 | require.Nil(t, res.Error) 26 | 27 | var bal ljsonrpc.AccountBalanceResponse 28 | err = ljsonrpc.Decode(res.Result, &bal) 29 | require.NoError(t, err) 30 | assert.GreaterOrEqual(t, bal.Available.Cmp(decimal.NewFromInt(1)), 0) 31 | 32 | assert.NoError(t, w.Unload()) 33 | assert.NoError(t, w.RemoveFile()) 34 | } 35 | -------------------------------------------------------------------------------- /internal/test/static_assets.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | "os" 7 | "path" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | var staticPath = "https://ik.imagekit.io/odystatic/" 14 | 15 | func StaticAsset(t *testing.T, fileName string) string { 16 | t.Helper() 17 | t.Logf("getting static asset %s", staticPath+fileName) 18 | r, err := http.Get(staticPath + fileName) 19 | require.NoError(t, err) 20 | f, err := os.Create(path.Join(t.TempDir(), fileName)) 21 | require.NoError(t, err) 22 | defer f.Close() 23 | _, err = io.Copy(f, r.Body) 24 | require.NoError(t, err) 25 | return f.Name() 26 | } 27 | -------------------------------------------------------------------------------- /internal/test/templates.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | var tmplWallet = ` 4 | { 5 | "accounts": [ 6 | { 7 | "address_generator": { 8 | "name": "single-address" 9 | }, 10 | "certificates": {}, 11 | "encrypted": false, 12 | "ledger": "lbc_mainnet", 13 | "modified_on": 1657898418, 14 | "name": "Account #bP2uhrhHgHLR6WNHCjQpWFDtb3V8aPpo6Q", 15 | "private_key": "{{.PrivateKey}}", 16 | "public_key": "{{.PublicKey}}" 17 | } 18 | ], 19 | "name": "My Wallet", 20 | "preferences": {}, 21 | "version": 1 22 | } 23 | ` 24 | -------------------------------------------------------------------------------- /internal/test/test_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "net/http" 7 | "testing" 8 | 9 | ljsonrpc "github.com/lbryio/lbry.go/v2/extras/jsonrpc" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestMockHTTPServer(t *testing.T) { 16 | reqChan := ReqChan() 17 | rpcServer := MockHTTPServer(reqChan) 18 | defer rpcServer.Close() 19 | 20 | rpcServer.NextResponse <- `{"result": {"items": [], "page": 1, "page_size": 2, "total_pages": 3}}` 21 | res, err := ljsonrpc.NewClient(rpcServer.URL).WalletList("", 1, 2) 22 | require.NoError(t, err) 23 | 24 | req := <-reqChan 25 | assert.Equal(t, req.R.Method, http.MethodPost) 26 | assert.Equal(t, req.Body, `{"method":"wallet_list","params":{"page":1,"page_size":2},"id":0,"jsonrpc":"2.0"}`) 27 | 28 | assert.Equal(t, res.Page, uint64(1)) 29 | assert.Equal(t, res.PageSize, uint64(2)) 30 | assert.Equal(t, res.TotalPages, uint64(3)) 31 | 32 | rpcServer.NextResponse <- `ok` 33 | c := &http.Client{} 34 | r, err := http.NewRequest(http.MethodPost, rpcServer.URL, bytes.NewBuffer([]byte("hello"))) 35 | require.NoError(t, err) 36 | res2, err := c.Do(r) 37 | require.NoError(t, err) 38 | 39 | req2 := <-reqChan 40 | assert.Equal(t, req2.R.Method, http.MethodPost) 41 | assert.Equal(t, req2.Body, `hello`) 42 | body, err := io.ReadAll(res2.Body) 43 | require.NoError(t, err) 44 | assert.Equal(t, string(body), "ok") 45 | } 46 | 47 | func TestAssertEqualJSON(t *testing.T) { 48 | testCases := []struct { 49 | a, b string 50 | same bool 51 | }{ 52 | {"{}", "12", false}, 53 | {"{}", "{}", true}, 54 | {"{}", "", false}, 55 | {`{"a":1,"b":2}`, `{"b":2,"a":1}`, true}, 56 | } 57 | 58 | for i, tc := range testCases { 59 | testT := &testing.T{} 60 | same := AssertEqualJSON(testT, tc.a, tc.b) 61 | if tc.same { 62 | assert.True(t, same, "Case %d same", i) 63 | assert.False(t, testT.Failed(), "Case %d failure", i) 64 | } else { 65 | assert.False(t, same, "Case %d same", i) 66 | assert.True(t, testT.Failed(), "Case %d failure", i) 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /internal/testdeps/testdeps.go: -------------------------------------------------------------------------------- 1 | package testdeps 2 | 3 | import ( 4 | "context" 5 | "strconv" 6 | "testing" 7 | 8 | "github.com/hibiken/asynq" 9 | "github.com/redis/go-redis/v9" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | const ( 14 | baseRedisTestURL = "redis://:odyredis@localhost:6379/" 15 | ) 16 | 17 | type RedisTestHelper struct { 18 | AsynqOpts asynq.RedisConnOpt 19 | Client *redis.Client 20 | Opts *redis.Options 21 | URL string 22 | } 23 | 24 | func NewRedisTestHelper(t *testing.T, args ...int) *RedisTestHelper { 25 | t.Helper() 26 | var db int 27 | 28 | if len(args) > 0 { 29 | db = args[0] 30 | } 31 | url := baseRedisTestURL + strconv.Itoa(db) 32 | redisOpts, err := redis.ParseURL(url) 33 | 34 | require.NoError(t, err) 35 | asynqOpts, err := asynq.ParseRedisURI(url) 36 | require.NoError(t, err) 37 | c := redis.NewClient(redisOpts) 38 | c.FlushDB(context.Background()) 39 | t.Cleanup(func() { 40 | c.FlushDB(context.Background()) 41 | c.Close() 42 | }) 43 | return &RedisTestHelper{ 44 | AsynqOpts: asynqOpts, 45 | Client: c, 46 | Opts: redisOpts, 47 | URL: url, 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/OdyseeTeam/odysee-api/apps/lbrytv/config" 7 | "github.com/OdyseeTeam/odysee-api/cmd" 8 | "github.com/OdyseeTeam/odysee-api/internal/monitor" 9 | "github.com/OdyseeTeam/odysee-api/internal/storage" 10 | "github.com/OdyseeTeam/odysee-api/pkg/migrator" 11 | "github.com/OdyseeTeam/odysee-api/version" 12 | 13 | "github.com/getsentry/sentry-go" 14 | ) 15 | 16 | func main() { 17 | monitor.IsProduction = config.IsProduction() 18 | monitor.ConfigureSentry(config.GetSentryDSN(), version.GetDevVersion(), monitor.LogMode()) 19 | defer sentry.Flush(2 * time.Second) 20 | 21 | db, err := migrator.ConnectDB(migrator.DBConfigFromApp(config.GetDatabase())) 22 | if err != nil { 23 | panic(err) 24 | } 25 | defer db.Close() 26 | storage.SetDB(db) 27 | go storage.WatchMetrics(10 * time.Second) 28 | 29 | cmd.Execute() 30 | } 31 | -------------------------------------------------------------------------------- /models/boil_queries.go: -------------------------------------------------------------------------------- 1 | // Code generated by SQLBoiler (https://github.com/volatiletech/sqlboiler). DO NOT EDIT. 2 | // This file is meant to be re-generated in place and/or deleted at any time. 3 | 4 | package models 5 | 6 | import ( 7 | "github.com/volatiletech/sqlboiler/drivers" 8 | "github.com/volatiletech/sqlboiler/queries" 9 | "github.com/volatiletech/sqlboiler/queries/qm" 10 | ) 11 | 12 | var dialect = drivers.Dialect{ 13 | LQ: 0x22, 14 | RQ: 0x22, 15 | 16 | UseIndexPlaceholders: true, 17 | UseLastInsertID: false, 18 | UseSchema: false, 19 | UseDefaultKeyword: true, 20 | UseAutoColumns: false, 21 | UseTopClause: false, 22 | UseOutputClause: false, 23 | UseCaseWhenExistsClause: false, 24 | } 25 | 26 | // NewQuery initializes a new Query using the passed in QueryMods 27 | func NewQuery(mods ...qm.QueryMod) *queries.Query { 28 | q := &queries.Query{} 29 | queries.SetDialect(q, &dialect) 30 | qm.Apply(q, mods...) 31 | 32 | return q 33 | } 34 | -------------------------------------------------------------------------------- /models/boil_table_names.go: -------------------------------------------------------------------------------- 1 | // Code generated by SQLBoiler (https://github.com/volatiletech/sqlboiler). DO NOT EDIT. 2 | // This file is meant to be re-generated in place and/or deleted at any time. 3 | 4 | package models 5 | 6 | var TableNames = struct { 7 | Asynqueries string 8 | GorpMigrations string 9 | LbrynetServers string 10 | PublishQueries string 11 | QueryLog string 12 | Uploads string 13 | Users string 14 | }{ 15 | Asynqueries: "asynqueries", 16 | GorpMigrations: "gorp_migrations", 17 | LbrynetServers: "lbrynet_servers", 18 | PublishQueries: "publish_queries", 19 | QueryLog: "query_log", 20 | Uploads: "uploads", 21 | Users: "users", 22 | } 23 | -------------------------------------------------------------------------------- /models/boil_types.go: -------------------------------------------------------------------------------- 1 | // Code generated by SQLBoiler (https://github.com/volatiletech/sqlboiler). DO NOT EDIT. 2 | // This file is meant to be re-generated in place and/or deleted at any time. 3 | 4 | package models 5 | 6 | import ( 7 | "strconv" 8 | 9 | "github.com/pkg/errors" 10 | "github.com/volatiletech/sqlboiler/boil" 11 | "github.com/volatiletech/sqlboiler/strmangle" 12 | ) 13 | 14 | // M type is for providing columns and column values to UpdateAll. 15 | type M map[string]interface{} 16 | 17 | // ErrSyncFail occurs during insert when the record could not be retrieved in 18 | // order to populate default value information. This usually happens when LastInsertId 19 | // fails or there was a primary key configuration that was not resolvable. 20 | var ErrSyncFail = errors.New("models: failed to synchronize data after insert") 21 | 22 | type insertCache struct { 23 | query string 24 | retQuery string 25 | valueMapping []uint64 26 | retMapping []uint64 27 | } 28 | 29 | type updateCache struct { 30 | query string 31 | valueMapping []uint64 32 | } 33 | 34 | func makeCacheKey(cols boil.Columns, nzDefaults []string) string { 35 | buf := strmangle.GetBuffer() 36 | 37 | buf.WriteString(strconv.Itoa(cols.Kind)) 38 | for _, w := range cols.Cols { 39 | buf.WriteString(w) 40 | } 41 | 42 | if len(nzDefaults) != 0 { 43 | buf.WriteByte('.') 44 | } 45 | for _, nz := range nzDefaults { 46 | buf.WriteString(nz) 47 | } 48 | 49 | str := buf.String() 50 | strmangle.PutBuffer(buf) 51 | return str 52 | } 53 | 54 | // Enum values for asynquery_status 55 | const ( 56 | AsynqueryStatusReceived = "received" 57 | AsynqueryStatusPreparing = "preparing" 58 | AsynqueryStatusForwarded = "forwarded" 59 | AsynqueryStatusFailed = "failed" 60 | AsynqueryStatusSucceeded = "succeeded" 61 | ) 62 | 63 | // Enum values for publish_query_status 64 | const ( 65 | PublishQueryStatusReceived = "received" 66 | PublishQueryStatusForwarded = "forwarded" 67 | PublishQueryStatusFailed = "failed" 68 | PublishQueryStatusSucceeded = "succeeded" 69 | ) 70 | 71 | // Enum values for upload_status 72 | const ( 73 | UploadStatusCreated = "created" 74 | UploadStatusUploading = "uploading" 75 | UploadStatusReceived = "received" 76 | UploadStatusTerminated = "terminated" 77 | UploadStatusAbandoned = "abandoned" 78 | UploadStatusFailed = "failed" 79 | UploadStatusFinished = "finished" 80 | ) 81 | -------------------------------------------------------------------------------- /models/psql_upsert.go: -------------------------------------------------------------------------------- 1 | // Code generated by SQLBoiler (https://github.com/volatiletech/sqlboiler). DO NOT EDIT. 2 | // This file is meant to be re-generated in place and/or deleted at any time. 3 | 4 | package models 5 | 6 | import ( 7 | "fmt" 8 | "strings" 9 | 10 | "github.com/volatiletech/sqlboiler/drivers" 11 | "github.com/volatiletech/sqlboiler/strmangle" 12 | ) 13 | 14 | // buildUpsertQueryPostgres builds a SQL statement string using the upsertData provided. 15 | func buildUpsertQueryPostgres(dia drivers.Dialect, tableName string, updateOnConflict bool, ret, update, conflict, whitelist []string) string { 16 | conflict = strmangle.IdentQuoteSlice(dia.LQ, dia.RQ, conflict) 17 | whitelist = strmangle.IdentQuoteSlice(dia.LQ, dia.RQ, whitelist) 18 | ret = strmangle.IdentQuoteSlice(dia.LQ, dia.RQ, ret) 19 | 20 | buf := strmangle.GetBuffer() 21 | defer strmangle.PutBuffer(buf) 22 | 23 | columns := "DEFAULT VALUES" 24 | if len(whitelist) != 0 { 25 | columns = fmt.Sprintf("(%s) VALUES (%s)", 26 | strings.Join(whitelist, ", "), 27 | strmangle.Placeholders(dia.UseIndexPlaceholders, len(whitelist), 1, 1)) 28 | } 29 | 30 | fmt.Fprintf( 31 | buf, 32 | "INSERT INTO %s %s ON CONFLICT ", 33 | tableName, 34 | columns, 35 | ) 36 | 37 | if !updateOnConflict || len(update) == 0 { 38 | buf.WriteString("DO NOTHING") 39 | } else { 40 | buf.WriteByte('(') 41 | buf.WriteString(strings.Join(conflict, ", ")) 42 | buf.WriteString(") DO UPDATE SET ") 43 | 44 | for i, v := range update { 45 | if i != 0 { 46 | buf.WriteByte(',') 47 | } 48 | quoted := strmangle.IdentQuote(dia.LQ, dia.RQ, v) 49 | buf.WriteString(quoted) 50 | buf.WriteString(" = EXCLUDED.") 51 | buf.WriteString(quoted) 52 | } 53 | } 54 | 55 | if len(ret) != 0 { 56 | buf.WriteString(" RETURNING ") 57 | buf.WriteString(strings.Join(ret, ", ")) 58 | } 59 | 60 | return buf.String() 61 | } 62 | -------------------------------------------------------------------------------- /oapi.yml: -------------------------------------------------------------------------------- 1 | LbrynetServers: 2 | default: http://localhost:15279/ 3 | lbrynet1: http://localhost:15279/ 4 | lbrynet2: http://localhost:15279/ 5 | 6 | Debug: 1 7 | 8 | BaseContentURL: https://player.odycdn.com/api 9 | FreeContentURL: https://player.odycdn.com/api/v4/streams/free/ 10 | PaidContentURL: https://player.odycdn.com/api/v3/streams/paid/ 11 | 12 | StreamsV5: 13 | Host: https://player.odycdn.com 14 | PaidHost: https://secure.odycdn.com 15 | StartPath: /v5/streams/start 16 | HLSPath: /v5/streams/hls 17 | PaidPass: paid-pass 18 | 19 | StreamsV6: 20 | Host: player.odycdn.com 21 | PaidHost: secure.odycdn.com 22 | StartPath: /v6/streams/%s/%s.mp4 23 | Token: cdn-paid-token 24 | 25 | InternalAPIHost: https://api.odysee.com 26 | ProjectURL: https://odysee.com 27 | 28 | ArfleetCDN: https://thumbnails-arfleet.odycdn.com 29 | 30 | DatabaseDSN: postgres://postgres:odyseeteam@localhost 31 | Database: 32 | DBName: oapi 33 | Options: sslmode=disable 34 | 35 | OAuth: 36 | ClientID: odysee-apis 37 | ProviderURL: https://sso.odysee.com/auth/realms/Users 38 | TokenPath: /protocol/openid-connect/token 39 | 40 | PublishSourceDir: ./rundata/storage/publish 41 | GeoPublishSourceDir: ./rundata/storage/geopublish 42 | 43 | PaidTokenPrivKey: token_privkey.rsa 44 | 45 | # Change this key for production! 46 | # You can re-generate the key by running: 47 | # openssl ecparam -genkey -name prime256v1 -noout -out private_key.pem && base64 -i private_key.pem 48 | UploadTokenPrivateKey: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUZZYWxQZkhySzNSQ1F2YmhRQ1h6cDZiWG9uODZWOGI5L3B0bjB3QTZxNkxvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFZjhyN3RlQWJwUlVldXZhVWRsNDQzVS9VZkpYZURDd051QkRrbmp5ZnRZaXZ2Tnl6cGt6ZgpYdDl3RE9rc1VZSmEzNVhvSndabjNHMmw2L2EvdVUvWmh3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= 49 | UploadServiceURL: http://localhost:8980/v1/ 50 | 51 | CORSDomains: 52 | - http://localhost:1337 53 | - http://localhost:9090 54 | 55 | RPCTimeouts: 56 | txo_spend: 4m 57 | txo_list: 4m 58 | transaction_list: 4m 59 | publish: 4m 60 | 61 | RedisLocker: redis://:odyredis@localhost:6379/1 62 | RedisBus: redis://:odyredis@localhost:6379/2 63 | 64 | # AsynqueryRequestsConnURL is Redis database where asynquery will be listening for finalized uploads requests. 65 | # This corresponds to AsynqueryRequestsConnURL in forklift.yml config. 66 | AsynqueryRequestsConnURL: redis://:odyredis@localhost:6379/3 67 | 68 | SturdyCache: 69 | Master: localhost:6379 70 | Replicas: 71 | - localhost:6379 72 | Password: odyredis 73 | 74 | ReflectorUpstream: 75 | DatabaseDSN: 'user:password@tcp(localhost:3306)/blobs' 76 | Destinations: 77 | Wasabi: 78 | Endpoint: https://s3.wasabisys.com 79 | Region: us-east-1 80 | Bucket: blobs1 81 | AWS_ID: key1 82 | AWS_Secret: secret1 83 | ShardingSize: 4 84 | Another: 85 | Endpoint: https://aws.another.com 86 | Region: us-east-2 87 | Bucket: blobs2 88 | AWS_ID: key2 89 | AWS_Secret: secret2 90 | ShardingSize: 0 91 | 92 | Logging: 93 | Level: debug 94 | Format: console 95 | -------------------------------------------------------------------------------- /pkg/blobs/blobs_test.go: -------------------------------------------------------------------------------- 1 | package blobs 2 | 3 | import ( 4 | "os" 5 | "path" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/OdyseeTeam/odysee-api/internal/test" 10 | "github.com/lbryio/lbry.go/v3/stream" 11 | 12 | "github.com/spf13/viper" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func TestSplit(t *testing.T) { 17 | filePath := test.StaticAsset(t, "doc.pdf") 18 | 19 | s := NewSource(filePath, t.TempDir(), "doc.pdf") 20 | pbs, err := s.Split() 21 | require.NoError(t, err) 22 | require.Equal(t, "doc.pdf", pbs.GetSource().Name) 23 | 24 | require.NoError(t, err) 25 | stream := make(stream.Stream, len(s.blobsManifest)) 26 | for i, b := range s.blobsManifest { 27 | data, err := os.ReadFile(path.Join(s.finalPath, b)) 28 | require.NoError(t, err) 29 | stream[i] = data 30 | } 31 | 32 | result, err := stream.Decode() 33 | 34 | require.NoError(t, err) 35 | original, err := os.ReadFile(filePath) 36 | require.NoError(t, err) 37 | require.Equal(t, original, result) 38 | } 39 | 40 | func TestConfig(t *testing.T) { 41 | require := require.New(t) 42 | 43 | v := viper.New() 44 | v.SetConfigType("yaml") 45 | 46 | cfgPath, _ := filepath.Abs("./testdata/config.yml") 47 | f, err := os.Open(cfgPath) 48 | require.NoError(err) 49 | err = v.ReadConfig(f) 50 | require.NoError(err) 51 | 52 | stores, err := CreateStoresFromConfig(v, "ReflectorStorage.Destinations") 53 | require.NoError(err) 54 | require.Len(stores, 2) 55 | require.Equal("s3-another", stores[0].Name()) 56 | require.Equal("s3-wasabi", stores[1].Name()) 57 | } 58 | -------------------------------------------------------------------------------- /pkg/blobs/testdata/config.yml: -------------------------------------------------------------------------------- 1 | ReflectorStorage: 2 | DatabaseDSN: 'user:password@tcp(host.com)/blobs' 3 | Destinations: 4 | Wasabi: 5 | Endpoint: https://s3.wasabisys.com 6 | Region: us-east-1 7 | Bucket: blobs1 8 | AWS_ID: key1 9 | AWS_Secret: secret1 10 | ShardingSize: 4 11 | Another: 12 | Endpoint: https://aws.another.com 13 | Region: us-east-2 14 | Bucket: blobs2 15 | AWS_ID: key2 16 | AWS_Secret: secret2 17 | ShardingSize: 0 18 | -------------------------------------------------------------------------------- /pkg/chainquery/chainquery.go: -------------------------------------------------------------------------------- 1 | package chainquery 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/url" 9 | "time" 10 | 11 | "github.com/friendsofgo/errors" 12 | ) 13 | 14 | const ( 15 | apiUrl = "https://chainquery.odysee.tv/api/sql" 16 | queryTimeout = 15 * time.Second 17 | ) 18 | 19 | type HttpDoer interface { 20 | Do(*http.Request) (*http.Response, error) 21 | } 22 | 23 | type HeightResponse struct { 24 | Success bool `json:"success"` 25 | Error *string `json:"error"` 26 | Data []HeightData `json:"data"` 27 | } 28 | 29 | type HeightData struct { 30 | Height int `json:"height"` 31 | } 32 | 33 | var client HttpDoer = &http.Client{ 34 | Timeout: queryTimeout, 35 | } 36 | 37 | func GetHeight() (int, error) { 38 | r := HeightResponse{} 39 | err := makeRequest(client, "select height from block order by id desc limit 1", &r) 40 | if err != nil { 41 | return 0, errors.Wrap(err, "error retrieving latest height") 42 | } 43 | if len(r.Data) != 1 { 44 | return 0, errors.Errorf("error retrieving latest height, expected %v items in response, got %v", 1, len(r.Data)) 45 | } 46 | return r.Data[0].Height, nil 47 | } 48 | 49 | func makeRequest(client HttpDoer, query string, target any) error { 50 | baseUrl, err := url.Parse(apiUrl) 51 | if err != nil { 52 | return err 53 | } 54 | params := url.Values{} 55 | params.Add("query", query) 56 | baseUrl.RawQuery = params.Encode() 57 | 58 | req, err := http.NewRequest(http.MethodGet, baseUrl.String(), nil) 59 | if err != nil { 60 | return fmt.Errorf("error creating request: %w", err) 61 | } 62 | 63 | resp, err := client.Do(req) 64 | if err != nil { 65 | return fmt.Errorf("error sending request: %w", err) 66 | } 67 | defer resp.Body.Close() 68 | 69 | body, err := io.ReadAll(resp.Body) 70 | if err != nil { 71 | return fmt.Errorf("error reading response: %w", err) 72 | } 73 | 74 | if resp.StatusCode != http.StatusOK { 75 | return fmt.Errorf("unexpected status code: got %v, want %v", resp.StatusCode, http.StatusOK) 76 | } 77 | 78 | err = json.Unmarshal(body, target) 79 | if err != nil { 80 | return err 81 | } 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /pkg/chainquery/chainquery_test.go: -------------------------------------------------------------------------------- 1 | package chainquery 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io" 7 | "net/http" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | type failureTestClient struct{} 15 | 16 | func (failureTestClient) Do(*http.Request) (*http.Response, error) { 17 | data := HeightResponse{ 18 | Success: true, 19 | Error: nil, 20 | Data: []HeightData{}, 21 | } 22 | 23 | body, _ := json.Marshal(data) 24 | 25 | return &http.Response{ 26 | Status: "200 OK", 27 | StatusCode: 200, 28 | Body: io.NopCloser(bytes.NewReader(body)), 29 | Header: http.Header{"Content-Type": []string{"application/json"}}, 30 | }, nil 31 | } 32 | 33 | func TestGetHeight(t *testing.T) { 34 | require := require.New(t) 35 | assert := assert.New(t) 36 | 37 | height, err := GetHeight() 38 | require.NoError(err) 39 | assert.Greater(height, 1500_000) 40 | } 41 | 42 | func TestGetHeightFailure(t *testing.T) { 43 | assert := assert.New(t) 44 | 45 | origClient := client 46 | client = failureTestClient{} 47 | defer func() { 48 | client = origClient 49 | }() 50 | 51 | height, err := GetHeight() 52 | assert.ErrorContains(err, "error retrieving latest height, expected 1 items in response, got 0") 53 | assert.Equal(0, height) 54 | } 55 | -------------------------------------------------------------------------------- /pkg/configng/client_s3.go: -------------------------------------------------------------------------------- 1 | package configng 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/credentials" 8 | "github.com/aws/aws-sdk-go/aws/session" 9 | "github.com/aws/aws-sdk-go/service/s3" 10 | ) 11 | 12 | func NewS3Client(s3cfg S3Config) (*s3.S3, error) { 13 | cfg := aws.NewConfig(). 14 | WithCredentials(credentials.NewStaticCredentials(s3cfg.Key, s3cfg.Secret, "")). 15 | WithRegion(s3cfg.Region) 16 | 17 | if s3cfg.Endpoint != "" { 18 | cfg = cfg.WithEndpoint(s3cfg.Endpoint) 19 | } 20 | 21 | if s3cfg.Flavor == "minio" || s3cfg.Flavor == "ovh" { 22 | cfg = cfg.WithS3ForcePathStyle(true) 23 | } 24 | 25 | sess, err := session.NewSession(cfg) 26 | if err != nil { 27 | panic(fmt.Errorf("unable to create AWS session: %w", err)) 28 | } 29 | client := s3.New(sess) 30 | return client, nil 31 | } 32 | -------------------------------------------------------------------------------- /pkg/configng/configng.go: -------------------------------------------------------------------------------- 1 | package configng 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/viper" 7 | ) 8 | 9 | type S3Config struct { 10 | Endpoint string 11 | Region string 12 | Bucket string 13 | Key, Secret string 14 | Flavor string 15 | VerifyBucket bool 16 | } 17 | 18 | type PostgresConfig struct { 19 | DSN string 20 | DBName string 21 | AutoMigrations bool 22 | } 23 | 24 | type Config struct { 25 | V *viper.Viper 26 | } 27 | 28 | func Read(path, name, format string) (*Config, error) { 29 | v := viper.New() 30 | v.SetConfigName(name) 31 | v.SetConfigType(format) 32 | v.AddConfigPath(path) 33 | err := v.ReadInConfig() 34 | if err != nil { 35 | return nil, fmt.Errorf("failed to read config: %w", err) 36 | } 37 | return &Config{V: v}, nil 38 | } 39 | 40 | func (c *Config) ReadS3Config(name string) (S3Config, error) { 41 | var s3cfg S3Config 42 | return s3cfg, c.V.UnmarshalKey(name, &s3cfg) 43 | } 44 | 45 | func (c *Config) ReadPostgresConfig(name string) PostgresConfig { 46 | var pcfg PostgresConfig 47 | c.V.UnmarshalKey(name, &pcfg) 48 | return pcfg 49 | } 50 | 51 | func (c PostgresConfig) GetFullDSN() string { 52 | return c.DSN 53 | } 54 | 55 | func (c PostgresConfig) GetDBName() string { 56 | return c.DBName 57 | } 58 | 59 | func (c PostgresConfig) MigrateOnConnect() bool { 60 | return c.AutoMigrations 61 | } 62 | -------------------------------------------------------------------------------- /pkg/keybox/keybox_test.go: -------------------------------------------------------------------------------- 1 | package keybox 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | "crypto/elliptic" 6 | "crypto/rand" 7 | "crypto/x509" 8 | "encoding/base64" 9 | "encoding/pem" 10 | "net/http" 11 | "net/http/httptest" 12 | "testing" 13 | "time" 14 | 15 | "github.com/Pallinder/go-randomdata" 16 | "github.com/stretchr/testify/assert" 17 | "github.com/stretchr/testify/require" 18 | ) 19 | 20 | var testPrivKey = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUZZYWxQZkhySzNSQ1F2YmhRQ1h6cDZiWG9uODZWOGI5L3B0bjB3QTZxNkxvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFZjhyN3RlQWJwUlVldXZhVWRsNDQzVS9VZkpYZURDd051QkRrbmp5ZnRZaXZ2Tnl6cGt6ZgpYdDl3RE9rc1VZSmEzNVhvSndabjNHMmw2L2EvdVUvWmh3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=" 21 | var testPubKey = "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEsDn7Awhhaw5iZ0Q1GVpzYuZavxH5b/AJS2b3FPFF2/NcN+MMll9lzdtHVo1RGsskGqDy0vII8GK6xxSJl4n1Ig==" 22 | 23 | func TestKeyfob(t *testing.T) { 24 | _, err := KeyfobFromString(testPrivKey) 25 | require.NoError(t, err) 26 | } 27 | 28 | func TestNewValidator(t *testing.T) { 29 | _, err := NewValidator("not a key") 30 | require.ErrorContains(t, err, "not an ECDSA public key") 31 | 32 | validator, err := ValidatorFromPublicKeyString(testPubKey) 33 | require.NoError(t, err) 34 | 35 | assert.NotNil(t, validator.publicKey) 36 | publicKeyBytes, err := x509.MarshalPKIXPublicKey(validator.publicKey) 37 | require.NoError(t, err) 38 | assert.Equal(t, base64.StdEncoding.EncodeToString(publicKeyBytes), testPubKey) 39 | } 40 | 41 | func TestKeyfobFromStringGenerateToken(t *testing.T) { 42 | privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) 43 | require.NoError(t, err) 44 | 45 | bkey, err := x509.MarshalECPrivateKey(privateKey) 46 | require.NoError(t, err) 47 | privateKeyPEM := pem.EncodeToMemory(&pem.Block{ 48 | Type: "ECDSA PRIVATE KEY", 49 | Bytes: bkey, 50 | }) 51 | 52 | km, err := KeyfobFromString(base64.StdEncoding.EncodeToString(privateKeyPEM)) 53 | require.NoError(t, err) 54 | 55 | upid := randomdata.RandStringRunes(32) 56 | token, err := km.GenerateToken(123, time.Now().Add(24*time.Second), "upload_id", upid) 57 | require.NoError(t, err) 58 | 59 | pt, err := km.Validator().ParseToken(token) 60 | require.NoError(t, err) 61 | assert.Equal(t, upid, pt.PrivateClaims()["upload_id"]) 62 | } 63 | 64 | func TestPublicKeyHandler(t *testing.T) { 65 | assert := assert.New(t) 66 | require := require.New(t) 67 | 68 | kf, err := KeyfobFromString(testPrivKey) 69 | require.NoError(err) 70 | 71 | ts := httptest.NewServer(http.HandlerFunc(kf.PublicKeyHandler)) 72 | defer ts.Close() 73 | 74 | pubKey, err := PublicKeyFromURL(ts.URL) 75 | require.NoError(err) 76 | 77 | assert.Equal(pubKey, kf.PublicKey(), "retrieved public key does not match parsed public key") 78 | } 79 | -------------------------------------------------------------------------------- /pkg/logging/logging.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "context" 5 | 6 | "logur.dev/logur" 7 | ) 8 | 9 | type ctxKey int 10 | 11 | const loggingContextKey ctxKey = iota 12 | 13 | var ( 14 | LevelDebug = "debug" 15 | LevelInfo = "info" 16 | ) 17 | 18 | type Logger interface { 19 | Debug(args ...interface{}) 20 | Info(args ...interface{}) 21 | Warn(args ...interface{}) 22 | Error(args ...interface{}) 23 | Fatal(args ...interface{}) 24 | With(keyvals ...interface{}) Logger 25 | } 26 | 27 | type KVLogger interface { 28 | Debug(msg string, keyvals ...interface{}) 29 | Info(msg string, keyvals ...interface{}) 30 | Warn(msg string, keyvals ...interface{}) 31 | Error(msg string, keyvals ...interface{}) 32 | Fatal(msg string, keyvals ...interface{}) 33 | With(keyvals ...interface{}) KVLogger 34 | } 35 | 36 | type LoggingOpts interface { 37 | Level() string 38 | Format() string 39 | } 40 | 41 | type NoopKVLogger struct { 42 | logur.NoopKVLogger 43 | } 44 | 45 | type NoopLogger struct{} 46 | 47 | func (NoopLogger) Debug(args ...interface{}) {} 48 | func (NoopLogger) Info(args ...interface{}) {} 49 | func (NoopLogger) Warn(args ...interface{}) {} 50 | func (NoopLogger) Error(args ...interface{}) {} 51 | func (NoopLogger) Fatal(args ...interface{}) {} 52 | 53 | func (l NoopLogger) With(args ...interface{}) Logger { 54 | return l 55 | } 56 | 57 | func (l NoopKVLogger) Fatal(msg string, keyvals ...interface{}) {} 58 | 59 | func (l NoopKVLogger) With(keyvals ...interface{}) KVLogger { 60 | return l 61 | } 62 | 63 | func AddLogRef(l KVLogger, sdHash string) KVLogger { 64 | if len(sdHash) >= 8 { 65 | return l.With("ref", sdHash[:8]) 66 | } 67 | return l.With("ref?", sdHash) 68 | } 69 | 70 | func AddToContext(ctx context.Context, l KVLogger) context.Context { 71 | return context.WithValue(ctx, loggingContextKey, l) 72 | } 73 | 74 | func GetFromContext(ctx context.Context) KVLogger { 75 | l, ok := ctx.Value(loggingContextKey).(KVLogger) 76 | if !ok { 77 | return NoopKVLogger{} 78 | } 79 | return l 80 | } 81 | -------------------------------------------------------------------------------- /pkg/logging/tracing.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | type TracedObject interface { 4 | GetTraceData() map[string]string 5 | } 6 | 7 | func TracedLogger(l KVLogger, t TracedObject) KVLogger { 8 | for k, v := range t.GetTraceData() { 9 | l = l.With(k, v) 10 | } 11 | return l 12 | } 13 | -------------------------------------------------------------------------------- /pkg/logging/zapadapter/zapadapter_test.go: -------------------------------------------------------------------------------- 1 | package zapadapter 2 | 3 | import ( 4 | stdlog "log" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | "go.uber.org/zap" 10 | "go.uber.org/zap/zapcore" 11 | "go.uber.org/zap/zaptest/observer" 12 | ) 13 | 14 | func TestLogger_LogLevels(t *testing.T) { 15 | observedLogs, logs := observer.New(zap.InfoLevel) 16 | logger := New(zap.New(observedLogs)) 17 | 18 | logger.Trace("trace message") 19 | logger.Debug("debug message") 20 | logger.Info("info message") 21 | logger.Warn("warn message") 22 | logger.Error("error message") 23 | 24 | assert.Equal(t, 3, logs.Len()) 25 | 26 | testCases := []struct { 27 | level zapcore.Level 28 | message string 29 | }{ 30 | {zap.InfoLevel, "info message"}, 31 | {zap.WarnLevel, "warn message"}, 32 | {zap.ErrorLevel, "error message"}, 33 | } 34 | 35 | for i, tc := range testCases { 36 | logEntry := logs.All()[i] 37 | assert.Equal(t, tc.level, logEntry.Level) 38 | assert.Equal(t, tc.message, logEntry.Message) 39 | } 40 | } 41 | 42 | func TestLogger_With(t *testing.T) { 43 | observedLogs, logs := observer.New(zap.InfoLevel) 44 | logger := New(zap.New(observedLogs)).With("key", "value") 45 | 46 | logger.Info("message") 47 | 48 | assert.Equal(t, 1, logs.Len()) 49 | 50 | logEntry := logs.All()[0] 51 | assert.Equal(t, "value", logEntry.ContextMap()["key"]) 52 | assert.Equal(t, "message", logEntry.Message) 53 | } 54 | 55 | func TestNewNamedKV(t *testing.T) { 56 | logger := NewNamedKV("testlogger", NewLoggingOpts("info", "json")) 57 | logger.Info("message") 58 | 59 | logger = NewNamedKV("testlogger", NewLoggingOpts("debug", "console")) 60 | logger.Info("message") 61 | 62 | assert.Panics(t, func() { 63 | NewNamedKV("testlogger", NewLoggingOpts("", "")) 64 | }) 65 | } 66 | 67 | func TestKVLogger_Write(t *testing.T) { 68 | core, recordedLogs := observer.New(zapcore.InfoLevel) 69 | zapLogger := zap.New(core) 70 | kvLogger := NewKV(zapLogger) 71 | stdLogger := stdlog.New(kvLogger, "", 0) 72 | logMessage := "This is a log message from the standard logger" 73 | structLogMessage := `event="ChunkWriteComplete" id="8a254f1b8061c2f74a76f6aa8ef59d8e" bytesWritten="26214400"` 74 | stdLogger.Output(2, logMessage) 75 | stdLogger.Output(2, structLogMessage) 76 | 77 | require.Equal(t, 2, recordedLogs.Len()) 78 | 79 | logEntry := recordedLogs.All()[0] 80 | assert.Equal(t, logMessage, logEntry.Message) 81 | structLogEntry := recordedLogs.All()[1] 82 | assert.Equal(t, "ChunkWriteComplete", structLogEntry.Message) 83 | assert.Equal(t, "8a254f1b8061c2f74a76f6aa8ef59d8e", structLogEntry.ContextMap()["id"]) 84 | assert.Equal(t, "26214400", structLogEntry.ContextMap()["bytesWritten"]) 85 | } 86 | -------------------------------------------------------------------------------- /pkg/migrator/cli.go: -------------------------------------------------------------------------------- 1 | package migrator 2 | 3 | type CLI struct { 4 | MigrateUp struct { 5 | } `cmd:"" help:"Apply database migrations"` 6 | MigrateDown struct { 7 | Max int `optional:"" help:"Max number of migrations to unapply" default:"0"` 8 | } `cmd:"" help:"Unapply database migrations"` 9 | } 10 | -------------------------------------------------------------------------------- /pkg/migrator/db.go: -------------------------------------------------------------------------------- 1 | package migrator 2 | 3 | import ( 4 | "database/sql" 5 | "embed" 6 | "fmt" 7 | ) 8 | 9 | type DSNConfig interface { 10 | GetFullDSN() string 11 | GetDBName() string 12 | } 13 | 14 | type DBConfig struct { 15 | appName, dsn, dbName, connOpts string 16 | } 17 | 18 | func DefaultDBConfig() *DBConfig { 19 | return &DBConfig{ 20 | dsn: "postgres://postgres:odyseeteam@localhost", 21 | dbName: "postgres", 22 | connOpts: "sslmode=disable", 23 | } 24 | } 25 | 26 | func (c *DBConfig) DSN(dsn string) *DBConfig { 27 | c.dsn = dsn 28 | return c 29 | } 30 | 31 | func (c *DBConfig) Name(dbName string) *DBConfig { 32 | c.dbName = dbName 33 | return c 34 | } 35 | 36 | func (c *DBConfig) AppName(appName string) *DBConfig { 37 | c.appName = appName 38 | return c 39 | } 40 | 41 | func (c *DBConfig) ConnOpts(connOpts string) *DBConfig { 42 | c.connOpts = connOpts 43 | return c 44 | } 45 | 46 | func (c *DBConfig) GetFullDSN() string { 47 | return fmt.Sprintf("%s/%s?%s", c.dsn, c.dbName, c.connOpts) 48 | } 49 | 50 | func (c *DBConfig) GetDBName() string { 51 | return c.dbName 52 | } 53 | 54 | func ConnectDB(cfg DSNConfig, migrationsFS ...embed.FS) (*sql.DB, error) { 55 | var err error 56 | db, err := sql.Open("postgres", cfg.GetFullDSN()) 57 | if err != nil { 58 | return nil, err 59 | } 60 | if len(migrationsFS) > 0 { 61 | _, err := New(db, migrationsFS[0]).MigrateUp(0) 62 | if err != nil { 63 | return nil, err 64 | } 65 | } 66 | 67 | return db, nil 68 | } 69 | 70 | func DBConfigFromApp(cfg DSNConfig) *DBConfig { 71 | return DefaultDBConfig().DSN(cfg.GetFullDSN()).Name(cfg.GetDBName()) 72 | } 73 | -------------------------------------------------------------------------------- /pkg/migrator/migrator.go: -------------------------------------------------------------------------------- 1 | package migrator 2 | 3 | import ( 4 | "database/sql" 5 | "embed" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/OdyseeTeam/odysee-api/pkg/logging" 10 | "github.com/OdyseeTeam/odysee-api/pkg/logging/zapadapter" 11 | 12 | "github.com/lib/pq" 13 | migrate "github.com/rubenv/sql-migrate" 14 | ) 15 | 16 | const dialect = "postgres" 17 | 18 | type Migrator struct { 19 | db *sql.DB 20 | ms migrate.MigrationSet 21 | source *migrate.EmbedFileSystemMigrationSource 22 | logger logging.KVLogger 23 | } 24 | 25 | func New(db *sql.DB, fs embed.FS) Migrator { 26 | return Migrator{ 27 | db: db, 28 | // ms: migrate.MigrationSet{TableName: migrTableName + "_gorp_migrations"}, 29 | ms: migrate.MigrationSet{TableName: "gorp_migrations"}, 30 | source: &migrate.EmbedFileSystemMigrationSource{ 31 | FileSystem: fs, 32 | Root: "migrations", 33 | }, 34 | logger: zapadapter.NewKV(nil), 35 | } 36 | } 37 | 38 | // MigrateUp executes forward migrations. 39 | func (m Migrator) MigrateUp(max int) (int, error) { 40 | n, err := m.ms.ExecMax(m.db, dialect, m.source, migrate.Up, max) 41 | if err != nil { 42 | return 0, err 43 | } 44 | m.logger.Info("migrations applied", "count", n) 45 | return n, nil 46 | } 47 | 48 | // MigrateDown undoes a specified number of migrations. 49 | func (m Migrator) MigrateDown(max int) (int, error) { 50 | n, err := m.ms.ExecMax(m.db, dialect, m.source, migrate.Down, max) 51 | if err != nil { 52 | return 0, err 53 | } 54 | m.logger.Info("migrations unapplied", "count", n) 55 | return n, nil 56 | } 57 | 58 | // Truncate purges records from the requested tables. 59 | func (m Migrator) Truncate(tables []string) error { 60 | _, err := m.db.Exec(fmt.Sprintf("TRUNCATE %s CASCADE;", strings.Join(tables, ", "))) 61 | return err 62 | } 63 | 64 | // CreateDB creates the requested database. 65 | func (m Migrator) CreateDB(dbName string) error { 66 | // fmt.Sprintf is used instead of query placeholders because postgres does not 67 | // handle them in schema-modifying queries. 68 | _, err := m.db.Exec(fmt.Sprintf("create database %s;", pq.QuoteIdentifier(dbName))) 69 | if err != nil { 70 | return err 71 | } 72 | m.logger.Info("migrations applied", "db", dbName) 73 | return nil 74 | } 75 | 76 | // DropDB drops the requested database. 77 | func (m Migrator) DropDB(dbName string) error { 78 | _, err := m.db.Exec(fmt.Sprintf("drop database %s;", pq.QuoteIdentifier(dbName))) 79 | if err != nil { 80 | return err 81 | } 82 | m.logger.Info("database dropped", "db", dbName) 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /pkg/migrator/testing.go: -------------------------------------------------------------------------------- 1 | package migrator 2 | 3 | import ( 4 | "database/sql" 5 | "embed" 6 | "fmt" 7 | 8 | "github.com/Pallinder/go-randomdata" 9 | ) 10 | 11 | type TestDBCleanup func() error 12 | 13 | func CreateTestDB(cfg *DBConfig, mfs embed.FS) (*sql.DB, TestDBCleanup, error) { 14 | db, err := ConnectDB(cfg) 15 | testDBName := fmt.Sprintf("test-%s-%s-%s", cfg.dbName, randomdata.Noun(), randomdata.Adjective()) 16 | if err != nil { 17 | return nil, nil, err 18 | } 19 | m := New(db, mfs) 20 | m.CreateDB(testDBName) 21 | 22 | tdb, err := ConnectDB(cfg.Name(testDBName), mfs) 23 | if err != nil { 24 | return nil, nil, err 25 | } 26 | tm := New(tdb, mfs) 27 | _, err = tm.MigrateUp(0) 28 | if err != nil { 29 | return nil, nil, err 30 | } 31 | return tdb, func() error { 32 | tdb.Close() 33 | err := m.DropDB(testDBName) 34 | db.Close() 35 | if err != nil { 36 | return err 37 | } 38 | return nil 39 | }, nil 40 | } 41 | -------------------------------------------------------------------------------- /pkg/queue/metrics.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | const ns = "queue" 8 | 9 | var ( 10 | queueTasks = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 11 | Namespace: ns, 12 | Name: "queue_tasks", 13 | }, []string{"status"}) 14 | ) 15 | 16 | func registerMetrics(registry prometheus.Registerer) { 17 | if registry == nil { 18 | registry = prometheus.DefaultRegisterer 19 | } 20 | registry.MustRegister( 21 | queueTasks, 22 | ) 23 | } 24 | -------------------------------------------------------------------------------- /pkg/queue/queue_test.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "testing" 7 | "time" 8 | 9 | "github.com/OdyseeTeam/odysee-api/internal/testdeps" 10 | "github.com/OdyseeTeam/odysee-api/pkg/logging/zapadapter" 11 | 12 | "github.com/hibiken/asynq" 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | const ( 18 | queueRequest = "test:request" 19 | queueResponse = "test:response" 20 | ) 21 | 22 | func TestQueueIntegration(t *testing.T) { 23 | assert := assert.New(t) 24 | require := require.New(t) 25 | 26 | redisRequestsHelper := testdeps.NewRedisTestHelper(t, 1) 27 | redisResponsesHelper := testdeps.NewRedisTestHelper(t, 2) 28 | queue, err := New( 29 | WithRequestsConnOpts(redisRequestsHelper.AsynqOpts), 30 | WithResponsesConnOpts(redisResponsesHelper.AsynqOpts), 31 | WithConcurrency(2), 32 | WithLogger(zapadapter.NewKV(nil)), 33 | ) 34 | require.NoError(err) 35 | defer queue.Shutdown() 36 | 37 | queueResponses, err := New( 38 | WithRequestsConnOpts(redisResponsesHelper.AsynqOpts), 39 | WithConcurrency(2), 40 | WithLogger(zapadapter.NewKV(nil)), 41 | ) 42 | require.NoError(err) 43 | defer queueResponses.Shutdown() 44 | 45 | requests := make(chan map[string]any, 1) 46 | responses := make(chan map[string]any, 1) 47 | reqPayload := map[string]any{ 48 | "request": "request", 49 | } 50 | respPayload := map[string]any{ 51 | "response": "response", 52 | } 53 | 54 | queue.AddHandler(queueRequest, func(ctx context.Context, task *asynq.Task) error { 55 | assert.Equal(queueRequest, task.Type()) 56 | var payload map[string]any 57 | err := json.Unmarshal(task.Payload(), &payload) 58 | assert.NoError(err) 59 | requests <- payload 60 | queue.SendResponse(queueResponse, payload) 61 | return nil 62 | }) 63 | 64 | queueResponses.AddHandler(queueResponse, func(ctx context.Context, task *asynq.Task) error { 65 | assert.Equal(queueResponse, task.Type()) 66 | var payload map[string]any 67 | err := json.Unmarshal(task.Payload(), &payload) 68 | assert.NoError(err) 69 | assert.Equal(reqPayload, payload) 70 | responses <- respPayload 71 | return nil 72 | }) 73 | 74 | go func() { 75 | err := queue.ServeUntilShutdown() 76 | require.NoError(err) 77 | }() 78 | go func() { 79 | err := queueResponses.ServeUntilShutdown() 80 | require.NoError(err) 81 | }() 82 | 83 | err = queue.SendRequest(queueRequest, reqPayload) 84 | require.NoError(err) 85 | 86 | select { 87 | case rcPayload := <-requests: 88 | require.Equal(reqPayload, rcPayload) 89 | case <-time.After(10 * time.Second): 90 | t.Fatal("timeout waiting for request to be processed") 91 | } 92 | 93 | select { 94 | case rcPayload := <-responses: 95 | require.Equal(respPayload, rcPayload) 96 | case <-time.After(10 * time.Second): 97 | t.Fatal("timeout waiting for response to be sent") 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /pkg/redislocker/metrics.go: -------------------------------------------------------------------------------- 1 | package redislocker 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | const ns = "redislocker" 8 | 9 | var ( 10 | locked = prometheus.NewCounter(prometheus.CounterOpts{ 11 | Namespace: ns, 12 | Name: "locked", 13 | }) 14 | unlocked = prometheus.NewCounter(prometheus.CounterOpts{ 15 | Namespace: ns, 16 | Name: "unlocked", 17 | }) 18 | 19 | fileLockedErrors = prometheus.NewCounter(prometheus.CounterOpts{ 20 | Namespace: ns, 21 | Subsystem: "errors", 22 | Name: "file_locked", 23 | }) 24 | unlockErrors = prometheus.NewCounter(prometheus.CounterOpts{ 25 | Namespace: ns, 26 | Subsystem: "errors", 27 | Name: "unlock", 28 | }) 29 | ) 30 | 31 | func RegisterMetrics(registry prometheus.Registerer) { 32 | if registry == nil { 33 | registry = prometheus.DefaultRegisterer 34 | } 35 | registry.MustRegister(locked, unlocked, fileLockedErrors, unlockErrors) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/redislocker/redislocker.go: -------------------------------------------------------------------------------- 1 | package redislocker 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/go-redsync/redsync/v4" 9 | "github.com/go-redsync/redsync/v4/redis/goredis/v9" 10 | goredislib "github.com/redis/go-redis/v9" 11 | "github.com/tus/tusd/pkg/handler" 12 | ) 13 | 14 | var ( 15 | lockTimeout = 100 * time.Second 16 | ) 17 | 18 | type Locker struct { 19 | rs *redsync.Redsync 20 | } 21 | 22 | type lock struct { 23 | name string 24 | mutex *redsync.Mutex 25 | } 26 | 27 | func New(redisOpts *goredislib.Options) (*Locker, error) { 28 | client := goredislib.NewClient(redisOpts) 29 | err := client.Ping(context.Background()).Err() 30 | if err != nil { 31 | return nil, err 32 | } 33 | pool := goredis.NewPool(client) 34 | rs := redsync.New(pool) 35 | return &Locker{rs}, nil 36 | } 37 | 38 | func (locker *Locker) NewLock(name string) (handler.Lock, error) { 39 | m := locker.rs.NewMutex(name, redsync.WithExpiry(lockTimeout)) 40 | return &lock{name, m}, nil 41 | } 42 | 43 | // UseIn adds this locker to the passed composer. 44 | func (locker *Locker) UseIn(composer *handler.StoreComposer) { 45 | composer.UseLocker(locker) 46 | } 47 | 48 | func (l lock) Lock() error { 49 | if err := l.mutex.Lock(); err != nil { 50 | fileLockedErrors.Inc() 51 | return fmt.Errorf("%w: file %s: %s", handler.ErrFileLocked, l.name, err) 52 | } 53 | locked.Inc() 54 | return nil 55 | } 56 | 57 | func (l lock) Unlock() error { 58 | if ok, err := l.mutex.Unlock(); !ok || err != nil { 59 | unlockErrors.Inc() 60 | return fmt.Errorf("cannot unlock file %s: %w", l.name, err) 61 | } 62 | unlocked.Inc() 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /pkg/redislocker/redislocker_test.go: -------------------------------------------------------------------------------- 1 | package redislocker 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "testing" 7 | 8 | "github.com/OdyseeTeam/odysee-api/pkg/testservices" 9 | "github.com/redis/go-redis/v9" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | "github.com/tus/tusd/pkg/handler" 13 | ) 14 | 15 | var redisOpts *redis.Options 16 | 17 | func TestMain(m *testing.M) { 18 | var err error 19 | var teardown testservices.Teardown 20 | redisOpts, teardown, err = testservices.Redis() 21 | if err != nil { 22 | log.Fatalf("failed to init redis: %s", err) 23 | } 24 | defer teardown() 25 | 26 | code := m.Run() 27 | os.Exit(code) 28 | } 29 | 30 | func TestLocker(t *testing.T) { 31 | a := assert.New(t) 32 | r := require.New(t) 33 | 34 | locker, err := New(redisOpts) 35 | r.NoError(err) 36 | 37 | lock1, err := locker.NewLock("one") 38 | a.NoError(err) 39 | 40 | a.NoError(lock1.Lock()) 41 | a.ErrorIs(lock1.Lock(), handler.ErrFileLocked) 42 | 43 | lock2, err := locker.NewLock("one") 44 | a.NoError(err) 45 | a.ErrorIs(lock2.Lock(), handler.ErrFileLocked) 46 | 47 | a.NoError(lock1.Unlock()) 48 | } 49 | -------------------------------------------------------------------------------- /pkg/rpcerrors/rpcerrors_test.go: -------------------------------------------------------------------------------- 1 | package rpcerrors 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | "github.com/ybbus/jsonrpc/v2" 11 | ) 12 | 13 | func TestWrite(t *testing.T) { 14 | w := new(bytes.Buffer) 15 | Write(w, errors.New("error!")) 16 | 17 | b, _ := json.MarshalIndent(jsonrpc.RPCResponse{ 18 | Error: &jsonrpc.RPCError{ 19 | Code: rpcErrorCodeInternal, 20 | Message: "error!", 21 | }, 22 | JSONRPC: "2.0", 23 | }, "", " ") 24 | require.Equal(t, b, w.Bytes()) 25 | } 26 | -------------------------------------------------------------------------------- /pkg/sturdycache/testing.go: -------------------------------------------------------------------------------- 1 | package sturdycache 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/alicebob/miniredis/v2" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | type teardownFunc func() 11 | 12 | func CreateTestCache(t *testing.T) (*ReplicatedCache, *miniredis.Miniredis, []*miniredis.Miniredis, teardownFunc) { 13 | require := require.New(t) 14 | master := miniredis.RunT(t) 15 | 16 | replicas := make([]*miniredis.Miniredis, 3) 17 | for i := range 3 { 18 | replicas[i] = miniredis.RunT(t) 19 | } 20 | 21 | replicaAddrs := make([]string, len(replicas)) 22 | for i, r := range replicas { 23 | replicaAddrs[i] = r.Addr() 24 | } 25 | 26 | cache, err := NewReplicatedCache( 27 | master.Addr(), 28 | replicaAddrs, 29 | "", 30 | ) 31 | require.NoError(err) 32 | return cache, master, replicas, func() { 33 | master.Close() 34 | for _, r := range replicas { 35 | r.Close() 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pkg/testservices/testservices.go: -------------------------------------------------------------------------------- 1 | package testservices 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/ory/dockertest/v3" 8 | "github.com/redis/go-redis/v9" 9 | ) 10 | 11 | type Teardown func() error 12 | 13 | // Redis will spin up a redis container and return a connection options 14 | // plus a tear down function that needs to be called to spin the container down. 15 | func Redis() (*redis.Options, Teardown, error) { 16 | var err error 17 | pool, err := dockertest.NewPool("") 18 | if err != nil { 19 | return nil, nil, fmt.Errorf("could not connect to docker: %w", err) 20 | } 21 | 22 | resource, err := pool.Run("redis", "7", nil) 23 | if err != nil { 24 | return nil, nil, fmt.Errorf("could not start resource: %w", err) 25 | } 26 | 27 | redisOpts := &redis.Options{ 28 | Addr: fmt.Sprintf("localhost:%s", resource.GetPort("6379/tcp")), 29 | } 30 | 31 | if err = pool.Retry(func() error { 32 | db := redis.NewClient(redisOpts) 33 | err := db.Ping(context.Background()).Err() 34 | return err 35 | }); err != nil { 36 | return nil, nil, fmt.Errorf("could not connect to redis: %w", err) 37 | } 38 | 39 | return redisOpts, func() error { 40 | if err = pool.Purge(resource); err != nil { 41 | return fmt.Errorf("could not purge resource: %w", err) 42 | } 43 | return nil 44 | }, nil 45 | } 46 | -------------------------------------------------------------------------------- /pkg/testservices/testservices_test.go: -------------------------------------------------------------------------------- 1 | package testservices 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | goredislib "github.com/redis/go-redis/v9" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestRedis(t *testing.T) { 12 | redisOpts, teardown, err := Redis() 13 | require.NoError(t, err) 14 | defer teardown() 15 | client := goredislib.NewClient(redisOpts) 16 | err = client.Ping(context.Background()).Err() 17 | require.NoError(t, err) 18 | } 19 | -------------------------------------------------------------------------------- /scripts/init_test_daemon_settings.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cat > /storage/data/daemon_settings.yml <