├── .cargo ├── config.toml └── nextest.toml ├── .dockerignore ├── .env ├── .gitignore ├── .vscode ├── launch.json └── settings.json ├── Cargo.lock ├── Cargo.toml ├── Dockerfile ├── FAQ.md ├── Jenkinsfile ├── LICENSE ├── README.md ├── TODO.md ├── bin ├── docker-compose-production └── docker-compose-production-deploy ├── config ├── development_polygon.toml ├── example.toml └── minimal.toml ├── deduped_broadcast ├── Cargo.toml └── src │ └── lib.rs ├── deferred-rate-limiter ├── Cargo.toml └── src │ └── lib.rs ├── docker-compose.common.yml ├── docker-compose.prod.yml ├── docker-compose.yml ├── docker └── cargo-config.toml ├── docs ├── curl login.md ├── faster perf.txt ├── histograms.txt ├── http routes.txt ├── influx notes.txt ├── misc curl.bash ├── requests to test.txt ├── top 100 rpc keys.sql └── tracing notes.txt ├── entities ├── Cargo.toml └── src │ ├── admin.rs │ ├── admin_increase_balance_receipt.rs │ ├── admin_trail.rs │ ├── balance.rs │ ├── increase_on_chain_balance_receipt.rs │ ├── login.rs │ ├── mod.rs │ ├── pending_login.rs │ ├── prelude.rs │ ├── referee.rs │ ├── referrer.rs │ ├── revert_log.rs │ ├── rpc_accounting.rs │ ├── rpc_accounting_v2.rs │ ├── rpc_key.rs │ ├── sea_orm_active_enums.rs │ ├── secondary_user.rs │ ├── serialization.rs │ ├── stripe_increase_balance_receipt.rs │ ├── user.rs │ └── user_tier.rs ├── example.sql ├── latency ├── Cargo.toml └── src │ ├── ewma.rs │ ├── lib.rs │ ├── peak_ewma │ ├── mod.rs │ └── rtt_estimate.rs │ ├── rolling_quantile.rs │ └── util │ ├── atomic_f32_pair.rs │ ├── mod.rs │ ├── nanos.rs │ └── span.rs ├── migration ├── Cargo.toml ├── README.md └── src │ ├── lib.rs │ ├── m20220101_000001_create_table.rs │ ├── m20220921_181610_log_reverts.rs │ ├── m20220928_015108_concurrency_limits.rs │ ├── m20221007_213828_accounting.rs │ ├── m20221025_210326_add_chain_id_to_reverts.rs │ ├── m20221026_230819_rename_user_keys.rs │ ├── m20221027_002407_user_tiers.rs │ ├── m20221031_211916_clean_up.rs │ ├── m20221101_222349_archive_request.rs │ ├── m20221108_200345_save_anon_stats.rs │ ├── m20221211_124002_request_method_privacy.rs │ ├── m20221213_134158_move_login_into_database.rs │ ├── m20230117_191358_admin_table.rs │ ├── m20230119_204135_better_free_tier.rs │ ├── m20230125_204810_stats_v2.rs │ ├── m20230130_124740_read_only_login_logic.rs │ ├── m20230130_165144_prepare_admin_imitation_pre_login.rs │ ├── m20230205_130035_create_balance.rs │ ├── m20230205_133755_create_referrals.rs │ ├── m20230214_134254_increase_balance_transactions.rs │ ├── m20230215_152254_admin_trail.rs │ ├── m20230221_230953_track_spend.rs │ ├── m20230307_002623_migrate_rpc_accounting_to_rpc_accounting_v2.rs │ ├── m20230412_171916_modify_secondary_user_add_primary_user.rs │ ├── m20230422_172555_premium_downgrade_logic.rs │ ├── m20230511_161214_remove_columns_statsv2_origin_and_method.rs │ ├── m20230512_220213_allow_null_rpc_key_id_in_stats_v2.rs │ ├── m20230514_114803_admin_add_credits.rs │ ├── m20230607_221917_total_deposits.rs │ ├── m20230615_221201_handle_payment_uncles.rs │ ├── m20230618_230611_longer_payload.rs │ ├── m20230619_172237_default_tracking.rs │ ├── m20230622_104142_stripe_deposits.rs │ ├── m20230705_214013_type_fixes.rs │ ├── m20230707_211936_premium_tier_changes.rs │ ├── m20230708_151756_rpc_accounting_free_usage_credits.rs │ ├── m20230708_152131_referral_track_one_time_bonus_bonus.rs │ ├── m20230713_144446_stripe_default_date_created.rs │ ├── m20230713_210511_deposit_add_date_created.rs │ ├── m20230726_072845_default_premium_user_tier.rs │ ├── m20230726_162138_drop_rpc_accounting_v2_fk.rs │ ├── m20230726_225124_reduce_out_of_funds_tier_limits.rs │ ├── m20230911_180520_high_concurrency_tier.rs │ └── main.rs ├── payment-contracts ├── .gitignore ├── Cargo.toml ├── abi │ ├── IERC20.json │ ├── PaymentFactory.json │ ├── PaymentProxy.json │ └── PaymentSweeper.json ├── build.rs └── src │ ├── contracts │ ├── ierc20.rs │ ├── mod.rs │ ├── payment_factory.rs │ ├── payment_proxy.rs │ └── payment_sweeper.rs │ └── lib.rs ├── quick_cache_ttl ├── Cargo.toml └── src │ ├── cache.rs │ ├── kq_cache.rs │ └── lib.rs ├── rate-counter ├── Cargo.toml └── src │ └── lib.rs ├── redis-rate-limiter ├── Cargo.toml └── src │ └── lib.rs ├── rust-toolchain.toml ├── scripts ├── apply-migrations.sh ├── brownie-tests │ ├── .gitattributes │ ├── .gitignore │ ├── brownie-config.yaml │ └── scripts │ │ └── make_payment.py ├── ethspam ├── generate-requests-and-stats.sh ├── get-stats-aggregated.sh ├── install-test-suite.sh ├── manual-tests │ ├── 101-balance-referral-stats.sh │ ├── 12-subusers-premium-account.sh │ ├── 123-get-key-roles.sh │ ├── 135-stripe-deposit.sh │ ├── 16-change-user-tier.sh │ ├── 19-admin-imitate-user.sh │ ├── 21-sql-migration-make-backup.sh │ ├── 21-sql-migration-verify-test-queries.sql │ ├── 24-simple-referral-program.sh │ ├── 42-simple-balance.sh │ ├── 45-admin-add-balance.sh │ ├── 48-balance-downgrade.sh │ └── 52-simple-get-deposits.sh ├── requirements.txt └── versus ├── web3_proxy ├── Cargo.toml └── src │ ├── admin_queries.rs │ ├── app │ ├── mod.rs │ └── ws.rs │ ├── balance.rs │ ├── block_number.rs │ ├── caches.rs │ ├── compute_units.rs │ ├── config.rs │ ├── errors.rs │ ├── frontend │ ├── admin.rs │ ├── authorization.rs │ ├── errors.rs │ ├── mod.rs │ ├── request_id.rs │ ├── rpc_proxy_http.rs │ ├── rpc_proxy_ws.rs │ ├── status.rs │ ├── streaming.rs │ └── users │ │ ├── authentication.rs │ │ ├── mod.rs │ │ ├── payment.rs │ │ ├── payment_stripe.rs │ │ ├── referral.rs │ │ ├── rpc_keys.rs │ │ ├── stats.rs │ │ └── subuser.rs │ ├── globals.rs │ ├── http_params.rs │ ├── jsonrpc │ ├── error.rs │ ├── id.rs │ ├── mod.rs │ ├── request.rs │ ├── request_builder.rs │ └── response.rs │ ├── kafka.rs │ ├── lib.rs │ ├── pagerduty.rs │ ├── prelude.rs │ ├── premium.rs │ ├── prometheus.rs │ ├── referral_code.rs │ ├── relational_db.rs │ ├── response_cache.rs │ ├── rpcs │ ├── blockchain.rs │ ├── consensus.rs │ ├── grpc_erigon.rs │ ├── http.rs │ ├── many.rs │ ├── mod.rs │ ├── one.rs │ ├── provider.rs │ └── request.rs │ ├── secrets.rs │ ├── stats │ ├── db_queries.rs │ ├── influxdb_queries.rs │ ├── mod.rs │ └── stat_buffer.rs │ ├── test_utils │ ├── anvil.rs │ ├── create_provider_with_rpc_key.rs │ ├── influx.rs │ ├── lib.rs │ ├── mod.rs │ └── mysql.rs │ └── user_token.rs ├── web3_proxy_cli ├── Cargo.toml ├── examples │ ├── subscribe_blocks.rs │ ├── tracing.rs │ └── watch_blocks.rs ├── src │ ├── bin │ │ └── wait_for_sync.rs │ ├── lib.rs │ ├── main.rs │ ├── sub_commands │ │ ├── change_admin_status.rs │ │ ├── change_user_address.rs │ │ ├── change_user_tier.rs │ │ ├── change_user_tier_by_address.rs │ │ ├── change_user_tier_by_key.rs │ │ ├── check_balance.rs │ │ ├── check_config.rs │ │ ├── count_users.rs │ │ ├── create_key.rs │ │ ├── create_user.rs │ │ ├── delete_user.rs │ │ ├── drop_migration_lock.rs │ │ ├── example.rs │ │ ├── grant_credits_to_address.rs │ │ ├── list_recent_users.rs │ │ ├── mass_grant_credits.rs │ │ ├── migrate_stats_to_v2.rs │ │ ├── mod.rs │ │ ├── pagerduty.rs │ │ ├── popularity_contest.rs │ │ ├── proxyd.rs │ │ ├── rpc_accounting.rs │ │ ├── search_kafka.rs │ │ ├── sentryd │ │ │ ├── compare.rs │ │ │ ├── mod.rs │ │ │ └── simple.rs │ │ ├── stat_age.rs │ │ ├── transfer_key.rs │ │ ├── user_export.rs │ │ └── user_import.rs │ └── test_utils │ │ ├── admin_deposits.rs │ │ ├── admin_increases_balance.rs │ │ ├── app.rs │ │ ├── create_admin.rs │ │ ├── create_provider_with_rpc_key.rs │ │ ├── create_user.rs │ │ ├── mod.rs │ │ ├── referral.rs │ │ ├── rpc_key.rs │ │ ├── stats_accounting.rs │ │ └── user_balance.rs └── tests │ ├── test_admins.rs │ ├── test_multiple_proxy.rs │ ├── test_proxy.rs │ ├── test_single_proxy.rs │ ├── test_sum_credits_used.rs │ └── test_users.rs └── wrk ├── getBlockNumber.lua └── getLatestBlockByNumber.lua /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = [ 3 | # potentially faster. https://github.com/aws/aws-graviton-getting-started/blob/main/rust.md 4 | "-C", "target-feature=+lse", 5 | # potentially faster. https://nnethercote.github.io/perf-book/build-configuration.html 6 | "-C", "target-cpu=native", 7 | # tokio unstable is needed for tokio-console 8 | "--cfg", "tokio_unstable", 9 | # uuid unstable is needed for zerocopy deserialize 10 | "--cfg", "uuid_unstable", 11 | # parallel build frontend 12 | "-Z", "threads=8", 13 | ] 14 | rustdocflags = [ 15 | # potentially faster. https://github.com/aws/aws-graviton-getting-started/blob/main/rust.md 16 | "-C", "target-feature=+lse", 17 | # potentially faster. https://nnethercote.github.io/perf-book/build-configuration.html 18 | "-C", "target-cpu=native", 19 | # tokio unstable is needed for tokio-console 20 | "--cfg", "tokio_unstable", 21 | # uuid unstable is needed for zerocopy deserialize 22 | "--cfg", "uuid_unstable", 23 | # parallel build frontend 24 | "-Z", "threads=8", 25 | ] 26 | 27 | [registries.crates-io] 28 | protocol = "sparse" 29 | -------------------------------------------------------------------------------- /.cargo/nextest.toml: -------------------------------------------------------------------------------- 1 | [profile.default] 2 | slow-timeout = { period = "60s", terminate-after = 2 } 3 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | flamegraph.svg 3 | perf.data 4 | perf.data.old 5 | 6 | /*.md 7 | /.env 8 | /.git 9 | /.gitignore 10 | /.vscode 11 | /Dockerfile 12 | /Jenkinsfile 13 | /bin 14 | /config/dev*.toml 15 | /config/prod*.toml 16 | /data 17 | /docker-compose* 18 | /docs 19 | /redis-cell-server/ 20 | /scripts 21 | /target*/ 22 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | DATABASE_URL=mysql://root:dev_web3_proxy@127.0.0.1:13306/dev_web3_proxy 2 | RUST_BACKTRACE=1 3 | RUST_LOG=web3_proxy=debug,info 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | cargo-flamegraph.stacks 3 | flamegraph.svg 4 | perf.data 5 | perf.data.old 6 | 7 | /config/*.toml 8 | /data 9 | /target 10 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "rust-analyzer.server.extraEnv": { 3 | "CARGO_TARGET_DIR": "target/rust-analyzer" 4 | }, 5 | "rust-analyzer.checkOnSave.extraArgs": [ 6 | "--target-dir", 7 | "target/rust-analyzer" 8 | ], 9 | "rust-analyzer.cargo.features": [ 10 | "tokio-console" 11 | ] 12 | } -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "deduped_broadcast", 4 | "deferred-rate-limiter", 5 | "entities", 6 | "latency", 7 | "migration", 8 | "payment-contracts", 9 | "rate-counter", 10 | "redis-rate-limiter", 11 | "web3_proxy", 12 | "web3_proxy_cli", 13 | ] 14 | resolver = "2" 15 | 16 | [profile.release] 17 | # `debug = true` so that sentry can give us line numbers 18 | debug = true 19 | 20 | [profile.faster_release] 21 | inherits = "release" 22 | 23 | # spend longer compiling for a faster binary 24 | codegen-units = 1 25 | # `link time optimization = true = fat` 26 | lto = true 27 | -------------------------------------------------------------------------------- /FAQ.md: -------------------------------------------------------------------------------- 1 | # Frequently Asked Questions 2 | 3 | ## Q) My wallet sits on "sending transaction" a lot longer than on other services. 4 | 5 | We send your transactions to multiple private relays to get them mined without exposing them to sandwich attacks. 6 | 7 | We have plans to return after the first successful response, but that won't get your transaction confirmed any faster. 8 | 9 | Soon, you can opt out of this behavior and we will broadcast your transactions publicly. 10 | 11 | ## !) How do I sign a login message with cURL? 12 | 13 | ``` 14 | curl -d '{ 15 | "address": "0x22fbd6248cb2837900c3fe69f725bc02dd3a3b33", 16 | "msg": "0x73746167696e672e6c6c616d616e6f6465732e636f6d2077616e747320796f7520746f207369676e20696e207769746820796f757220457468657265756d206163636f756e743a0a3078323246624436323438634232383337393030633366653639663732356263303244643341334233330a0af09fa699f09fa699f09fa699f09fa699f09fa6990a0a5552493a2068747470733a2f2f73746167696e672e6c6c616d616e6f6465732e636f6d2f0a56657273696f6e3a20310a436861696e2049443a20310a4e6f6e63653a203031474654345052584342444157355844544643575957354a360a4973737565642041743a20323032322d31302d32305430373a32383a34342e3937323233343730385a0a45787069726174696f6e2054696d653a20323032322d31302d32305430373a34383a34342e3937323233343730385a", 17 | "sig": "08478ba4646423d67b36b26d60d31b8a54c7b133a5260045b484df687c1fe8f4196dc69792019852c282fb2a1b030be130ef5b78864fff216cdd0c71929351761b", 18 | "version": "3", 19 | "signer": "MEW" 20 | }' -H "Content-Type: application/json" --verbose "http://127.0.0.1:8544/user/login?invite_code=XYZ" 21 | ``` 22 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | @Library('jenkins_lib@main') _ 2 | 3 | pipeline { 4 | agent { 5 | // not strictly required, but we only build graviton2 right now so this keeps the jenkins-agent count down 6 | label 'arm64_graviton2' 7 | } 8 | options { 9 | ansiColor('xterm') 10 | } 11 | environment { 12 | // AWS_ECR_URL needs to be set in jenkin's config. 13 | // AWS_ECR_URL could really be any docker registry. we just use ECR so that we don't have to manage it 14 | 15 | REPO_NAME="web3-proxy" 16 | 17 | // branch that should get tagged with "latest_$arch" (stable, main, master, etc.) 18 | LATEST_BRANCH="main" 19 | } 20 | stages { 21 | stage('Check and Cancel Old Builds') { 22 | steps { 23 | script { 24 | myCancelRunning.cancelRunning() 25 | } 26 | } 27 | } 28 | stage('build and push') { 29 | parallel { 30 | stage('Build and push arm64_graviton2 image') { 31 | agent { 32 | label 'arm64_graviton2' 33 | } 34 | environment { 35 | ARCH="arm64_graviton2" 36 | } 37 | steps { 38 | script { 39 | myBuildAndPush.buildAndPush() 40 | } 41 | } 42 | } 43 | stage('Build and push intel_xeon1 image') { 44 | agent { 45 | label 'intel_xeon1' 46 | } 47 | environment { 48 | ARCH="intel_xeon1" 49 | } 50 | steps { 51 | script { 52 | myBuildAndPush.buildAndPush() 53 | } 54 | } 55 | } 56 | } 57 | } 58 | stage('push latest') { 59 | parallel { 60 | stage('maybe push latest_arm64_graviton2 tag') { 61 | agent any 62 | environment { 63 | ARCH="arm64_graviton2" 64 | } 65 | steps { 66 | script { 67 | myPushLatest.maybePushLatest() 68 | } 69 | } 70 | } 71 | stage('maybe push latest_intel_xeon1 tag') { 72 | agent any 73 | environment { 74 | ARCH="intel_xeon1" 75 | } 76 | steps { 77 | script { 78 | myPushLatest.maybePushLatest() 79 | } 80 | } 81 | } 82 | } 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /bin/docker-compose-production: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | # change to the project's root directory 4 | cd "${0%/*}/.." 5 | 6 | export DOCKER_BUILDKIT=1 7 | 8 | exec docker-compose -f docker-compose.prod.yml "$@" 9 | -------------------------------------------------------------------------------- /bin/docker-compose-production-deploy: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | # change to the project's root directory 4 | cd "${0%/*}/.." 5 | 6 | # put our scripts on the path 7 | export PATH="$(pwd)/bin:$PATH" 8 | 9 | # build the image 10 | docker-compose-production build --pull 11 | 12 | # run the image 13 | docker-compose-production up -d --remove-orphans "$@" 14 | 15 | sleep 3 16 | 17 | docker-compose-production ps 18 | 19 | echo "$(pwd)/bin/docker-compose-production logs -f" 20 | -------------------------------------------------------------------------------- /config/minimal.toml: -------------------------------------------------------------------------------- 1 | [app] 2 | chain_id = 1 3 | 4 | # no database 5 | # no influxdb 6 | # no redis 7 | # no sentry 8 | # no public limits means anon gets full access 9 | 10 | # no thundering herd protection 11 | min_sum_soft_limit = 1 12 | min_synced_rpcs = 1 13 | 14 | # 1GB of cache 15 | response_cache_max_bytes = 1_000_000_000 16 | 17 | [balanced_rpcs] 18 | 19 | [balanced_rpcs.llama_public_both] 20 | disabled = false 21 | display_name = "LlamaNodes Both" 22 | ws_url = "wss://eth.llamarpc.com/" 23 | http_url = "https://eth.llamarpc.com/" 24 | soft_limit = 1_000 25 | 26 | [balanced_rpcs.llama_public_https] 27 | disabled = false 28 | display_name = "LlamaNodes HTTPS" 29 | http_url = "https://eth.llamarpc.com/" 30 | soft_limit = 1_000 31 | 32 | [balanced_rpcs.llama_public_wss] 33 | # TODO: what should we do if all rpcs are disabled? warn and wait for a config change? 34 | disabled = false 35 | display_name = "LlamaNodes WSS" 36 | ws_url = "wss://eth.llamarpc.com/" 37 | soft_limit = 1_000 38 | -------------------------------------------------------------------------------- /deduped_broadcast/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "deduped_broadcast" 3 | version = "0.2.2" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | moka = { version = "0.12.1", features = ["future"] } 10 | serde = "1" 11 | tokio = { version = "1.34.0", features = ["full"] } 12 | -------------------------------------------------------------------------------- /deferred-rate-limiter/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "deferred-rate-limiter" 3 | version = "0.2.0" 4 | authors = ["Bryan Stitt "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | redis-rate-limiter = { path = "../redis-rate-limiter" } 9 | 10 | anyhow = "1.0.75" 11 | moka = { version = "0.12.1", features = ["future"] } 12 | tokio = "1.34.0" 13 | tracing = "0.1.40" 14 | -------------------------------------------------------------------------------- /docker-compose.common.yml: -------------------------------------------------------------------------------- 1 | services: 2 | web3-proxy: 3 | # TODO: build in dev but use docker hub in prod? 4 | build: . 5 | init: true 6 | restart: unless-stopped 7 | command: --config /config.toml --workers 16 proxyd 8 | # rust's tokio crate expects a SIGINT https://tokio.rs/tokio/topics/shutdown 9 | stop_signal: SIGINT 10 | environment: 11 | RUST_LOG: "info,ethers_providers::rpc=off,web3_proxy=info" 12 | volumes: 13 | - /etc/ssl/certs/:/etc/ssl/certs/:ro 14 | - /usr/local/share/ca-certificates/:/usr/local/share/ca-certificates/:ro 15 | 16 | volatile_redis: 17 | image: redis:6.2-alpine 18 | command: [ "redis-server" ] 19 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # development config 3 | version: "3.4" 4 | 5 | services: 6 | # manage the databases with a user friendly interface 7 | # it is slightly dangerous with "drop all" as a single click 8 | dev-adminer: 9 | image: adminer 10 | ports: 11 | - 18306:8080 12 | environment: 13 | ADMINER_DEFAULT_SERVER: dev-db 14 | 15 | # in dev we use mysql, but production will use RDS or similar 16 | dev-db: 17 | image: mysql 18 | environment: 19 | MYSQL_ROOT_PASSWORD: dev_web3_proxy 20 | MYSQL_DATABASE: dev_web3_proxy 21 | ports: 22 | - 127.0.0.1:13306:3306 23 | volumes: 24 | - ./data/dev_mysql:/var/lib/mysql 25 | 26 | # influxdb for stats 27 | dev-influxdb: 28 | image: influxdb:2.6.1-alpine 29 | environment: 30 | DOCKER_INFLUXDB_INIT_MODE: setup 31 | DOCKER_INFLUXDB_INIT_USERNAME: dev_web3_proxy 32 | DOCKER_INFLUXDB_INIT_PASSWORD: dev_web3_proxy 33 | DOCKER_INFLUXDB_INIT_ORG: dev_org 34 | DOCKER_INFLUXDB_INIT_BUCKET: dev_web3_proxy 35 | DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: dev_web3_proxy_auth_token 36 | ports: 37 | - 127.0.0.1:18086:8086 38 | volumes: 39 | - ./data/dev_influxdb/data:/var/lib/influxdb2 40 | - ./data/dev_influxdb/config:/etc/influxdb2 41 | 42 | dev-kafka: 43 | image: bitnami/kafka:3.4 44 | ports: 45 | - "127.0.0.1:19092:9092" 46 | volumes: 47 | - "./data/dev_kafka:/bitnami" 48 | environment: 49 | - KAFKA_CFG_ZOOKEEPER_CONNECT=dev-zookeeper:2181 50 | - ALLOW_PLAINTEXT_LISTENER=yes 51 | depends_on: 52 | - dev-zookeeper 53 | 54 | # volatile redis for storing rate limits 55 | dev-vredis: 56 | extends: 57 | file: docker-compose.common.yml 58 | service: volatile_redis 59 | ports: 60 | - 127.0.0.1:16379:6379 61 | 62 | # TODO: kafka doesn't need zookeeper anymore, but all the docs still use it 63 | dev-zookeeper: 64 | image: bitnami/zookeeper:3.8 65 | ports: 66 | - "127.0.0.1:12181:2181" 67 | volumes: 68 | - "./data/zookeeper:/bitnami" 69 | environment: 70 | - ALLOW_ANONYMOUS_LOGIN=yes 71 | -------------------------------------------------------------------------------- /docker/cargo-config.toml: -------------------------------------------------------------------------------- 1 | [target.aarch64-unknown-linux-gnu] 2 | linker = "clang" 3 | rustflags = [ 4 | # faster linker 5 | "-C", "link-arg=-fuse-ld=/usr/local/bin/mold", 6 | # potentially faster. https://github.com/aws/aws-graviton-getting-started/blob/main/rust.md 7 | "-C", "target-feature=+lse", 8 | # potentially faster. https://nnethercote.github.io/perf-book/build-configuration.html 9 | "-C", "target-cpu=native", 10 | # tokio unstable is needed for tokio-console 11 | "--cfg", "tokio_unstable", 12 | # uuid unstable is needed for zerocopy deserialize 13 | "--cfg", "uuid_unstable", 14 | # parallel build frontend 15 | "-Z", "threads=8", 16 | ] 17 | 18 | [target.aarch64-unknown-linux-musl] 19 | linker = "clang" 20 | rustflags = [ 21 | # faster linker 22 | "-C", "link-arg=-fuse-ld=/usr/local/bin/mold", 23 | # potentially faster. https://github.com/aws/aws-graviton-getting-started/blob/main/rust.md 24 | "-C", "target-feature=+lse", 25 | # potentially faster. https://nnethercote.github.io/perf-book/build-configuration.html 26 | "-C", "target-cpu=native", 27 | # tokio unstable is needed for tokio-console 28 | "--cfg", "tokio_unstable", 29 | # uuid unstable is needed for zerocopy deserialize 30 | "--cfg", "uuid_unstable", 31 | # parallel build frontend 32 | "-Z", "threads=8", 33 | ] 34 | 35 | [target.x86_64-unknown-linux-gnu] 36 | linker = "clang" 37 | rustflags = [ 38 | # faster linker 39 | "-C", "link-arg=-fuse-ld=/usr/local/bin/mold", 40 | # potentially faster. https://nnethercote.github.io/perf-book/build-configuration.html 41 | "-C", "target-cpu=native", 42 | # tokio unstable is needed for tokio-console 43 | "--cfg", "tokio_unstable", 44 | # uuid unstable is needed for zerocopy deserialize 45 | "--cfg", "uuid_unstable", 46 | # parallel build frontend 47 | "-Z", "threads=8", 48 | ] 49 | -------------------------------------------------------------------------------- /docs/curl login.md: -------------------------------------------------------------------------------- 1 | # log in with curl 2 | 3 | 1. curl http://127.0.0.1:8544/user/login/$ADDRESS 4 | 2. Sign the text with a site like https://www.myetherwallet.com/wallet/sign 5 | 3. POST the signed data: 6 | 7 | curl -X POST http://127.0.0.1:8544/user/login -H 'Content-Type: application/json' -d 8 | '{ "address": "0x9eb9e3dc2543dc9ff4058e2a2da43a855403f1fd", "msg": "0x6c6c616d616e6f6465732e636f6d2077616e747320796f7520746f207369676e20696e207769746820796f757220457468657265756d206163636f756e743a0a3078396562396533646332353433646339464634303538653241324441343341383535343033463166440a0af09fa699f09fa699f09fa699f09fa699f09fa6990a0a5552493a2068747470733a2f2f6c6c616d616e6f6465732e636f6d2f0a56657273696f6e3a20310a436861696e2049443a20310a4e6f6e63653a203031474d37373330375344324448333854454d3957545156454a0a4973737565642041743a20323032322d31322d31345430323a32333a31372e3735333736335a0a45787069726174696f6e2054696d653a20323032322d31322d31345430323a34333a31372e3735333736335a", "sig": "16bac055345279723193737c6c67cf995e821fd7c038d31fd6f671102088c7b85ab4b13069fd2ed02da186cf549530e315d8d042d721bf81289b3ffdbe8cf9ce1c", "version": "3", "signer": "MEW" }' 9 | 10 | 4. The response will include a bearer token. Use it with curl ... -H 'Authorization: Bearer $TOKEN' 11 | -------------------------------------------------------------------------------- /docs/faster perf.txt: -------------------------------------------------------------------------------- 1 | sudo apt install bison flex 2 | wget https://eighty-twenty.org/files/0001-tools-perf-Use-long-running-addr2line-per-dso.patch 3 | git clone https://github.com/torvalds/linux.git 4 | cd linux 5 | git checkout v5.15 6 | git apply ../0001-tools-perf-Use-long-running-addr2line-per-dso.patch 7 | cd tools/perf 8 | make prefix=$HOME/.local VERSION=5.15 install-bin 9 | -------------------------------------------------------------------------------- /docs/histograms.txt: -------------------------------------------------------------------------------- 1 | [2023-06-09T18:45:50Z WARN web3_proxy::rpcs::consensus] TODO: find the troughs in the histogram: HISTFAAAADV4nC3GoQ0AIAxE0eNAkRAEwbELQxE2QGG7aEeoaL95fz0ZACq8HKbwb/U5bGXystMAZl8EMw== 2 | 3 | Paste the HIST data here: 4 | 5 | Save the "Decoded histogram data" to a file and upload it here: 6 | -------------------------------------------------------------------------------- /docs/influx notes.txt: -------------------------------------------------------------------------------- 1 | influx config set -n llamanodes_staging -a 2 | influx delete --bucket web3_proxy --start 2023-01-01T00:00:00-00:00 --stop "$(date +"%Y-%m-%dT%H:%M:%SZ")" --predicate _measurement="global_proxy" 3 | influx delete --bucket web3_proxy --start 2023-01-01T00:00:00-00:00 --stop "$(date +"%Y-%m-%dT%H:%M:%SZ")" --predicate _measurement="opt_in_proxy" 4 | 5 | influx config set -n llamanodes_production -a 6 | influx delete --bucket web3_proxy --start 2023-07-18T18:00:00-07:00 --stop "$(date +"%Y-%m-%dT%H:%M:%SZ")" --predicate _measurement="global_proxy" 7 | influx delete --bucket web3_proxy --start 2023-07-18T18:00:00-07:00 --stop "$(date +"%Y-%m-%dT%H:%M:%SZ")" --predicate _measurement="opt_in_proxy" 8 | -------------------------------------------------------------------------------- /docs/requests to test.txt: -------------------------------------------------------------------------------- 1 | { 2 | "jsonrpc": "2.0", 3 | "id": 1, 4 | "method": "eth_call", 5 | "params": [ 6 | { 7 | "from": null, 8 | "to": "0x72a19342e8F1838460eBFCCEf09F6585e32db86E", 9 | "data": "0xaa33fedb0000000000000000000000005668ead1edb8e2a4d724c8fb9cb5ffeabeb422dc0000000000000000000000000000000000000000000000000000000000000002" 10 | }, 11 | "latest" 12 | ] 13 | } 14 | { 15 | "jsonrpc": "2.0", 16 | "id": 2, 17 | "method": "eth_getBlockByNumber", 18 | "params": [ 19 | "0x111cdd4", 20 | false 21 | ] 22 | } 23 | { 24 | "jsonrpc": "2.0", 25 | "id": 3, 26 | "method": "eth_getBlockByNumber", 27 | "params": [ 28 | "0x111cdd5", 29 | false 30 | ] 31 | } 32 | { 33 | "jsonrpc": "2.0", 34 | "id": 4, 35 | "method": "eth_call", 36 | "params": [ 37 | { 38 | "from": null, 39 | "to": "0xC51D3287A8c20dc865dF7b5Da6f5b2d596eE694B", 40 | "data": "0x38fff2d0" 41 | }, 42 | "latest" 43 | ] 44 | } -------------------------------------------------------------------------------- /docs/top 100 rpc keys.sql: -------------------------------------------------------------------------------- 1 | SELECT SUM(`sum_credits_used`) as sum, rpc_accounting_v2.rpc_key_id, rpc_key.user_id, concat("0x", hex(user.address)) as address, user.email 2 | FROM `rpc_accounting_v2` 3 | JOIN rpc_key ON rpc_accounting_v2.rpc_key_id=rpc_key.id 4 | JOIN user on rpc_key.user_id=user.id 5 | GROUP BY `rpc_key_id` 6 | ORDER BY sum DESC 7 | LIMIT 100 -------------------------------------------------------------------------------- /docs/tracing notes.txt: -------------------------------------------------------------------------------- 1 | Hello, I'm pretty new to tracing so my vocabulary might be wrong. I've got my app using tracing to log to stdout. I have a bunch of fields including user_id and ip_addr that make telling where logs are from nice and easy. 2 | 3 | Now there is one part of my code where I want to save a log to a database. I'm not sure of the best/correct way to do this. I can get the current span with tracing::Span::current(), but AFAICT that doesn't have a way to get to the values. I think I need to write my own Subscriber or Visitor (or both) and then tell tracing to use it only in this one part of the code. Am I on the right track? Is there a place in the docs that explains something similar? 4 | 5 | https://burgers.io/custom-logging-in-rust-using-tracing 6 | 7 | if you are doing it learn how to write a subscriber then you should write a custom layer. If you are simply trying to work on your main project there are several subscribers that already do this work for you. 8 | 9 | look at opentelemetry_otlp .. this will let you connect opentelemetry collector to your tracing using tracing_opentelemetry 10 | 11 | I'd suggest using the Registry subscriber because it can take multiple layers ... and use a filtered_layer to filter out the messages (look at env_filter, it can take the filtering params from an environment variable or a config string) and then have your collector be the second layer. e.... Registery can take in a vector of layers that are also-in-turn multi-layered. 12 | let me see if i can pull up an example 13 | On the https://docs.rs/tracing-subscriber/latest/tracing_subscriber/layer/ page about half-way down there is an example of boxed layers 14 | 15 | you basically end up composing different layers that output to different trace stores and also configure each using per-layer filtering (see https://docs.rs/tracing-subscriber/latest/tracing_subscriber/layer/#per-layer-filtering) 16 | -------------------------------------------------------------------------------- /entities/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "entities" 3 | version = "0.43.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | name = "entities" 8 | path = "src/mod.rs" 9 | 10 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 11 | 12 | [dependencies] 13 | ethers = { version = "2.0.11", default-features = false } 14 | sea-orm = "0.12.8" 15 | serde = "1.0.193" 16 | ulid = "1.1.0" 17 | -------------------------------------------------------------------------------- /entities/src/admin.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 7 | #[sea_orm(table_name = "admin")] 8 | pub struct Model { 9 | #[sea_orm(primary_key)] 10 | pub id: u64, 11 | #[sea_orm(unique)] 12 | pub user_id: u64, 13 | } 14 | 15 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 16 | pub enum Relation { 17 | #[sea_orm(has_many = "super::admin_increase_balance_receipt::Entity")] 18 | AdminIncreaseBalanceReceipt, 19 | #[sea_orm( 20 | belongs_to = "super::user::Entity", 21 | from = "Column::UserId", 22 | to = "super::user::Column::Id", 23 | on_update = "NoAction", 24 | on_delete = "NoAction" 25 | )] 26 | User, 27 | } 28 | 29 | impl Related for Entity { 30 | fn to() -> RelationDef { 31 | Relation::AdminIncreaseBalanceReceipt.def() 32 | } 33 | } 34 | 35 | impl Related for Entity { 36 | fn to() -> RelationDef { 37 | Relation::User.def() 38 | } 39 | } 40 | 41 | impl ActiveModelBehavior for ActiveModel {} 42 | -------------------------------------------------------------------------------- /entities/src/admin_increase_balance_receipt.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 7 | #[sea_orm(table_name = "admin_increase_balance_receipt")] 8 | pub struct Model { 9 | #[sea_orm(primary_key)] 10 | pub id: i32, 11 | #[sea_orm(column_type = "Decimal(Some((20, 10)))")] 12 | pub amount: Decimal, 13 | pub admin_id: u64, 14 | pub deposit_to_user_id: u64, 15 | pub note: String, 16 | pub date_created: DateTimeUtc, 17 | } 18 | 19 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 20 | pub enum Relation { 21 | #[sea_orm( 22 | belongs_to = "super::admin::Entity", 23 | from = "Column::AdminId", 24 | to = "super::admin::Column::Id", 25 | on_update = "NoAction", 26 | on_delete = "NoAction" 27 | )] 28 | Admin, 29 | #[sea_orm( 30 | belongs_to = "super::user::Entity", 31 | from = "Column::DepositToUserId", 32 | to = "super::user::Column::Id", 33 | on_update = "NoAction", 34 | on_delete = "NoAction" 35 | )] 36 | User, 37 | } 38 | 39 | impl Related for Entity { 40 | fn to() -> RelationDef { 41 | Relation::Admin.def() 42 | } 43 | } 44 | 45 | impl Related for Entity { 46 | fn to() -> RelationDef { 47 | Relation::User.def() 48 | } 49 | } 50 | 51 | impl ActiveModelBehavior for ActiveModel {} 52 | -------------------------------------------------------------------------------- /entities/src/admin_trail.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 7 | #[sea_orm(table_name = "admin_trail")] 8 | pub struct Model { 9 | #[sea_orm(primary_key)] 10 | pub id: i32, 11 | pub caller: u64, 12 | pub imitating_user: Option, 13 | #[sea_orm(column_type = "Text")] 14 | pub endpoint: String, 15 | #[sea_orm(column_type = "Text")] 16 | pub payload: String, 17 | pub timestamp: DateTimeUtc, 18 | } 19 | 20 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 21 | pub enum Relation { 22 | #[sea_orm( 23 | belongs_to = "super::user::Entity", 24 | from = "Column::Caller", 25 | to = "super::user::Column::Id", 26 | on_update = "NoAction", 27 | on_delete = "NoAction" 28 | )] 29 | User2, 30 | #[sea_orm( 31 | belongs_to = "super::user::Entity", 32 | from = "Column::ImitatingUser", 33 | to = "super::user::Column::Id", 34 | on_update = "NoAction", 35 | on_delete = "NoAction" 36 | )] 37 | User1, 38 | } 39 | 40 | impl ActiveModelBehavior for ActiveModel {} 41 | -------------------------------------------------------------------------------- /entities/src/balance.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Clone, Debug, Default, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 7 | #[sea_orm(table_name = "balance")] 8 | pub struct Model { 9 | #[sea_orm(primary_key)] 10 | pub id: i32, 11 | #[sea_orm(column_type = "Decimal(Some((20, 10)))")] 12 | pub total_spent_including_free_tier: Decimal, 13 | #[sea_orm(unique)] 14 | pub user_id: u64, 15 | #[sea_orm(column_type = "Decimal(Some((20, 10)))")] 16 | pub total_spent_outside_free_tier: Decimal, 17 | #[sea_orm(column_type = "Decimal(Some((20, 10)))")] 18 | pub total_deposits: Decimal, 19 | } 20 | 21 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 22 | pub enum Relation { 23 | #[sea_orm( 24 | belongs_to = "super::user::Entity", 25 | from = "Column::UserId", 26 | to = "super::user::Column::Id", 27 | on_update = "NoAction", 28 | on_delete = "NoAction" 29 | )] 30 | User, 31 | } 32 | 33 | impl Related for Entity { 34 | fn to() -> RelationDef { 35 | Relation::User.def() 36 | } 37 | } 38 | 39 | impl ActiveModelBehavior for ActiveModel {} 40 | -------------------------------------------------------------------------------- /entities/src/increase_on_chain_balance_receipt.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 7 | #[sea_orm(table_name = "increase_on_chain_balance_receipt")] 8 | pub struct Model { 9 | #[sea_orm(primary_key)] 10 | pub id: i32, 11 | pub tx_hash: String, 12 | pub chain_id: u64, 13 | #[sea_orm(column_type = "Decimal(Some((20, 10)))")] 14 | pub amount: Decimal, 15 | pub deposit_to_user_id: u64, 16 | pub block_hash: String, 17 | pub log_index: u64, 18 | pub token_address: String, 19 | pub date_created: DateTimeUtc, 20 | } 21 | 22 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 23 | pub enum Relation { 24 | #[sea_orm( 25 | belongs_to = "super::user::Entity", 26 | from = "Column::DepositToUserId", 27 | to = "super::user::Column::Id", 28 | on_update = "NoAction", 29 | on_delete = "NoAction" 30 | )] 31 | User, 32 | } 33 | 34 | impl Related for Entity { 35 | fn to() -> RelationDef { 36 | Relation::User.def() 37 | } 38 | } 39 | 40 | impl ActiveModelBehavior for ActiveModel {} 41 | -------------------------------------------------------------------------------- /entities/src/login.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use crate::serialization; 4 | use sea_orm::entity::prelude::*; 5 | use serde::{Deserialize, Serialize}; 6 | 7 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 8 | #[sea_orm(table_name = "login")] 9 | pub struct Model { 10 | #[sea_orm(primary_key)] 11 | pub id: u64, 12 | #[sea_orm(column_type = "Binary(BlobSize::Blob(Some(16)))", unique)] 13 | #[serde( 14 | serialize_with = "serialization::uuid_as_ulid", 15 | deserialize_with = "serialization::ulid_to_uuid" 16 | )] 17 | pub bearer_token: Uuid, 18 | pub user_id: u64, 19 | pub expires_at: DateTimeUtc, 20 | pub read_only: bool, 21 | } 22 | 23 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 24 | pub enum Relation { 25 | #[sea_orm( 26 | belongs_to = "super::user::Entity", 27 | from = "Column::UserId", 28 | to = "super::user::Column::Id", 29 | on_update = "NoAction", 30 | on_delete = "NoAction" 31 | )] 32 | User, 33 | } 34 | 35 | impl Related for Entity { 36 | fn to() -> RelationDef { 37 | Relation::User.def() 38 | } 39 | } 40 | 41 | impl ActiveModelBehavior for ActiveModel {} 42 | -------------------------------------------------------------------------------- /entities/src/mod.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | pub mod prelude; 4 | 5 | pub mod admin; 6 | pub mod admin_increase_balance_receipt; 7 | pub mod admin_trail; 8 | pub mod balance; 9 | pub mod increase_on_chain_balance_receipt; 10 | pub mod login; 11 | pub mod pending_login; 12 | pub mod referee; 13 | pub mod referrer; 14 | pub mod revert_log; 15 | pub mod rpc_accounting; 16 | pub mod rpc_accounting_v2; 17 | pub mod rpc_key; 18 | pub mod sea_orm_active_enums; 19 | pub mod secondary_user; 20 | pub mod serialization; 21 | pub mod stripe_increase_balance_receipt; 22 | pub mod user; 23 | pub mod user_tier; 24 | -------------------------------------------------------------------------------- /entities/src/pending_login.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use crate::serialization; 4 | use sea_orm::entity::prelude::*; 5 | use serde::{Deserialize, Serialize}; 6 | 7 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 8 | #[sea_orm(table_name = "pending_login")] 9 | pub struct Model { 10 | #[sea_orm(primary_key)] 11 | pub id: u64, 12 | #[sea_orm(column_type = "Binary(BlobSize::Blob(Some(16)))", unique)] 13 | #[serde( 14 | serialize_with = "serialization::uuid_as_ulid", 15 | deserialize_with = "serialization::ulid_to_uuid" 16 | )] 17 | pub nonce: Uuid, 18 | #[sea_orm(column_type = "Text")] 19 | pub message: String, 20 | pub expires_at: DateTimeUtc, 21 | pub imitating_user: Option, 22 | } 23 | 24 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 25 | pub enum Relation { 26 | #[sea_orm( 27 | belongs_to = "super::user::Entity", 28 | from = "Column::ImitatingUser", 29 | to = "super::user::Column::Id", 30 | on_update = "NoAction", 31 | on_delete = "NoAction" 32 | )] 33 | User, 34 | } 35 | 36 | impl Related for Entity { 37 | fn to() -> RelationDef { 38 | Relation::User.def() 39 | } 40 | } 41 | 42 | impl ActiveModelBehavior for ActiveModel {} 43 | -------------------------------------------------------------------------------- /entities/src/prelude.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | pub use super::admin::Entity as Admin; 4 | pub use super::admin_increase_balance_receipt::Entity as AdminIncreaseBalanceReceipt; 5 | pub use super::admin_trail::Entity as AdminTrail; 6 | pub use super::balance::Entity as Balance; 7 | pub use super::increase_on_chain_balance_receipt::Entity as IncreaseOnChainBalanceReceipt; 8 | pub use super::login::Entity as Login; 9 | pub use super::pending_login::Entity as PendingLogin; 10 | pub use super::referee::Entity as Referee; 11 | pub use super::referrer::Entity as Referrer; 12 | pub use super::revert_log::Entity as RevertLog; 13 | pub use super::rpc_accounting::Entity as RpcAccounting; 14 | pub use super::rpc_accounting_v2::Entity as RpcAccountingV2; 15 | pub use super::rpc_key::Entity as RpcKey; 16 | pub use super::secondary_user::Entity as SecondaryUser; 17 | pub use super::stripe_increase_balance_receipt::Entity as StripeIncreaseBalanceReceipt; 18 | pub use super::user::Entity as User; 19 | pub use super::user_tier::Entity as UserTier; 20 | -------------------------------------------------------------------------------- /entities/src/referee.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 7 | #[sea_orm(table_name = "referee")] 8 | pub struct Model { 9 | #[sea_orm(primary_key)] 10 | pub id: i32, 11 | #[sea_orm(column_type = "Decimal(Some((20, 10)))")] 12 | pub one_time_bonus_applied_for_referee: Decimal, 13 | #[sea_orm(column_type = "Decimal(Some((20, 10)))")] 14 | pub credits_applied_for_referrer: Decimal, 15 | pub referral_start_date: DateTime, 16 | pub used_referral_code: i32, 17 | #[sea_orm(unique)] 18 | pub user_id: u64, 19 | } 20 | 21 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 22 | pub enum Relation { 23 | #[sea_orm( 24 | belongs_to = "super::referrer::Entity", 25 | from = "Column::UsedReferralCode", 26 | to = "super::referrer::Column::Id", 27 | on_update = "NoAction", 28 | on_delete = "NoAction" 29 | )] 30 | Referrer, 31 | #[sea_orm( 32 | belongs_to = "super::user::Entity", 33 | from = "Column::UserId", 34 | to = "super::user::Column::Id", 35 | on_update = "NoAction", 36 | on_delete = "NoAction" 37 | )] 38 | User, 39 | } 40 | 41 | impl Related for Entity { 42 | fn to() -> RelationDef { 43 | Relation::Referrer.def() 44 | } 45 | } 46 | 47 | impl Related for Entity { 48 | fn to() -> RelationDef { 49 | Relation::User.def() 50 | } 51 | } 52 | 53 | impl ActiveModelBehavior for ActiveModel {} 54 | -------------------------------------------------------------------------------- /entities/src/referrer.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 7 | #[sea_orm(table_name = "referrer")] 8 | pub struct Model { 9 | #[sea_orm(primary_key)] 10 | pub id: i32, 11 | #[sea_orm(unique)] 12 | pub referral_code: String, 13 | #[sea_orm(unique)] 14 | pub user_id: u64, 15 | } 16 | 17 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 18 | pub enum Relation { 19 | #[sea_orm(has_many = "super::referee::Entity")] 20 | Referee, 21 | #[sea_orm( 22 | belongs_to = "super::user::Entity", 23 | from = "Column::UserId", 24 | to = "super::user::Column::Id", 25 | on_update = "NoAction", 26 | on_delete = "NoAction" 27 | )] 28 | User, 29 | } 30 | 31 | impl Related for Entity { 32 | fn to() -> RelationDef { 33 | Relation::Referee.def() 34 | } 35 | } 36 | 37 | impl Related for Entity { 38 | fn to() -> RelationDef { 39 | Relation::User.def() 40 | } 41 | } 42 | 43 | impl ActiveModelBehavior for ActiveModel {} 44 | -------------------------------------------------------------------------------- /entities/src/revert_log.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use super::sea_orm_active_enums::Method; 4 | use crate::serialization; 5 | use sea_orm::entity::prelude::*; 6 | use serde::{Deserialize, Serialize}; 7 | 8 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 9 | #[sea_orm(table_name = "revert_log")] 10 | pub struct Model { 11 | #[sea_orm(primary_key)] 12 | pub id: u64, 13 | pub rpc_key_id: u64, 14 | pub timestamp: DateTimeUtc, 15 | pub method: Method, 16 | #[sea_orm(column_type = "Binary(BlobSize::Blob(Some(20)))")] 17 | #[serde( 18 | serialize_with = "serialization::vec_as_address", 19 | deserialize_with = "serialization::address_to_vec" 20 | )] 21 | pub to: Vec, 22 | #[sea_orm(column_type = "Text", nullable)] 23 | pub call_data: Option, 24 | pub chain_id: u64, 25 | } 26 | 27 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 28 | pub enum Relation { 29 | #[sea_orm( 30 | belongs_to = "super::rpc_key::Entity", 31 | from = "Column::RpcKeyId", 32 | to = "super::rpc_key::Column::Id", 33 | on_update = "NoAction", 34 | on_delete = "NoAction" 35 | )] 36 | RpcKey, 37 | } 38 | 39 | impl Related for Entity { 40 | fn to() -> RelationDef { 41 | Relation::RpcKey.def() 42 | } 43 | } 44 | 45 | impl ActiveModelBehavior for ActiveModel {} 46 | -------------------------------------------------------------------------------- /entities/src/rpc_accounting.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] 7 | #[sea_orm(table_name = "rpc_accounting")] 8 | pub struct Model { 9 | #[sea_orm(primary_key)] 10 | pub id: u64, 11 | pub rpc_key_id: Option, 12 | pub chain_id: u64, 13 | pub method: Option, 14 | pub error_response: bool, 15 | pub period_datetime: DateTimeUtc, 16 | pub frontend_requests: u64, 17 | pub backend_requests: u64, 18 | pub cache_misses: u64, 19 | pub cache_hits: u64, 20 | pub sum_request_bytes: u64, 21 | pub min_request_bytes: u64, 22 | #[sea_orm(column_type = "Double")] 23 | pub mean_request_bytes: f64, 24 | pub p50_request_bytes: u64, 25 | pub p90_request_bytes: u64, 26 | pub p99_request_bytes: u64, 27 | pub max_request_bytes: u64, 28 | pub sum_response_millis: u64, 29 | pub min_response_millis: u64, 30 | #[sea_orm(column_type = "Double")] 31 | pub mean_response_millis: f64, 32 | pub p50_response_millis: u64, 33 | pub p90_response_millis: u64, 34 | pub p99_response_millis: u64, 35 | pub max_response_millis: u64, 36 | pub sum_response_bytes: u64, 37 | pub min_response_bytes: u64, 38 | #[sea_orm(column_type = "Double")] 39 | pub mean_response_bytes: f64, 40 | pub p50_response_bytes: u64, 41 | pub p90_response_bytes: u64, 42 | pub p99_response_bytes: u64, 43 | pub max_response_bytes: u64, 44 | pub archive_request: bool, 45 | pub origin: Option, 46 | pub migrated: Option, 47 | } 48 | 49 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 50 | pub enum Relation { 51 | #[sea_orm( 52 | belongs_to = "super::rpc_key::Entity", 53 | from = "Column::RpcKeyId", 54 | to = "super::rpc_key::Column::Id", 55 | on_update = "NoAction", 56 | on_delete = "NoAction" 57 | )] 58 | RpcKey, 59 | } 60 | 61 | impl Related for Entity { 62 | fn to() -> RelationDef { 63 | Relation::RpcKey.def() 64 | } 65 | } 66 | 67 | impl ActiveModelBehavior for ActiveModel {} 68 | -------------------------------------------------------------------------------- /entities/src/rpc_accounting_v2.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 7 | #[sea_orm(table_name = "rpc_accounting_v2")] 8 | pub struct Model { 9 | #[sea_orm(primary_key)] 10 | pub id: u64, 11 | pub rpc_key_id: Option, 12 | pub chain_id: u64, 13 | pub period_datetime: DateTimeUtc, 14 | pub archive_needed: bool, 15 | pub error_response: bool, 16 | pub frontend_requests: u64, 17 | pub backend_requests: u64, 18 | pub backend_retries: u64, 19 | pub no_servers: u64, 20 | pub cache_misses: u64, 21 | pub cache_hits: u64, 22 | pub sum_request_bytes: u64, 23 | pub sum_response_millis: u64, 24 | pub sum_response_bytes: u64, 25 | #[sea_orm(column_type = "Decimal(Some((20, 10)))")] 26 | pub sum_credits_used: Decimal, 27 | #[sea_orm(column_type = "Decimal(Some((20, 10)))")] 28 | pub sum_incl_free_credits_used: Decimal, 29 | } 30 | 31 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 32 | pub enum Relation { 33 | #[sea_orm( 34 | belongs_to = "super::rpc_key::Entity", 35 | from = "Column::RpcKeyId", 36 | to = "super::rpc_key::Column::Id", 37 | on_update = "NoAction", 38 | on_delete = "NoAction" 39 | )] 40 | RpcKey, 41 | } 42 | 43 | impl Related for Entity { 44 | fn to() -> RelationDef { 45 | Relation::RpcKey.def() 46 | } 47 | } 48 | 49 | impl ActiveModelBehavior for ActiveModel {} 50 | -------------------------------------------------------------------------------- /entities/src/rpc_key.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use crate::serialization; 4 | use sea_orm::entity::prelude::*; 5 | use serde::{Deserialize, Serialize}; 6 | 7 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] 8 | #[sea_orm(table_name = "rpc_key")] 9 | pub struct Model { 10 | #[sea_orm(primary_key)] 11 | pub id: u64, 12 | pub user_id: u64, 13 | #[sea_orm(column_type = "Binary(BlobSize::Blob(Some(16)))", unique)] 14 | #[serde( 15 | serialize_with = "serialization::uuid_as_ulid", 16 | deserialize_with = "serialization::ulid_to_uuid" 17 | )] 18 | pub secret_key: Uuid, 19 | pub description: Option, 20 | pub private_txs: bool, 21 | pub active: bool, 22 | #[sea_orm(column_type = "Text", nullable)] 23 | pub allowed_ips: Option, 24 | #[sea_orm(column_type = "Text", nullable)] 25 | pub allowed_origins: Option, 26 | #[sea_orm(column_type = "Text", nullable)] 27 | pub allowed_referers: Option, 28 | #[sea_orm(column_type = "Text", nullable)] 29 | pub allowed_user_agents: Option, 30 | #[sea_orm(column_type = "Double")] 31 | pub log_revert_chance: f64, 32 | } 33 | 34 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 35 | pub enum Relation { 36 | #[sea_orm(has_many = "super::revert_log::Entity")] 37 | RevertLog, 38 | #[sea_orm(has_many = "super::rpc_accounting::Entity")] 39 | RpcAccounting, 40 | #[sea_orm(has_many = "super::rpc_accounting_v2::Entity")] 41 | RpcAccountingV2, 42 | #[sea_orm(has_many = "super::secondary_user::Entity")] 43 | SecondaryUser, 44 | #[sea_orm( 45 | belongs_to = "super::user::Entity", 46 | from = "Column::UserId", 47 | to = "super::user::Column::Id", 48 | on_update = "NoAction", 49 | on_delete = "NoAction" 50 | )] 51 | User, 52 | } 53 | 54 | impl Related for Entity { 55 | fn to() -> RelationDef { 56 | Relation::RevertLog.def() 57 | } 58 | } 59 | 60 | impl Related for Entity { 61 | fn to() -> RelationDef { 62 | Relation::RpcAccounting.def() 63 | } 64 | } 65 | 66 | impl Related for Entity { 67 | fn to() -> RelationDef { 68 | Relation::RpcAccountingV2.def() 69 | } 70 | } 71 | 72 | impl Related for Entity { 73 | fn to() -> RelationDef { 74 | Relation::SecondaryUser.def() 75 | } 76 | } 77 | 78 | impl Related for Entity { 79 | fn to() -> RelationDef { 80 | Relation::User.def() 81 | } 82 | } 83 | 84 | impl ActiveModelBehavior for ActiveModel {} 85 | -------------------------------------------------------------------------------- /entities/src/sea_orm_active_enums.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] 7 | #[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "method")] 8 | pub enum Method { 9 | #[sea_orm(string_value = "eth_call")] 10 | EthCall, 11 | #[sea_orm(string_value = "eth_estimateGas")] 12 | EthEstimateGas, 13 | #[sea_orm(string_value = "eth_sendRawTransaction")] 14 | EthSendRawTransaction, 15 | } 16 | #[derive(Debug, Clone, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)] 17 | #[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "role")] 18 | pub enum Role { 19 | #[sea_orm(string_value = "owner")] 20 | Owner, 21 | #[sea_orm(string_value = "admin")] 22 | Admin, 23 | #[sea_orm(string_value = "collaborator")] 24 | Collaborator, 25 | } 26 | -------------------------------------------------------------------------------- /entities/src/secondary_user.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use super::sea_orm_active_enums::Role; 4 | use sea_orm::entity::prelude::*; 5 | use serde::{Deserialize, Serialize}; 6 | 7 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 8 | #[sea_orm(table_name = "secondary_user")] 9 | pub struct Model { 10 | #[sea_orm(primary_key)] 11 | pub id: u64, 12 | pub user_id: u64, 13 | pub description: Option, 14 | pub role: Role, 15 | pub rpc_secret_key_id: u64, 16 | } 17 | 18 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 19 | pub enum Relation { 20 | #[sea_orm( 21 | belongs_to = "super::rpc_key::Entity", 22 | from = "Column::RpcSecretKeyId", 23 | to = "super::rpc_key::Column::Id", 24 | on_update = "NoAction", 25 | on_delete = "NoAction" 26 | )] 27 | RpcKey, 28 | #[sea_orm( 29 | belongs_to = "super::user::Entity", 30 | from = "Column::UserId", 31 | to = "super::user::Column::Id", 32 | on_update = "NoAction", 33 | on_delete = "NoAction" 34 | )] 35 | User, 36 | } 37 | 38 | impl Related for Entity { 39 | fn to() -> RelationDef { 40 | Relation::RpcKey.def() 41 | } 42 | } 43 | 44 | impl Related for Entity { 45 | fn to() -> RelationDef { 46 | Relation::User.def() 47 | } 48 | } 49 | 50 | impl ActiveModelBehavior for ActiveModel {} 51 | -------------------------------------------------------------------------------- /entities/src/serialization.rs: -------------------------------------------------------------------------------- 1 | //! sea-orm types don't always serialize how we want. this helps that, though it won't help every case. 2 | use ethers::prelude::Address; 3 | use sea_orm::prelude::Uuid; 4 | use serde::{Deserialize, Deserializer, Serialize, Serializer}; 5 | use std::convert::TryInto; 6 | use ulid::Ulid; 7 | 8 | pub fn to_fixed_length(v: Vec) -> [T; N] { 9 | v.try_into() 10 | .unwrap_or_else(|v: Vec| panic!("Expected a Vec of length {} but it was {}", N, v.len())) 11 | } 12 | 13 | pub fn vec_as_address(x: &[u8], s: S) -> Result 14 | where 15 | S: Serializer, 16 | { 17 | let x = Address::from_slice(x); 18 | 19 | x.serialize(s) 20 | } 21 | 22 | pub fn address_to_vec<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { 23 | let address = Address::deserialize(deserializer)?; 24 | 25 | Ok(address.to_fixed_bytes().into()) 26 | } 27 | 28 | pub fn uuid_as_ulid(x: &Uuid, s: S) -> Result 29 | where 30 | S: Serializer, 31 | { 32 | let x = Ulid::from(x.as_u128()); 33 | 34 | // TODO: to_string shouldn't be needed, but i'm still seeing Uuid length 35 | x.to_string().serialize(s) 36 | } 37 | 38 | pub fn ulid_to_uuid<'de, D: Deserializer<'de>>(deserializer: D) -> Result { 39 | let ulid = Ulid::deserialize(deserializer)?; 40 | 41 | Ok(ulid.into()) 42 | } 43 | -------------------------------------------------------------------------------- /entities/src/stripe_increase_balance_receipt.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 7 | #[sea_orm(table_name = "stripe_increase_balance_receipt")] 8 | pub struct Model { 9 | #[sea_orm(primary_key)] 10 | pub id: u64, 11 | pub stripe_payment_intend_id: String, 12 | pub deposit_to_user_id: Option, 13 | #[sea_orm(column_type = "Decimal(Some((20, 10)))")] 14 | pub amount: Decimal, 15 | pub currency: String, 16 | pub status: String, 17 | pub description: Option, 18 | pub date_created: DateTimeUtc, 19 | } 20 | 21 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 22 | pub enum Relation { 23 | #[sea_orm( 24 | belongs_to = "super::user::Entity", 25 | from = "Column::DepositToUserId", 26 | to = "super::user::Column::Id", 27 | on_update = "NoAction", 28 | on_delete = "NoAction" 29 | )] 30 | User, 31 | } 32 | 33 | impl Related for Entity { 34 | fn to() -> RelationDef { 35 | Relation::User.def() 36 | } 37 | } 38 | 39 | impl ActiveModelBehavior for ActiveModel {} 40 | -------------------------------------------------------------------------------- /entities/src/user_tier.rs: -------------------------------------------------------------------------------- 1 | //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 2 | 3 | use sea_orm::entity::prelude::*; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] 7 | #[sea_orm(table_name = "user_tier")] 8 | pub struct Model { 9 | #[sea_orm(primary_key)] 10 | pub id: u64, 11 | pub title: String, 12 | pub max_requests_per_period: Option, 13 | pub max_concurrent_requests: Option, 14 | pub downgrade_tier_id: Option, 15 | } 16 | 17 | #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] 18 | pub enum Relation { 19 | #[sea_orm(has_many = "super::user::Entity")] 20 | User, 21 | #[sea_orm( 22 | belongs_to = "Entity", 23 | from = "Column::DowngradeTierId", 24 | to = "Column::Id", 25 | on_update = "NoAction", 26 | on_delete = "NoAction" 27 | )] 28 | SelfRef, 29 | } 30 | 31 | impl Related for Entity { 32 | fn to() -> RelationDef { 33 | Relation::User.def() 34 | } 35 | } 36 | 37 | impl ActiveModelBehavior for ActiveModel {} 38 | -------------------------------------------------------------------------------- /example.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | user.id AS user_id, 3 | COALESCE(SUM(admin_receipt.amount), 0) + COALESCE(SUM(chain_receipt.amount), 0) + COALESCE(SUM(stripe_receipt.amount), 0) + COALESCE(SUM(referee.one_time_bonus_applied_for_referee), 0) + COALESCE(referrer_bonus.total_bonus, 0) AS total_deposits, 4 | COALESCE(SUM(accounting.sum_credits_used), 0) AS total_spent_including_free_tier, 5 | COALESCE(SUM(accounting.sum_incl_free_credits_used), 0) AS total_spent_outside_free_tier 6 | FROM 7 | user 8 | LEFT JOIN 9 | admin_increase_balance_receipt AS admin_receipt ON user.id = admin_receipt.deposit_to_user_id 10 | LEFT JOIN 11 | increase_on_chain_balance_receipt AS chain_receipt ON user.id = chain_receipt.deposit_to_user_id 12 | LEFT JOIN 13 | stripe_increase_balance_receipt AS stripe_receipt ON user.id = stripe_receipt.deposit_to_user_id 14 | LEFT JOIN 15 | referee ON user.id = referee.user_id 16 | LEFT JOIN 17 | (SELECT referrer.user_id, SUM(referee.credits_applied_for_referrer) AS total_bonus 18 | FROM referrer 19 | JOIN referee ON referrer.id = referee.used_referral_code 20 | GROUP BY referrer.user_id) AS referrer_bonus ON user.id = referrer_bonus.user_id 21 | LEFT JOIN 22 | rpc_key ON user.id = rpc_key.user_id 23 | LEFT JOIN 24 | rpc_accounting_v2 AS accounting ON rpc_key.id = accounting.rpc_key_id 25 | LEFT JOIN 26 | user_tier ON user.user_tier_id = user_tier.id 27 | WHERE 28 | user.id = 1; -------------------------------------------------------------------------------- /latency/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "latency" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | portable-atomic = { version = "1.5.1", features = ["float"] } 10 | serde = { version = "1.0.193", features = [] } 11 | tokio = { version = "1.34.0", features = ["full"] } 12 | tracing = "0.1.40" 13 | watermill = "0.1.1" 14 | 15 | [dev-dependencies] 16 | tokio = { version = "1.34.0", features = ["full", "test-util"] } 17 | -------------------------------------------------------------------------------- /latency/src/ewma.rs: -------------------------------------------------------------------------------- 1 | use crate::util::span::span_to_alpha; 2 | use serde::ser::Serializer; 3 | use serde::Serialize; 4 | use tokio::time::Duration; 5 | use watermill::ewmean::EWMean; 6 | use watermill::stats::Univariate; 7 | 8 | pub struct EwmaLatency { 9 | /// exponentially weighted of some latency in milliseconds 10 | /// TODO: compare crates: ewma vs watermill 11 | seconds: EWMean, 12 | } 13 | 14 | /// serialize as milliseconds 15 | impl Serialize for EwmaLatency { 16 | fn serialize(&self, serializer: S) -> Result 17 | where 18 | S: Serializer, 19 | { 20 | serializer.serialize_f32(self.seconds.get() * 1000.0) 21 | } 22 | } 23 | 24 | impl EwmaLatency { 25 | #[inline] 26 | pub fn record(&mut self, duration: Duration) { 27 | self.record_secs(duration.as_secs_f32()); 28 | } 29 | 30 | #[inline] 31 | pub fn record_secs(&mut self, secs: f32) { 32 | // TODO: we could change this to use a channel like the peak_ewma and rolling_quantile code, but this is fine if it updates on insert instead of async 33 | self.seconds.update(secs); 34 | } 35 | 36 | /// Current EWMA value in seconds 37 | #[inline] 38 | pub fn value(&self) -> f32 { 39 | self.seconds.get() 40 | } 41 | 42 | /// Current EWMA value in seconds 43 | #[inline] 44 | pub fn latency(&self) -> Duration { 45 | let x = self.seconds.get(); 46 | 47 | Duration::from_secs_f32(x) 48 | } 49 | } 50 | 51 | impl Default for EwmaLatency { 52 | fn default() -> Self { 53 | // TODO: what should the default span be? 10 requests? 54 | let span = 10.0; 55 | 56 | // TODO: what should the defautt start be? 57 | let start = 60.0; 58 | 59 | Self::new(span, start) 60 | } 61 | } 62 | 63 | impl EwmaLatency { 64 | // depending on the span, start might not be perfect 65 | pub fn new(span: f32, start_ms: f32) -> Self { 66 | let alpha = span_to_alpha(span); 67 | 68 | let mut seconds = EWMean::new(alpha); 69 | 70 | if start_ms > 0.0 { 71 | for _ in 0..(span as u64) { 72 | seconds.update(start_ms); 73 | } 74 | } 75 | 76 | Self { seconds } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /latency/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod ewma; 2 | mod peak_ewma; 3 | mod rolling_quantile; 4 | mod util; 5 | 6 | pub use self::ewma::EwmaLatency; 7 | pub use self::peak_ewma::PeakEwmaLatency; 8 | pub use self::rolling_quantile::RollingQuantileLatency; 9 | -------------------------------------------------------------------------------- /latency/src/util/atomic_f32_pair.rs: -------------------------------------------------------------------------------- 1 | use std::sync::atomic::{AtomicU64, Ordering}; 2 | 3 | /// Implements an atomic pair of f32s 4 | /// 5 | /// This uses an AtomicU64 internally. 6 | #[derive(Debug)] 7 | pub struct AtomicF32Pair(AtomicU64); 8 | 9 | impl AtomicF32Pair { 10 | /// Creates a new atomic pair. 11 | pub fn new(pair: [f32; 2]) -> Self { 12 | Self(AtomicU64::new(to_bits(pair))) 13 | } 14 | 15 | /// Loads a value from the atomic pair. 16 | pub fn load(&self, ordering: Ordering) -> [f32; 2] { 17 | from_bits(self.0.load(ordering)) 18 | } 19 | 20 | /// Fetches the value, and applies a function to it that returns an 21 | /// optional new value. Returns a Result of Ok(previous_value) if 22 | /// the function returned Some(_), else Err(previous_value). 23 | pub fn fetch_update( 24 | &self, 25 | set_order: Ordering, 26 | fetch_order: Ordering, 27 | mut f: F, 28 | ) -> Result<[f32; 2], [f32; 2]> 29 | where 30 | F: FnMut([f32; 2]) -> Option<[f32; 2]>, 31 | { 32 | self.0 33 | .fetch_update(set_order, fetch_order, |bits| { 34 | f(from_bits(bits)).map(to_bits) 35 | }) 36 | .map(from_bits) 37 | .map_err(from_bits) 38 | } 39 | } 40 | 41 | /// Convert a f32 pair to its bit-representation as u64 42 | fn to_bits(pair: [f32; 2]) -> u64 { 43 | let f1 = pair[0].to_bits() as u64; 44 | let f2 = pair[1].to_bits() as u64; 45 | (f1 << 32) | f2 46 | } 47 | 48 | /// Build a f32 pair from its bit-representation as u64 49 | fn from_bits(bits: u64) -> [f32; 2] { 50 | let f1 = f32::from_bits((bits >> 32) as u32); 51 | let f2 = f32::from_bits(bits as u32); 52 | [f1, f2] 53 | } 54 | 55 | #[cfg(test)] 56 | mod tests { 57 | use std::f32; 58 | use std::sync::atomic::Ordering; 59 | 60 | use super::{from_bits, to_bits, AtomicF32Pair}; 61 | 62 | #[test] 63 | fn test_f32_pair_bit_conversions() { 64 | let pair = [f32::consts::PI, f32::consts::E]; 65 | assert_eq!(pair, from_bits(to_bits(pair))); 66 | } 67 | 68 | #[test] 69 | fn test_atomic_f32_pair_load() { 70 | let pair = [f32::consts::PI, f32::consts::E]; 71 | let atomic = AtomicF32Pair::new(pair); 72 | assert_eq!(pair, atomic.load(Ordering::Relaxed)); 73 | } 74 | 75 | #[test] 76 | fn test_atomic_f32_pair_fetch_update() { 77 | let pair = [f32::consts::PI, f32::consts::E]; 78 | let atomic = AtomicF32Pair::new(pair); 79 | atomic 80 | .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |[f1, f2]| { 81 | Some([f1 + 1.0, f2 + 1.0]) 82 | }) 83 | .unwrap(); 84 | assert_eq!( 85 | [pair[0] + 1.0, pair[1] + 1.0], 86 | atomic.load(Ordering::Relaxed) 87 | ); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /latency/src/util/mod.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod atomic_f32_pair; 2 | pub(crate) mod nanos; 3 | pub(crate) mod span; 4 | -------------------------------------------------------------------------------- /latency/src/util/nanos.rs: -------------------------------------------------------------------------------- 1 | use tokio::time::Duration; 2 | 3 | pub const NANOS_PER_MILLI: f64 = 1_000_000.0; 4 | 5 | /// Utility that converts durations to nanos in f64. 6 | /// 7 | /// Due to a lossy transformation, the maximum value that can be represented is ~585 years, 8 | /// which, I hope, is more than enough to represent request latencies. 9 | pub fn nanos(d: Duration) -> f64 { 10 | const NANOS_PER_SEC: u64 = 1_000_000_000; 11 | let n = f64::from(d.subsec_nanos()); 12 | let s = d.as_secs().saturating_mul(NANOS_PER_SEC) as f64; 13 | n + s 14 | } 15 | 16 | #[cfg(test)] 17 | mod tests { 18 | use tokio::time::Duration; 19 | 20 | #[test] 21 | fn nanos() { 22 | assert_eq!(super::nanos(Duration::new(0, 0)), 0.0); 23 | assert_eq!(super::nanos(Duration::new(0, 123)), 123.0); 24 | assert_eq!(super::nanos(Duration::new(1, 23)), 1_000_000_023.0); 25 | assert_eq!( 26 | super::nanos(Duration::new(::std::u64::MAX, 999_999_999)), 27 | 18446744074709553000.0 28 | ); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /latency/src/util/span.rs: -------------------------------------------------------------------------------- 1 | // TODO: generic for any float 2 | pub fn span_to_alpha(span: f32) -> f32 { 3 | 2.0 / (span + 1.0) 4 | } 5 | -------------------------------------------------------------------------------- /migration/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "migration" 3 | version = "0.43.0" 4 | edition = "2021" 5 | publish = false 6 | 7 | [lib] 8 | name = "migration" 9 | path = "src/lib.rs" 10 | 11 | [dependencies] 12 | tokio = { version = "1.34.0", features = ["full", "tracing"] } 13 | chrono = "0.4.31" 14 | 15 | [dependencies.sea-orm-migration] 16 | version = "0.12.6" 17 | features = [ 18 | # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. 19 | # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. 20 | "runtime-tokio-rustls", # `ASYNC_RUNTIME` feature 21 | "sqlx-mysql", # `DATABASE_DRIVER` feature 22 | ] 23 | -------------------------------------------------------------------------------- /migration/README.md: -------------------------------------------------------------------------------- 1 | # Running Migrator CLI 2 | 3 | - Generate a new migration file 4 | ```sh 5 | cargo run -- generate MIGRATION_NAME 6 | ``` 7 | - Apply all pending migrations 8 | ```sh 9 | cargo run 10 | ``` 11 | ```sh 12 | cargo run -- up 13 | ``` 14 | - Apply first 10 pending migrations 15 | ```sh 16 | cargo run -- up -n 10 17 | ``` 18 | - Rollback last applied migrations 19 | ```sh 20 | cargo run -- down 21 | ``` 22 | - Rollback last 10 applied migrations 23 | ```sh 24 | cargo run -- down -n 10 25 | ``` 26 | - Drop all tables from the database, then reapply all migrations 27 | ```sh 28 | cargo run -- fresh 29 | ``` 30 | - Rollback all applied migrations, then reapply all migrations 31 | ```sh 32 | cargo run -- refresh 33 | ``` 34 | - Rollback all applied migrations 35 | ```sh 36 | cargo run -- reset 37 | ``` 38 | - Check the status of all migrations 39 | ```sh 40 | cargo run -- status 41 | ``` 42 | -------------------------------------------------------------------------------- /migration/src/m20220928_015108_concurrency_limits.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | use sea_orm_migration::sea_query::table::ColumnDef; 3 | 4 | #[derive(DeriveMigrationName)] 5 | pub struct Migration; 6 | 7 | #[async_trait::async_trait] 8 | impl MigrationTrait for Migration { 9 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 10 | // add a field to the UserKeys table 11 | manager 12 | .alter_table( 13 | sea_query::Table::alter() 14 | .table(UserKeys::Table) 15 | // add column for a better version of rate limiting 16 | .add_column( 17 | ColumnDef::new(UserKeys::MaxConcurrentRequests) 18 | .big_unsigned() 19 | .null() 20 | .default(200), 21 | ) 22 | .to_owned(), 23 | ) 24 | .await 25 | } 26 | 27 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 28 | // put the UserKeys back to how it was before our migrations 29 | manager 30 | .alter_table( 31 | sea_query::Table::alter() 32 | .table(UserKeys::Table) 33 | .drop_column(UserKeys::MaxConcurrentRequests) 34 | .to_owned(), 35 | ) 36 | .await 37 | } 38 | } 39 | 40 | // copied from *_log_reverts.rs, but added new columns 41 | #[derive(Iden)] 42 | pub enum UserKeys { 43 | Table, 44 | // we don't touch some of the columns 45 | // Id, 46 | // UserId, 47 | // ApiKey, 48 | // Description, 49 | // PrivateTxs, 50 | // Active, 51 | // RequestsPerMinute, 52 | // LogRevertChance, 53 | // AllowedIps, 54 | // AllowedOrigins, 55 | // AllowedReferers, 56 | // AllowedUserAgents, 57 | MaxConcurrentRequests, 58 | } 59 | -------------------------------------------------------------------------------- /migration/src/m20221025_210326_add_chain_id_to_reverts.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::table::ColumnDef; 2 | use sea_orm_migration::prelude::*; 3 | 4 | #[derive(DeriveMigrationName)] 5 | pub struct Migration; 6 | 7 | #[async_trait::async_trait] 8 | impl MigrationTrait for Migration { 9 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 10 | // add a field to the UserKeys table 11 | manager 12 | .alter_table( 13 | sea_query::Table::alter() 14 | .table(RevertLogs::Table) 15 | // add column for a better version of rate limiting 16 | .add_column( 17 | ColumnDef::new(RevertLogs::ChainId) 18 | .big_unsigned() 19 | .not_null() 20 | // create it with a default of 1 21 | .default(1), 22 | ) 23 | .to_owned(), 24 | ) 25 | .await?; 26 | 27 | manager 28 | .alter_table( 29 | sea_query::Table::alter() 30 | .table(RevertLogs::Table) 31 | // remove the default 32 | .modify_column( 33 | ColumnDef::new(RevertLogs::ChainId) 34 | .big_unsigned() 35 | .not_null(), 36 | ) 37 | .to_owned(), 38 | ) 39 | .await?; 40 | 41 | Ok(()) 42 | } 43 | 44 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 45 | // put the RevertLogs back to how it was before our migrations 46 | manager 47 | .alter_table( 48 | sea_query::Table::alter() 49 | .table(RevertLogs::Table) 50 | .drop_column(RevertLogs::ChainId) 51 | .to_owned(), 52 | ) 53 | .await 54 | } 55 | } 56 | 57 | #[derive(Iden)] 58 | enum RevertLogs { 59 | Table, 60 | // Id, 61 | // UserKeyId, 62 | // Method, 63 | // CallData, 64 | // To, 65 | // Timestamp, 66 | ChainId, 67 | } 68 | -------------------------------------------------------------------------------- /migration/src/m20221026_230819_rename_user_keys.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .rename_table( 11 | Table::rename() 12 | .table(Alias::new("user_keys"), Alias::new("rpc_keys")) 13 | .to_owned(), 14 | ) 15 | .await?; 16 | 17 | manager 18 | .alter_table( 19 | Table::alter() 20 | .table(Alias::new("rpc_keys")) 21 | .rename_column(Alias::new("api_key"), Alias::new("rpc_key")) 22 | .to_owned(), 23 | ) 24 | .await?; 25 | 26 | Ok(()) 27 | } 28 | 29 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 30 | manager 31 | .alter_table( 32 | Table::alter() 33 | .table(Alias::new("rpc_keys")) 34 | .rename_column(Alias::new("rpc_key"), Alias::new("api_key")) 35 | .to_owned(), 36 | ) 37 | .await?; 38 | 39 | manager 40 | .rename_table( 41 | Table::rename() 42 | .table(Alias::new("rpc_keys"), Alias::new("user_keys")) 43 | .to_owned(), 44 | ) 45 | .await?; 46 | 47 | Ok(()) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /migration/src/m20221101_222349_archive_request.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .alter_table( 11 | Table::alter() 12 | .table(Alias::new("rpc_accounting")) 13 | .add_column( 14 | ColumnDef::new(Alias::new("archive_request")) 15 | .boolean() 16 | .not_null(), 17 | ) 18 | .drop_column(Alias::new("backend_retries")) 19 | .drop_column(Alias::new("no_servers")) 20 | .to_owned(), 21 | ) 22 | .await?; 23 | 24 | Ok(()) 25 | } 26 | 27 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 28 | manager 29 | .alter_table( 30 | Table::alter() 31 | .table(Alias::new("rpc_accounting")) 32 | .drop_column(Alias::new("archive_request")) 33 | .to_owned(), 34 | ) 35 | .await?; 36 | 37 | Ok(()) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /migration/src/m20221108_200345_save_anon_stats.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | // note: somehow this column got added in prod, but the migration wasn't marked as complete 10 | let _ = manager 11 | .alter_table( 12 | Table::alter() 13 | .table(RpcAccounting::Table) 14 | .drop_column(RpcAccounting::Origin) 15 | .to_owned(), 16 | ) 17 | .await; 18 | 19 | manager 20 | .alter_table( 21 | Table::alter() 22 | .table(RpcAccounting::Table) 23 | .modify_column( 24 | ColumnDef::new(RpcAccounting::RpcKeyId) 25 | .big_unsigned() 26 | .null(), 27 | ) 28 | .add_column(ColumnDef::new(RpcAccounting::Origin).string().null()) 29 | .to_owned(), 30 | ) 31 | .await 32 | } 33 | 34 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 35 | manager 36 | .alter_table( 37 | Table::alter() 38 | .table(RpcAccounting::Table) 39 | .modify_column( 40 | ColumnDef::new(RpcAccounting::RpcKeyId) 41 | .big_unsigned() 42 | .not_null(), 43 | ) 44 | .drop_column(RpcAccounting::Origin) 45 | .to_owned(), 46 | ) 47 | .await 48 | } 49 | } 50 | 51 | /// Learn more at https://docs.rs/sea-query#iden 52 | #[derive(Iden)] 53 | enum RpcAccounting { 54 | Table, 55 | RpcKeyId, 56 | Origin, 57 | } 58 | -------------------------------------------------------------------------------- /migration/src/m20221211_124002_request_method_privacy.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | // allow null method 10 | manager 11 | .alter_table( 12 | Table::alter() 13 | .table(Alias::new("rpc_accounting")) 14 | .modify_column(ColumnDef::new(Alias::new("method")).string().null()) 15 | .to_owned(), 16 | ) 17 | .await?; 18 | 19 | // existing keys get set to detailed logging 20 | manager 21 | .alter_table( 22 | Table::alter() 23 | .table(RpcKey::Table) 24 | .add_column( 25 | ColumnDef::new(RpcKey::LogLevel) 26 | .enumeration( 27 | Alias::new("log_level"), 28 | [ 29 | Alias::new("none"), 30 | Alias::new("aggregated"), 31 | Alias::new("detailed"), 32 | ], 33 | ) 34 | .not_null() 35 | .default("detailed"), 36 | ) 37 | .to_owned(), 38 | ) 39 | .await?; 40 | 41 | // new keys get set to no logging 42 | manager 43 | .alter_table( 44 | Table::alter() 45 | .table(RpcKey::Table) 46 | .modify_column( 47 | ColumnDef::new(RpcKey::LogLevel) 48 | .enumeration( 49 | Alias::new("log_level"), 50 | [ 51 | Alias::new("none"), 52 | Alias::new("aggregated"), 53 | Alias::new("detailed"), 54 | ], 55 | ) 56 | .not_null() 57 | .default("none"), 58 | ) 59 | .to_owned(), 60 | ) 61 | .await?; 62 | 63 | Ok(()) 64 | } 65 | 66 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 67 | manager 68 | .alter_table( 69 | Table::alter() 70 | .table(RpcKey::Table) 71 | .drop_column(RpcKey::LogLevel) 72 | .to_owned(), 73 | ) 74 | .await?; 75 | 76 | Ok(()) 77 | } 78 | } 79 | 80 | /// Learn more at https://docs.rs/sea-query#iden 81 | #[derive(Iden)] 82 | enum RpcKey { 83 | Table, 84 | LogLevel, 85 | } 86 | -------------------------------------------------------------------------------- /migration/src/m20230117_191358_admin_table.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .create_table( 11 | Table::create() 12 | .table(Admin::Table) 13 | .col( 14 | ColumnDef::new(Admin::Id) 15 | .big_unsigned() 16 | .not_null() 17 | .auto_increment() 18 | .primary_key(), 19 | ) 20 | .col( 21 | ColumnDef::new(Admin::UserId) 22 | .big_unsigned() 23 | .unique_key() 24 | .not_null(), 25 | ) 26 | .foreign_key( 27 | ForeignKey::create() 28 | .name("fk-admin-user_id") 29 | .from(Admin::Table, Admin::UserId) 30 | .to(User::Table, User::Id), 31 | ) 32 | .to_owned(), 33 | ) 34 | .await 35 | } 36 | 37 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 38 | manager 39 | .drop_table(Table::drop().table(Admin::Table).to_owned()) 40 | .await 41 | } 42 | } 43 | 44 | /// Learn more at https://docs.rs/sea-query#iden 45 | #[derive(Iden)] 46 | enum User { 47 | Table, 48 | Id, 49 | } 50 | 51 | #[derive(Iden)] 52 | enum Admin { 53 | Table, 54 | Id, 55 | UserId, 56 | } 57 | -------------------------------------------------------------------------------- /migration/src/m20230119_204135_better_free_tier.rs: -------------------------------------------------------------------------------- 1 | //! Increase requests per minute for the free tier to be better than our public tier (which has 3900/min) 2 | use sea_orm_migration::{prelude::*, sea_orm::ConnectionTrait}; 3 | 4 | #[derive(DeriveMigrationName)] 5 | pub struct Migration; 6 | 7 | #[async_trait::async_trait] 8 | impl MigrationTrait for Migration { 9 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 10 | let db_conn = manager.get_connection(); 11 | let db_backend = manager.get_database_backend(); 12 | 13 | let update_free = Query::update() 14 | .table(UserTier::Table) 15 | .value(UserTier::MaxRequestsPerPeriod, 6000) 16 | .and_where(Expr::col(UserTier::Title).eq("Free")) 17 | .limit(1) 18 | .to_owned(); 19 | 20 | let x = db_backend.build(&update_free); 21 | 22 | let rows_affected = db_conn.execute(x).await?.rows_affected(); 23 | 24 | assert_eq!(rows_affected, 1, "unable to update free tier"); 25 | 26 | Ok(()) 27 | } 28 | 29 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 30 | unimplemented!(); 31 | } 32 | } 33 | 34 | #[derive(Iden)] 35 | enum UserTier { 36 | Table, 37 | Title, 38 | MaxRequestsPerPeriod, 39 | } 40 | -------------------------------------------------------------------------------- /migration/src/m20230130_124740_read_only_login_logic.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | // Add a read-only column to the table 10 | manager 11 | .alter_table( 12 | Table::alter() 13 | .table(Login::Table) 14 | .add_column(ColumnDef::new(Login::ReadOnly).boolean().not_null()) 15 | .to_owned(), 16 | ) 17 | .await 18 | } 19 | 20 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 21 | // Drop the column from the table ... 22 | manager 23 | .alter_table( 24 | Table::alter() 25 | .table(Login::Table) 26 | .drop_column(Login::ReadOnly) 27 | .to_owned(), 28 | ) 29 | .await 30 | } 31 | } 32 | 33 | /// Learn more at https://docs.rs/sea-query#iden 34 | #[derive(Iden)] 35 | enum Login { 36 | Table, 37 | // Id, 38 | // BearerToken, 39 | ReadOnly, 40 | // UserId, 41 | } 42 | -------------------------------------------------------------------------------- /migration/src/m20230130_165144_prepare_admin_imitation_pre_login.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .alter_table( 11 | Table::alter() 12 | .table(PendingLogin::Table) 13 | .add_column(ColumnDef::new(PendingLogin::ImitatingUser).big_unsigned()) 14 | .add_foreign_key( 15 | TableForeignKey::new() 16 | .name("fk-pending_login-imitating_user") 17 | .from_tbl(PendingLogin::Table) 18 | .to_tbl(User::Table) 19 | .from_col(PendingLogin::ImitatingUser) 20 | .to_col(User::Id), 21 | ) 22 | .to_owned(), 23 | ) 24 | .await 25 | } 26 | 27 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 28 | manager 29 | .alter_table( 30 | Table::alter() 31 | .table(PendingLogin::Table) 32 | .drop_foreign_key(Alias::new("fk-pending_login-imitating_user")) 33 | .drop_column(PendingLogin::ImitatingUser) 34 | .to_owned(), 35 | ) 36 | .await 37 | } 38 | } 39 | 40 | /// Learn more at https://docs.rs/sea-query#iden 41 | #[derive(Iden)] 42 | enum PendingLogin { 43 | Table, 44 | // Id, 45 | // Nonce, 46 | // Message, 47 | // ExpiresAt, 48 | ImitatingUser, 49 | } 50 | 51 | /// Learn more at https://docs.rs/sea-query#iden 52 | #[derive(Iden)] 53 | enum User { 54 | Table, 55 | Id, 56 | } 57 | -------------------------------------------------------------------------------- /migration/src/m20230205_130035_create_balance.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .create_table( 11 | Table::create() 12 | .table(Balance::Table) 13 | .if_not_exists() 14 | .col( 15 | ColumnDef::new(Balance::Id) 16 | .integer() 17 | .not_null() 18 | .auto_increment() 19 | .primary_key(), 20 | ) 21 | .col( 22 | ColumnDef::new(Balance::AvailableBalance) 23 | .decimal_len(20, 10) 24 | .not_null() 25 | .default(0.0), 26 | ) 27 | .col( 28 | ColumnDef::new(Balance::UsedBalance) 29 | .decimal_len(20, 10) 30 | .not_null() 31 | .default(0.0), 32 | ) 33 | .col( 34 | ColumnDef::new(Balance::UserId) 35 | .big_unsigned() 36 | .unique_key() 37 | .not_null(), 38 | ) 39 | .foreign_key( 40 | sea_query::ForeignKey::create() 41 | .from(Balance::Table, Balance::UserId) 42 | .to(User::Table, User::Id), 43 | ) 44 | .to_owned(), 45 | ) 46 | .await 47 | } 48 | 49 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 50 | manager 51 | .drop_table(Table::drop().table(Balance::Table).to_owned()) 52 | .await 53 | } 54 | } 55 | 56 | /// Learn more at https://docs.rs/sea-query#iden 57 | #[derive(Iden)] 58 | enum User { 59 | Table, 60 | Id, 61 | } 62 | 63 | #[allow(clippy::enum_variant_names)] 64 | #[derive(Iden)] 65 | enum Balance { 66 | Table, 67 | Id, 68 | UserId, 69 | AvailableBalance, 70 | UsedBalance, 71 | } 72 | -------------------------------------------------------------------------------- /migration/src/m20230215_152254_admin_trail.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .create_table( 11 | Table::create() 12 | .table(AdminTrail::Table) 13 | .if_not_exists() 14 | .col( 15 | ColumnDef::new(AdminTrail::Id) 16 | .integer() 17 | .not_null() 18 | .auto_increment() 19 | .primary_key(), 20 | ) 21 | .col( 22 | ColumnDef::new(AdminTrail::Caller).big_unsigned().not_null(), // TODO: Add Foreign Key 23 | ) 24 | .foreign_key( 25 | sea_query::ForeignKey::create() 26 | .from(AdminTrail::Table, AdminTrail::Caller) 27 | .to(User::Table, User::Id), 28 | ) 29 | .col( 30 | ColumnDef::new(AdminTrail::ImitatingUser).big_unsigned(), // Can be null bcs maybe we're just logging in / using endpoints that don't imitate a user 31 | // TODO: Add Foreign Key 32 | ) 33 | .foreign_key( 34 | sea_query::ForeignKey::create() 35 | .from(AdminTrail::Table, AdminTrail::ImitatingUser) 36 | .to(User::Table, User::Id), 37 | ) 38 | .col(ColumnDef::new(AdminTrail::Endpoint).string().not_null()) 39 | .col(ColumnDef::new(AdminTrail::Payload).string().not_null()) 40 | .col( 41 | ColumnDef::new(AdminTrail::Timestamp) 42 | .timestamp() 43 | .not_null() 44 | .extra("DEFAULT CURRENT_TIMESTAMP".to_string()), 45 | ) 46 | .to_owned(), 47 | ) 48 | .await 49 | } 50 | 51 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 52 | manager 53 | .drop_table(Table::drop().table(AdminTrail::Table).to_owned()) 54 | .await 55 | } 56 | } 57 | 58 | /// Learn more at https://docs.rs/sea-query#iden 59 | #[derive(Iden)] 60 | enum AdminTrail { 61 | Table, 62 | Id, 63 | Caller, 64 | ImitatingUser, 65 | Endpoint, 66 | Payload, 67 | Timestamp, 68 | } 69 | 70 | /// Learn more at https://docs.rs/sea-query#iden 71 | #[derive(Iden)] 72 | enum User { 73 | Table, 74 | Id, 75 | // Address, 76 | // Description, 77 | // Email, 78 | } 79 | -------------------------------------------------------------------------------- /migration/src/m20230221_230953_track_spend.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | // Track spend inside the RPC accounting v2 table 10 | manager 11 | .alter_table( 12 | Table::alter() 13 | .table(RpcAccountingV2::Table) 14 | .add_column( 15 | ColumnDef::new(RpcAccountingV2::SumCreditsUsed) 16 | .decimal_len(20, 10) 17 | .not_null(), 18 | ) 19 | .to_owned(), 20 | ) 21 | .await 22 | } 23 | 24 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 25 | manager 26 | .alter_table( 27 | sea_query::Table::alter() 28 | .table(RpcAccountingV2::Table) 29 | .drop_column(RpcAccountingV2::SumCreditsUsed) 30 | .to_owned(), 31 | ) 32 | .await 33 | } 34 | } 35 | 36 | /// Learn more at https://docs.rs/sea-query#iden 37 | #[derive(Iden)] 38 | enum RpcAccountingV2 { 39 | Table, 40 | SumCreditsUsed, 41 | } 42 | -------------------------------------------------------------------------------- /migration/src/m20230307_002623_migrate_rpc_accounting_to_rpc_accounting_v2.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | // Add a nullable timestamp column to check if things were migrated in the rpc_accounting table 10 | manager 11 | .alter_table( 12 | Table::alter() 13 | .table(RpcAccounting::Table) 14 | .add_column(ColumnDef::new(RpcAccounting::Migrated).timestamp()) 15 | .to_owned(), 16 | ) 17 | .await 18 | } 19 | 20 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 21 | manager 22 | .alter_table( 23 | Table::alter() 24 | .table(RpcAccounting::Table) 25 | .drop_column(RpcAccounting::Migrated) 26 | .to_owned(), 27 | ) 28 | .await 29 | } 30 | } 31 | 32 | /// partial table for RpcAccounting 33 | #[derive(Iden)] 34 | enum RpcAccounting { 35 | Table, 36 | Migrated, 37 | } 38 | -------------------------------------------------------------------------------- /migration/src/m20230412_171916_modify_secondary_user_add_primary_user.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .alter_table( 11 | Table::alter() 12 | .table(SecondaryUser::Table) 13 | .add_column( 14 | ColumnDef::new(SecondaryUser::RpcSecretKeyId) 15 | .big_unsigned() 16 | .not_null(), // add foreign key to user table ..., 17 | ) 18 | .add_foreign_key( 19 | TableForeignKey::new() 20 | .name("FK_secondary_user-rpc_key") 21 | .from_tbl(SecondaryUser::Table) 22 | .from_col(SecondaryUser::RpcSecretKeyId) 23 | .to_tbl(RpcKey::Table) 24 | .to_col(RpcKey::Id) 25 | .on_delete(ForeignKeyAction::NoAction) 26 | .on_update(ForeignKeyAction::NoAction), 27 | ) 28 | .to_owned(), 29 | ) 30 | .await 31 | 32 | // TODO: Add a unique index on RpcKey + Subuser 33 | } 34 | 35 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 36 | manager 37 | .alter_table( 38 | sea_query::Table::alter() 39 | .table(SecondaryUser::Table) 40 | .drop_column(SecondaryUser::RpcSecretKeyId) 41 | .to_owned(), 42 | ) 43 | .await 44 | } 45 | } 46 | 47 | /// Learn more at https://docs.rs/sea-query#iden 48 | #[derive(Iden)] 49 | enum SecondaryUser { 50 | Table, 51 | RpcSecretKeyId, 52 | } 53 | 54 | #[derive(Iden)] 55 | enum RpcKey { 56 | Table, 57 | Id, 58 | } 59 | -------------------------------------------------------------------------------- /migration/src/m20230511_161214_remove_columns_statsv2_origin_and_method.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .alter_table( 11 | Table::alter() 12 | .table(RpcAccountingV2::Table) 13 | .drop_column(RpcAccountingV2::Origin) 14 | .drop_column(RpcAccountingV2::Method) 15 | .to_owned(), 16 | ) 17 | .await 18 | } 19 | 20 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 21 | manager 22 | .alter_table( 23 | Table::alter() 24 | .table(RpcAccountingV2::Table) 25 | .add_column( 26 | ColumnDef::new(RpcAccountingV2::Method) 27 | .string() 28 | .not_null() 29 | .default(""), 30 | ) 31 | .add_column( 32 | ColumnDef::new(RpcAccountingV2::Origin) 33 | .string() 34 | .not_null() 35 | .default(""), 36 | ) 37 | .to_owned(), 38 | ) 39 | .await 40 | } 41 | } 42 | 43 | /// Learn more at https://docs.rs/sea-query#iden 44 | #[derive(Iden)] 45 | enum RpcAccountingV2 { 46 | Table, 47 | Origin, 48 | Method, 49 | } 50 | -------------------------------------------------------------------------------- /migration/src/m20230512_220213_allow_null_rpc_key_id_in_stats_v2.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .alter_table( 11 | sea_query::Table::alter() 12 | .table(RpcAccountingV2::Table) 13 | .to_owned() 14 | // allow rpc_key_id to be null. Needed for public rpc stat tracking 15 | .modify_column( 16 | ColumnDef::new(RpcAccountingV2::RpcKeyId) 17 | .big_unsigned() 18 | .null(), 19 | ) 20 | .to_owned(), 21 | ) 22 | .await 23 | } 24 | 25 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 26 | manager 27 | .alter_table( 28 | sea_query::Table::alter() 29 | .table(RpcAccountingV2::Table) 30 | .to_owned() 31 | .modify_column( 32 | ColumnDef::new(RpcAccountingV2::RpcKeyId) 33 | .big_unsigned() 34 | .not_null() 35 | .default(0), 36 | ) 37 | .to_owned(), 38 | ) 39 | .await 40 | } 41 | } 42 | 43 | /// Learn more at https://docs.rs/sea-query#iden 44 | #[derive(Iden)] 45 | enum RpcAccountingV2 { 46 | Table, 47 | RpcKeyId, 48 | } 49 | -------------------------------------------------------------------------------- /migration/src/m20230607_221917_total_deposits.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .alter_table( 11 | Table::alter() 12 | .table(Balance::Table) 13 | .add_column( 14 | ColumnDef::new(Balance::TotalSpentOutsideFreeTier) 15 | .decimal_len(20, 10) 16 | .not_null() 17 | .default(0.0), 18 | ) 19 | .add_column( 20 | ColumnDef::new(Balance::TotalDeposits) 21 | .decimal_len(20, 10) 22 | .not_null() 23 | .default(0.0), 24 | ) 25 | .rename_column(Balance::UsedBalance, Balance::TotalSpentIncludingFreeTier) 26 | .drop_column(Balance::AvailableBalance) 27 | .to_owned(), 28 | ) 29 | .await 30 | } 31 | 32 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 33 | // Remove the column again I suppose, but this will delete data, needless to say 34 | manager 35 | .alter_table( 36 | Table::alter() 37 | .table(Balance::Table) 38 | .rename_column(Balance::TotalSpentIncludingFreeTier, Balance::UsedBalance) 39 | .drop_column(Balance::TotalSpentOutsideFreeTier) 40 | .drop_column(Balance::TotalDeposits) 41 | .add_column( 42 | ColumnDef::new(Balance::AvailableBalance) 43 | .decimal_len(20, 10) 44 | .not_null() 45 | .default(0.0), 46 | ) 47 | .add_column( 48 | ColumnDef::new(Balance::UsedBalance) 49 | .decimal_len(20, 10) 50 | .not_null() 51 | .default(0.0), 52 | ) 53 | .to_owned(), 54 | ) 55 | .await 56 | } 57 | } 58 | 59 | #[allow(clippy::enum_variant_names)] 60 | #[derive(Iden)] 61 | enum Balance { 62 | Table, 63 | TotalSpentIncludingFreeTier, 64 | TotalSpentOutsideFreeTier, 65 | TotalDeposits, 66 | AvailableBalance, 67 | UsedBalance, 68 | } 69 | -------------------------------------------------------------------------------- /migration/src/m20230615_221201_handle_payment_uncles.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | // TODO: also alter the index to include the BlockHash? or 10 | manager 11 | .alter_table( 12 | Table::alter() 13 | .table(IncreaseOnChainBalanceReceipt::Table) 14 | .add_column( 15 | ColumnDef::new(IncreaseOnChainBalanceReceipt::BlockHash) 16 | .string() 17 | .not_null(), 18 | ) 19 | .add_column( 20 | ColumnDef::new(IncreaseOnChainBalanceReceipt::LogIndex) 21 | .big_integer() 22 | .unsigned() 23 | .not_null(), 24 | ) 25 | .add_column( 26 | ColumnDef::new(IncreaseOnChainBalanceReceipt::TokenAddress) 27 | .string() 28 | .not_null(), 29 | ) 30 | .drop_foreign_key(Alias::new("fk-deposit_to_user_id")) 31 | .add_foreign_key( 32 | TableForeignKey::new() 33 | .name("fk-deposit_to_user_id-v2") 34 | .from_col(IncreaseOnChainBalanceReceipt::DepositToUserId) 35 | .to_tbl(User::Table) 36 | .to_col(User::Id), 37 | ) 38 | .to_owned(), 39 | ) 40 | .await 41 | } 42 | 43 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 44 | manager 45 | .alter_table( 46 | Table::alter() 47 | .table(IncreaseOnChainBalanceReceipt::Table) 48 | .drop_column(IncreaseOnChainBalanceReceipt::BlockHash) 49 | .drop_column(IncreaseOnChainBalanceReceipt::LogIndex) 50 | .drop_column(IncreaseOnChainBalanceReceipt::TokenAddress) 51 | .to_owned(), 52 | ) 53 | .await 54 | } 55 | } 56 | 57 | /// Learn more at https://docs.rs/sea-query#iden 58 | #[derive(Iden)] 59 | enum IncreaseOnChainBalanceReceipt { 60 | Table, 61 | BlockHash, 62 | LogIndex, 63 | TokenAddress, 64 | DepositToUserId, 65 | } 66 | 67 | #[derive(Iden)] 68 | enum User { 69 | Table, 70 | Id, 71 | } 72 | -------------------------------------------------------------------------------- /migration/src/m20230618_230611_longer_payload.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .alter_table( 11 | Table::alter() 12 | .table(AdminTrail::Table) 13 | .modify_column(ColumnDef::new(AdminTrail::Endpoint).text().not_null()) 14 | .modify_column(ColumnDef::new(AdminTrail::Payload).text().not_null()) 15 | .to_owned(), 16 | ) 17 | .await 18 | } 19 | 20 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 21 | manager 22 | .alter_table( 23 | Table::alter() 24 | .table(AdminTrail::Table) 25 | .modify_column(ColumnDef::new(AdminTrail::Endpoint).string().not_null()) 26 | .modify_column(ColumnDef::new(AdminTrail::Payload).string().not_null()) 27 | .to_owned(), 28 | ) 29 | .await 30 | } 31 | } 32 | 33 | /// Learn more at https://docs.rs/sea-query#iden 34 | #[derive(Iden)] 35 | enum AdminTrail { 36 | Table, 37 | Endpoint, 38 | Payload, 39 | } 40 | -------------------------------------------------------------------------------- /migration/src/m20230619_172237_default_tracking.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | // new keys get set to aggregated logging 10 | // TODO: rename "none" to "minimal" 11 | manager 12 | .alter_table( 13 | Table::alter() 14 | .table(RpcKey::Table) 15 | .modify_column( 16 | ColumnDef::new(RpcKey::LogLevel) 17 | .enumeration( 18 | Alias::new("log_level"), 19 | [ 20 | Alias::new("none"), 21 | Alias::new("aggregated"), 22 | Alias::new("detailed"), 23 | ], 24 | ) 25 | .not_null() 26 | .default("detailed"), 27 | ) 28 | .to_owned(), 29 | ) 30 | .await 31 | } 32 | 33 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 34 | // new keys get set to none logging 35 | manager 36 | .alter_table( 37 | Table::alter() 38 | .table(RpcKey::Table) 39 | .modify_column( 40 | ColumnDef::new(RpcKey::LogLevel) 41 | .enumeration( 42 | Alias::new("log_level"), 43 | [ 44 | Alias::new("none"), 45 | Alias::new("aggregated"), 46 | Alias::new("detailed"), 47 | ], 48 | ) 49 | .not_null() 50 | .default("none"), 51 | ) 52 | .to_owned(), 53 | ) 54 | .await 55 | } 56 | } 57 | 58 | /// Learn more at https://docs.rs/sea-query#iden 59 | #[derive(Iden)] 60 | enum RpcKey { 61 | Table, 62 | LogLevel, 63 | } 64 | -------------------------------------------------------------------------------- /migration/src/m20230705_214013_type_fixes.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | // Replace the sample below with your own migration scripts 10 | manager 11 | .alter_table( 12 | Table::alter() 13 | .table(IncreaseOnChainBalanceReceipt::Table) 14 | .modify_column( 15 | ColumnDef::new(IncreaseOnChainBalanceReceipt::LogIndex) 16 | .big_unsigned() 17 | .not_null(), 18 | ) 19 | .to_owned(), 20 | ) 21 | .await 22 | } 23 | 24 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 25 | manager 26 | .alter_table( 27 | Table::alter() 28 | .table(IncreaseOnChainBalanceReceipt::Table) 29 | .modify_column( 30 | ColumnDef::new(IncreaseOnChainBalanceReceipt::LogIndex) 31 | .big_integer() 32 | .unsigned() 33 | .not_null(), 34 | ) 35 | .to_owned(), 36 | ) 37 | .await 38 | } 39 | } 40 | 41 | /// Learn more at https://docs.rs/sea-query#iden 42 | #[derive(Iden)] 43 | enum IncreaseOnChainBalanceReceipt { 44 | Table, 45 | LogIndex, 46 | } 47 | -------------------------------------------------------------------------------- /migration/src/m20230707_211936_premium_tier_changes.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | // Replace the sample below with your own migration scripts 10 | let update_out_of_funds = Query::update() 11 | .table(UserTier::Table) 12 | .limit(1) 13 | .values([ 14 | (UserTier::MaxRequestsPerPeriod, Some("3900").into()), 15 | (UserTier::MaxConcurrentRequests, Some("3").into()), 16 | ]) 17 | .and_where(Expr::col(UserTier::Title).eq("Premium Out Of Funds")) 18 | .to_owned(); 19 | 20 | manager.exec_stmt(update_out_of_funds).await?; 21 | 22 | let update_premium = Query::update() 23 | .table(UserTier::Table) 24 | .limit(1) 25 | .values([ 26 | (UserTier::MaxRequestsPerPeriod, None::<&str>.into()), 27 | (UserTier::MaxConcurrentRequests, Some("20").into()), 28 | ]) 29 | .and_where(Expr::col(UserTier::Title).eq("Premium")) 30 | .to_owned(); 31 | 32 | manager.exec_stmt(update_premium).await?; 33 | 34 | Ok(()) 35 | } 36 | 37 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 38 | let update_out_of_funds = Query::update() 39 | .table(UserTier::Table) 40 | .limit(1) 41 | .values([ 42 | (UserTier::MaxRequestsPerPeriod, Some("6000").into()), 43 | (UserTier::MaxConcurrentRequests, Some("5").into()), 44 | ]) 45 | .and_where(Expr::col(UserTier::Title).eq("Premium Out Of Funds")) 46 | .to_owned(); 47 | 48 | manager.exec_stmt(update_out_of_funds).await?; 49 | 50 | let update_premium = Query::update() 51 | .table(UserTier::Table) 52 | .limit(1) 53 | .values([ 54 | (UserTier::MaxRequestsPerPeriod, None::<&str>.into()), 55 | (UserTier::MaxConcurrentRequests, Some("100").into()), 56 | ]) 57 | .and_where(Expr::col(UserTier::Title).eq("Premium")) 58 | .to_owned(); 59 | 60 | manager.exec_stmt(update_premium).await?; 61 | 62 | Ok(()) 63 | } 64 | } 65 | 66 | /// Learn more at https://docs.rs/sea-query#iden 67 | #[derive(Iden)] 68 | enum UserTier { 69 | Table, 70 | Title, 71 | MaxRequestsPerPeriod, 72 | MaxConcurrentRequests, 73 | } 74 | -------------------------------------------------------------------------------- /migration/src/m20230708_151756_rpc_accounting_free_usage_credits.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .alter_table( 11 | Table::alter() 12 | .table(RpcAccountingV2::Table) 13 | .add_column( 14 | ColumnDef::new(RpcAccountingV2::SumInclFreeCreditsUsed) 15 | .decimal_len(20, 10) 16 | .default("0.0") 17 | .not_null(), 18 | ) 19 | .to_owned(), 20 | ) 21 | .await 22 | } 23 | 24 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 25 | manager 26 | .alter_table( 27 | sea_query::Table::alter() 28 | .table(RpcAccountingV2::Table) 29 | .drop_column(RpcAccountingV2::SumInclFreeCreditsUsed) 30 | .to_owned(), 31 | ) 32 | .await 33 | } 34 | } 35 | 36 | /// Learn more at https://docs.rs/sea-query#iden 37 | #[derive(Iden)] 38 | enum RpcAccountingV2 { 39 | Table, 40 | SumInclFreeCreditsUsed, 41 | } 42 | -------------------------------------------------------------------------------- /migration/src/m20230708_152131_referral_track_one_time_bonus_bonus.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | manager 10 | .alter_table( 11 | Table::alter() 12 | .table(Referee::Table) 13 | .drop_column(Referee::CreditsAppliedForReferee) 14 | .add_column( 15 | ColumnDef::new(Referee::OneTimeBonusAppliedForReferee) 16 | .decimal_len(20, 10) 17 | .default("0.0") 18 | .not_null(), 19 | ) 20 | .to_owned(), 21 | ) 22 | .await 23 | } 24 | 25 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 26 | manager 27 | .alter_table( 28 | Table::alter() 29 | .table(Referee::Table) 30 | .drop_column(Referee::OneTimeBonusAppliedForReferee) 31 | .add_column( 32 | ColumnDef::new(Referee::CreditsAppliedForReferee) 33 | .boolean() 34 | .not_null(), 35 | ) 36 | .to_owned(), 37 | ) 38 | .await 39 | } 40 | } 41 | 42 | #[allow(clippy::enum_variant_names)] 43 | #[derive(Iden)] 44 | enum Referee { 45 | Table, 46 | CreditsAppliedForReferee, 47 | OneTimeBonusAppliedForReferee, 48 | } 49 | -------------------------------------------------------------------------------- /migration/src/m20230713_144446_stripe_default_date_created.rs: -------------------------------------------------------------------------------- 1 | // TODO: Try to re-export timestamp from within sea-orm 2 | use sea_orm_migration::prelude::*; 3 | 4 | #[derive(DeriveMigrationName)] 5 | pub struct Migration; 6 | 7 | #[async_trait::async_trait] 8 | impl MigrationTrait for Migration { 9 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 10 | manager 11 | .alter_table( 12 | Table::alter() 13 | .table(StripeIncreaseBalanceReceipt::Table) 14 | .modify_column( 15 | ColumnDef::new(StripeIncreaseBalanceReceipt::DateCreated) 16 | .timestamp() 17 | .extra("DEFAULT CURRENT_TIMESTAMP".to_string()) 18 | .not_null(), 19 | ) 20 | .to_owned(), 21 | ) 22 | .await?; 23 | 24 | let now = chrono::offset::Utc::now(); 25 | 26 | // Then change all columns to "now" 27 | let update_to_current_timestamp = Query::update() 28 | .table(StripeIncreaseBalanceReceipt::Table) 29 | .values([(StripeIncreaseBalanceReceipt::DateCreated, Some(now).into())]) 30 | .to_owned(); 31 | 32 | manager.exec_stmt(update_to_current_timestamp).await 33 | } 34 | 35 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 36 | // Do nothing ... 37 | Ok(()) 38 | } 39 | } 40 | 41 | /// Learn more at https://docs.rs/sea-query#iden 42 | #[derive(Iden)] 43 | enum StripeIncreaseBalanceReceipt { 44 | Table, 45 | DateCreated, 46 | } 47 | -------------------------------------------------------------------------------- /migration/src/m20230726_072845_default_premium_user_tier.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | /// change default to premium tier 9 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 10 | let db_conn = manager.get_connection(); 11 | let db_backend = manager.get_database_backend(); 12 | 13 | let select_premium_id = Query::select() 14 | .column(UserTier::Id) 15 | .column(UserTier::Title) 16 | .from(UserTier::Table) 17 | .and_having(Expr::col(UserTier::Title).eq("Premium")) 18 | .to_owned(); 19 | 20 | let premium_id: u64 = db_conn 21 | .query_one(db_backend.build(&select_premium_id)) 22 | .await? 23 | .expect("Premium tier should exist") 24 | .try_get("", &UserTier::Id.to_string())?; 25 | 26 | manager 27 | .alter_table( 28 | Table::alter() 29 | .table(User::Table) 30 | .modify_column( 31 | ColumnDef::new(User::UserTierId) 32 | .big_unsigned() 33 | .default(premium_id) 34 | .not_null(), 35 | ) 36 | .to_owned(), 37 | ) 38 | .await?; 39 | 40 | Ok(()) 41 | } 42 | 43 | /// change default to free tier 44 | async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { 45 | let db_conn = manager.get_connection(); 46 | let db_backend = manager.get_database_backend(); 47 | 48 | let select_free_id = Query::select() 49 | .column(UserTier::Id) 50 | .column(UserTier::Title) 51 | .from(UserTier::Table) 52 | .and_having(Expr::col(UserTier::Title).eq("Free")) 53 | .to_owned(); 54 | 55 | let free_id: u64 = db_conn 56 | .query_one(db_backend.build(&select_free_id)) 57 | .await? 58 | .expect("Free tier should exist") 59 | .try_get("", &UserTier::Id.to_string())?; 60 | 61 | manager 62 | .alter_table( 63 | Table::alter() 64 | .table(User::Table) 65 | .modify_column( 66 | ColumnDef::new(User::UserTierId) 67 | .big_unsigned() 68 | .default(free_id) 69 | .not_null(), 70 | ) 71 | .to_owned(), 72 | ) 73 | .await?; 74 | 75 | Ok(()) 76 | } 77 | } 78 | 79 | #[derive(Iden)] 80 | enum User { 81 | Table, 82 | UserTierId, 83 | } 84 | 85 | #[derive(Iden)] 86 | enum UserTier { 87 | Table, 88 | Id, 89 | Title, 90 | } 91 | -------------------------------------------------------------------------------- /migration/src/m20230726_162138_drop_rpc_accounting_v2_fk.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | let _ = manager 10 | .drop_foreign_key( 11 | ForeignKey::drop() 12 | .name("rpc_accounting_v2_ibfk_1") 13 | .table(RpcAccountingV2::Table) 14 | .to_owned(), 15 | ) 16 | .await; 17 | 18 | Ok(()) 19 | } 20 | 21 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 22 | Ok(()) 23 | } 24 | } 25 | 26 | /// Learn more at https://docs.rs/sea-query#iden 27 | #[derive(Iden)] 28 | enum RpcAccountingV2 { 29 | Table, 30 | } 31 | -------------------------------------------------------------------------------- /migration/src/m20230726_225124_reduce_out_of_funds_tier_limits.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | let update_out_of_funds_tier = Query::update() 10 | .table(UserTier::Table) 11 | .values([ 12 | (UserTier::MaxRequestsPerPeriod, Some("3000").into()), 13 | (UserTier::MaxConcurrentRequests, Some("3").into()), 14 | ]) 15 | .and_where(Expr::col(UserTier::Title).eq("Premium Out Of Funds")) 16 | .to_owned(); 17 | 18 | manager.exec_stmt(update_out_of_funds_tier).await?; 19 | 20 | Ok(()) 21 | } 22 | 23 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 24 | Ok(()) 25 | } 26 | } 27 | 28 | #[derive(Iden)] 29 | enum UserTier { 30 | Table, 31 | Title, 32 | MaxRequestsPerPeriod, 33 | MaxConcurrentRequests, 34 | } 35 | -------------------------------------------------------------------------------- /migration/src/m20230911_180520_high_concurrency_tier.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[derive(DeriveMigrationName)] 4 | pub struct Migration; 5 | 6 | #[async_trait::async_trait] 7 | impl MigrationTrait for Migration { 8 | async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { 9 | let db_conn = manager.get_connection(); 10 | let db_backend = manager.get_database_backend(); 11 | 12 | let select_free_id = Query::select() 13 | .column(UserTier::Id) 14 | .column(UserTier::Title) 15 | .from(UserTier::Table) 16 | .and_having(Expr::col(UserTier::Title).eq("Free")) 17 | .to_owned(); 18 | 19 | let free_id: u64 = db_conn 20 | .query_one(db_backend.build(&select_free_id)) 21 | .await? 22 | .expect("Free tier should always exist") 23 | .try_get("", &UserTier::Id.to_string())?; 24 | 25 | let user_tiers = Query::insert() 26 | .into_table(UserTier::Table) 27 | .columns([ 28 | UserTier::Title, 29 | UserTier::MaxRequestsPerPeriod, 30 | UserTier::MaxConcurrentRequests, 31 | UserTier::DowngradeTierId, 32 | ]) 33 | .values_panic([ 34 | "High Concurrency".into(), 35 | None::.into(), 36 | Some(300).into(), 37 | Some(free_id).into(), 38 | ]) 39 | .to_owned(); 40 | 41 | manager.exec_stmt(user_tiers).await?; 42 | 43 | Ok(()) 44 | } 45 | 46 | async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { 47 | Ok(()) 48 | } 49 | } 50 | 51 | #[derive(Iden)] 52 | enum UserTier { 53 | Table, 54 | Id, 55 | Title, 56 | MaxRequestsPerPeriod, 57 | MaxConcurrentRequests, 58 | DowngradeTierId, 59 | } 60 | -------------------------------------------------------------------------------- /migration/src/main.rs: -------------------------------------------------------------------------------- 1 | use sea_orm_migration::prelude::*; 2 | 3 | #[tokio::main] 4 | async fn main() { 5 | cli::run_cli(migration::Migrator).await; 6 | } 7 | -------------------------------------------------------------------------------- /payment-contracts/.gitignore: -------------------------------------------------------------------------------- 1 | /src/contracts/*.rs -------------------------------------------------------------------------------- /payment-contracts/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "payment-contracts" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [build-dependencies] 9 | ethers = { version = "2.0.11", default-features = false } 10 | glob = "0.3.1" 11 | 12 | [dependencies] 13 | ethers = { version = "2.0.11", default-features = false } 14 | -------------------------------------------------------------------------------- /payment-contracts/abi/PaymentProxy.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "stateMutability": "nonpayable", 4 | "type": "constructor", 5 | "inputs": [ 6 | { 7 | "name": "_factory", 8 | "type": "address" 9 | } 10 | ], 11 | "outputs": [] 12 | }, 13 | { 14 | "stateMutability": "nonpayable", 15 | "type": "fallback" 16 | }, 17 | { 18 | "stateMutability": "view", 19 | "type": "function", 20 | "name": "implementation", 21 | "inputs": [], 22 | "outputs": [ 23 | { 24 | "name": "", 25 | "type": "address" 26 | } 27 | ] 28 | } 29 | ] -------------------------------------------------------------------------------- /payment-contracts/abi/PaymentSweeper.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "TokenBalanceRecovered", 4 | "inputs": [ 5 | { 6 | "name": "receiver", 7 | "type": "address", 8 | "indexed": false 9 | }, 10 | { 11 | "name": "token", 12 | "type": "address", 13 | "indexed": false 14 | }, 15 | { 16 | "name": "amount", 17 | "type": "uint256", 18 | "indexed": false 19 | } 20 | ], 21 | "anonymous": false, 22 | "type": "event" 23 | }, 24 | { 25 | "stateMutability": "nonpayable", 26 | "type": "constructor", 27 | "inputs": [ 28 | { 29 | "name": "_factory", 30 | "type": "address" 31 | } 32 | ], 33 | "outputs": [] 34 | }, 35 | { 36 | "stateMutability": "nonpayable", 37 | "type": "function", 38 | "name": "send_token", 39 | "inputs": [ 40 | { 41 | "name": "_token", 42 | "type": "address" 43 | }, 44 | { 45 | "name": "_amount", 46 | "type": "uint256" 47 | } 48 | ], 49 | "outputs": [] 50 | }, 51 | { 52 | "stateMutability": "nonpayable", 53 | "type": "function", 54 | "name": "sweep_token_balance", 55 | "inputs": [ 56 | { 57 | "name": "_token", 58 | "type": "address" 59 | } 60 | ], 61 | "outputs": [] 62 | }, 63 | { 64 | "stateMutability": "nonpayable", 65 | "type": "function", 66 | "name": "recover_token_balance", 67 | "inputs": [ 68 | { 69 | "name": "_token", 70 | "type": "address" 71 | } 72 | ], 73 | "outputs": [] 74 | }, 75 | { 76 | "stateMutability": "view", 77 | "type": "function", 78 | "name": "FACTORY", 79 | "inputs": [], 80 | "outputs": [ 81 | { 82 | "name": "", 83 | "type": "address" 84 | } 85 | ] 86 | } 87 | ] -------------------------------------------------------------------------------- /payment-contracts/build.rs: -------------------------------------------------------------------------------- 1 | use ethers::contract::MultiAbigen; 2 | use glob::glob; 3 | 4 | fn main() { 5 | // Tell Cargo that if the given file changes, to rerun this build script. 6 | println!("cargo:rerun-if-changed=build.rs"); 7 | println!("cargo:rerun-if-changed=Cargo.toml"); 8 | glob("./abi/*.json").unwrap().for_each(|x| { 9 | if let Ok(x) = x { 10 | println!("cargo:rerun-if-changed={}", x.display()); 11 | } 12 | }); 13 | 14 | let gen = MultiAbigen::from_json_files("./abi").unwrap(); 15 | 16 | let bindings = gen.build().unwrap(); 17 | 18 | bindings.write_to_module("./src/contracts", false).unwrap(); 19 | } 20 | -------------------------------------------------------------------------------- /payment-contracts/src/contracts/mod.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::all)] 2 | //! This module contains abigen! generated bindings for solidity contracts. 3 | //! This is autogenerated code. 4 | //! Do not manually edit these files. 5 | //! These files may be overwritten by the codegen system at any time. 6 | pub mod ierc20; 7 | pub mod payment_factory; 8 | pub mod payment_proxy; 9 | pub mod payment_sweeper; 10 | -------------------------------------------------------------------------------- /payment-contracts/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod contracts; 2 | 3 | pub use contracts::*; 4 | -------------------------------------------------------------------------------- /quick_cache_ttl/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "quick_cache_ttl" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | quick_cache = "0.3.0" 10 | serde = "1" 11 | tokio = { version = "1.28.2", features = ["full"] } 12 | 13 | [dev-dependencies] 14 | tokio = { version = "1.28.2", features = ["full", "test-util"] } 15 | -------------------------------------------------------------------------------- /quick_cache_ttl/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod cache; 2 | mod kq_cache; 3 | 4 | pub use cache::CacheWithTTL; 5 | pub use kq_cache::{KQCacheWithTTL, PlaceholderGuardWithTTL}; 6 | pub use quick_cache::sync::{Cache, KQCache}; 7 | pub use quick_cache::{DefaultHashBuilder, UnitWeighter, Weighter}; 8 | 9 | #[cfg(test)] 10 | mod tests { 11 | use std::time::Duration; 12 | use tokio::task::yield_now; 13 | use tokio::time; 14 | 15 | use crate::CacheWithTTL; 16 | 17 | #[tokio::test(start_paused = true)] 18 | async fn test_time_passing() { 19 | let x = CacheWithTTL::::new("test", 2, Duration::from_secs(2)).await; 20 | 21 | assert!(x.get(&0).is_none()); 22 | 23 | x.try_insert(0, 0).unwrap(); 24 | 25 | assert!(x.get(&0).is_some()); 26 | 27 | time::advance(Duration::from_secs(1)).await; 28 | 29 | assert!(x.get(&0).is_some()); 30 | 31 | time::advance(Duration::from_secs(1)).await; 32 | 33 | // yield so that the expiration code gets a chance to run 34 | yield_now().await; 35 | 36 | assert!(x.get(&0).is_none()); 37 | } 38 | 39 | #[tokio::test(start_paused = true)] 40 | async fn test_capacity_based_eviction() { 41 | let x = CacheWithTTL::::new("test", 1, Duration::from_secs(2)).await; 42 | 43 | assert!(x.get(&0).is_none()); 44 | 45 | x.try_insert(0, ()).unwrap(); 46 | 47 | assert!(x.get(&0).is_some()); 48 | 49 | x.try_insert(1, ()).unwrap(); 50 | 51 | assert!(x.get(&1).is_some()); 52 | assert!(x.get(&0).is_none()); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /rate-counter/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rate-counter" 3 | version = "0.1.0" 4 | authors = ["Bryan Stitt "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | tokio = { version = "1.34.0", features = ["time"] } 9 | -------------------------------------------------------------------------------- /rate-counter/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! A counter of events in a time period. 2 | use std::collections::VecDeque; 3 | use tokio::time::{Duration, Instant}; 4 | 5 | /// Measures ticks in a time period. 6 | #[derive(Debug)] 7 | pub struct RateCounter { 8 | period: Duration, 9 | items: VecDeque, 10 | } 11 | 12 | impl RateCounter { 13 | pub fn new(period: Duration) -> Self { 14 | let items = VecDeque::new(); 15 | 16 | Self { period, items } 17 | } 18 | 19 | /// update the counter and return the rate for the current period 20 | /// true if the current time should be counted 21 | pub fn update(&mut self, tick: bool) -> usize { 22 | let now = Instant::now(); 23 | let too_old = now - self.period; 24 | 25 | while self.items.front().map_or(false, |t| *t < too_old) { 26 | self.items.pop_front(); 27 | } 28 | 29 | if tick { 30 | self.items.push_back(now); 31 | } 32 | 33 | self.items.len() 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /redis-rate-limiter/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "redis-rate-limiter" 3 | version = "0.2.0" 4 | authors = ["Bryan Stitt "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | anyhow = "1.0.75" 9 | chrono = "0.4.31" 10 | deadpool-redis = { version = "0.13.0", features = ["rt_tokio_1", "serde"] } 11 | tokio = "1.34.0" 12 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "nightly-2023-11-09" 3 | -------------------------------------------------------------------------------- /scripts/apply-migrations.sh: -------------------------------------------------------------------------------- 1 | # sea-orm-cli migrate up 2 | # sea-orm-cli generate entity -u mysql://root:dev_web3_proxy@127.0.0.1:13306/dev_web3_proxy -o entities/src --with-serde both 3 | # sea-orm-cli generate entity -t -------------------------------------------------------------------------------- /scripts/brownie-tests/.gitattributes: -------------------------------------------------------------------------------- 1 | *.sol linguist-language=Solidity 2 | *.vy linguist-language=Python 3 | -------------------------------------------------------------------------------- /scripts/brownie-tests/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .env 3 | .history 4 | .hypothesis/ 5 | build/ 6 | reports/ 7 | -------------------------------------------------------------------------------- /scripts/brownie-tests/brownie-config.yaml: -------------------------------------------------------------------------------- 1 | dotenv: .env -------------------------------------------------------------------------------- /scripts/brownie-tests/scripts/make_payment.py: -------------------------------------------------------------------------------- 1 | from brownie import Contract, Sweeper, accounts 2 | from brownie.network import priority_fee 3 | 4 | def main(): 5 | print("Hello") 6 | 7 | 8 | print("accounts are") 9 | token = Contract.from_explorer("0xC9fCFA7e28fF320C49967f4522EBc709aa1fDE7c") 10 | factory = Contract.from_explorer("0x4e3bc2054788de923a04936c6addb99a05b0ea36") 11 | user = accounts.load("david") 12 | # user = accounts.load("david-main") 13 | 14 | print("Llama token") 15 | print(token) 16 | 17 | print("Factory token") 18 | print(factory) 19 | 20 | print("User addr") 21 | print(user) 22 | 23 | # Sweeper and Proxy are deployed by us, as the user, by calling factory 24 | # Already been called before ... 25 | # factory.create_payment_address({'from': user}) 26 | sweeper = Sweeper.at(factory.account_to_payment_address(user)) 27 | print("Sweeper is at") 28 | print(sweeper) 29 | 30 | priority_fee("auto") 31 | token._mint_for_testing(user, (10_000)*(10**18), {'from': user}) 32 | # token.approve(sweeper, 2**256-1, {'from': user}) 33 | sweeper.send_token(token, (5_000)*(10**18), {'from': user}) 34 | # sweeper.send_token(token, (47)*(10**13), {'from': user}) 35 | -------------------------------------------------------------------------------- /scripts/ethspam: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llamanodes/web3-proxy/f4a71ee0b621c461828956436f6d55372228ec18/scripts/ethspam -------------------------------------------------------------------------------- /scripts/generate-requests-and-stats.sh: -------------------------------------------------------------------------------- 1 | # Got eth spam from here 2 | # https://github.com/shazow/ethspam 3 | 4 | # Got versus from here 5 | # https://github.com/INFURA/versus 6 | # ./ethspam | ./versus --stop-after 100 "http://localhost:8544/" # Pipe into the endpoint ..., add a bearer token and all that 7 | 8 | ./ethspam http://127.0.0.1:8544/rpc/01H2D5DN4D423VR2KFWBZE46TR | ./versus --concurrency=4 --stop-after 10000 http://localhost:8544/rpc/01H2D5DN4D423VR2KFWBZE46TR 9 | 10 | ./ethspam http://127.0.0.1:8544/rpc/01H2D5CAP1KF2NKRS30SGATDSD | ./versus --concurrency=4 --stop-after 10000 http://localhost:8544/rpc/01H2D5CAP1KF2NKRS30SGATDSD 11 | -------------------------------------------------------------------------------- /scripts/get-stats-aggregated.sh: -------------------------------------------------------------------------------- 1 | # Make a get request to get the stats in an aggregated fashion 2 | 3 | # I dont think we need a user id ... 4 | 5 | 6 | curl -X GET \ 7 | "http://localhost:8544/user/stats/aggregate?query_start=1678780033&query_window_seconds=1000" 8 | 9 | curl -X GET \ 10 | -H "Authorization: Bearer 01GZK8MHHGQWK4VPGF97HS91MB" \ 11 | "http://localhost:8544/user/stats/detailed?query_start=1678780033&query_window_seconds=1000" 12 | -------------------------------------------------------------------------------- /scripts/install-test-suite.sh: -------------------------------------------------------------------------------- 1 | cargo install cargo-binstall 2 | cargo binstall cargo-nextest 3 | # cargo nextest run -------------------------------------------------------------------------------- /scripts/manual-tests/123-get-key-roles.sh: -------------------------------------------------------------------------------- 1 | 2 | curl \ 3 | -H "Authorization: Bearer 01H2D5CAQJF7P80222P4ZAFQ26" \ 4 | -X GET "127.0.0.1:8544/user/keys" -------------------------------------------------------------------------------- /scripts/manual-tests/135-stripe-deposit.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llamanodes/web3-proxy/f4a71ee0b621c461828956436f6d55372228ec18/scripts/manual-tests/135-stripe-deposit.sh -------------------------------------------------------------------------------- /scripts/manual-tests/16-change-user-tier.sh: -------------------------------------------------------------------------------- 1 | # rm -rf data/ 2 | # docker-compose up -d 3 | # sea-orm-cli migrate up 4 | 5 | # Use CLI to create the admin that will call the endpoint 6 | cargo run create_user --address 0xeB3E928A2E54BE013EF8241d4C9EaF4DfAE94D5a 7 | cargo run change_admin_status 0xeB3E928A2E54BE013EF8241d4C9EaF4DfAE94D5a true 8 | 9 | # Use CLI to create the user whose role will be changed via the endpoint 10 | cargo run create_user --address 0x077e43dcca20da9859daa3fd78b5998b81f794f7 11 | 12 | # Run the proxyd instance 13 | cargo run --release -- proxyd 14 | 15 | # Check if the instance is running 16 | curl --verbose -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"web3_clientVersion","id":1}' 127.0.0.1:8544 17 | 18 | curl --verbose -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params": ["latest", false], "id":1}' 127.0.0.1:8544 19 | 20 | # Open this website to get the nonce to log in 21 | curl -X GET "http://127.0.0.1:8544/user/login/0xeB3E928A2E54BE013EF8241d4C9EaF4DfAE94D5a" 22 | 23 | # Use this site to sign a message 24 | # https://www.myetherwallet.com/wallet/sign (whatever is output with the above code) 25 | curl -X POST http://127.0.0.1:8544/user/login \ 26 | -H 'Content-Type: application/json' \ 27 | -d '{"address": "0xeb3e928a2e54be013ef8241d4c9eaf4dfae94d5a", "msg": "0x6c6c616d616e6f6465732e636f6d2077616e747320796f7520746f207369676e20696e207769746820796f757220457468657265756d206163636f756e743a0a3078654233453932384132453534424530313345463832343164344339456146344466414539344435610a0af09fa699f09fa699f09fa699f09fa699f09fa6990a0a5552493a2068747470733a2f2f6c6c616d616e6f6465732e636f6d2f0a56657273696f6e3a20310a436861696e2049443a20310a4e6f6e63653a2030314753414e37464d47574335314e50544737343338384a44350a4973737565642041743a20323032332d30322d31355431333a34363a33372e3037323739335a0a45787069726174696f6e2054696d653a20323032332d30322d31355431343a30363a33372e3037323739335a", "sig": "2d2eb576b2e6d05845710b7229f2a1ff9707e928fdcf571d1ce0ae094577e4310873fa1376c69440b60d6a1c76c62a4586b9d6426fb6559dee371e490d708f3e1b", "version": "3", "signer": "MEW"}' 28 | 29 | ## Login in the user first (add a random bearer token into the database) 30 | ## (This segment was not yet tested, but should next time you run the query) 31 | #INSERT INTO login (bearer_token, user_id, expires_at, read_only) VALUES ( 32 | # "01GSAMZ6QY7KH9AQ", 33 | # 1, 34 | # "2024-01-01", 35 | # FALSE 36 | #); 37 | 38 | #curl -X POST -H "Content-Type: application/json" --data '{}' 127.0.0.1:8544/user/login 39 | #curl -X GET "127.0.0.1:8544/user/login/0xeB3E928A2E54BE013EF8241d4C9EaF4DfAE94D5a/" 40 | #curl -X GET "127.0.0.1:8544/admin/modify_role?user_address=0xeB3E928A2E54BE013EF8241d4C9EaF4DfAE94D5a&user_tier_title=Unlimited" 41 | 42 | # Now modify the user role and check this in the database 43 | curl \ 44 | -H "Authorization: Bearer 01GSANKVBB22D5P2351P4Y42NV" \ 45 | -X GET "127.0.0.1:8544/admin/modify_role?user_address=0x077e43dcca20da9859daa3fd78b5998b81f794f7&user_tier_title=Unlimited&user_id=1" 46 | 47 | -------------------------------------------------------------------------------- /scripts/manual-tests/19-admin-imitate-user.sh: -------------------------------------------------------------------------------- 1 | # rm -rf data/ 2 | # docker-compose up -d 3 | # sea-orm-cli migrate up 4 | 5 | # Use CLI to create the admin that will call the endpoint 6 | RUSTFLAGS="--cfg tokio_unstable" cargo run create_user --address 0xeB3E928A2E54BE013EF8241d4C9EaF4DfAE94D5a 7 | RUSTFLAGS="--cfg tokio_unstable" cargo run change_admin_status 0xeB3E928A2E54BE013EF8241d4C9EaF4DfAE94D5a true 8 | 9 | # Use CLI to create the user whose role will be changed via the endpoint 10 | RUSTFLAGS="--cfg tokio_unstable" cargo run create_user --address 0x077e43dcca20da9859daa3fd78b5998b81f794f7 11 | 12 | # Run the proxyd instance 13 | RUSTFLAGS="--cfg tokio_unstable" cargo run --release -- proxyd 14 | 15 | # Check if the instance is running 16 | curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"web3_clientVersion","id":1}' 127.0.0.1:8544 17 | 18 | # Open this website to get the nonce to log in 19 | curl \ 20 | -H "Authorization: Bearer 01GSANKVBB22D5P2351P4Y42NV" \ 21 | -X GET "http://127.0.0.1:8544/admin/imitate-login/0xeB3E928A2E54BE013EF8241d4C9EaF4DfAE94D5a/0x077e43dcca20da9859daa3fd78b5998b81f794f7" 22 | 23 | # Use this site to sign a message 24 | # https://www.myetherwallet.com/wallet/sign (whatever is output with the above code) 25 | curl -X POST http://127.0.0.1:8544/admin/imitate-login \ 26 | -H 'Content-Type: application/json' \ 27 | -H "Authorization: Bearer 01GSANKVBB22D5P2351P4Y42NV" \ 28 | -d '{"address": "0xeb3e928a2e54be013ef8241d4c9eaf4dfae94d5a", "msg": "0x6c6c616d616e6f6465732e636f6d2077616e747320796f7520746f207369676e20696e207769746820796f757220457468657265756d206163636f756e743a0a3078654233453932384132453534424530313345463832343164344339456146344466414539344435610a0af09fa699f09fa699f09fa699f09fa699f09fa6990a0a5552493a2068747470733a2f2f6c6c616d616e6f6465732e636f6d2f0a56657273696f6e3a20310a436861696e2049443a20310a4e6f6e63653a20303147534150545132413932415332435752563158504d4347470a4973737565642041743a20323032332d30322d31355431343a31343a33352e3835303636385a0a45787069726174696f6e2054696d653a20323032332d30322d31355431343a33343a33352e3835303636385a", "sig": "d5fed789e98769b8b726a79f222f2e06476de15948d35c167c4f294bb98edf42244edc703b6d729e5d08bd73c318fc9729b985022229c7669a945d64da47ab641c", "version": "3", "signer": "MEW"}' 29 | 30 | # Now modify the user role and check this in the database 31 | # 01GSAMMWQ41TVVH3DH8MSEP8X6 32 | # Now we can get a bearer-token to imitate the user 33 | curl \ 34 | -H "Authorization: Bearer 01GSAPZNVZ96ADJAEZ1VTRSA5T" \ 35 | -X GET "127.0.0.1:8544/user/keys" 36 | 37 | 38 | # docker-compose down 39 | -------------------------------------------------------------------------------- /scripts/manual-tests/21-sql-migration-make-backup.sh: -------------------------------------------------------------------------------- 1 | mysqldump -u root --password=dev_web3_proxy -h 127.0.0.1 --port 13306 2 | -------------------------------------------------------------------------------- /scripts/manual-tests/21-sql-migration-verify-test-queries.sql: -------------------------------------------------------------------------------- 1 | SELECT COUNT(*) FROM rpc_accounting WHERE migrated IS NULL; 2 | UPDATE rpc_accounting SET migrated = NULL; 3 | 4 | SELECT SUM(frontend_requests) FROM rpc_accounting; 5 | SELECT SUM(frontend_requests) FROM rpc_accounting_v2; 6 | 7 | SELECT SUM(backend_requests) FROM rpc_accounting; 8 | SELECT SUM(backend_requests) FROM rpc_accounting_v2; 9 | 10 | SELECT SUM(sum_request_bytes) FROM rpc_accounting; 11 | SELECT SUM(sum_request_bytes) FROM rpc_accounting_v2; 12 | 13 | SELECT SUM(sum_response_millis) FROM rpc_accounting; 14 | SELECT SUM(sum_response_millis) FROM rpc_accounting_v2; 15 | 16 | SELECT SUM(sum_response_bytes) FROM rpc_accounting; 17 | SELECT SUM(sum_response_bytes) FROM rpc_accounting_v2; 18 | -------------------------------------------------------------------------------- /scripts/manual-tests/45-admin-add-balance.sh: -------------------------------------------------------------------------------- 1 | 2 | # Create / Login user1 3 | curl -X GET "http://127.0.0.1:8544/user/login/0xeb3e928a2e54be013ef8241d4c9eaf4dfae94d5a" 4 | curl -X POST http://127.0.0.1:8544/user/login \ 5 | -H 'Content-Type: application/json' \ 6 | -d '{ 7 | "address": "0xeb3e928a2e54be013ef8241d4c9eaf4dfae94d5a", 8 | "msg": "0x6c6c616d616e6f6465732e636f6d2077616e747320796f7520746f207369676e20696e207769746820796f757220457468657265756d206163636f756e743a0a3078654233453932384132453534424530313345463832343164344339456146344466414539344435610a0af09fa699f09fa699f09fa699f09fa699f09fa6990a0a5552493a2068747470733a2f2f6c6c616d616e6f6465732e636f6d2f0a56657273696f6e3a20310a436861696e2049443a20310a4e6f6e63653a203031483044573642334a48355a4b384a4e3947504d594e4d4b370a4973737565642041743a20323032332d30352d31345431393a33353a35352e3736323632395a0a45787069726174696f6e2054696d653a20323032332d30352d31345431393a35353a35352e3736323632395a", 9 | "sig": "f88b42d638246f8e51637c753052cab3a13b2a138faf3107c921ce2f0027d6506b9adcd3a7b72af830cdf50d20e6e9cb3f9f456dd1be47f6543990ea050909791c", 10 | "version": "3", 11 | "signer": "MEW" 12 | }' 13 | 14 | # 01H0DW6VFCP365B9TXVQVVMHHY 15 | # 01H0DVZNDJWQ7YG8RBHXQHJ301 16 | 17 | # Make user1 an admin 18 | cargo run change_admin_status 0xeB3E928A2E54BE013EF8241d4C9EaF4DfAE94D5a true 19 | 20 | # Create/Login user2 21 | curl -X GET "http://127.0.0.1:8544/user/login/0x762390ae7a3c4D987062a398C1eA8767029AB08E" 22 | 23 | curl -X POST http://127.0.0.1:8544/user/login \ 24 | -H 'Content-Type: application/json' \ 25 | -d '{ 26 | "address": "0x762390ae7a3c4d987062a398c1ea8767029ab08e", 27 | "msg": "0x6c6c616d616e6f6465732e636f6d2077616e747320796f7520746f207369676e20696e207769746820796f757220457468657265756d206163636f756e743a0a3078373632333930616537613363344439383730363261333938433165413837363730323941423038450a0af09fa699f09fa699f09fa699f09fa699f09fa6990a0a5552493a2068747470733a2f2f6c6c616d616e6f6465732e636f6d2f0a56657273696f6e3a20310a436861696e2049443a20310a4e6f6e63653a20303148304457384233304e534447594e484d33514d4a31434e530a4973737565642041743a20323032332d30352d31345431393a33373a30312e3238303338355a0a45787069726174696f6e2054696d653a20323032332d30352d31345431393a35373a30312e3238303338355a", 28 | "sig": "c545235557b7952a789dffa2af153af5cf663dcc05449bcc4b651b04cda57de05bcef55c0f5cbf6aa2432369582eb6a40927d14ad0a2d15f48fa45f32fbf273f1c", 29 | "version": "3", 30 | "signer": "MEW" 31 | }' 32 | 33 | # 01H0DWPXRQA7XX2VFSNR02CG1N 34 | # 01H0DWPXQQ951Y3R90QMF6MYGE 35 | 36 | curl \ 37 | -H "Authorization: Bearer 01H0DWPXRQA7XX2VFSNR02CG1N" \ 38 | -X GET "127.0.0.1:8544/user/balance" 39 | 40 | 41 | # Admin add balance 42 | curl \ 43 | -H "Authorization: Bearer 01H0DW6VFCP365B9TXVQVVMHHY" \ 44 | -X GET "127.0.0.1:8544/admin/increase_balance?user_address=0x762390ae7a3c4D987062a398C1eA8767029AB08E&amount=100.0" 45 | -------------------------------------------------------------------------------- /scripts/manual-tests/52-simple-get-deposits.sh: -------------------------------------------------------------------------------- 1 | # Check the balance of the user 2 | # Balance seems to be returning properly (0, in this test case) 3 | curl \ 4 | -H "Authorization: Bearer 01GZHMCXHXHPGAABAQQTXKMSM3" \ 5 | -X GET "127.0.0.1:8544/user/deposits" 6 | -------------------------------------------------------------------------------- /scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | python-dotenv 2 | eth-brownie 3 | ensurepath 4 | brownie-token-tester -------------------------------------------------------------------------------- /scripts/versus: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llamanodes/web3-proxy/f4a71ee0b621c461828956436f6d55372228ec18/scripts/versus -------------------------------------------------------------------------------- /web3_proxy/src/caches.rs: -------------------------------------------------------------------------------- 1 | use crate::balance::Balance; 2 | use crate::errors::{Web3ProxyError, Web3ProxyResult}; 3 | use crate::frontend::authorization::AuthorizationChecks; 4 | use crate::secrets::RpcSecretKey; 5 | use derive_more::From; 6 | use entities::rpc_key; 7 | use migration::sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; 8 | use moka::future::Cache; 9 | use std::fmt; 10 | use std::net::IpAddr; 11 | use std::sync::Arc; 12 | use tokio::sync::RwLock as AsyncRwLock; 13 | use tracing::trace; 14 | 15 | /// Cache data from the database about rpc keys 16 | /// TODO: try Ulid/u128 instead of RpcSecretKey in case my hash method is broken 17 | pub type RpcSecretKeyCache = Cache; 18 | 19 | #[derive(Clone, Copy, Hash, Eq, PartialEq)] 20 | pub struct RegisteredUserRateLimitKey(pub u64, pub IpAddr); 21 | 22 | impl std::fmt::Display for RegisteredUserRateLimitKey { 23 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 24 | write!(f, "{}-{}", self.0, self.1) 25 | } 26 | } 27 | 28 | /// Cache data from the database about user balances 29 | #[derive(Clone, From)] 30 | pub struct UserBalanceCache(pub Cache>>); 31 | 32 | impl UserBalanceCache { 33 | pub async fn get_or_insert( 34 | &self, 35 | db_conn: &DatabaseConnection, 36 | user_id: u64, 37 | ) -> Web3ProxyResult>> { 38 | if user_id == 0 { 39 | return Ok(Arc::new(AsyncRwLock::new(Balance::default()))); 40 | } 41 | 42 | let x = self 43 | .0 44 | .try_get_with(user_id, async move { 45 | let x = match Balance::try_from_db(db_conn, user_id).await? { 46 | None => Balance { 47 | user_id, 48 | ..Default::default() 49 | }, 50 | Some(x) => x, 51 | }; 52 | trace!(?x, "from database"); 53 | 54 | Ok::<_, Web3ProxyError>(Arc::new(AsyncRwLock::new(x))) 55 | }) 56 | .await?; 57 | 58 | Ok(x) 59 | } 60 | 61 | pub async fn invalidate( 62 | &self, 63 | user_id: &u64, 64 | db_conn: &DatabaseConnection, 65 | rpc_secret_key_cache: &RpcSecretKeyCache, 66 | ) -> Web3ProxyResult<()> { 67 | self.0.invalidate(user_id).await; 68 | 69 | trace!(%user_id, "invalidating"); 70 | 71 | // Remove all RPC-keys owned by this user from the cache, s.t. rate limits are re-calculated 72 | let rpc_keys = rpc_key::Entity::find() 73 | .filter(rpc_key::Column::UserId.eq(*user_id)) 74 | .all(db_conn) 75 | .await?; 76 | 77 | for rpc_key_entity in rpc_keys { 78 | let rpc_key_id = rpc_key_entity.id; 79 | let secret_key = rpc_key_entity.secret_key.into(); 80 | 81 | trace!(%user_id, %rpc_key_id, ?secret_key, "invalidating"); 82 | 83 | rpc_secret_key_cache.invalidate(&secret_key).await; 84 | } 85 | 86 | Ok(()) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /web3_proxy/src/frontend/errors.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::Web3ProxyError; 2 | use axum::response::{IntoResponse, Response}; 3 | 4 | #[inline] 5 | pub async fn handler_404() -> Response { 6 | Web3ProxyError::NotFound.into_response() 7 | } 8 | -------------------------------------------------------------------------------- /web3_proxy/src/frontend/request_id.rs: -------------------------------------------------------------------------------- 1 | use std::task::{Context, Poll}; 2 | 3 | use http::Request; 4 | use tower_service::Service; 5 | use ulid::Ulid; 6 | 7 | /// RequestId from x-amzn-trace-id header or new Ulid 8 | #[derive(Clone, Debug)] 9 | pub struct RequestId(pub String); 10 | 11 | /// Middleware layer for adding RequestId as an Extension 12 | #[derive(Clone, Debug)] 13 | pub struct RequestIdLayer; 14 | 15 | impl tower_layer::Layer for RequestIdLayer { 16 | type Service = RequestIdService; 17 | 18 | fn layer(&self, inner: S) -> Self::Service { 19 | RequestIdService { inner } 20 | } 21 | } 22 | 23 | /// Service used by RequestIdLayer to inject RequestId 24 | #[derive(Clone, Debug)] 25 | pub struct RequestIdService { 26 | inner: S, 27 | } 28 | 29 | impl Service> for RequestIdService 30 | where 31 | S: Service>, 32 | { 33 | type Response = S::Response; 34 | type Error = S::Error; 35 | type Future = S::Future; 36 | 37 | #[inline] 38 | fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { 39 | self.inner.poll_ready(cx) 40 | } 41 | 42 | fn call(&mut self, mut req: Request) -> Self::Future { 43 | let request_id = req 44 | .headers() 45 | .get("x-amzn-trace-id") 46 | .and_then(|x| x.to_str().ok()) 47 | .map(ToString::to_string) 48 | .unwrap_or_else(|| Ulid::new().to_string()); 49 | req.extensions_mut().insert(RequestId(request_id)); 50 | self.inner.call(req) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /web3_proxy/src/frontend/streaming.rs: -------------------------------------------------------------------------------- 1 | use axum::{body::BoxBody, response::IntoResponse}; 2 | use bytes::Bytes; 3 | use futures::StreamExt; 4 | use http::Response; 5 | use std::pin::Pin; 6 | use std::task::{Context, Poll}; 7 | use tokio::stream::Stream; 8 | 9 | struct SizingBody { 10 | inner: B, 11 | web3_request: RequestMetadata, 12 | } 13 | 14 | impl SizingBody { 15 | fn new(inner: B) -> Self { 16 | Self { inner, size: 0 } 17 | } 18 | } 19 | 20 | impl Stream for SizingBody 21 | where 22 | B: Stream>> + Unpin, 23 | { 24 | type Item = Result>; 25 | 26 | fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { 27 | match Pin::new(&mut self.inner).poll_next(cx) { 28 | Poll::Ready(Some(Ok(chunk))) => { 29 | self.size += chunk.len(); 30 | Poll::Ready(Some(Ok(chunk))) 31 | } 32 | Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))), 33 | Poll::Ready(None) => { 34 | println!("Final response size: {}", self.size); 35 | Poll::Ready(None) 36 | } 37 | Poll::Pending => Poll::Pending, 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /web3_proxy/src/globals.rs: -------------------------------------------------------------------------------- 1 | // TODO: think a lot more about this 2 | 3 | use crate::{app::App, errors::Web3ProxyError, relational_db::DatabaseReplica}; 4 | use derivative::Derivative; 5 | use migration::{ 6 | sea_orm::{DatabaseConnection, DatabaseTransaction, TransactionTrait}, 7 | DbErr, 8 | }; 9 | use parking_lot::RwLock; 10 | use std::sync::{Arc, LazyLock, OnceLock}; 11 | 12 | pub static APP: OnceLock> = OnceLock::new(); 13 | 14 | pub static DB_CONN: LazyLock>> = 15 | LazyLock::new(|| RwLock::new(Err(DatabaseError::NotConfigured))); 16 | 17 | pub static DB_REPLICA: LazyLock>> = 18 | LazyLock::new(|| RwLock::new(Err(DatabaseError::NotConfigured))); 19 | 20 | #[derive(Clone, Debug, Derivative)] 21 | pub enum DatabaseError { 22 | /// no database configured. depending on what you need, this may or may not be a problem 23 | NotConfigured, 24 | /// an error that happened when creating the connection pool 25 | Connect(Arc), 26 | /// an error that just happened 27 | Begin(Arc), 28 | } 29 | 30 | impl From for Web3ProxyError { 31 | fn from(value: DatabaseError) -> Self { 32 | match value { 33 | DatabaseError::NotConfigured => Self::NoDatabaseConfigured, 34 | DatabaseError::Connect(err) | DatabaseError::Begin(err) => Self::DatabaseArc(err), 35 | } 36 | } 37 | } 38 | 39 | /// TODO: do we need this clone? should we just do DB_CONN.read() whenever we need a Connection? 40 | #[inline] 41 | pub fn global_db_conn() -> Result { 42 | DB_CONN.read().clone() 43 | } 44 | 45 | #[inline] 46 | pub async fn global_db_transaction() -> Result { 47 | let x = global_db_conn()?; 48 | 49 | let x = x 50 | .begin() 51 | .await 52 | .map_err(|x| DatabaseError::Begin(Arc::new(x)))?; 53 | 54 | Ok(x) 55 | } 56 | 57 | /// TODO: do we need this clone? 58 | #[inline] 59 | pub fn global_db_replica_conn() -> Result { 60 | DB_REPLICA.read().clone() 61 | } 62 | -------------------------------------------------------------------------------- /web3_proxy/src/jsonrpc/error.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use std::borrow::Cow; 3 | 4 | // TODO: impl Error on this? 5 | /// All jsonrpc errors use this structure 6 | #[derive(Clone, Debug, Deserialize, Serialize)] 7 | pub struct JsonRpcErrorData { 8 | /// The error code 9 | pub code: i64, 10 | /// The error message 11 | pub message: Cow<'static, str>, 12 | /// Additional data 13 | #[serde(skip_serializing_if = "Option::is_none")] 14 | pub data: Option, 15 | } 16 | 17 | impl JsonRpcErrorData { 18 | pub fn num_bytes(&self) -> u64 { 19 | serde_json::to_string(self) 20 | .expect("should always serialize") 21 | .len() as u64 22 | } 23 | 24 | // pub fn is_retryable(&self) -> bool { 25 | // // TODO: move stuff from request to here 26 | // todo!() 27 | // } 28 | } 29 | 30 | impl From<&'static str> for JsonRpcErrorData { 31 | fn from(value: &'static str) -> Self { 32 | Self { 33 | code: -32000, 34 | message: value.into(), 35 | data: None, 36 | } 37 | } 38 | } 39 | 40 | impl From for JsonRpcErrorData { 41 | fn from(value: String) -> Self { 42 | Self { 43 | code: -32000, 44 | message: value.into(), 45 | data: None, 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /web3_proxy/src/jsonrpc/id.rs: -------------------------------------------------------------------------------- 1 | use derive_more::From; 2 | use serde_json::{json, value::RawValue}; 3 | 4 | /// being strict on id doesn't really help much. just accept anything 5 | #[derive(From)] 6 | pub enum LooseId { 7 | None, 8 | Number(u64), 9 | String(String), 10 | Raw(Box), 11 | } 12 | 13 | impl LooseId { 14 | pub fn to_raw_value(self) -> Box { 15 | // TODO: is this a good way to do this? maybe also have `as_raw_value`? 16 | match self { 17 | Self::None => Default::default(), 18 | Self::Number(x) => { 19 | serde_json::from_value(json!(x)).expect("number id should always work") 20 | } 21 | Self::String(x) => serde_json::from_str(&x).expect("string id should always work"), 22 | Self::Raw(x) => x, 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /web3_proxy/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![feature(lazy_cell)] 2 | #![feature(let_chains)] 3 | #![feature(trait_alias)] 4 | #![feature(result_flattening)] 5 | #![forbid(unsafe_code)] 6 | 7 | pub mod admin_queries; 8 | pub mod app; 9 | pub mod balance; 10 | pub mod block_number; 11 | pub mod caches; 12 | pub mod compute_units; 13 | pub mod config; 14 | pub mod errors; 15 | pub mod frontend; 16 | pub mod globals; 17 | pub mod http_params; 18 | pub mod jsonrpc; 19 | pub mod pagerduty; 20 | pub mod prelude; 21 | pub mod premium; 22 | pub mod prometheus; 23 | pub mod referral_code; 24 | pub mod relational_db; 25 | pub mod response_cache; 26 | pub mod rpcs; 27 | pub mod secrets; 28 | pub mod stats; 29 | pub mod test_utils; 30 | pub mod user_token; 31 | 32 | #[cfg(feature = "rdkafka")] 33 | pub mod kafka; 34 | -------------------------------------------------------------------------------- /web3_proxy/src/prelude.rs: -------------------------------------------------------------------------------- 1 | pub use anyhow; 2 | pub use argh; 3 | pub use chrono; 4 | pub use entities; 5 | pub use ethers; 6 | pub use ethers::prelude::rand; 7 | pub use fdlimit; 8 | pub use futures; 9 | pub use glob; 10 | pub use hashbrown; 11 | pub use http; 12 | pub use influxdb2; 13 | pub use migration; 14 | pub use migration::sea_orm; 15 | pub use moka; 16 | pub use num; 17 | pub use ordered_float; 18 | pub use pagerduty_rs; 19 | pub use parking_lot; 20 | pub use reqwest; 21 | pub use rust_decimal; 22 | pub use sentry; 23 | pub use sentry_tracing; 24 | pub use serde; 25 | pub use serde_inline_default; 26 | pub use serde_json; 27 | pub use tokio; 28 | pub use toml; 29 | pub use tracing; 30 | pub use ulid; 31 | pub use url; 32 | pub use uuid; 33 | 34 | #[cfg(feature = "rdkafka")] 35 | pub use rdkafka; 36 | -------------------------------------------------------------------------------- /web3_proxy/src/premium.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::Web3ProxyResult; 2 | use anyhow::Context; 3 | use entities::{user, user_tier}; 4 | use ethers::prelude::Address; 5 | use migration::sea_orm::{ 6 | self, ActiveModelTrait, ColumnTrait, DatabaseTransaction, EntityTrait, IntoActiveModel, 7 | QueryFilter, 8 | }; 9 | use tracing::info; 10 | 11 | pub async fn get_user_and_tier_from_address( 12 | user_address: &Address, 13 | txn: &DatabaseTransaction, 14 | ) -> Web3ProxyResult)>> { 15 | let x = user::Entity::find() 16 | .filter(user::Column::Address.eq(user_address.as_bytes())) 17 | .find_also_related(user_tier::Entity) 18 | .one(txn) 19 | .await?; 20 | 21 | Ok(x) 22 | } 23 | 24 | pub async fn get_user_and_tier_from_id( 25 | user_id: u64, 26 | txn: &DatabaseTransaction, 27 | ) -> Web3ProxyResult)>> { 28 | let x = user::Entity::find_by_id(user_id) 29 | .find_also_related(user_tier::Entity) 30 | .one(txn) 31 | .await?; 32 | 33 | Ok(x) 34 | } 35 | 36 | /// TODO: improve this so that funding an account that has an "unlimited" key is left alone 37 | pub async fn grant_premium_tier( 38 | user: &user::Model, 39 | user_tier: Option<&user_tier::Model>, 40 | txn: &DatabaseTransaction, 41 | ) -> Web3ProxyResult<()> { 42 | if user_tier.is_none() || user_tier.and_then(|x| x.downgrade_tier_id).is_none() { 43 | if user_tier.map(|x| x.title.as_str()) == Some("Premium") { 44 | // user is already premium 45 | } else { 46 | info!("upgrading {} to Premium", user.id); 47 | 48 | // switch the user to the premium tier 49 | let new_user_tier = user_tier::Entity::find() 50 | .filter(user_tier::Column::Title.like("Premium")) 51 | .one(txn) 52 | .await? 53 | .context("premium tier not found")?; 54 | 55 | let mut user = user.clone().into_active_model(); 56 | 57 | user.user_tier_id = sea_orm::Set(new_user_tier.id); 58 | 59 | user.save(txn).await?; 60 | } 61 | } 62 | 63 | Ok(()) 64 | } 65 | -------------------------------------------------------------------------------- /web3_proxy/src/prometheus.rs: -------------------------------------------------------------------------------- 1 | use axum::extract::State; 2 | use axum::headers::HeaderName; 3 | use axum::http::HeaderValue; 4 | use axum::response::{IntoResponse, Response}; 5 | use axum::{routing::get, Router}; 6 | use std::net::SocketAddr; 7 | use std::sync::atomic::Ordering; 8 | use std::sync::Arc; 9 | use tokio::sync::broadcast; 10 | use tracing::info; 11 | 12 | use crate::app::App; 13 | use crate::errors::Web3ProxyResult; 14 | 15 | /// Run a prometheus metrics server on the given port. 16 | pub async fn serve( 17 | app: Arc, 18 | mut shutdown_receiver: broadcast::Receiver<()>, 19 | ) -> Web3ProxyResult<()> { 20 | // routes should be ordered most to least common 21 | let router = Router::new().route("/", get(root)).with_state(app.clone()); 22 | 23 | // note: the port here might be 0 24 | let port = app.prometheus_port.load(Ordering::SeqCst); 25 | // TODO: config for the host? 26 | let addr = SocketAddr::from(([0, 0, 0, 0], port)); 27 | 28 | let service = router.into_make_service(); 29 | 30 | // `axum::Server` is a re-export of `hyper::Server` 31 | let server = axum::Server::bind(&addr).serve(service); 32 | 33 | let port = server.local_addr().port(); 34 | info!("prometheus listening on port {}", port); 35 | 36 | app.prometheus_port.store(port, Ordering::SeqCst); 37 | 38 | server 39 | .with_graceful_shutdown(async move { 40 | let _ = shutdown_receiver.recv().await; 41 | }) 42 | .await 43 | .map_err(Into::into) 44 | } 45 | 46 | async fn root(State(app): State>) -> Response { 47 | let serialized = app.prometheus_metrics().await; 48 | 49 | let mut r = serialized.into_response(); 50 | 51 | // // TODO: is there an easier way to do this? 52 | r.headers_mut().insert( 53 | HeaderName::from_static("content-type"), 54 | HeaderValue::from_static("application/openmetrics-text; version=1.0.0; charset=utf-8"), 55 | ); 56 | 57 | r 58 | } 59 | -------------------------------------------------------------------------------- /web3_proxy/src/referral_code.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Display; 2 | 3 | use anyhow::{self, Result}; 4 | use ulid::Ulid; 5 | 6 | pub struct ReferralCode(String); 7 | 8 | impl Default for ReferralCode { 9 | fn default() -> Self { 10 | let out = Ulid::new(); 11 | Self(format!("{}", out)) 12 | } 13 | } 14 | 15 | impl Display for ReferralCode { 16 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 17 | f.write_str(&self.0) 18 | } 19 | } 20 | 21 | impl TryFrom for ReferralCode { 22 | type Error = anyhow::Error; 23 | 24 | fn try_from(x: String) -> Result { 25 | // TODO: Check if string is a valid ULID 26 | Ok(Self(x)) 27 | // match Ulid::try_from(x.clone()) { 28 | // Ok(_) => , 29 | // Err(_) => Err(anyhow::anyhow!( 30 | // "Referral Code does not have the right format" 31 | // )), 32 | // } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /web3_proxy/src/rpcs/grpc_erigon.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llamanodes/web3-proxy/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/rpcs/grpc_erigon.rs -------------------------------------------------------------------------------- /web3_proxy/src/rpcs/http.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llamanodes/web3-proxy/f4a71ee0b621c461828956436f6d55372228ec18/web3_proxy/src/rpcs/http.rs -------------------------------------------------------------------------------- /web3_proxy/src/rpcs/mod.rs: -------------------------------------------------------------------------------- 1 | // TODO: all pub, or export useful things here instead? 2 | pub mod blockchain; 3 | pub mod consensus; 4 | pub mod many; 5 | pub mod one; 6 | pub mod provider; 7 | pub mod request; 8 | -------------------------------------------------------------------------------- /web3_proxy/src/secrets.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use std::fmt; 3 | use ulid::Ulid; 4 | use uuid::Uuid; 5 | 6 | /// This lets us use UUID and ULID while we transition to only ULIDs 7 | /// TODO: custom deserialize that can also go from String to Ulid 8 | #[derive(Copy, Clone, Deserialize)] 9 | pub enum RpcSecretKey { 10 | Ulid(Ulid), 11 | Uuid(Uuid), 12 | } 13 | 14 | impl RpcSecretKey { 15 | pub fn new() -> Self { 16 | Ulid::new().into() 17 | } 18 | 19 | pub fn as_128(&self) -> u128 { 20 | match self { 21 | Self::Ulid(x) => x.0, 22 | Self::Uuid(x) => x.as_u128(), 23 | } 24 | } 25 | } 26 | 27 | impl PartialEq for RpcSecretKey { 28 | fn eq(&self, other: &Self) -> bool { 29 | self.as_128() == other.as_128() 30 | } 31 | } 32 | 33 | impl Eq for RpcSecretKey {} 34 | 35 | impl fmt::Debug for RpcSecretKey { 36 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 37 | match self { 38 | Self::Ulid(x) => fmt::Debug::fmt(x, f), 39 | Self::Uuid(x) => { 40 | let x = Ulid::from(x.as_u128()); 41 | 42 | fmt::Debug::fmt(&x, f) 43 | } 44 | } 45 | } 46 | } 47 | 48 | /// always serialize as a ULID. 49 | impl Serialize for RpcSecretKey { 50 | fn serialize(&self, serializer: S) -> Result 51 | where 52 | S: serde::Serializer, 53 | { 54 | match self { 55 | Self::Ulid(x) => x.serialize(serializer), 56 | Self::Uuid(x) => { 57 | let x: Ulid = x.to_owned().into(); 58 | 59 | x.serialize(serializer) 60 | } 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /web3_proxy/src/test_utils/anvil.rs: -------------------------------------------------------------------------------- 1 | // TODO: option to spawn in a dedicated thread? 2 | // TODO: option to subscribe to another anvil and copy blocks 3 | 4 | use crate::rpcs::provider::EthersHttpProvider; 5 | use ethers::{ 6 | signers::LocalWallet, 7 | utils::{Anvil, AnvilInstance}, 8 | }; 9 | use tracing::info; 10 | 11 | /// on drop, the anvil instance will be shut down 12 | pub struct TestAnvil { 13 | pub instance: AnvilInstance, 14 | pub provider: EthersHttpProvider, 15 | } 16 | 17 | impl TestAnvil { 18 | pub async fn new(chain_id: Option, fork_rpc: Option<&str>) -> Self { 19 | info!(?chain_id); 20 | 21 | let mut instance = Anvil::new(); 22 | 23 | if let Some(chain_id) = chain_id { 24 | instance = instance.chain_id(chain_id); 25 | } 26 | 27 | if let Some(fork_rpc) = fork_rpc { 28 | instance = instance.fork(fork_rpc); 29 | } 30 | 31 | let instance = instance.spawn(); 32 | 33 | let provider = EthersHttpProvider::try_from(instance.endpoint()).unwrap(); 34 | 35 | Self { instance, provider } 36 | } 37 | 38 | pub async fn spawn(chain_id: u64) -> Self { 39 | Self::new(Some(chain_id), None).await 40 | } 41 | 42 | pub async fn spawn_fork(fork_rpc: &str) -> Self { 43 | Self::new(None, Some(fork_rpc)).await 44 | } 45 | 46 | pub fn wallet(&self, id: usize) -> LocalWallet { 47 | self.instance.keys()[id].clone().into() 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /web3_proxy/src/test_utils/create_provider_with_rpc_key.rs: -------------------------------------------------------------------------------- 1 | use crate::rpcs::provider::EthersHttpProvider; 2 | use ulid::Ulid; 3 | use url::Url; 4 | 5 | pub async fn create_provider_for_user(url: &Url, user_secret_key: &Ulid) -> EthersHttpProvider { 6 | // Then generate a provider 7 | let proxy_endpoint = format!("{}rpc/{}", url, user_secret_key); 8 | 9 | EthersHttpProvider::try_from(proxy_endpoint).unwrap() 10 | } 11 | -------------------------------------------------------------------------------- /web3_proxy/src/test_utils/lib.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | println!("Hello, world!"); 3 | } 4 | -------------------------------------------------------------------------------- /web3_proxy/src/test_utils/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod anvil; 2 | pub mod create_provider_with_rpc_key; 3 | pub mod influx; 4 | pub mod mysql; 5 | 6 | pub use self::anvil::TestAnvil; 7 | pub use self::influx::TestInflux; 8 | pub use self::mysql::TestMysql; 9 | -------------------------------------------------------------------------------- /web3_proxy/src/user_token.rs: -------------------------------------------------------------------------------- 1 | use axum::headers::authorization::Bearer; 2 | use migration::sea_orm::prelude::Uuid; 3 | use serde::{Deserialize, Serialize}; 4 | use std::fmt; 5 | use std::str::FromStr; 6 | use ulid::Ulid; 7 | 8 | /// Key used for caching the user's login 9 | #[derive(Copy, Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)] 10 | #[serde(transparent)] 11 | pub struct UserBearerToken(pub Ulid); 12 | 13 | impl UserBearerToken { 14 | pub fn redis_key(&self) -> String { 15 | format!("bearer:{}", self.0) 16 | } 17 | 18 | pub fn uuid(&self) -> Uuid { 19 | Uuid::from_u128(self.0.into()) 20 | } 21 | } 22 | 23 | impl Default for UserBearerToken { 24 | fn default() -> Self { 25 | Self(Ulid::new()) 26 | } 27 | } 28 | 29 | impl FromStr for UserBearerToken { 30 | type Err = ulid::DecodeError; 31 | 32 | fn from_str(s: &str) -> Result { 33 | let ulid = Ulid::from_str(s)?; 34 | 35 | Ok(Self(ulid)) 36 | } 37 | } 38 | 39 | impl From for UserBearerToken { 40 | fn from(x: Ulid) -> Self { 41 | Self(x) 42 | } 43 | } 44 | 45 | impl From for Uuid { 46 | fn from(x: UserBearerToken) -> Self { 47 | x.uuid() 48 | } 49 | } 50 | 51 | impl fmt::Display for UserBearerToken { 52 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 53 | self.0.fmt(f) 54 | } 55 | } 56 | 57 | impl TryFrom for UserBearerToken { 58 | type Error = ulid::DecodeError; 59 | 60 | fn try_from(b: Bearer) -> Result { 61 | let u = Ulid::from_string(b.token())?; 62 | 63 | Ok(UserBearerToken(u)) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /web3_proxy_cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "web3_proxy_cli" 3 | version = "1.43.100" 4 | edition = "2021" 5 | default-run = "web3_proxy_cli" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [features] 10 | default = ["tokio-console"] 11 | 12 | deadlock_detection = ["parking_lot/deadlock_detection"] 13 | mimalloc = ["web3_proxy/mimalloc"] 14 | stripe = ["web3_proxy/stripe"] 15 | rdkafka-src = ["web3_proxy/rdkafka-src"] 16 | tests-needing-docker = ["web3_proxy/tests-needing-docker"] 17 | tokio-console = ["dep:tokio-console", "dep:console-subscriber"] 18 | 19 | [dependencies] 20 | web3_proxy = { path = "../web3_proxy" } 21 | 22 | console-subscriber = { version = "0.2.0", features = ["env-filter", "parking_lot"], optional = true } 23 | parking_lot = { version = "0.12.1", features = ["arc_lock", "nightly"] } 24 | prettytable = { version = "0.10.0", default-features = false } 25 | serde = { version = "1.0.193" } 26 | serde_json = { version = "1.0.108", default-features = false, features = ["raw_value"] } 27 | tokio-console = { version = "0.1.10", optional = true } 28 | tracing = "0.1" 29 | tracing-subscriber = { version = "0.3", features = ["env-filter"] } 30 | 31 | [dev-dependencies] 32 | env_logger = { version ="0.10", default-features = false, features = ["auto-color"] } 33 | test-log = { version ="0.2.13", default-features = false, features = ["trace"] } 34 | -------------------------------------------------------------------------------- /web3_proxy_cli/examples/subscribe_blocks.rs: -------------------------------------------------------------------------------- 1 | /// subscribe to a websocket rpc 2 | use std::time::Duration; 3 | use web3_proxy::prelude::anyhow; 4 | use web3_proxy::prelude::ethers::prelude::*; 5 | use web3_proxy::prelude::fdlimit; 6 | use web3_proxy::prelude::tokio; 7 | 8 | #[tokio::main] 9 | async fn main() -> anyhow::Result<()> { 10 | // install global collector configured based on RUST_LOG env var. 11 | fdlimit::raise_fd_limit()?; 12 | 13 | // erigon 14 | let url = "ws://10.11.12.16:8548"; 15 | // geth 16 | // let url = "ws://10.11.12.16:8546"; 17 | 18 | println!("Subscribing to blocks from {}", url); 19 | 20 | let provider = Ws::connect(url).await?; 21 | 22 | let provider = Provider::new(provider).interval(Duration::from_secs(1)); 23 | 24 | let mut stream = provider.subscribe_blocks().await?; 25 | while let Some(block) = stream.next().await { 26 | println!( 27 | "{:?} = Ts: {:?}, block number: {}", 28 | block.hash.unwrap(), 29 | block.timestamp, 30 | block.number.unwrap(), 31 | ); 32 | } 33 | 34 | Ok(()) 35 | } 36 | -------------------------------------------------------------------------------- /web3_proxy_cli/examples/tracing.rs: -------------------------------------------------------------------------------- 1 | use tracing::info; 2 | 3 | fn main() { 4 | // install global collector configured based on RUST_LOG env var. 5 | tracing_subscriber::fmt::init(); 6 | 7 | let number_of_yaks = 3; 8 | // this creates a new event, outside of any spans. 9 | info!(number_of_yaks, "preparing to shave yaks"); 10 | 11 | let number_shaved = 3; 12 | info!( 13 | all_yaks_shaved = number_shaved == number_of_yaks, 14 | "yak shaving completed." 15 | ); 16 | } 17 | -------------------------------------------------------------------------------- /web3_proxy_cli/examples/watch_blocks.rs: -------------------------------------------------------------------------------- 1 | /// poll an http rpc 2 | use std::{str::FromStr, time::Duration}; 3 | use web3_proxy::prelude::anyhow; 4 | use web3_proxy::prelude::ethers::prelude::*; 5 | use web3_proxy::prelude::fdlimit; 6 | use web3_proxy::prelude::tokio; 7 | 8 | #[tokio::main] 9 | async fn main() -> anyhow::Result<()> { 10 | fdlimit::raise_fd_limit()?; 11 | 12 | // erigon does not support most filters 13 | // let url = "http://10.11.12.16:8545"; 14 | // geth 15 | let url = "http://10.11.12.16:8545"; 16 | 17 | println!("Watching blocks from {:?}", url); 18 | 19 | let provider = Http::from_str(url)?; 20 | 21 | let provider = Provider::new(provider).interval(Duration::from_secs(1)); 22 | 23 | let mut stream = provider.watch_blocks().await?; 24 | while let Some(block_number) = stream.next().await { 25 | let block = provider.get_block(block_number).await?.unwrap(); 26 | println!( 27 | "{:?} = Ts: {:?}, block number: {}", 28 | block.hash.unwrap(), 29 | block.timestamp, 30 | block.number.unwrap(), 31 | ); 32 | } 33 | 34 | Ok(()) 35 | } 36 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod sub_commands; 2 | pub mod test_utils; 3 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/change_admin_status.rs: -------------------------------------------------------------------------------- 1 | use web3_proxy::prelude::anyhow::{self, Context}; 2 | use web3_proxy::prelude::argh::{self, FromArgs}; 3 | use web3_proxy::prelude::entities::{admin, login, user}; 4 | use web3_proxy::prelude::ethers::types::Address; 5 | use web3_proxy::prelude::migration::sea_orm::{ 6 | self, ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, ModelTrait, QueryFilter, 7 | }; 8 | use web3_proxy::prelude::serde_json::json; 9 | use web3_proxy::prelude::tracing::{debug, info}; 10 | 11 | /// change a user's admin status. eiter they are an admin, or they aren't 12 | #[derive(FromArgs, PartialEq, Eq, Debug)] 13 | #[argh(subcommand, name = "change_admin_status")] 14 | pub struct ChangeAdminStatusSubCommand { 15 | /// the address of the user whose admin status you want to modify 16 | #[argh(positional)] 17 | pub address: String, 18 | 19 | /// true if the user should be an admin, false otherwise 20 | #[argh(positional)] 21 | pub should_be_admin: bool, 22 | } 23 | 24 | impl ChangeAdminStatusSubCommand { 25 | pub async fn main(self, db_conn: &DatabaseConnection) -> anyhow::Result<()> { 26 | let address: Address = self.address.parse()?; 27 | let should_be_admin: bool = self.should_be_admin; 28 | 29 | // Find user in database 30 | let user = user::Entity::find() 31 | .filter(user::Column::Address.eq(address.as_bytes())) 32 | .one(db_conn) 33 | .await? 34 | .context(format!("No user with this address found {:?}", address))?; 35 | 36 | debug!("user: {:#}", json!(&user)); 37 | 38 | // Check if there is a record in the database 39 | match admin::Entity::find() 40 | .filter(admin::Column::UserId.eq(user.id)) 41 | .one(db_conn) 42 | .await? 43 | { 44 | Some(old_admin) if !should_be_admin => { 45 | // User is already an admin, but shouldn't be 46 | old_admin.delete(db_conn).await?; 47 | info!("revoked admin status"); 48 | } 49 | None if should_be_admin => { 50 | // User is not an admin yet, but should be 51 | let new_admin = admin::ActiveModel { 52 | user_id: sea_orm::Set(user.id), 53 | ..Default::default() 54 | }; 55 | new_admin.insert(db_conn).await?; 56 | info!("granted admin status"); 57 | } 58 | _ => { 59 | info!("no change needed for: {:#}", json!(user)); 60 | // Since no change happened, we do not want to delete active logins. Return now. 61 | return Ok(()); 62 | } 63 | } 64 | 65 | // Remove any user logins from the database (incl. bearer tokens) 66 | let delete_result = login::Entity::delete_many() 67 | .filter(login::Column::UserId.eq(user.id)) 68 | .exec(db_conn) 69 | .await?; 70 | 71 | debug!("cleared modified logins: {:?}", delete_result); 72 | 73 | Ok(()) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/change_user_address.rs: -------------------------------------------------------------------------------- 1 | use web3_proxy::prelude::anyhow::{self, Context}; 2 | use web3_proxy::prelude::argh::{self, FromArgs}; 3 | use web3_proxy::prelude::entities::user; 4 | use web3_proxy::prelude::ethers::types::Address; 5 | use web3_proxy::prelude::migration::sea_orm::{ 6 | self, ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, IntoActiveModel, 7 | QueryFilter, 8 | }; 9 | use web3_proxy::prelude::serde_json::json; 10 | use web3_proxy::prelude::tracing::{debug, info}; 11 | 12 | /// change a user's address. 13 | #[derive(FromArgs, PartialEq, Eq, Debug)] 14 | #[argh(subcommand, name = "change_user_address")] 15 | pub struct ChangeUserAddressSubCommand { 16 | /// the address of the user you want to change 17 | #[argh(positional)] 18 | old_address: String, 19 | 20 | /// the address of the user you want to change 21 | #[argh(positional)] 22 | new_address: String, 23 | } 24 | 25 | impl ChangeUserAddressSubCommand { 26 | pub async fn main(self, db_conn: &DatabaseConnection) -> anyhow::Result<()> { 27 | let old_address: Address = self.old_address.parse()?; 28 | let new_address: Address = self.new_address.parse()?; 29 | 30 | let u = user::Entity::find() 31 | .filter(user::Column::Address.eq(old_address.as_bytes())) 32 | .one(db_conn) 33 | .await? 34 | .context("No user found with that address")?; 35 | 36 | debug!("initial user: {:#}", json!(&u)); 37 | 38 | if u.address == new_address.as_bytes() { 39 | info!("user already has this address"); 40 | } else { 41 | let mut u = u.into_active_model(); 42 | 43 | u.address = sea_orm::Set(new_address.as_bytes().to_vec()); 44 | 45 | let u = u.save(db_conn).await?; 46 | 47 | info!("updated user: {:#?}", u); 48 | } 49 | 50 | Ok(()) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/change_user_tier.rs: -------------------------------------------------------------------------------- 1 | use web3_proxy::prelude::anyhow::{self, Context}; 2 | use web3_proxy::prelude::argh::{self, FromArgs}; 3 | use web3_proxy::prelude::entities::user_tier; 4 | use web3_proxy::prelude::migration::sea_orm::{ 5 | self, ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, IntoActiveModel, 6 | QueryFilter, 7 | }; 8 | use web3_proxy::prelude::serde_json::json; 9 | use web3_proxy::prelude::tracing::{debug, info}; 10 | 11 | /// change a user's tier. 12 | #[derive(FromArgs, PartialEq, Eq, Debug)] 13 | #[argh(subcommand, name = "change_user_tier")] 14 | pub struct ChangeUserTierSubCommand { 15 | /// the title of the user tier you are going to modify. 16 | #[argh(positional)] 17 | user_tier_title: String, 18 | 19 | /// the amount of requests to allow per rate limit period 20 | #[argh(option)] 21 | max_requests_per_period: Option, 22 | 23 | /// the amount of concurret requests to allow from a single user 24 | #[argh(option)] 25 | max_concurrent_requests: Option, 26 | } 27 | 28 | impl ChangeUserTierSubCommand { 29 | // TODO: don't expose the RpcSecretKeys at all. Better to take a user/key id. this is definitely most convenient 30 | 31 | pub async fn main(self, db_conn: &DatabaseConnection) -> anyhow::Result<()> { 32 | let user_tier = user_tier::Entity::find() 33 | .filter(user_tier::Column::Title.eq(self.user_tier_title)) 34 | .one(db_conn) 35 | .await? 36 | .context("No user tier found with that name")?; 37 | 38 | debug!("initial user_tier: {:#}", json!(&user_tier)); 39 | 40 | let mut user_tier = user_tier.into_active_model(); 41 | 42 | if let Some(max_requests_per_period) = self.max_requests_per_period { 43 | if user_tier.max_requests_per_period == sea_orm::Set(Some(max_requests_per_period)) { 44 | info!("max_requests_per_period already has this value"); 45 | } else { 46 | user_tier.max_requests_per_period = sea_orm::Set(Some(max_requests_per_period)); 47 | 48 | info!("changed max_requests_per_period") 49 | } 50 | } 51 | 52 | if let Some(max_concurrent_requests) = self.max_concurrent_requests { 53 | if user_tier.max_concurrent_requests == sea_orm::Set(Some(max_concurrent_requests)) { 54 | info!("max_concurrent_requests already has this value"); 55 | } else { 56 | user_tier.max_concurrent_requests = sea_orm::Set(Some(max_concurrent_requests)); 57 | 58 | info!("changed max_concurrent_requests") 59 | } 60 | } 61 | 62 | let user_tier = user_tier.save(db_conn).await?; 63 | 64 | debug!("new user_tier: {:#?}", user_tier); 65 | 66 | Ok(()) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/change_user_tier_by_address.rs: -------------------------------------------------------------------------------- 1 | use web3_proxy::prelude::anyhow::{self, Context}; 2 | use web3_proxy::prelude::argh::{self, FromArgs}; 3 | use web3_proxy::prelude::entities::{user, user_tier}; 4 | use web3_proxy::prelude::ethers::types::Address; 5 | use web3_proxy::prelude::migration::sea_orm::{ 6 | self, ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, IntoActiveModel, 7 | QueryFilter, 8 | }; 9 | use web3_proxy::prelude::serde_json::json; 10 | use web3_proxy::prelude::tracing::{debug, info}; 11 | 12 | /// change a user's tier. 13 | #[derive(FromArgs, PartialEq, Eq, Debug)] 14 | #[argh(subcommand, name = "change_user_tier_by_address")] 15 | pub struct ChangeUserTierByAddressSubCommand { 16 | #[argh(positional)] 17 | /// the address of the user you want to change. 18 | user_address: Address, 19 | 20 | /// the title of the desired user tier. 21 | #[argh(positional)] 22 | user_tier_title: String, 23 | } 24 | 25 | impl ChangeUserTierByAddressSubCommand { 26 | pub async fn main(self, db_conn: &DatabaseConnection) -> anyhow::Result<()> { 27 | // use the address to get the user 28 | let user = user::Entity::find() 29 | .filter(user::Column::Address.eq(self.user_address.as_bytes())) 30 | .one(db_conn) 31 | .await? 32 | .context("No user found with that key")?; 33 | 34 | // TODO: don't serialize the rpc key 35 | debug!("user: {:#}", json!(&user)); 36 | 37 | // use the title to get the user tier 38 | let user_tier = user_tier::Entity::find() 39 | .filter(user_tier::Column::Title.eq(self.user_tier_title)) 40 | .one(db_conn) 41 | .await? 42 | .context("No user tier found with that name")?; 43 | 44 | debug!("user_tier: {:#}", json!(&user_tier)); 45 | 46 | if user.user_tier_id == user_tier.id { 47 | info!("user already has that tier"); 48 | } else { 49 | let mut user = user.into_active_model(); 50 | 51 | user.user_tier_id = sea_orm::Set(user_tier.id); 52 | 53 | user.save(db_conn).await?; 54 | 55 | info!("user's tier changed"); 56 | } 57 | 58 | Ok(()) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/change_user_tier_by_key.rs: -------------------------------------------------------------------------------- 1 | use web3_proxy::prelude::anyhow::{self, Context}; 2 | use web3_proxy::prelude::argh::{self, FromArgs}; 3 | use web3_proxy::prelude::entities::{rpc_key, user, user_tier}; 4 | use web3_proxy::prelude::migration::sea_orm::{ 5 | self, ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, IntoActiveModel, 6 | QueryFilter, 7 | }; 8 | use web3_proxy::prelude::serde_json::json; 9 | use web3_proxy::prelude::tracing::{debug, info}; 10 | use web3_proxy::prelude::uuid::Uuid; 11 | use web3_proxy::secrets::RpcSecretKey; 12 | 13 | /// change a user's tier. 14 | #[derive(FromArgs, PartialEq, Eq, Debug)] 15 | #[argh(subcommand, name = "change_user_tier_by_key")] 16 | pub struct ChangeUserTierByKeySubCommand { 17 | #[argh(positional)] 18 | /// the RPC key owned by the user you want to change. 19 | rpc_secret_key: RpcSecretKey, 20 | 21 | /// the title of the desired user tier. 22 | #[argh(positional)] 23 | user_tier_title: String, 24 | } 25 | 26 | impl ChangeUserTierByKeySubCommand { 27 | // TODO: don't expose the RpcSecretKeys at all. Better to take a user/key id. this is definitely most convenient 28 | 29 | pub async fn main(self, db_conn: &DatabaseConnection) -> anyhow::Result<()> { 30 | let rpc_secret_key: Uuid = self.rpc_secret_key.into(); 31 | 32 | let user_tier = user_tier::Entity::find() 33 | .filter(user_tier::Column::Title.eq(self.user_tier_title)) 34 | .one(db_conn) 35 | .await? 36 | .context("No user tier found with that name")?; 37 | 38 | debug!("user_tier: {:#}", json!(&user_tier)); 39 | 40 | // use the rpc secret key to get the user 41 | let user = user::Entity::find() 42 | .inner_join(rpc_key::Entity) 43 | .filter(rpc_key::Column::SecretKey.eq(rpc_secret_key)) 44 | .one(db_conn) 45 | .await? 46 | .context("No user found with that key")?; 47 | 48 | debug!("user: {:#}", json!(&user)); 49 | 50 | if user.user_tier_id == user_tier.id { 51 | info!("user already has that tier"); 52 | } else { 53 | let mut user = user.into_active_model(); 54 | 55 | user.user_tier_id = sea_orm::Set(user_tier.id); 56 | 57 | user.save(db_conn).await?; 58 | 59 | info!("user's tier changed"); 60 | } 61 | 62 | Ok(()) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/check_balance.rs: -------------------------------------------------------------------------------- 1 | use web3_proxy::balance::Balance; 2 | use web3_proxy::prelude::anyhow::{self, Context}; 3 | use web3_proxy::prelude::argh::{self, FromArgs}; 4 | use web3_proxy::prelude::entities::user; 5 | use web3_proxy::prelude::ethers::types::Address; 6 | use web3_proxy::prelude::migration::sea_orm::{ 7 | ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, 8 | }; 9 | use web3_proxy::prelude::serde_json::json; 10 | use web3_proxy::prelude::tracing::debug; 11 | 12 | /// change a user's tier. 13 | #[derive(FromArgs, PartialEq, Eq, Debug)] 14 | #[argh(subcommand, name = "check_balance")] 15 | pub struct CheckBalanceSubCommand { 16 | #[argh(positional)] 17 | /// the address of the user you want to check. 18 | user_address: Address, 19 | } 20 | 21 | impl CheckBalanceSubCommand { 22 | pub async fn main(self, db_conn: &DatabaseConnection) -> anyhow::Result<()> { 23 | // use the address to get the user 24 | let user = user::Entity::find() 25 | .filter(user::Column::Address.eq(self.user_address.as_bytes())) 26 | .one(db_conn) 27 | .await? 28 | .context("No user found with that key")?; 29 | 30 | // TODO: don't serialize the rpc key 31 | debug!("user: {:#}", json!(&user)); 32 | 33 | let balance = Balance::try_from_db(db_conn, user.id).await?; 34 | debug!("balance: {:#}", json!(&balance)); 35 | 36 | Ok(()) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/count_users.rs: -------------------------------------------------------------------------------- 1 | use tracing::info; 2 | use web3_proxy::prelude::anyhow; 3 | use web3_proxy::prelude::argh::{self, FromArgs}; 4 | use web3_proxy::prelude::entities::user; 5 | use web3_proxy::prelude::migration::sea_orm::{self, EntityTrait, PaginatorTrait}; 6 | 7 | #[derive(FromArgs, PartialEq, Debug, Eq)] 8 | /// Create a new user and api key 9 | #[argh(subcommand, name = "count_users")] 10 | pub struct CountUsersSubCommand {} 11 | 12 | impl CountUsersSubCommand { 13 | pub async fn main(self, db: &sea_orm::DatabaseConnection) -> anyhow::Result<()> { 14 | let count = user::Entity::find().count(db).await?; 15 | 16 | info!("user count: {}", count); 17 | 18 | Ok(()) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/create_key.rs: -------------------------------------------------------------------------------- 1 | use web3_proxy::prelude::anyhow::{self, Context}; 2 | use web3_proxy::prelude::argh::{self, FromArgs}; 3 | use web3_proxy::prelude::entities::{rpc_key, user}; 4 | use web3_proxy::prelude::ethers::prelude::Address; 5 | use web3_proxy::prelude::migration::sea_orm::{ 6 | self, ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, 7 | }; 8 | use web3_proxy::prelude::tracing::info; 9 | use web3_proxy::prelude::ulid::Ulid; 10 | use web3_proxy::prelude::uuid::Uuid; 11 | use web3_proxy::secrets::RpcSecretKey; 12 | 13 | #[derive(FromArgs, PartialEq, Debug, Eq)] 14 | /// Create a new user and api key 15 | #[argh(subcommand, name = "create_key")] 16 | pub struct CreateKeySubCommand { 17 | /// the user's ethereum address or descriptive string. 18 | /// If a string is given, it will be converted to hex and potentially truncated. 19 | /// Users from strings are only for testing since they won't be able to log in. 20 | #[argh(positional)] 21 | address: Address, 22 | 23 | /// the user's api ULID or UUID key. 24 | /// If none given, one will be created. 25 | #[argh(option)] 26 | rpc_secret_key: Option, 27 | 28 | /// an optional short description of the key's purpose. 29 | #[argh(option)] 30 | description: Option, 31 | } 32 | 33 | impl CreateKeySubCommand { 34 | pub async fn main(self, db: &sea_orm::DatabaseConnection) -> anyhow::Result<()> { 35 | // TODO: get existing or create a new one 36 | let u = user::Entity::find() 37 | .filter(user::Column::Address.eq(self.address.as_bytes())) 38 | .one(db) 39 | .await? 40 | .context("No user found with that address")?; 41 | 42 | info!("user #{}", u.id); 43 | 44 | let rpc_secret_key = self.rpc_secret_key.unwrap_or_default(); 45 | 46 | // create a key for the new user 47 | let uk = rpc_key::ActiveModel { 48 | user_id: sea_orm::Set(u.id), 49 | secret_key: sea_orm::Set(rpc_secret_key.into()), 50 | description: sea_orm::Set(self.description), 51 | ..Default::default() 52 | }; 53 | 54 | let _uk = uk.save(db).await.context("Failed saving new user key")?; 55 | 56 | info!("user key as ULID: {}", Ulid::from(rpc_secret_key)); 57 | info!("user key as UUID: {}", Uuid::from(rpc_secret_key)); 58 | 59 | Ok(()) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/delete_user.rs: -------------------------------------------------------------------------------- 1 | //! delete user. don't delete rpc_accounting because we need that for our own accounting. have a "deleted" user that takes them over 2 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/drop_migration_lock.rs: -------------------------------------------------------------------------------- 1 | use web3_proxy::prelude::anyhow; 2 | use web3_proxy::prelude::argh::{self, FromArgs}; 3 | use web3_proxy::prelude::migration::sea_orm::DatabaseConnection; 4 | use web3_proxy::relational_db::{drop_migration_lock, migrate_db}; 5 | 6 | #[derive(FromArgs, PartialEq, Debug, Eq)] 7 | /// In case of emergency, break glass. 8 | #[argh(subcommand, name = "drop_migration_lock")] 9 | pub struct DropMigrationLockSubCommand { 10 | #[argh(option)] 11 | /// run migrations after dropping the lock 12 | and_migrate: bool, 13 | } 14 | 15 | impl DropMigrationLockSubCommand { 16 | pub async fn main(&self, db_conn: &DatabaseConnection) -> anyhow::Result<()> { 17 | if self.and_migrate { 18 | migrate_db(db_conn, true).await?; 19 | } else { 20 | // just drop the lock 21 | drop_migration_lock(db_conn).await?; 22 | } 23 | 24 | Ok(()) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/example.rs: -------------------------------------------------------------------------------- 1 | use web3_proxy::prelude::argh::{self, FromArgs}; 2 | 3 | #[derive(FromArgs, PartialEq, Debug)] 4 | /// An example subcommand. Copy paste this into a new file. 5 | #[argh(subcommand, name = "example")] 6 | pub struct ExampleSubcommand { 7 | #[argh(switch)] 8 | /// whether to fooey 9 | fooey: bool, 10 | } 11 | 12 | impl ExampleSubcommand { 13 | pub async fn main(self) -> anyhow::Result<()> { 14 | todo!() 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/list_recent_users.rs: -------------------------------------------------------------------------------- 1 | //! List users that have recently made a request 2 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/mod.rs: -------------------------------------------------------------------------------- 1 | mod change_admin_status; 2 | mod change_user_address; 3 | mod change_user_tier; 4 | mod change_user_tier_by_address; 5 | mod change_user_tier_by_key; 6 | mod check_balance; 7 | mod check_config; 8 | mod count_users; 9 | mod create_key; 10 | mod create_user; 11 | mod drop_migration_lock; 12 | mod grant_credits_to_address; 13 | mod mass_grant_credits; 14 | mod migrate_stats_to_v2; 15 | mod pagerduty; 16 | mod popularity_contest; 17 | mod proxyd; 18 | mod rpc_accounting; 19 | mod sentryd; 20 | mod transfer_key; 21 | mod user_export; 22 | mod user_import; 23 | 24 | #[cfg(feature = "rdkafka")] 25 | mod search_kafka; 26 | 27 | pub use self::change_admin_status::ChangeAdminStatusSubCommand; 28 | pub use self::change_user_address::ChangeUserAddressSubCommand; 29 | pub use self::change_user_tier::ChangeUserTierSubCommand; 30 | pub use self::change_user_tier_by_address::ChangeUserTierByAddressSubCommand; 31 | pub use self::change_user_tier_by_key::ChangeUserTierByKeySubCommand; 32 | pub use self::check_balance::CheckBalanceSubCommand; 33 | pub use self::check_config::CheckConfigSubCommand; 34 | pub use self::count_users::CountUsersSubCommand; 35 | pub use self::create_key::CreateKeySubCommand; 36 | pub use self::create_user::CreateUserSubCommand; 37 | pub use self::drop_migration_lock::DropMigrationLockSubCommand; 38 | pub use self::grant_credits_to_address::GrantCreditsToAddress; 39 | pub use self::mass_grant_credits::MassGrantCredits; 40 | pub use self::migrate_stats_to_v2::MigrateStatsToV2SubCommand; 41 | pub use self::pagerduty::PagerdutySubCommand; 42 | pub use self::popularity_contest::PopularityContestSubCommand; 43 | pub use self::proxyd::ProxydSubCommand; 44 | pub use self::rpc_accounting::RpcAccountingSubCommand; 45 | pub use self::sentryd::SentrydSubCommand; 46 | pub use self::transfer_key::TransferKeySubCommand; 47 | pub use self::user_export::UserExportSubCommand; 48 | pub use self::user_import::UserImportSubCommand; 49 | 50 | #[cfg(feature = "rdkafka")] 51 | pub use self::search_kafka::SearchKafkaSubCommand; 52 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/pagerduty.rs: -------------------------------------------------------------------------------- 1 | use tracing::{error, info}; 2 | use web3_proxy::prelude::anyhow; 3 | use web3_proxy::prelude::argh::{self, FromArgs}; 4 | use web3_proxy::prelude::pagerduty_rs; 5 | use web3_proxy::prelude::pagerduty_rs::{ 6 | eventsv2async::EventsV2 as PagerdutyAsyncEventsV2, types::Event, 7 | }; 8 | use web3_proxy::prelude::serde_json::json; 9 | use web3_proxy::{ 10 | config::TopConfig, 11 | pagerduty::{pagerduty_alert, pagerduty_alert_for_config}, 12 | }; 13 | 14 | #[derive(FromArgs, PartialEq, Debug, Eq)] 15 | /// Quickly create a pagerduty alert 16 | #[argh(subcommand, name = "pagerduty")] 17 | pub struct PagerdutySubCommand { 18 | #[argh(positional)] 19 | /// short description of the alert 20 | summary: String, 21 | 22 | /// the chain id to require. Only used if not using --config. 23 | #[argh(option)] 24 | chain_id: Option, 25 | 26 | #[argh(option)] 27 | /// the class/type of the event 28 | class: Option, 29 | 30 | #[argh(option)] 31 | /// the component of the event 32 | component: Option, 33 | 34 | #[argh(option)] 35 | /// deduplicate alerts based on this key. 36 | /// If there are no open incidents with this key, a new incident will be created. 37 | /// If there is an open incident with a matching key, the new event will be appended to that incident's Alerts log as an additional Trigger log entry. 38 | dedup_key: Option, 39 | } 40 | 41 | impl PagerdutySubCommand { 42 | pub async fn main( 43 | self, 44 | pagerduty_async: Option, 45 | top_config: Option, 46 | ) -> anyhow::Result<()> { 47 | // TODO: allow customizing severity 48 | let event = top_config 49 | .map(|top_config| { 50 | pagerduty_alert_for_config( 51 | self.class.clone(), 52 | self.component.clone(), 53 | None::<()>, 54 | pagerduty_rs::types::Severity::Error, 55 | self.summary.clone(), 56 | None, 57 | top_config, 58 | ) 59 | }) 60 | .unwrap_or_else(|| { 61 | pagerduty_alert( 62 | self.chain_id, 63 | self.class, 64 | None, 65 | None, 66 | self.component, 67 | None::<()>, 68 | pagerduty_rs::types::Severity::Error, 69 | None, 70 | self.summary, 71 | None, 72 | ) 73 | }); 74 | 75 | if let Some(pagerduty_async) = pagerduty_async { 76 | info!("sending to pagerduty: {:#}", json!(&event)); 77 | 78 | if let Err(err) = pagerduty_async.event(Event::AlertTrigger(event)).await { 79 | error!("Failed sending to pagerduty: {}", err); 80 | } 81 | } else { 82 | info!( 83 | "would send to pagerduty if PAGERDUTY_INTEGRATION_KEY were set: {:#}", 84 | json!(&event) 85 | ); 86 | } 87 | 88 | Ok(()) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/sentryd/simple.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use super::{SentrydErrorBuilder, SentrydResult}; 4 | use tracing::{debug, trace}; 5 | use web3_proxy::prelude::anyhow::{self, Context}; 6 | use web3_proxy::prelude::reqwest; 7 | use web3_proxy::prelude::tokio::time::Instant; 8 | 9 | /// GET the url and return an error if it wasn't a success 10 | pub async fn main( 11 | error_builder: SentrydErrorBuilder, 12 | url: String, 13 | timeout: Duration, 14 | ) -> SentrydResult { 15 | let start = Instant::now(); 16 | 17 | let r = reqwest::get(&url) 18 | .await 19 | .context(format!("Failed GET {}", &url)) 20 | .map_err(|x| error_builder.build(x))?; 21 | 22 | let elapsed = start.elapsed(); 23 | 24 | if elapsed > timeout { 25 | return error_builder.result( 26 | anyhow::anyhow!( 27 | "query took longer than {}ms ({}ms): {:#?}", 28 | timeout.as_millis(), 29 | elapsed.as_millis(), 30 | r 31 | ) 32 | .context(format!("fetching {} took too long", &url)), 33 | ); 34 | } 35 | 36 | // TODO: what should we do if we get rate limited here? 37 | 38 | if r.status().is_success() { 39 | debug!("{} is healthy", &url); 40 | trace!("Successful {:#?}", r); 41 | return Ok(()); 42 | } 43 | 44 | // TODO: capture headers? or is that already part of r? 45 | let detail = format!("{:#?}", r); 46 | 47 | let summary = format!("{} is unhealthy: {}", &url, r.status()); 48 | 49 | let body = r 50 | .text() 51 | .await 52 | .context(detail.clone()) 53 | .context(summary.clone()) 54 | .map_err(|x| error_builder.build(x))?; 55 | 56 | error_builder.result( 57 | anyhow::anyhow!("body: {}", body) 58 | .context(detail) 59 | .context(summary), 60 | ) 61 | } 62 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/stat_age.rs: -------------------------------------------------------------------------------- 1 | //! show how old the most recently saved stat is 2 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/transfer_key.rs: -------------------------------------------------------------------------------- 1 | use tracing::{debug, info}; 2 | use web3_proxy::prelude::anyhow::{self, Context}; 3 | use web3_proxy::prelude::argh::{self, FromArgs}; 4 | use web3_proxy::prelude::entities::{rpc_key, user}; 5 | use web3_proxy::prelude::ethers::types::Address; 6 | use web3_proxy::prelude::sea_orm::{ 7 | self, ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, IntoActiveModel, 8 | QueryFilter, 9 | }; 10 | use web3_proxy::prelude::uuid::Uuid; 11 | use web3_proxy::secrets::RpcSecretKey; 12 | 13 | /// change a key's owner. 14 | #[derive(FromArgs, PartialEq, Eq, Debug)] 15 | #[argh(subcommand, name = "transfer_key")] 16 | pub struct TransferKeySubCommand { 17 | #[argh(positional)] 18 | /// the RPC key that you want to transfer. 19 | rpc_secret_key: RpcSecretKey, 20 | 21 | /// the new owner for the key. 22 | #[argh(positional)] 23 | new_address: String, 24 | } 25 | 26 | impl TransferKeySubCommand { 27 | pub async fn main(self, db_conn: &DatabaseConnection) -> anyhow::Result<()> { 28 | let rpc_secret_key: Uuid = self.rpc_secret_key.into(); 29 | 30 | let new_address: Address = self.new_address.parse()?; 31 | 32 | let uk = rpc_key::Entity::find() 33 | .filter(rpc_key::Column::SecretKey.eq(rpc_secret_key)) 34 | .one(db_conn) 35 | .await? 36 | .context("No key found")?; 37 | 38 | debug!("user key: {}", serde_json::to_string(&uk)?); 39 | 40 | let new_u = user::Entity::find() 41 | .filter(user::Column::Address.eq(new_address.as_bytes())) 42 | .one(db_conn) 43 | .await? 44 | .context("No user found with that key")?; 45 | 46 | debug!("new user: {}", serde_json::to_string(&new_u)?); 47 | 48 | if new_u.id == uk.user_id { 49 | info!("user already owns that key"); 50 | } else { 51 | let mut uk = uk.into_active_model(); 52 | 53 | uk.user_id = sea_orm::Set(new_u.id); 54 | 55 | uk.save(db_conn).await?; 56 | 57 | info!("changed the key's owner"); 58 | } 59 | 60 | Ok(()) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/sub_commands/user_export.rs: -------------------------------------------------------------------------------- 1 | use argh::FromArgs; 2 | use entities::{rpc_key, user}; 3 | use migration::sea_orm::{DatabaseConnection, EntityTrait, PaginatorTrait}; 4 | use std::fs::{self, create_dir_all}; 5 | use std::path::Path; 6 | use tracing::info; 7 | 8 | use web3_proxy::prelude::*; 9 | 10 | #[derive(FromArgs, PartialEq, Eq, Debug)] 11 | /// Export users from the database. 12 | #[argh(subcommand, name = "user_export")] 13 | pub struct UserExportSubCommand { 14 | /// where to write the file 15 | /// TODO: validate this is a valid path here? 16 | #[argh(positional, default = "\"./data/users\".to_string()")] 17 | output_dir: String, 18 | } 19 | 20 | impl UserExportSubCommand { 21 | pub async fn main(self, db_conn: &DatabaseConnection) -> anyhow::Result<()> { 22 | // create the output dir if it does not exist 23 | create_dir_all(&self.output_dir)?; 24 | 25 | let now = chrono::Utc::now().timestamp(); 26 | 27 | let export_dir = Path::new(&self.output_dir); 28 | 29 | // get all the users from the database (paged) 30 | let mut user_pages = user::Entity::find().paginate(db_conn, 1000); 31 | 32 | // TODO: for now all user_tier tables match in all databases, but in the future we might need to export/import this 33 | 34 | // save all users to a file 35 | let mut user_file_count = 0; 36 | while let Some(users) = user_pages.fetch_and_next().await? { 37 | let export_file = export_dir.join(format!("{}-users-{}.json", now, user_file_count)); 38 | 39 | fs::write( 40 | export_file, 41 | serde_json::to_string_pretty(&users).expect("users should serialize"), 42 | )?; 43 | 44 | user_file_count += 1; 45 | } 46 | 47 | info!( 48 | "Saved {} user file(s) to {}", 49 | user_file_count, 50 | export_dir.to_string_lossy() 51 | ); 52 | 53 | // get all the rpc keys from the database (paged) 54 | let mut rpc_key_pages = rpc_key::Entity::find().paginate(db_conn, 1000); 55 | 56 | let mut rpc_key_file_count = 0; 57 | while let Some(rpc_keys) = rpc_key_pages.fetch_and_next().await? { 58 | let export_file = 59 | export_dir.join(format!("{}-rpc_keys-{}.json", now, rpc_key_file_count)); 60 | 61 | fs::write( 62 | export_file, 63 | serde_json::to_string_pretty(&rpc_keys).expect("rpc_keys should serialize"), 64 | )?; 65 | 66 | rpc_key_file_count += 1; 67 | } 68 | 69 | info!( 70 | "Saved {} rpc key file(s) to {}", 71 | rpc_key_file_count, 72 | export_dir.to_string_lossy() 73 | ); 74 | 75 | Ok(()) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/test_utils/admin_deposits.rs: -------------------------------------------------------------------------------- 1 | use super::TestApp; 2 | use tracing::trace; 3 | use web3_proxy::frontend::users::authentication::LoginPostResponse; 4 | use web3_proxy::prelude::reqwest; 5 | 6 | /// Helper function to increase the balance of a user, from an admin 7 | #[allow(unused)] 8 | pub async fn get_admin_deposits( 9 | x: &TestApp, 10 | r: &reqwest::Client, 11 | user: &LoginPostResponse, 12 | ) -> serde_json::Value { 13 | let increase_balance_post_url = format!("{}user/deposits/admin", x.proxy_provider.url()); 14 | trace!("Get admin increase deposits"); 15 | // Login the user 16 | // Use the bearer token of admin to increase user balance 17 | let admin_balance_deposits = r 18 | .get(increase_balance_post_url) 19 | .bearer_auth(user.bearer_token) 20 | .send() 21 | .await 22 | .unwrap(); 23 | trace!(?admin_balance_deposits, "http response"); 24 | let admin_balance_deposits = admin_balance_deposits 25 | .json::() 26 | .await 27 | .unwrap(); 28 | trace!(?admin_balance_deposits, "json response"); 29 | 30 | admin_balance_deposits 31 | } 32 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/test_utils/admin_increases_balance.rs: -------------------------------------------------------------------------------- 1 | use super::TestApp; 2 | use tracing::info; 3 | use web3_proxy::frontend::admin::AdminIncreaseBalancePost; 4 | use web3_proxy::frontend::users::authentication::LoginPostResponse; 5 | use web3_proxy::prelude::ethers::prelude::{LocalWallet, Signer}; 6 | use web3_proxy::prelude::migration::sea_orm::prelude::Decimal; 7 | use web3_proxy::prelude::reqwest; 8 | 9 | /// Helper function to increase the balance of a user, from an admin 10 | #[allow(unused)] 11 | pub async fn admin_increase_balance( 12 | x: &TestApp, 13 | r: &reqwest::Client, 14 | admin_login_response: &LoginPostResponse, 15 | target_wallet: &LocalWallet, 16 | amount: Decimal, 17 | ) -> serde_json::Value { 18 | let increase_balance_post_url = format!("{}admin/increase_balance", x.proxy_provider.url()); 19 | info!("Increasing balance"); 20 | // Login the user 21 | // Use the bearer token of admin to increase user balance 22 | let increase_balance_data = AdminIncreaseBalancePost { 23 | user_address: target_wallet.address(), // set user address to increase balance 24 | amount, // set amount to increase 25 | note: Some("Test increasing balance".to_string()), 26 | }; 27 | info!(?increase_balance_post_url); 28 | info!(?increase_balance_data); 29 | info!(?admin_login_response.bearer_token); 30 | 31 | let increase_balance_response = r 32 | .post(increase_balance_post_url) 33 | .json(&increase_balance_data) 34 | .bearer_auth(admin_login_response.bearer_token) 35 | .send() 36 | .await 37 | .unwrap(); 38 | info!(?increase_balance_response, "http response"); 39 | 40 | // TODO: use a struct here 41 | let increase_balance_response = increase_balance_response 42 | .json::() 43 | .await 44 | .unwrap(); 45 | info!(?increase_balance_response, "json response"); 46 | 47 | increase_balance_response 48 | } 49 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/test_utils/create_provider_with_rpc_key.rs: -------------------------------------------------------------------------------- 1 | use web3_proxy::prelude::ulid::Ulid; 2 | use web3_proxy::prelude::url::Url; 3 | use web3_proxy::rpcs::provider::EthersHttpProvider; 4 | 5 | #[allow(unused)] 6 | pub async fn create_provider_for_user(url: &Url, user_secret_key: &Ulid) -> EthersHttpProvider { 7 | // Then generate a provider 8 | let proxy_endpoint = format!("{}rpc/{}", url, user_secret_key); 9 | 10 | EthersHttpProvider::try_from(proxy_endpoint).unwrap() 11 | } 12 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/test_utils/create_user.rs: -------------------------------------------------------------------------------- 1 | use super::TestApp; 2 | use tracing::{info, trace}; 3 | use web3_proxy::errors::Web3ProxyResult; 4 | use web3_proxy::frontend::users::authentication::{LoginPostResponse, PostLogin}; 5 | use web3_proxy::prelude::entities::{user, user_tier}; 6 | use web3_proxy::prelude::ethers::prelude::{LocalWallet, Signer}; 7 | use web3_proxy::prelude::ethers::types::Signature; 8 | use web3_proxy::prelude::http::StatusCode; 9 | use web3_proxy::prelude::migration::sea_orm::{ 10 | self, ActiveModelTrait, ColumnTrait, DatabaseConnection, EntityTrait, IntoActiveModel, 11 | QueryFilter, 12 | }; 13 | use web3_proxy::prelude::reqwest; 14 | 15 | /// Helper function to create an "ordinary" user 16 | #[allow(unused)] 17 | pub async fn create_user( 18 | x: &TestApp, 19 | r: &reqwest::Client, 20 | user_wallet: &LocalWallet, 21 | referral_code: Option, 22 | ) -> (LoginPostResponse) { 23 | let login_post_url = format!("{}user/login", x.proxy_provider.url()); 24 | let user_login_get_url = format!( 25 | "{}user/login/{:?}", 26 | x.proxy_provider.url(), 27 | user_wallet.address() 28 | ); 29 | let user_login_message = r 30 | .get(user_login_get_url) 31 | .send() 32 | .await 33 | .unwrap() 34 | .error_for_status() 35 | .unwrap(); 36 | let user_login_message = user_login_message.text().await.unwrap(); 37 | 38 | // Sign the message and POST it to login as the user 39 | let user_signed: Signature = user_wallet.sign_message(&user_login_message).await.unwrap(); 40 | info!(?user_signed); 41 | 42 | let user_post_login_data = PostLogin { 43 | msg: user_login_message, 44 | sig: user_signed.to_string(), 45 | referral_code, 46 | }; 47 | info!(?user_post_login_data); 48 | 49 | let user_login_response = r 50 | .post(&login_post_url) 51 | .json(&user_post_login_data) 52 | .send() 53 | .await 54 | .unwrap(); 55 | trace!(?user_login_response); 56 | 57 | assert_eq!(user_login_response.status(), StatusCode::CREATED); 58 | 59 | let user_login_text = user_login_response.text().await.unwrap(); 60 | trace!("user_login_text: {:#}", user_login_text); 61 | 62 | let user_login_response: LoginPostResponse = serde_json::from_str(&user_login_text).unwrap(); 63 | info!(?user_login_response); 64 | 65 | user_login_response 66 | } 67 | 68 | /// TODO: use an admin user to do this instead 69 | #[allow(unused)] 70 | pub async fn set_user_tier( 71 | x: &TestApp, 72 | db_conn: &DatabaseConnection, 73 | user: user::Model, 74 | tier_name: &str, 75 | ) -> Web3ProxyResult { 76 | let ut = user_tier::Entity::find() 77 | .filter(user_tier::Column::Title.like(tier_name)) 78 | .one(db_conn) 79 | .await? 80 | .unwrap(); 81 | 82 | let mut user = user.into_active_model(); 83 | 84 | user.user_tier_id = sea_orm::Set(ut.id); 85 | 86 | user.save(db_conn).await?; 87 | 88 | Ok(ut) 89 | } 90 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/test_utils/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod admin_deposits; 2 | pub mod admin_increases_balance; 3 | pub mod app; 4 | pub mod create_admin; 5 | pub mod create_provider_with_rpc_key; 6 | pub mod create_user; 7 | pub mod referral; 8 | pub mod rpc_key; 9 | pub mod stats_accounting; 10 | pub mod user_balance; 11 | 12 | pub use self::app::TestApp; 13 | pub use web3_proxy::test_utils::anvil::TestAnvil; 14 | pub use web3_proxy::test_utils::influx::TestInflux; 15 | pub use web3_proxy::test_utils::mysql::TestMysql; 16 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/test_utils/rpc_key.rs: -------------------------------------------------------------------------------- 1 | use super::TestApp; 2 | use std::time::Duration; 3 | use tracing::info; 4 | use web3_proxy::prelude::reqwest; 5 | use web3_proxy::prelude::serde::Deserialize; 6 | use web3_proxy::prelude::ulid::Ulid; 7 | use web3_proxy::{ 8 | errors::Web3ProxyResult, 9 | frontend::users::authentication::LoginPostResponse, 10 | rpcs::provider::{connect_http, EthersHttpProvider}, 11 | }; 12 | 13 | #[derive(Debug, Deserialize)] 14 | pub struct RpcKeyResponse { 15 | pub user_id: u64, 16 | pub user_rpc_keys: std::collections::HashMap, 17 | } 18 | 19 | #[derive(Debug, Deserialize)] 20 | pub struct RpcKey { 21 | pub active: bool, 22 | pub allowed_ips: Option, 23 | pub allowed_origins: Option, 24 | pub allowed_referers: Option, 25 | pub allowed_user_agents: Option, 26 | pub description: Option, 27 | pub id: u64, 28 | pub log_revert_chance: f64, 29 | pub private_txs: bool, 30 | pub role: String, 31 | pub secret_key: Ulid, 32 | pub user_id: u64, 33 | } 34 | 35 | /// Helper function to get the user's balance 36 | #[allow(unused)] 37 | pub async fn user_get_first_rpc_key( 38 | x: &TestApp, 39 | r: &reqwest::Client, 40 | login_response: &LoginPostResponse, 41 | ) -> RpcKey { 42 | // TODO: refactor to use login_response? or compare? 43 | let get_keys = format!("{}user/keys", x.proxy_provider.url()); 44 | 45 | info!("Get balance"); 46 | let rpc_key_response = r 47 | .get(get_keys) 48 | .bearer_auth(login_response.bearer_token) 49 | .send() 50 | .await 51 | .unwrap() 52 | .error_for_status() 53 | .unwrap(); 54 | info!(?rpc_key_response); 55 | 56 | let rpc_key_response = rpc_key_response.json::().await.unwrap(); 57 | info!(?rpc_key_response); 58 | 59 | info!("Parsing rpc key as json"); 60 | let rpc_key: RpcKeyResponse = serde_json::from_value(rpc_key_response).unwrap(); 61 | info!(?rpc_key); 62 | 63 | rpc_key.user_rpc_keys.into_iter().next().unwrap().1 64 | } 65 | 66 | #[allow(unused)] 67 | pub async fn user_get_provider( 68 | x: &TestApp, 69 | r: &reqwest::Client, 70 | login_response: &LoginPostResponse, 71 | ) -> Web3ProxyResult { 72 | let first_key = login_response.rpc_keys.iter().next().unwrap().1; 73 | 74 | let rpc_url = format!( 75 | "{}rpc/{}", 76 | x.proxy_provider.url(), 77 | Ulid::from(first_key.secret_key) 78 | ); 79 | 80 | connect_http( 81 | rpc_url.parse().unwrap(), 82 | Some(r.clone()), 83 | Duration::from_secs(1), 84 | ) 85 | } 86 | -------------------------------------------------------------------------------- /web3_proxy_cli/src/test_utils/user_balance.rs: -------------------------------------------------------------------------------- 1 | use super::TestApp; 2 | use tracing::{info, trace}; 3 | use web3_proxy::balance::Balance; 4 | use web3_proxy::frontend::users::authentication::LoginPostResponse; 5 | use web3_proxy::prelude::reqwest; 6 | use web3_proxy::prelude::serde_json::json; 7 | 8 | /// Helper function to get the user's balance 9 | #[allow(unused)] 10 | pub async fn user_get_balance( 11 | x: &TestApp, 12 | r: &reqwest::Client, 13 | login_response: &LoginPostResponse, 14 | ) -> Balance { 15 | let get_user_balance = format!("{}user/balance", x.proxy_provider.url()); 16 | 17 | let balance_response = r 18 | .get(get_user_balance) 19 | .bearer_auth(login_response.bearer_token) 20 | .send() 21 | .await 22 | .unwrap() 23 | .error_for_status() 24 | .unwrap(); 25 | trace!( 26 | ?balance_response, 27 | "get balance for user #{}", 28 | login_response.user.id 29 | ); 30 | 31 | let balance = balance_response.json().await.unwrap(); 32 | 33 | info!("balance: {:#}", json!(&balance)); 34 | 35 | balance 36 | } 37 | -------------------------------------------------------------------------------- /web3_proxy_cli/tests/test_admins.rs: -------------------------------------------------------------------------------- 1 | use std::str::FromStr; 2 | use std::time::Duration; 3 | use tracing::info; 4 | use web3_proxy::prelude::migration::sea_orm::prelude::Decimal; 5 | use web3_proxy::prelude::reqwest; 6 | use web3_proxy::prelude::tokio; 7 | use web3_proxy::test_utils::mysql::TestMysql; 8 | use web3_proxy::test_utils::TestAnvil; 9 | use web3_proxy_cli::test_utils::admin_increases_balance::admin_increase_balance; 10 | use web3_proxy_cli::test_utils::create_admin::create_user_as_admin; 11 | use web3_proxy_cli::test_utils::create_user::create_user; 12 | use web3_proxy_cli::test_utils::user_balance::user_get_balance; 13 | use web3_proxy_cli::test_utils::TestApp; 14 | 15 | // #[cfg_attr(not(feature = "tests-needing-docker"), ignore)] 16 | #[ignore = "under construction"] 17 | #[test_log::test(tokio::test)] 18 | async fn test_admin_imitate_user() { 19 | todo!(); 20 | } 21 | 22 | #[cfg_attr(not(feature = "tests-needing-docker"), ignore)] 23 | #[test_log::test(tokio::test)] 24 | async fn test_admin_grant_credits() { 25 | info!("Starting admin grant credits test"); 26 | 27 | let a: TestAnvil = TestAnvil::spawn(31337).await; 28 | 29 | let db = TestMysql::spawn().await; 30 | 31 | let x = TestApp::spawn(&a, Some(&db), None, None).await; 32 | 33 | let r = reqwest::Client::builder() 34 | .timeout(Duration::from_secs(3)) 35 | .build() 36 | .unwrap(); 37 | 38 | // Setup variables that will be used 39 | let user_wallet = a.wallet(0); 40 | let admin_wallet = a.wallet(1); 41 | info!(?admin_wallet); 42 | 43 | let user_login_response = create_user(&x, &r, &user_wallet, None).await; 44 | let admin_login_response = create_user_as_admin(&x, &db, &r, &admin_wallet).await; 45 | info!(?admin_login_response); 46 | 47 | let increase_balance_response = admin_increase_balance( 48 | &x, 49 | &r, 50 | &admin_login_response, 51 | &user_wallet, 52 | Decimal::from(100), 53 | ) 54 | .await; 55 | 56 | assert_eq!( 57 | Decimal::from_str(increase_balance_response["amount"].as_str().unwrap()).unwrap(), 58 | Decimal::from(100) 59 | ); 60 | 61 | let user_balance = user_get_balance(&x, &r, &user_login_response).await; 62 | assert_eq!(user_balance.remaining(), Decimal::from(100)); 63 | 64 | x.wait_for_stop(); 65 | } 66 | 67 | // #[cfg_attr(not(feature = "tests-needing-docker"), ignore)] 68 | #[ignore = "under construction"] 69 | #[test_log::test(tokio::test)] 70 | async fn test_admin_change_user_tier() { 71 | todo!(); 72 | } 73 | -------------------------------------------------------------------------------- /wrk/getBlockNumber.lua: -------------------------------------------------------------------------------- 1 | wrk.method = "POST" 2 | wrk.body = "{\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\",\"params\":[],\"id\":420}" 3 | wrk.headers["Content-Type"] = "application/json" 4 | 5 | response = function(status, headers, body) 6 | if status ~= 200 then 7 | io.write("Status: ".. status .."\n") 8 | io.write("Body:\n") 9 | io.write(body .. "\n") 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /wrk/getLatestBlockByNumber.lua: -------------------------------------------------------------------------------- 1 | wrk.method = "POST" 2 | wrk.body = "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getBlockByNumber\",\"params\":[\"latest\", false],\"id\":420}" 3 | wrk.headers["Content-Type"] = "application/json" 4 | 5 | response = function(status, headers, body) 6 | if status ~= 200 then 7 | io.write("Status: ".. status .."\n") 8 | io.write("Body:\n") 9 | io.write(body .. "\n") 10 | end 11 | end 12 | --------------------------------------------------------------------------------