├── .cargo └── audit.toml ├── .circleci └── config.yml ├── .clog.toml ├── .config └── nextest.toml ├── .dockerignore ├── .github ├── dependabot.yml └── workflows │ └── glean-probe-scraper.yml ├── .gitignore ├── .gitmodules ├── .sentryclirc.example ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Cargo.lock ├── Cargo.toml ├── Dockerfile ├── LICENSE ├── Makefile ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── config └── local.example.toml ├── docker-compose.e2e.mysql.yaml ├── docker-compose.e2e.spanner.yaml ├── docker-compose.mysql.yaml ├── docker-compose.spanner.yaml ├── docs ├── adr │ ├── 0001-daily-active-use-server-side-metrics-glean.md │ ├── index.md │ └── template.md ├── config.md ├── tokenserver │ ├── tokenserver.md │ └── tokenserver_api.md └── tools │ ├── process_account_events.md │ ├── purge_old_records_tokenserver.md │ └── spanner_purge_ttl.md ├── glean ├── Cargo.toml ├── data-review │ └── request.md ├── metrics.yaml ├── pings.yaml └── src │ ├── lib.rs │ └── server_events.rs ├── requirements.txt ├── scripts ├── build-docs.sh ├── id_rsa.enc ├── prepare-spanner.sh ├── sentry-release.sh └── start_mock_fxa_server.sh ├── shell.nix ├── syncserver-common ├── Cargo.toml └── src │ ├── lib.rs │ ├── metrics.rs │ ├── middleware │ ├── mod.rs │ └── sentry.rs │ └── tags.rs ├── syncserver-db-common ├── Cargo.toml └── src │ ├── error.rs │ ├── lib.rs │ └── test.rs ├── syncserver-settings ├── Cargo.toml └── src │ └── lib.rs ├── syncserver ├── Cargo.toml ├── src │ ├── db │ │ └── mod.rs │ ├── error.rs │ ├── lib.rs │ ├── logging.rs │ ├── main.rs │ ├── server │ │ ├── mod.rs │ │ ├── test.rs │ │ └── user_agent.rs │ ├── tokenserver │ │ ├── README.md │ │ ├── extractors.rs │ │ ├── handlers.rs │ │ ├── logging.rs │ │ └── mod.rs │ └── web │ │ ├── auth.rs │ │ ├── error.rs │ │ ├── extractors.rs │ │ ├── handlers.rs │ │ ├── middleware │ │ ├── mod.rs │ │ ├── rejectua.rs │ │ └── weave.rs │ │ ├── mod.rs │ │ └── transaction.rs └── version.json ├── syncstorage-db-common ├── Cargo.toml └── src │ ├── error.rs │ ├── lib.rs │ ├── params.rs │ ├── results.rs │ └── util.rs ├── syncstorage-db ├── Cargo.toml └── src │ ├── lib.rs │ ├── mock.rs │ └── tests │ ├── batch.rs │ ├── db.rs │ ├── mod.rs │ └── support.rs ├── syncstorage-mysql ├── Cargo.toml ├── migrations │ ├── 2018-08-28-010336_init │ │ ├── down.sql │ │ └── up.sql │ ├── 2019-09-11-164500 │ │ ├── down.sql │ │ └── up.sql │ ├── 2019-09-25-174347_min_collection_id │ │ ├── down.sql │ │ └── up.sql │ ├── 2020-04-03-102015_change_userid │ │ ├── down.sql │ │ └── up.sql │ ├── 2020-06-12-231034_new_batch │ │ ├── down.sql │ │ └── up.sql │ └── 2020-08-24-091401_add_quota │ │ ├── down.sql │ │ └── up.sql └── src │ ├── batch.rs │ ├── batch_commit.sql │ ├── diesel_ext.rs │ ├── error.rs │ ├── lib.rs │ ├── models.rs │ ├── pool.rs │ ├── schema.rs │ └── test.rs ├── syncstorage-settings ├── Cargo.toml └── src │ └── lib.rs ├── syncstorage-spanner ├── Cargo.toml └── src │ ├── BATCH_COMMIT.txt │ ├── batch.rs │ ├── batch_commit_insert.sql │ ├── batch_commit_update.sql │ ├── batch_index.sql │ ├── bin │ └── purge_ttl.rs │ ├── error.rs │ ├── insert_standard_collections.sql │ ├── lib.rs │ ├── macros.rs │ ├── manager │ ├── bb8.rs │ ├── deadpool.rs │ ├── mod.rs │ └── session.rs │ ├── metadata.rs │ ├── models.rs │ ├── pool.rs │ ├── schema.ddl │ └── support.rs ├── tests.ini ├── tokenserver-auth ├── Cargo.toml └── src │ ├── crypto.rs │ ├── lib.rs │ ├── oauth.rs │ ├── oauth │ ├── native.rs │ ├── py.rs │ └── verify.py │ ├── token.rs │ └── token │ ├── native.rs │ └── py.rs ├── tokenserver-common ├── Cargo.toml └── src │ ├── error.rs │ └── lib.rs ├── tokenserver-db ├── Cargo.toml ├── migrations │ ├── 2021-07-16-001122_init │ │ ├── down.sql │ │ └── up.sql │ ├── 2021-08-03-234845_populate_services │ │ ├── down.sql │ │ └── up.sql │ ├── 2021-09-30-142643_remove_foreign_key_constraints │ │ ├── down.sql │ │ └── up.sql │ ├── 2021-09-30-142654_remove_node_defaults │ │ ├── down.sql │ │ └── up.sql │ ├── 2021-09-30-142746_add_indexes │ │ ├── down.sql │ │ └── up.sql │ ├── 2021-09-30-144043_remove_nodes_service_key │ │ ├── down.sql │ │ └── up.sql │ ├── 2021-09-30-144225_remove_users_nodeid_key │ │ ├── down.sql │ │ └── up.sql │ └── 2021-12-22-160451_remove_services │ │ ├── down.sql │ │ └── up.sql └── src │ ├── error.rs │ ├── lib.rs │ ├── mock.rs │ ├── models.rs │ ├── params.rs │ ├── pool.rs │ └── results.rs ├── tokenserver-settings ├── Cargo.toml └── src │ └── lib.rs └── tools ├── README.md ├── examples ├── README.md └── put.bash ├── hawk ├── README.md ├── make_hawk_token.py └── requirements.txt ├── integration_tests ├── conftest.py ├── requirements.txt ├── test_storage.py ├── test_support.py ├── tests-no-batch.ini ├── tests-paginated.ini ├── tests.ini └── tokenserver │ ├── __init__.py │ ├── mock_fxa_server.py │ ├── test_authorization.py │ ├── test_e2e.py │ ├── test_misc.py │ ├── test_node_assignment.py │ └── test_support.py ├── spanner ├── Dockerfile ├── README.md ├── count_expired_rows.py ├── count_users.py ├── purge_ttl.py ├── requirements.txt └── write_batch.py ├── tokenserver ├── __init__.py ├── add_node.py ├── allocate_user.py ├── conftest.py ├── count_users.py ├── database.py ├── loadtests │ ├── README.md │ ├── generate-keys.sh │ ├── get_jwk.py │ ├── locustfile.py │ ├── populate_db.py │ └── requirements.txt ├── process_account_events.py ├── purge_old_records.py ├── pytest.ini ├── remove_node.py ├── requirements.txt ├── test_database.py ├── test_process_account_events.py ├── test_purge_old_records.py ├── test_scripts.py ├── unassign_node.py ├── update_node.py └── util.py └── user_migration ├── README.md ├── fix_collections.sql ├── gen_bso_users.py ├── gen_fxa_users.py ├── migrate_node.py ├── old ├── dump_avro.py ├── dump_mysql.py ├── migrate_user.py ├── requirements.txt └── sync.avsc └── requirements.txt /.cargo/audit.toml: -------------------------------------------------------------------------------- 1 | [advisories] 2 | ignore = [ 3 | "RUSTSEC-2024-0365", # Bound by diesel 1.4 (4GB limit n/a to tokenserver) 4 | "RUSTSEC-2024-0421", # Bound by diesel 1.4, `idna` < 0.1.5, Upgrade to >=1.0.0 5 | "RUSTSEC-2024-0437", # Bound by grpcio 0.13, 6 | ] 7 | -------------------------------------------------------------------------------- /.clog.toml: -------------------------------------------------------------------------------- 1 | [clog] 2 | repository = "https://github.com/mozilla-services/syncstorage-rs" 3 | changelog = "CHANGELOG.md" 4 | from-latest-tag = true 5 | link-style = "github" 6 | 7 | [sections] 8 | Refactor = ["refactor"] 9 | Test = ["test"] 10 | Doc = ["docs"] 11 | Chore = ["chore"] 12 | Features = ["feat", "feature"] 13 | "Bug Fixes" = ["fix", "bug"] 14 | -------------------------------------------------------------------------------- /.config/nextest.toml: -------------------------------------------------------------------------------- 1 | [store] 2 | dir = "target/nextest" 3 | 4 | [profile.default] 5 | retries = 0 6 | test-threads = 1 7 | threads-required = 1 8 | status-level = "pass" 9 | final-status-level = "flaky" 10 | failure-output = "immediate" 11 | success-output = "never" 12 | fail-fast = false 13 | slow-timeout = { period = "300s" } 14 | 15 | [profile.ci] 16 | fail-fast = false 17 | 18 | [profile.ci.junit] 19 | path = "junit.xml" 20 | 21 | report-name = "syncstorage-unit-tests" 22 | store-success-output = false 23 | store-failure-output = true 24 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | CHANGELOG.md 3 | CODE_OF_CONDUCT.md 4 | config/*.example.toml 5 | CONTRIBUTING.md 6 | Dockerfile 7 | docs/ 8 | Makefile 9 | PULL_REQUEST_TEMPLATE.md 10 | README.md 11 | target 12 | tools/examples 13 | db/ 14 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: cargo 4 | directory: "/" 5 | schedule: 6 | interval: monthly 7 | timezone: UCT 8 | open-pull-requests-limit: 0 9 | labels: 10 | - dependencies 11 | ignore: 12 | - dependency-name: actix-rt 13 | versions: 14 | - ">= 2.a, < 3" 15 | - dependency-name: protobuf 16 | versions: 17 | - ">= 2.14.a, < 2.15" 18 | - dependency-name: tokio 19 | versions: 20 | - ">= 0.3.a, < 0.4" 21 | - dependency-name: tokio 22 | versions: 23 | - ">= 1.a, < 2" 24 | - dependency-name: futures 25 | versions: 26 | - 0.3.12 27 | - 0.3.13 28 | - dependency-name: serde_json 29 | versions: 30 | - 1.0.64 31 | - dependency-name: hyper 32 | versions: 33 | - 0.14.4 34 | - dependency-name: url 35 | versions: 36 | - 2.2.1 37 | - dependency-name: cadence 38 | versions: 39 | - 0.24.0 40 | - dependency-name: slog-async 41 | versions: 42 | - 2.6.0 43 | - dependency-name: log 44 | versions: 45 | - 0.4.14 46 | - dependency-name: serde 47 | versions: 48 | - 1.0.121 49 | - dependency-name: sha2 50 | versions: 51 | - 0.9.3 52 | - dependency-name: slog-scope 53 | versions: 54 | - 4.4.0 55 | -------------------------------------------------------------------------------- /.github/workflows/glean-probe-scraper.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Glean probe-scraper 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master] 8 | jobs: 9 | glean-probe-scraper: 10 | uses: mozilla/probe-scraper/.github/workflows/glean.yaml@main 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | syntax:glob 3 | .svn 4 | .coverage 5 | *.pyc 6 | *.egg-info 7 | *.egg 8 | *~ 9 | build 10 | ddb 11 | dist 12 | docs/_build 13 | *.xml 14 | html_coverage 15 | .hgignore 16 | .idea 17 | *.iml 18 | site-packages/* 19 | lib-python/* 20 | bin/* 21 | include/* 22 | lib_pypy/* 23 | *.swp 24 | pypy/ 25 | ./src/ 26 | .tox/ 27 | .eggs/ 28 | target 29 | *.rs.bk 30 | .#* 31 | service-account.json 32 | .sentryclirc 33 | .envrc 34 | 35 | config/local.toml 36 | tools/tokenserver/loadtests/*.pem 37 | tools/tokenserver/loadtests/*.pub 38 | venv 39 | .vscode/settings.json 40 | 41 | # circleci 42 | workspace -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/mozilla-rust-sdk/googleapis-raw/grpc"] 2 | path = vendor/mozilla-rust-sdk/googleapis-raw/grpc 3 | url = https://github.com/grpc/grpc.git 4 | -------------------------------------------------------------------------------- /.sentryclirc.example: -------------------------------------------------------------------------------- 1 | [defaults] 2 | project=syncstorage-prod 3 | org=your-org-name-here 4 | url=https://self.hosted.sentry-url 5 | 6 | [auth] 7 | token=super_secret_token 8 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Community Participation Guidelines 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 4 | For more details, please read the 5 | [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 6 | 7 | ## How to Report 8 | For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. 9 | 10 | 16 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contribution Guidelines 2 | 3 | Anyone is welcome to contribute to this project. Feel free to get in touch with 4 | other community members on IRC, the mailing list or through issues here on 5 | GitHub. 6 | 7 | [See the README](/README.md) for contact information. 8 | 9 | ## Bug Reports 10 | 11 | You can file issues here on GitHub. Please try to include as much information as 12 | you can and under what conditions you saw the issue. 13 | 14 | ## Sending Pull Requests 15 | 16 | Patches should be submitted as pull requests (PR). 17 | 18 | Before submitting a PR: 19 | - Your code must run and pass all the automated tests before you submit your PR 20 | for review. "Work in progress" pull requests are allowed to be submitted, but 21 | should be clearly labeled as such and should not be merged until all tests 22 | pass and the code has been reviewed. 23 | - Your patch should include new tests that cover your changes. It is your and 24 | your reviewer's responsibility to ensure your patch includes adequate tests. 25 | 26 | When submitting a PR: 27 | - **[Sign all your git commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification#ssh-commit-verification)**. 28 | We cannot accept any PR that does not have all commits signed. This is a policy 29 | put in place by our Security Operations team and is enforced by our CI processes. 30 | - You agree to license your code under the project's open source license 31 | ([MPL 2.0](/LICENSE)). 32 | - Base your branch off the current `master`. 33 | - Add both your code and new tests if relevant. 34 | - Run the test suite to make sure your code passes linting and tests. 35 | - Ensure your changes do not reduce code coverage of the test suite. 36 | - Please do not include merge commits in pull requests; include only commits 37 | with the new relevant code. 38 | - PR naming conventions - begins with type (fix, feature, doc, chore, etc) and a short description with no period. 39 | 40 | See the main [README.md](/README.md) for information on prerequisites, 41 | installing, running and testing. 42 | 43 | ## Code Review 44 | 45 | This project is production Mozilla code and subject to our [engineering practices and quality standards](https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Committing_Rules_and_Responsibilities). Every patch must be peer reviewed. 46 | 47 | ## Git Commit Guidelines 48 | 49 | We loosely follow the [Angular commit guidelines](https://github.com/angular/angular.js/blob/master/CONTRIBUTING.md#type) 50 | of `: ` where `type` must be one of: 51 | 52 | * **feat**: A new feature 53 | * **fix**: A bug fix 54 | * **docs**: Documentation only changes 55 | * **style**: Changes that do not affect the meaning of the code (white-space, formatting, missing 56 | semi-colons, etc) 57 | * **refactor**: A code change that neither fixes a bug or adds a feature 58 | * **perf**: A code change that improves performance 59 | * **test**: Adding missing tests 60 | * **chore**: Changes to the build process or auxiliary tools and libraries such as documentation 61 | generation 62 | 63 | ### Subject 64 | 65 | The subject contains succinct description of the change: 66 | 67 | * use the imperative, present tense: "change" not "changed" nor "changes" 68 | * don't capitalize first letter 69 | * no dot (.) at the end 70 | 71 | ### Body 72 | 73 | In order to maintain a reference to the context of the commit, add 74 | `Closes #` if it closes a related issue or `Issue #` 75 | if it's a partial fix. 76 | 77 | You can also write a detailed description of the commit: Just as in the 78 | **subject**, use the imperative, present tense: "change" not "changed" nor 79 | "changes" It should include the motivation for the change and contrast this with 80 | previous behavior. 81 | 82 | ### Footer 83 | 84 | The footer should contain any information about **Breaking Changes** and is also 85 | the place to reference GitHub issues that this commit **Closes**. 86 | 87 | ### Example 88 | 89 | A properly formatted commit message should look like: 90 | 91 | ``` 92 | feat: give the developers a delicious cookie 93 | 94 | Properly formatted commit messages provide understandable history and 95 | documentation. This patch will provide a delicious cookie when all tests have 96 | passed and the commit message is properly formatted. 97 | 98 | BREAKING CHANGE: This patch requires developer to lower expectations about 99 | what "delicious" and "cookie" may mean. Some sadness may result. 100 | 101 | Closes #3.14, #9.75 102 | ``` 103 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = [ 4 | "syncserver-common", 5 | "syncserver-db-common", 6 | "syncserver-settings", 7 | "syncstorage-db", 8 | "syncstorage-db-common", 9 | "syncstorage-mysql", 10 | "syncstorage-settings", 11 | "syncstorage-spanner", 12 | "tokenserver-auth", 13 | "tokenserver-common", 14 | "tokenserver-db", 15 | "tokenserver-settings", 16 | "syncserver", 17 | ] 18 | default-members = ["syncserver"] 19 | 20 | [workspace.package] 21 | version = "0.18.3" 22 | authors = [ 23 | "Ben Bangert ", 24 | "Phil Jenvey ", 25 | "Mozilla Services Engineering ", 26 | ] 27 | edition = "2021" 28 | rust-version = "1.86" 29 | license = "MPL-2.0" 30 | 31 | [workspace.dependencies] 32 | actix-web = "4" 33 | 34 | docopt = "1.1" 35 | base64 = "0.22" 36 | 37 | # Updating to 2.* requires changes to the Connection code for logging. 38 | # (Adding an `instrumentation()` and `set_instrumentation()` method.) 39 | # More investigation required. 40 | diesel = "1.4" 41 | diesel_migrations = "1.4" 42 | diesel_logger = "0.1" 43 | 44 | cadence = "1.3" 45 | backtrace = "0.3" 46 | chrono = "0.4" 47 | deadpool = { version = "0.12", features = ["rt_tokio_1"] } 48 | env_logger = "0.11" 49 | futures = { version = "0.3", features = ["compat"] } 50 | futures-util = { version = "0.3", features = [ 51 | "async-await", 52 | "compat", 53 | "sink", 54 | "io", 55 | ] } 56 | hex = "0.4" 57 | hostname = "0.4" 58 | hkdf = "0.12" 59 | hmac = "0.12" 60 | http = "1.1" 61 | jsonwebtoken = { version = "9.2", default-features = false } 62 | lazy_static = "1.4" 63 | protobuf = "=2.25.2" # pin to 2.25.2 to prevent side updating 64 | rand = "0.8" 65 | regex = "1.4" 66 | reqwest = { version = "0.12", default-features = false, features = [ 67 | "rustls-tls", 68 | ] } 69 | sentry = { version = "0.35", default-features = false, features = [ 70 | "curl", 71 | "backtrace", 72 | "contexts", 73 | "debug-images", 74 | ] } 75 | sentry-backtrace = "0.35" 76 | serde = "1.0" 77 | serde_derive = "1.0" 78 | serde_json = { version = "1.0", features = ["arbitrary_precision"] } 79 | sha2 = "0.10" 80 | slog = { version = "2.5", features = [ 81 | "max_level_trace", 82 | "release_max_level_info", 83 | "dynamic-keys", 84 | ] } 85 | slog-async = "2.5" 86 | slog-envlogger = "2.2.0" 87 | slog-mozlog-json = "0.1" 88 | slog-scope = "4.3" 89 | slog-stdlog = "4.1" 90 | slog-term = "2.6" 91 | tokio = "1" 92 | thiserror = "1.0.26" 93 | uuid = { version = "1.11", features = ["serde", "v4"] } 94 | 95 | [profile.release] 96 | # Enables line numbers in Sentry reporting 97 | debug = 1 98 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | Describe these changes. 4 | 5 | > **NOTE:** We can only accept PRS with all commits [signed](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification#ssh-commit-verification). PRs that contain _any_ unsigned commits will not be accepted and the PR _must_ be resubmitted. If this is something you cannot provide, please disclose and a team member _may_ duplicate the PR as signed for you (depending on availablity and priority. Thank you for your understanding and cooperation.) 6 | 7 | ## Testing 8 | 9 | How should reviewers test? 10 | 11 | ## Issue(s) 12 | 13 | Closes [link](link). 14 | -------------------------------------------------------------------------------- /config/local.example.toml: -------------------------------------------------------------------------------- 1 | master_secret = "INSERT_SECRET_KEY_HERE" 2 | 3 | # removing this line will default to moz_json formatted logs (which is preferred for production envs) 4 | human_logs = 1 5 | 6 | # Example Syncstorage settings: 7 | # Example MySQL DSN: 8 | syncstorage.database_url = "mysql://sample_user:sample_password@localhost/syncstorage_rs" 9 | # Example Spanner DSN: 10 | # database_url="spanner://projects/SAMPLE_GCP_PROJECT/instances/SAMPLE_SPANNER_INSTANCE/databases/SAMPLE_SPANNER_DB" 11 | # enable quota limits 12 | syncstorage.enable_quota = 0 13 | # set the quota limit to 2GB. 14 | # max_quota_limit = 200000000 15 | syncstorage.enabled = true 16 | syncstorage.limits.max_total_records = 1666 # See issues #298/#333 17 | 18 | # Example Tokenserver settings: 19 | tokenserver.database_url = "mysql://sample_user:sample_password@localhost/tokenserver_rs" 20 | tokenserver.enabled = true 21 | tokenserver.fxa_email_domain = "api-accounts.stage.mozaws.net" 22 | tokenserver.fxa_metrics_hash_secret = "INSERT_SECRET_KEY_HERE" 23 | tokenserver.fxa_oauth_server_url = "https://oauth.stage.mozaws.net" 24 | tokenserver.fxa_browserid_audience = "https://token.stage.mozaws.net" 25 | tokenserver.fxa_browserid_issuer = "https://api-accounts.stage.mozaws.net" 26 | tokenserver.fxa_browserid_server_url = "https://verifier.stage.mozaws.net/v2" 27 | 28 | # cors settings 29 | # cors_allowed_origin = "localhost" 30 | # cors_max_age = 86400 31 | -------------------------------------------------------------------------------- /docker-compose.e2e.mysql.yaml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | mysql-e2e-tests: 4 | container_name: mysql-e2e-tests 5 | depends_on: 6 | sync-db: 7 | condition: service_healthy 8 | mock-fxa-server: 9 | condition: service_started 10 | tokenserver-db: 11 | condition: service_healthy 12 | # this depend is to avoid migration collisions. 13 | # the syncserver isn't actually used for the tests, 14 | # but collisions can happen particularly in CI. 15 | syncserver: 16 | condition: service_started 17 | image: app:build 18 | privileged: true 19 | user: root 20 | environment: 21 | JWK_CACHE_DISABLED: false 22 | MOCK_FXA_SERVER_URL: http://mock-fxa-server:6000 23 | SYNC_HOST: 0.0.0.0 24 | SYNC_MASTER_SECRET: secret0 25 | SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:test@sync-db:3306/syncstorage 26 | SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver 27 | SYNC_TOKENSERVER__ENABLED: "true" 28 | SYNC_TOKENSERVER__FXA_BROWSERID_AUDIENCE: "https://token.stage.mozaws.net/" 29 | SYNC_TOKENSERVER__FXA_BROWSERID_ISSUER: "api-accounts.stage.mozaws.net" 30 | SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: api-accounts.stage.mozaws.net 31 | SYNC_TOKENSERVER__FXA_METRICS_HASH_SECRET: secret0 32 | SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" 33 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__KTY: "RSA" 34 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__ALG: "RS256" 35 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__KID: "20190730-15e473fd" 36 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__FXA_CREATED_AT: "1564502400" 37 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__USE: "sig" 38 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__N: "15OpVGC7ws_SlU0gRbRh1Iwo8_gR8ElX2CDnbN5blKyXLg-ll0ogktoDXc-tDvTabRTxi7AXU0wWQ247odhHT47y5uz0GASYXdfPponynQ_xR9CpNn1eEL1gvDhQN9rfPIzfncl8FUi9V4WMd5f600QC81yDw9dX-Z8gdkru0aDaoEKF9-wU2TqrCNcQdiJCX9BISotjz_9cmGwKXFEekQNJWBeRQxH2bUmgwUK0HaqwW9WbYOs-zstNXXWFsgK9fbDQqQeGehXLZM4Cy5Mgl_iuSvnT3rLzPo2BmlxMLUvRqBx3_v8BTtwmNGA0v9O0FJS_mnDq0Iue0Dz8BssQCQ" 39 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__E: "AQAB" 40 | TOKENSERVER_HOST: http://localhost:8000 41 | entrypoint: > 42 | /bin/sh -c " 43 | exit_code=0; 44 | pytest /app/tools/integration_tests/ --junit-xml=/mysql_integration_results.xml || exit_code=$$?; 45 | export JWK_CACHE_DISABLED=true; 46 | pytest /app/tools/integration_tests/ --junit-xml=/mysql_no_jwk_integration_results.xml || exit_code=$$?; 47 | exit $$exit_code; 48 | " 49 | -------------------------------------------------------------------------------- /docker-compose.e2e.spanner.yaml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | spanner-e2e-tests: 4 | container_name: spanner-e2e-tests 5 | depends_on: 6 | mock-fxa-server: 7 | condition: service_started 8 | syncserver: 9 | condition: service_started 10 | tokenserver-db: 11 | condition: service_healthy 12 | image: app:build 13 | privileged: true 14 | user: root 15 | environment: 16 | # Some tests can run without the `FXA_OAUTH...` vars. 17 | # Setting this to false will delete any of those keys before starting 18 | # the syncserver and startging the test. This can be set/passed 19 | # in from CircleCI when calling `docker-compose -f docker-compose.e2e.spanner.yaml` 20 | JWK_CACHE_DISABLED: false 21 | MOCK_FXA_SERVER_URL: http://mock-fxa-server:6000 22 | SYNC_HOST: 0.0.0.0 23 | SYNC_MASTER_SECRET: secret0 24 | SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database 25 | SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9010 26 | SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver 27 | SYNC_TOKENSERVER__ENABLED: "true" 28 | SYNC_TOKENSERVER__FXA_BROWSERID_AUDIENCE: "https://token.stage.mozaws.net/" 29 | SYNC_TOKENSERVER__FXA_BROWSERID_ISSUER: "api-accounts.stage.mozaws.net" 30 | SYNC_TOKENSERVER__FXA_EMAIL_DOMAIN: api-accounts.stage.mozaws.net 31 | SYNC_TOKENSERVER__FXA_METRICS_HASH_SECRET: secret0 32 | SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" 33 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__KTY: "RSA" 34 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__ALG: "RS256" 35 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__KID: "20190730-15e473fd" 36 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__FXA_CREATED_AT: "1564502400" 37 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__USE: "sig" 38 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__N: "15OpVGC7ws_SlU0gRbRh1Iwo8_gR8ElX2CDnbN5blKyXLg-ll0ogktoDXc-tDvTabRTxi7AXU0wWQ247odhHT47y5uz0GASYXdfPponynQ_xR9CpNn1eEL1gvDhQN9rfPIzfncl8FUi9V4WMd5f600QC81yDw9dX-Z8gdkru0aDaoEKF9-wU2TqrCNcQdiJCX9BISotjz_9cmGwKXFEekQNJWBeRQxH2bUmgwUK0HaqwW9WbYOs-zstNXXWFsgK9fbDQqQeGehXLZM4Cy5Mgl_iuSvnT3rLzPo2BmlxMLUvRqBx3_v8BTtwmNGA0v9O0FJS_mnDq0Iue0Dz8BssQCQ" 39 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__E: "AQAB" 40 | TOKENSERVER_HOST: http://localhost:8000 41 | entrypoint: > 42 | /bin/sh -c " 43 | exit_code=0; 44 | pytest /app/tools/integration_tests/ --junit-xml=/spanner_integration_results.xml || exit_code=$$?; 45 | export JWK_CACHE_DISABLED=true; 46 | pytest /app/tools/integration_tests/ --junit-xml=/spanner_no_jwk_integration_results.xml || exit_code=$$?; 47 | exit $$exit_code; 48 | " 49 | -------------------------------------------------------------------------------- /docker-compose.mysql.yaml: -------------------------------------------------------------------------------- 1 | # NOTE: This docker-compose file was constructed to create a base for 2 | # use by the End-to-end tests. It has not been fully tested for use in 3 | # constructing a true, stand-alone sync server. 4 | # If you're interested in doing that, please join our community in the 5 | # github issues and comments. 6 | # 7 | # Application runs off of port 8000. 8 | # you can test if it's available with 9 | # curl "http://localhost:8000/__heartbeat__" 10 | 11 | version: "3" 12 | services: 13 | sync-db: 14 | image: docker.io/library/mysql:5.7 15 | volumes: 16 | - sync_db_data:/var/lib/mysql 17 | restart: always 18 | ports: 19 | - "3306" 20 | command: --explicit_defaults_for_timestamp 21 | environment: 22 | #MYSQL_RANDOM_ROOT_PASSWORD: yes 23 | MYSQL_ROOT_PASSWORD: random 24 | MYSQL_DATABASE: syncstorage 25 | MYSQL_USER: test 26 | MYSQL_PASSWORD: test 27 | healthcheck: 28 | test: ["CMD-SHELL", "mysqladmin -uroot -p$${MYSQL_ROOT_PASSWORD} version"] 29 | interval: 2s 30 | retries: 10 31 | start_period: 20s 32 | timeout: 2s 33 | 34 | tokenserver-db: 35 | image: docker.io/library/mysql:5.7 36 | volumes: 37 | - tokenserver_db_data:/var/lib/mysql 38 | restart: always 39 | ports: 40 | - "3306" 41 | command: --explicit_defaults_for_timestamp 42 | environment: 43 | #MYSQL_RANDOM_ROOT_PASSWORD: yes 44 | MYSQL_ROOT_PASSWORD: random 45 | MYSQL_DATABASE: tokenserver 46 | MYSQL_USER: test 47 | MYSQL_PASSWORD: test 48 | healthcheck: 49 | test: ["CMD-SHELL", "mysqladmin -uroot -p$${MYSQL_ROOT_PASSWORD} version"] 50 | interval: 2s 51 | retries: 10 52 | start_period: 20s 53 | timeout: 2s 54 | 55 | mock-fxa-server: 56 | image: app:build 57 | restart: "no" 58 | entrypoint: "sh scripts/start_mock_fxa_server.sh" 59 | environment: 60 | MOCK_FXA_SERVER_HOST: 0.0.0.0 61 | MOCK_FXA_SERVER_PORT: 6000 62 | 63 | syncserver: 64 | # NOTE: The naming in the rest of this repository has been updated to reflect the fact 65 | # that Syncstorage and Tokenserver are now part of one repository/server called 66 | # "Syncserver" (updated from "syncstorage-rs"). We keep the legacy naming below for 67 | # backwards compatibility with previous Docker images. 68 | image: ${SYNCSTORAGE_RS_IMAGE:-syncstorage-rs:latest} 69 | restart: always 70 | ports: 71 | - "8000:8000" 72 | depends_on: 73 | sync-db: 74 | condition: service_healthy 75 | tokenserver-db: 76 | condition: service_healthy 77 | environment: 78 | SYNC_HOST: 0.0.0.0 79 | SYNC_MASTER_SECRET: secret0 80 | SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:test@sync-db:3306/syncstorage 81 | SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver 82 | SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" 83 | 84 | volumes: 85 | sync_db_data: 86 | tokenserver_db_data: 87 | -------------------------------------------------------------------------------- /docker-compose.spanner.yaml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | sync-db: 4 | # Getting sporadic errors in spanner. 5 | # These errors are "INTERNAL: ZETASQL_RET_CHECK failure" 6 | # in the `backend/query/query_engine.cc` file for spanner. 7 | # These result in a 500 error, which causes the test to fail. 8 | # I believe come from the SQL parser, but am not sure. I am 9 | # unable to produce these errors locally, and am using the cited 10 | # version. 11 | image: gcr.io/cloud-spanner-emulator/emulator:1.5.13 12 | ports: 13 | - "9010:9010" 14 | - "9020:9020" 15 | environment: 16 | PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 17 | sync-db-setup: 18 | image: app:build 19 | depends_on: 20 | - sync-db 21 | restart: "no" 22 | entrypoint: "/app/scripts/prepare-spanner.sh" 23 | environment: 24 | SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9020 25 | tokenserver-db: 26 | image: docker.io/library/mysql:5.7 27 | volumes: 28 | - tokenserver_db_data:/var/lib/mysql 29 | restart: always 30 | ports: 31 | - "3306" 32 | environment: 33 | #MYSQL_RANDOM_ROOT_PASSWORD: yes 34 | MYSQL_ROOT_PASSWORD: random 35 | MYSQL_DATABASE: tokenserver 36 | MYSQL_USER: test 37 | MYSQL_PASSWORD: test 38 | healthcheck: 39 | test: ["CMD-SHELL", "mysqladmin -uroot -p$${MYSQL_ROOT_PASSWORD} version"] 40 | interval: 2s 41 | retries: 10 42 | start_period: 20s 43 | timeout: 2s 44 | mock-fxa-server: 45 | image: app:build 46 | restart: "no" 47 | entrypoint: "sh /app/scripts/start_mock_fxa_server.sh" 48 | environment: 49 | MOCK_FXA_SERVER_HOST: 0.0.0.0 50 | MOCK_FXA_SERVER_PORT: 6000 51 | syncserver: 52 | # NOTE: The naming in the rest of this repository has been updated to reflect the fact 53 | # that Syncstorage and Tokenserver are now part of one repository/server called 54 | # "Syncserver" (updated from "syncstorage-rs"). We keep the legacy naming below for 55 | # backwards compatibility with previous Docker images. 56 | image: ${SYNCSTORAGE_RS_IMAGE:-syncstorage-rs:latest} 57 | restart: always 58 | ports: 59 | - "8000:8000" 60 | depends_on: 61 | - sync-db-setup 62 | environment: 63 | SYNC_HOST: 0.0.0.0 64 | SYNC_MASTER_SECRET: secret0 65 | SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database 66 | SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: sync-db:9010 67 | SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@tokenserver-db:3306/tokenserver 68 | SYNC_TOKENSERVER__RUN_MIGRATIONS: "true" 69 | 70 | volumes: 71 | tokenserver_db_data: 72 | 73 | # Application runs off of port 8000. 74 | # you can test if it's available with 75 | # curl "http://localhost:8000/__heartbeat__" 76 | -------------------------------------------------------------------------------- /docs/adr/index.md: -------------------------------------------------------------------------------- 1 | # Synsctorage-rs ADRs 2 | 3 | This directory archives all the Architectural Decision Records (ADRs) for Syncstorage-rs. 4 | -------------------------------------------------------------------------------- /docs/adr/template.md: -------------------------------------------------------------------------------- 1 | # [short title of solved problem and solution] 2 | 3 | * Status: [proposed | rejected | accepted | deprecated | … | superseded by [ADR-0005](0005-example.md)] 4 | * Deciders: [list everyone involved in the decision] 5 | * Date: [YYYY-MM-DD when the decision was last updated] 6 | 7 | Technical Story: [description | ticket/issue URL] 8 | 9 | ## Context and Problem Statement 10 | 11 | [Describe the context and problem statement, e.g., in free form using two to three sentences. You may want to articulate the problem in form of a question.] 12 | 13 | ## Decision Drivers 14 | 15 | 1. [primary driver, e.g., a force, facing concern, …] 16 | 2. [secondary driver, e.g., a force, facing concern, …] 17 | 3. … 18 | 19 | ## Considered Options 20 | 21 | * A. [option A] 22 | * B. [option B] 23 | * C. [option C] 24 | * D. … 25 | 26 | ## Decision Outcome 27 | 28 | Chosen option: 29 | 30 | * A. "[option A]" 31 | 32 | [justification. e.g., only option, which meets primary decision driver | which resolves a force or facing concern | … | comes out best (see below)]. 33 | 34 | ### Positive Consequences 35 | 36 | * [e.g., improvement of quality attribute satisfaction, follow-up decisions required, …] 37 | * … 38 | 39 | ### Negative Consequences 40 | 41 | * [e.g., compromising quality attribute, follow-up decisions required, …] 42 | * … 43 | 44 | ## Pros and Cons of the Options 45 | 46 | ### [option A] 47 | 48 | [example | description | pointer to more information | …] 49 | 50 | #### Pros 51 | 52 | * [argument for] 53 | * [argument for] 54 | * … 55 | 56 | #### Cons 57 | 58 | * [argument against] 59 | * … 60 | 61 | ### [option B] 62 | 63 | [example | description | pointer to more information | …] 64 | 65 | #### Pros 66 | 67 | * [argument for] 68 | * [argument for] 69 | * … 70 | 71 | #### Cons 72 | 73 | * [argument against] 74 | * … 75 | 76 | ### [option C] 77 | 78 | [example | description | pointer to more information | …] 79 | 80 | #### Pros 81 | 82 | * [argument for] 83 | * [argument for] 84 | * … 85 | 86 | #### Cons 87 | 88 | * [argument against] 89 | * … 90 | 91 | ## Links 92 | 93 | * [Link type] [Link to ADR] 94 | * … 95 | -------------------------------------------------------------------------------- /docs/config.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | Rust uses environment variables for a number of configuration options. Some of these include: 3 | 4 | | variable | value | description | 5 | | --- | --- | --- | 6 | | **RUST_LOG** | *debug*, *info*, *warn*, *error* | minimum Rust error logging level | 7 | | **RUST_TEST_THREADS** | 1 | maximum number of concurrent threads for testing. | 8 | 9 | In addition, durable sync configuration options can either be specified as environment variables (prefixed with **SYNC_***) or in a configuration file using the `--config` option. 10 | 11 | For example the following are equivalent: 12 | ```bash 13 | $ SYNC_HOST=0.0.0.0 SYNC_MASTER_SECRET="SuperSikkr3t" SYNC_SYNCSTORAGE__DATABASE_URL=mysql://scott:tiger@localhost/syncstorage cargo run 14 | ``` 15 | 16 | ```bash 17 | $ cat sync.ini 18 | HOST=0.0.0.0 19 | MASTER_SECRET=SuperSikkr3t 20 | 21 | [syncstorage] 22 | DATABASE_URL=mysql://scott:tiger@localhost/syncstorage 23 | $ cargo run -- --config sync.ini 24 | ``` 25 | 26 | Options can be mixed between environment and configuration. 27 | 28 | ## Options 29 | The following configuration options are available. 30 | 31 | | Option | Default value |Description | 32 | | --- | --- | --- | 33 | | debug | false | _unused_ | 34 | | port | 8000 | connection port | 35 | | host | 127.0.0.1 | host to listen for connections | 36 | | database_url | mysql://root@127.0.0.1/syncstorage | database DSN | 37 | | database_pool_max_size | _None_ | Max pool of database connections | 38 | | master_secret| _None_ | Sync master encryption secret | 39 | | limits.max_post_bytes | 2,097,152‬ | Largest record post size | 40 | | limits.max_post_records | 100 | Largest number of records per post | 41 | | limits.max_records_payload_bytes | 2,097,152‬ | Largest ... | 42 | | limits.max_request_bytes | 2,101,248 | Largest ... | 43 | | limits.max_total_bytes | 209,715,200 | Largest ... | 44 | | limits.max_total_records | 100,000 | Largest ... | 45 | 46 | -------------------------------------------------------------------------------- /docs/tokenserver/tokenserver.md: -------------------------------------------------------------------------------- 1 | # Tokenserver 2 | 3 | ## What is Tokenserver? 4 | Tokenserver is responsible for allocating Firefox Sync users to Sync Storage nodes hosted in our Spanner GCP Backend. 5 | Tokenserver provides the "glue" between [Firefox Accounts](https://github.com/mozilla/fxa/) and the 6 | [SyncStorage API](https://mozilla-services.readthedocs.io/en/latest/storage/apis-1.5.html). 7 | 8 | Broadly, Tokenserver is responsible for: 9 | 10 | * Checking the user's credentials as provided by FxA. 11 | * Sharding users across storage nodes in a way that evenly distributes server load. 12 | * Re-assigning the user to a new storage node if their FxA encryption key changes. 13 | * Cleaning up old data from deleted accounts. 14 | 15 | The service was originally conceived to be a general-purpose mechanism for connecting users 16 | to multiple different Mozilla-run services, and you can see some of the historical context 17 | for that original design [here](https://wiki.mozilla.org/Services/Sagrada/TokenServer) 18 | and [here](https://mozilla-services.readthedocs.io/en/latest/token/index.html). 19 | 20 | In practice today, it is only used for connecting to Sync. 21 | 22 | ## Tokenserver Crates & Their Purpose 23 | 24 | ### `tokenserver-auth` 25 | Handles authentication logic, including: 26 | - Token generation and validation. 27 | - Ensuring clients are authorized before accessing Sync services. 28 | 29 | ### `tokenserver-common` 30 | Provides shared functionality and types used across the Tokenserver ecosystem: 31 | - Common utility functions. 32 | - Structs and traits reused in other Tokenserver modules. 33 | 34 | ### `tokenserver-db` 35 | Responsible for persisting and retrieving authentication/session-related data securely and efficiently. 36 | Manages all database interactions for Tokenserver: 37 | - Database schema definitions. 38 | - Connection pooling and querying logic. 39 | 40 | ### `tokenserver-settings` 41 | Handles configuration management: 42 | - Loads and validates settings for Tokenserver. 43 | - Supports integration with different deployment environments. 44 | 45 | ## How Tokenserver Handles Failure Cases 46 | 47 | ### Token Expiry 48 | When a Tokenserver token expires, Sync Storage returns a 401 code, requiring clients to get a new token. Then, clients would use their FxA OAuth Access tokens to generate a new token, if the FxA Access Token is itself expired, then Tokenserver returns a 401 itself. 49 | 50 | ### User revoking access token 51 | The user could revoke the access token by signing out using the Mozilla Account’s Manage Account settings. In that case, clients continue to sync up to the expiry time, which is one hour. To mitigate against this case, Firefox clients currently receive push notifications from FxA instructing them to disconnect. Additionally, any requests done against FxA itself (for example to get the user’s profile data, connected devices, etc) will also trigger the client to disconnect. 52 | 53 | ### User Changes Their Password 54 | This is similar to the case where users revoke their access tokens. Any devices with a not-expired access token will continue to sync until expiry, but clients will likely disconnect those clients faster than the 1 hour - however, a malicious user might be able to sync upwards of 1 hour. 55 | 56 | ### User Forgetting Their Password (without a recovery key) 57 | When a user forgets and resets their password without a recovery key, their Sync keys change. The Tokenserver request includes the key ID (which is a hash of the sync key). Thus, on the next sync, Tokenserver recognizes that the password changed, and ensures that the tokens it issues point users to a new location on Sync Storage. In practice, it does that by including the Key ID itself in the Tokenserver token, which is then sent to Sync Storage. 58 | 59 | ### User Forgetting Their Password (with a recovery key) 60 | When a user forgets and resets their password, but has their recovery key, the behavior is similar to the password change and user revoking token cases. 61 | 62 | 63 | ## Utilities 64 | Tokenserver has two regular running utility scripts: 65 | 1 - [Process Account Events](../tools/process_account_events.md) 66 | 2 - [Purge Old Records](../tools/purge_old_records_tokenserver.md) 67 | 68 | For context on these processes, their purpose, and how to run them, please review their documentation pages. -------------------------------------------------------------------------------- /glean/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "glean" 3 | version.workspace = true 4 | license.workspace = true 5 | authors.workspace = true 6 | edition.workspace = true 7 | 8 | [dependencies] 9 | chrono.workspace = true 10 | serde.workspace = true 11 | serde_json.workspace = true 12 | uuid.workspace = true 13 | 14 | [lib] 15 | name = "glean" 16 | path = "src/lib.rs" -------------------------------------------------------------------------------- /glean/metrics.yaml: -------------------------------------------------------------------------------- 1 | ## This file describes the syncserver-rs daily active user (DAU) metrics. 2 | ## This defines the various allowed metrics that are to be captured. 3 | ## Each metric is written as a JSON blob to the default logger output. 4 | 5 | --- 6 | # Schema 7 | $schema: moz://mozilla.org/schemas/glean/metrics/2-0-0 8 | 9 | # Category 10 | syncstorage: 11 | get_collections: 12 | type: event 13 | description: | 14 | Event to record an instance of sync backend activity initiated by client. 15 | notification_emails: 16 | - sync-backend@mozilla.com 17 | bugs: 18 | - https://github.com/mozilla-services/syncstorage-rs/issues 19 | data_reviews: 20 | - https://bugzilla.mozilla.org/show_bug.cgi?id=1923967 21 | expires: never 22 | 23 | hashed_fxa_uid: 24 | type: string 25 | # yamllint disable 26 | description: > 27 | User identifier. Uses `hashed_fxa_uid` for accurate count of sync actions. 28 | Used to determine which user has initiated sync activity. 29 | This is the Firefox Accounts (FxA) User Identifier (UID) value passed through 30 | a SHA-256 hash to render a value that is unique, but ensures the privacy of the original UID. 31 | A single user could make numerous sync actions in a given time 32 | and this id is required to ensure only a single count of daily active use 33 | is made, given a number of actions. Sync_id is not used due to possibility 34 | of new keys being generated during resets or timeouts, whenever encryption 35 | keys change. 36 | # yamllint enable 37 | lifetime: application 38 | send_in_pings: 39 | - events 40 | notification_emails: 41 | - sync-backend@mozilla.com 42 | bugs: 43 | - https://github.com/mozilla-services/syncstorage-rs/issues 44 | data_reviews: 45 | - https://bugzilla.mozilla.org/show_bug.cgi?id=1923967 46 | expires: never 47 | 48 | platform: 49 | type: string 50 | # yamllint disable 51 | description: | 52 | Platform from which sync action was initiated. 53 | Firefox Desktop, Fenix, or Firefox iOS. 54 | # yamllint enable 55 | lifetime: application 56 | send_in_pings: 57 | - events 58 | notification_emails: 59 | - sync-backend@mozilla.com 60 | bugs: 61 | - https://github.com/mozilla-services/syncstorage-rs/issues 62 | data_reviews: 63 | - https://bugzilla.mozilla.org/show_bug.cgi?id=1923967 64 | expires: never 65 | 66 | device_family: 67 | type: string 68 | # yamllint disable 69 | description: | 70 | Device family from which sync action was initiated. 71 | Desktop PC, Tablet, Mobile, and Other. 72 | # yamllint enable 73 | lifetime: application 74 | send_in_pings: 75 | - events 76 | notification_emails: 77 | - sync-backend@mozilla.com 78 | bugs: 79 | - https://github.com/mozilla-services/syncstorage-rs/issues 80 | data_reviews: 81 | - https://bugzilla.mozilla.org/show_bug.cgi?id=1923967 82 | expires: never 83 | 84 | hashed_device_id: 85 | type: string 86 | # yamllint disable 87 | description: | 88 | Hashed device id that is associated with a given account. This is used 89 | entirely to associate opt-out or removal requests, as they make use of 90 | the "deletion-request" ping associated with the client side of Sync. 91 | # yamllint enable 92 | lifetime: application 93 | send_in_pings: 94 | - events 95 | notification_emails: 96 | - sync-backend@mozilla.com 97 | bugs: 98 | - https://github.com/mozilla-services/syncstorage-rs/issues 99 | data_reviews: 100 | - https://bugzilla.mozilla.org/show_bug.cgi?id=1923967 101 | expires: never 102 | -------------------------------------------------------------------------------- /glean/pings.yaml: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | --- 6 | ## Describes the pings being sent out to Glean. 7 | 8 | # Schema 9 | $schema: moz://mozilla.org/schemas/glean/pings/2-0-0 10 | -------------------------------------------------------------------------------- /glean/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Module related to Glean server metrics. 2 | 3 | pub mod server_events; 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cryptography==44.0.2 2 | pyfxa==0.8.1 3 | tokenlib==2.0.0 4 | -------------------------------------------------------------------------------- /scripts/build-docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copied from: https://github.com/kmcallister/travis-doc-upload 4 | # License: CC0 1.0 Universal 5 | # https://creativecommons.org/publicdomain/zero/1.0/legalcode 6 | 7 | set -e 8 | 9 | echo "Branch: $TRAVIS_BRANCH Pull request: $TRAVIS_PULL_REQUEST" 10 | 11 | if [ "$TRAVIS_BRANCH" = "master" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then 12 | echo "Building docs for deployment." 13 | cargo doc --document-private-items 14 | 15 | eval key=\$encrypted_ad173ee39c4b_key 16 | eval iv=\$encrypted_ad173ee39c4b_iv 17 | 18 | mkdir -p ~/.ssh 19 | openssl aes-256-cbc -K $key -iv $iv -in scripts/id_rsa.enc -out ~/.ssh/id_rsa -d 20 | chmod 600 ~/.ssh/id_rsa 21 | 22 | git clone --branch gh-pages git@github.com:mozilla-services/syncstorage-rs.git docs 23 | cd docs 24 | rm -rf * 25 | mv ../target/doc/* . 26 | 27 | CHANGES=`git status --porcelain` 28 | 29 | if [ "$CHANGES" = "" ]; then 30 | echo "Docs are unchanged, not deploying to GitHub Pages." 31 | exit 0 32 | fi 33 | 34 | echo "Deploying docs to GitHub Pages." 35 | 36 | git config user.name "app-services-devs" 37 | git config user.email "application-services@mozilla.com" 38 | git add -A . 39 | git commit -qm "chore: rebuild developer docs" 40 | git push -q origin gh-pages 41 | else 42 | echo "Not building docs for deployment, omitting dependencies." 43 | cargo doc --document-private-items --no-deps 44 | fi 45 | -------------------------------------------------------------------------------- /scripts/id_rsa.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/syncstorage-rs/0768d4975de555b2c7af64dfef1ba40e5e2f99df/scripts/id_rsa.enc -------------------------------------------------------------------------------- /scripts/prepare-spanner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright 2020 Google LLC 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # This file contains modifications of the original version. 19 | # 20 | 21 | sleep 5 22 | 23 | set -e 24 | 25 | PROJECT_ID=test-project 26 | INSTANCE_ID=test-instance 27 | DATABASE_ID=test-database 28 | 29 | DDL_STATEMENTS=$( 30 | grep -v ^-- schema.ddl \ 31 | | sed -n 's/ \+/ /gp' \ 32 | | tr -d '\n' \ 33 | | sed 's/\(.*\);/\1/' \ 34 | | jq -R -s -c 'split(";")' 35 | ) 36 | 37 | curl -sS --request POST \ 38 | "$SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST/v1/projects/$PROJECT_ID/instances" \ 39 | --header 'Accept: application/json' \ 40 | --header 'Content-Type: application/json' \ 41 | --data "{\"instance\":{\"config\":\"emulator-test-config\",\"nodeCount\":1,\"displayName\":\"Test Instance\"},\"instanceId\":\"$INSTANCE_ID\"}" 42 | 43 | curl -sS --request POST \ 44 | "$SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST/v1/projects/$PROJECT_ID/instances/$INSTANCE_ID/databases" \ 45 | --header 'Accept: application/json' \ 46 | --header 'Content-Type: application/json' \ 47 | --data "{\"createStatement\":\"CREATE DATABASE \`$DATABASE_ID\`\",\"extraStatements\":$DDL_STATEMENTS}" 48 | 49 | sleep infinity 50 | -------------------------------------------------------------------------------- /scripts/sentry-release.sh: -------------------------------------------------------------------------------- 1 | sentry-cli releases set-commits --auto $VERSION 2 | sentry-cli releases new -p syncstorage-prod $VERSION 3 | sentry-cli releases set-commits --auto $VERSION 4 | sentry-cli releases finalize $VERSION 5 | -------------------------------------------------------------------------------- /scripts/start_mock_fxa_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | python3 /app/tools/integration_tests/tokenserver/mock_fxa_server.py 4 | 5 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | # Nix is a powerful package manager for Linux and other Unix systems that makes 2 | # package management reliable and reproducible: https://nixos.org/nix/. 3 | # This file is intended to be used with `nix-shell` 4 | # (https://nixos.org/nix/manual/#sec-nix-shell) to setup a fully-functional 5 | # syncstorage-rs build environment by installing all required dependencies. 6 | with import {}; 7 | stdenv.mkDerivation { 8 | name = "syncstorage-rs"; 9 | buildInputs = [ 10 | rustc 11 | cargo 12 | libmysqlclient 13 | pkg-config 14 | openssl 15 | cmake 16 | protobuf 17 | go 18 | ]; 19 | NIX_LDFLAGS = "-L${libmysqlclient}/lib/mysql"; 20 | } 21 | -------------------------------------------------------------------------------- /syncserver-common/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "syncserver-common" 3 | version.workspace = true 4 | license.workspace = true 5 | authors.workspace = true 6 | edition.workspace = true 7 | 8 | [dependencies] 9 | actix-web.workspace = true 10 | backtrace.workspace = true 11 | cadence.workspace = true 12 | futures.workspace = true 13 | futures-util.workspace = true 14 | sha2.workspace = true 15 | sentry.workspace = true 16 | sentry-backtrace.workspace = true 17 | serde_json.workspace = true 18 | slog.workspace = true 19 | slog-scope.workspace = true 20 | hkdf.workspace = true 21 | 22 | scopeguard = "1.2" 23 | -------------------------------------------------------------------------------- /syncserver-common/src/middleware/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod sentry; 2 | -------------------------------------------------------------------------------- /syncserver-common/src/tags.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use actix_web::HttpMessage; 4 | use serde_json::Value; 5 | 6 | pub trait Taggable { 7 | /// Adds a tag to be included in any metric or Sentry error emitted from this point in the 8 | /// request lifecycle onwards. Tags **must** have low cardinality, meaning that the number of 9 | /// distinct possible values associated with a given tag must be small. 10 | fn add_tag(&self, key: String, value: String); 11 | 12 | /// Gets all the tags associated with `Self`. 13 | fn get_tags(&self) -> HashMap; 14 | 15 | /// Adds an extra to be included in any Sentry error emitted from this point in the request 16 | /// lifecycle onwards. Extras are intended to be used to report additional metadata that have 17 | /// cardinality that is too high for tags. Note that extras will not be included with metrics. 18 | fn add_extra(&self, key: String, value: String); 19 | 20 | /// Gets all the extras associated with `Self`. This converts the values to `serde_json::Value` 21 | /// because the only caller / consumer for this function is the Sentry middleware, which uses 22 | /// `Value` for extras. 23 | fn get_extras(&self) -> HashMap; 24 | } 25 | 26 | impl Taggable for T 27 | where 28 | T: HttpMessage, 29 | { 30 | fn add_tag(&self, key: String, value: String) { 31 | let mut exts = self.extensions_mut(); 32 | 33 | match exts.get_mut::() { 34 | Some(tags) => { 35 | tags.0.insert(key, value); 36 | } 37 | None => { 38 | let mut tags = Tags::default(); 39 | tags.0.insert(key, value); 40 | exts.insert(tags); 41 | } 42 | } 43 | } 44 | 45 | fn get_tags(&self) -> HashMap { 46 | self.extensions() 47 | .get::() 48 | .map(|tags_ref| tags_ref.0.clone()) 49 | .unwrap_or_default() 50 | } 51 | 52 | fn add_extra(&self, key: String, value: String) { 53 | let mut exts = self.extensions_mut(); 54 | 55 | match exts.get_mut::() { 56 | Some(extras) => { 57 | extras.0.insert(key, value); 58 | } 59 | None => { 60 | let mut extras = Extras::default(); 61 | extras.0.insert(key, value); 62 | exts.insert(extras); 63 | } 64 | } 65 | } 66 | 67 | fn get_extras(&self) -> HashMap { 68 | self.extensions() 69 | .get::() 70 | .map(|extras_ref| { 71 | extras_ref 72 | .0 73 | .clone() 74 | .into_iter() 75 | .map(|(k, v)| (k, Value::from(v))) 76 | .collect() 77 | }) 78 | .unwrap_or_default() 79 | } 80 | } 81 | 82 | /// Tags are metadata that will be included in both Sentry errors and metric emissions. Given that 83 | /// InfluxDB requires that tags have low cardinality, tags **must** have low cardinality. This 84 | /// means that the number of distinct values for a given tag across every request must be low. 85 | #[derive(Default)] 86 | struct Tags(HashMap); 87 | 88 | // "Extras" are pieces of metadata with high cardinality to be included in Sentry errors. 89 | #[derive(Default)] 90 | struct Extras(HashMap); 91 | -------------------------------------------------------------------------------- /syncserver-db-common/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "syncserver-db-common" 3 | version.workspace = true 4 | license.workspace = true 5 | authors.workspace = true 6 | edition.workspace = true 7 | 8 | [dependencies] 9 | backtrace.workspace = true 10 | deadpool.workspace = true 11 | futures.workspace = true 12 | http.workspace = true 13 | thiserror.workspace = true 14 | 15 | diesel = { workspace = true, features = ["mysql", "r2d2"] } 16 | diesel_migrations = { workspace = true, features = ["mysql"] } 17 | syncserver-common = { path = "../syncserver-common" } 18 | -------------------------------------------------------------------------------- /syncserver-db-common/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use backtrace::Backtrace; 4 | use http::StatusCode; 5 | use syncserver_common::{from_error, impl_fmt_display, ReportableError}; 6 | use thiserror::Error; 7 | 8 | /// Error specific to any SQL database backend. These errors are not related to the syncstorage 9 | /// or tokenserver application logic; rather, they are lower-level errors arising from diesel. 10 | #[derive(Debug)] 11 | pub struct SqlError { 12 | kind: SqlErrorKind, 13 | pub status: StatusCode, 14 | pub backtrace: Backtrace, 15 | } 16 | 17 | #[derive(Debug, Error)] 18 | enum SqlErrorKind { 19 | #[error("A database error occurred: {}", _0)] 20 | DieselQuery(#[from] diesel::result::Error), 21 | 22 | #[error("An error occurred while establishing a db connection: {}", _0)] 23 | DieselConnection(#[from] diesel::result::ConnectionError), 24 | 25 | #[error("A database pool error occurred: {}", _0)] 26 | Pool(diesel::r2d2::PoolError), 27 | 28 | #[error("Error migrating the database: {}", _0)] 29 | Migration(diesel_migrations::RunMigrationsError), 30 | } 31 | 32 | impl From for SqlError { 33 | fn from(kind: SqlErrorKind) -> Self { 34 | Self { 35 | kind, 36 | status: StatusCode::INTERNAL_SERVER_ERROR, 37 | backtrace: Backtrace::new(), 38 | } 39 | } 40 | } 41 | 42 | impl ReportableError for SqlError { 43 | fn is_sentry_event(&self) -> bool { 44 | #[allow(clippy::match_like_matches_macro)] 45 | match &self.kind { 46 | SqlErrorKind::Pool(_) => false, 47 | _ => true, 48 | } 49 | } 50 | 51 | fn metric_label(&self) -> Option<&'static str> { 52 | Some(match self.kind { 53 | SqlErrorKind::DieselQuery(_) => "storage.sql.error.diesel_query", 54 | SqlErrorKind::DieselConnection(_) => "storage.sql.error.diesel_connection", 55 | SqlErrorKind::Pool(_) => "storage.sql.error.pool", 56 | SqlErrorKind::Migration(_) => "storage.sql.error.migration", 57 | }) 58 | } 59 | 60 | fn backtrace(&self) -> Option<&Backtrace> { 61 | Some(&self.backtrace) 62 | } 63 | } 64 | 65 | impl_fmt_display!(SqlError, SqlErrorKind); 66 | 67 | from_error!(diesel::result::Error, SqlError, SqlErrorKind::DieselQuery); 68 | from_error!( 69 | diesel::result::ConnectionError, 70 | SqlError, 71 | SqlErrorKind::DieselConnection 72 | ); 73 | from_error!(diesel::r2d2::PoolError, SqlError, SqlErrorKind::Pool); 74 | from_error!( 75 | diesel_migrations::RunMigrationsError, 76 | SqlError, 77 | SqlErrorKind::Migration 78 | ); 79 | -------------------------------------------------------------------------------- /syncserver-db-common/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod error; 2 | pub mod test; 3 | 4 | use std::fmt::Debug; 5 | 6 | use futures::future::LocalBoxFuture; 7 | 8 | pub type DbFuture<'a, T, E> = LocalBoxFuture<'a, Result>; 9 | 10 | /// A trait to be implemented by database pool data structures. It provides an interface to 11 | /// derive the current state of the pool, as represented by the `PoolState` struct. 12 | pub trait GetPoolState { 13 | fn state(&self) -> PoolState; 14 | } 15 | 16 | #[derive(Debug, Default)] 17 | /// A mockable r2d2::State 18 | pub struct PoolState { 19 | pub connections: u32, 20 | pub idle_connections: u32, 21 | } 22 | 23 | impl From for PoolState { 24 | fn from(state: diesel::r2d2::State) -> PoolState { 25 | PoolState { 26 | connections: state.connections, 27 | idle_connections: state.idle_connections, 28 | } 29 | } 30 | } 31 | impl From for PoolState { 32 | fn from(status: deadpool::Status) -> PoolState { 33 | PoolState { 34 | connections: status.size as u32, 35 | idle_connections: status.available.max(0) as u32, 36 | } 37 | } 38 | } 39 | 40 | #[macro_export] 41 | macro_rules! sync_db_method { 42 | ($name:ident, $sync_name:ident, $type:ident) => { 43 | sync_db_method!($name, $sync_name, $type, results::$type); 44 | }; 45 | ($name:ident, $sync_name:ident, $type:ident, $result:ty) => { 46 | fn $name(&self, params: params::$type) -> DbFuture<'_, $result, DbError> { 47 | let db = self.clone(); 48 | Box::pin( 49 | self.blocking_threadpool 50 | .spawn(move || db.$sync_name(params)), 51 | ) 52 | } 53 | }; 54 | } 55 | -------------------------------------------------------------------------------- /syncserver-db-common/src/test.rs: -------------------------------------------------------------------------------- 1 | use diesel::{ 2 | mysql::MysqlConnection, 3 | r2d2::{CustomizeConnection, Error as PoolError}, 4 | Connection, 5 | }; 6 | 7 | #[derive(Debug)] 8 | pub struct TestTransactionCustomizer; 9 | 10 | impl CustomizeConnection for TestTransactionCustomizer { 11 | fn on_acquire(&self, conn: &mut MysqlConnection) -> Result<(), PoolError> { 12 | conn.begin_test_transaction().map_err(PoolError::QueryError) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /syncserver-settings/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "syncserver-settings" 3 | version.workspace = true 4 | license.workspace = true 5 | authors.workspace = true 6 | edition.workspace = true 7 | 8 | [dependencies] 9 | serde.workspace = true 10 | slog-scope.workspace = true 11 | 12 | config = "0.11" # pin to 11, 12+ introduces a breaking change for env vars. 13 | num_cpus = "1" 14 | syncserver-common = { path = "../syncserver-common" } 15 | syncstorage-settings = { path = "../syncstorage-settings" } 16 | tokenserver-settings = { path = "../tokenserver-settings" } 17 | url = "2.1" 18 | -------------------------------------------------------------------------------- /syncserver/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "syncserver" 3 | default-run = "syncserver" 4 | version.workspace = true 5 | license.workspace = true 6 | authors.workspace = true 7 | edition.workspace = true 8 | 9 | [dependencies] 10 | actix-web.workspace = true 11 | backtrace.workspace = true 12 | base64.workspace = true 13 | cadence.workspace = true 14 | chrono.workspace = true 15 | docopt.workspace = true 16 | futures.workspace = true 17 | hex.workspace = true 18 | hostname.workspace = true 19 | http.workspace = true 20 | lazy_static.workspace = true 21 | rand.workspace = true 22 | regex.workspace = true 23 | sentry.workspace = true 24 | serde.workspace = true 25 | serde_json.workspace = true 26 | sha2.workspace = true 27 | slog.workspace = true 28 | slog-async.workspace = true 29 | slog-envlogger.workspace = true 30 | slog-mozlog-json.workspace = true 31 | slog-scope.workspace = true 32 | slog-stdlog.workspace = true 33 | slog-term.workspace = true 34 | hmac.workspace = true 35 | thiserror.workspace = true 36 | 37 | actix-http = "3" 38 | actix-rt = "2" 39 | actix-cors = "0.7" 40 | glean = { path = "../glean" } 41 | hawk = "5.0" 42 | mime = "0.3" 43 | # pin to 0.19: https://github.com/getsentry/sentry-rust/issues/277 44 | syncserver-common = { path = "../syncserver-common" } 45 | syncserver-db-common = { path = "../syncserver-db-common" } 46 | syncserver-settings = { path = "../syncserver-settings" } 47 | syncstorage-db = { path = "../syncstorage-db" } 48 | syncstorage-settings = { path = "../syncstorage-settings" } 49 | time = "^0.3" 50 | tokenserver-auth = { path = "../tokenserver-auth", default-features = false } 51 | tokenserver-common = { path = "../tokenserver-common" } 52 | tokenserver-db = { path = "../tokenserver-db" } 53 | tokenserver-settings = { path = "../tokenserver-settings" } 54 | tokio = { workspace = true, features = ["macros", "sync"] } 55 | urlencoding = "2.1" 56 | validator = "0.19" 57 | validator_derive = "0.19" 58 | woothee = "0.13" 59 | 60 | [features] 61 | default = ["mysql", "py_verifier"] 62 | no_auth = [] 63 | py_verifier = ["tokenserver-auth/py"] 64 | mysql = ["syncstorage-db/mysql"] 65 | spanner = ["syncstorage-db/spanner"] 66 | -------------------------------------------------------------------------------- /syncserver/src/db/mod.rs: -------------------------------------------------------------------------------- 1 | //! Generic db abstration. 2 | 3 | pub mod mock; 4 | pub mod mysql; 5 | pub mod spanner; 6 | #[cfg(test)] 7 | mod tests; 8 | pub mod transaction; 9 | 10 | use std::sync::Arc; 11 | 12 | use syncserver_db_common::{ 13 | error::{DbError, DbErrorKind}, 14 | results, DbPool, 15 | }; 16 | use syncstorage_settings::Settings; 17 | use url::Url; 18 | 19 | use crate::server::{metrics::Metrics, BlockingThreadpool}; 20 | 21 | /// Create/initialize a pool of managed Db connections 22 | pub async fn pool_from_settings( 23 | settings: &Settings, 24 | metrics: &Metrics, 25 | blocking_threadpool: Arc, 26 | ) -> Result, DbError> { 27 | let url = 28 | Url::parse(&settings.database_url).map_err(|e| DbErrorKind::InvalidUrl(e.to_string()))?; 29 | Ok(match url.scheme() { 30 | "mysql" => Box::new(mysql::pool::MysqlDbPool::new( 31 | settings, 32 | metrics, 33 | blocking_threadpool, 34 | )?), 35 | "spanner" => Box::new( 36 | spanner::pool::SpannerDbPool::new(settings, metrics, blocking_threadpool).await?, 37 | ), 38 | _ => Err(DbErrorKind::InvalidUrl(settings.database_url.to_owned()))?, 39 | }) 40 | } 41 | -------------------------------------------------------------------------------- /syncserver/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![warn(rust_2018_idioms)] 2 | #![allow(clippy::try_err)] 3 | 4 | #[macro_use] 5 | extern crate slog_scope; 6 | #[macro_use] 7 | extern crate validator_derive; 8 | 9 | #[macro_use] 10 | pub mod error; 11 | pub mod logging; 12 | pub mod server; 13 | pub mod tokenserver; 14 | pub mod web; 15 | -------------------------------------------------------------------------------- /syncserver/src/logging.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | 3 | use crate::error::ApiResult; 4 | 5 | use slog::{self, slog_o, Drain}; 6 | use slog_mozlog_json::MozLogJson; 7 | 8 | pub fn init_logging(json: bool) -> ApiResult<()> { 9 | let logger = if json { 10 | let hostname = hostname::get() 11 | .expect("Couldn't get hostname") 12 | .into_string() 13 | .expect("Couldn't get hostname"); 14 | 15 | let drain = MozLogJson::new(io::stdout()) 16 | .logger_name(format!( 17 | "{}-{}", 18 | env!("CARGO_PKG_NAME"), 19 | env!("CARGO_PKG_VERSION") 20 | )) 21 | .msg_type(format!("{}:log", env!("CARGO_PKG_NAME"))) 22 | .hostname(hostname) 23 | .build() 24 | .fuse(); 25 | let drain = slog_envlogger::new(drain); 26 | let drain = slog_async::Async::new(drain).build().fuse(); 27 | slog::Logger::root(drain, slog_o!()) 28 | } else { 29 | let decorator = slog_term::TermDecorator::new().build(); 30 | let drain = slog_term::FullFormat::new(decorator).build().fuse(); 31 | let drain = slog_envlogger::new(drain); 32 | let drain = slog_async::Async::new(drain).build().fuse(); 33 | slog::Logger::root(drain, slog_o!()) 34 | }; 35 | // XXX: cancel slog_scope's NoGlobalLoggerSet for now, it's difficult to 36 | // prevent it from potentially panicing during tests. reset_logging resets 37 | // the global logger during shutdown anyway: 38 | // https://github.com/slog-rs/slog/issues/169 39 | slog_scope::set_global_logger(logger).cancel_reset(); 40 | slog_stdlog::init().ok(); 41 | Ok(()) 42 | } 43 | 44 | pub fn reset_logging() { 45 | let logger = slog::Logger::root(slog::Discard, slog_o!()); 46 | slog_scope::set_global_logger(logger).cancel_reset(); 47 | } 48 | -------------------------------------------------------------------------------- /syncserver/src/main.rs: -------------------------------------------------------------------------------- 1 | //! Sync Storage Server for Sync 1.5 2 | #[macro_use] 3 | extern crate slog_scope; 4 | 5 | use std::{error::Error, sync::Arc}; 6 | 7 | use docopt::Docopt; 8 | use serde::Deserialize; 9 | 10 | use logging::init_logging; 11 | use syncserver::{logging, server}; 12 | use syncserver_settings::Settings; 13 | 14 | const USAGE: &str = " 15 | Usage: syncstorage [options] 16 | 17 | Options: 18 | -h, --help Show this message. 19 | --config=CONFIGFILE Syncstorage configuration file path. 20 | "; 21 | 22 | #[derive(Debug, Deserialize)] 23 | struct Args { 24 | flag_config: Option, 25 | } 26 | 27 | #[actix_web::main] 28 | async fn main() -> Result<(), Box> { 29 | let args: Args = Docopt::new(USAGE) 30 | .and_then(|d| d.deserialize()) 31 | .unwrap_or_else(|e| e.exit()); 32 | let settings = Settings::with_env_and_config_file(args.flag_config.as_deref())?; 33 | init_logging(!settings.human_logs).expect("Logging failed to initialize"); 34 | debug!("Starting up..."); 35 | 36 | // Set SENTRY_DSN environment variable to enable Sentry. 37 | // Avoid its default reqwest transport for now due to issues w/ 38 | // likely grpcio's boringssl 39 | let curl_transport_factory = |options: &sentry::ClientOptions| { 40 | Arc::new(sentry::transports::CurlHttpTransport::new(options)) as Arc 41 | }; 42 | // debug-images conflicts w/ our debug = 1 rustc build option: 43 | // https://github.com/getsentry/sentry-rust/issues/574 44 | let mut opts = sentry::apply_defaults(sentry::ClientOptions { 45 | // Note: set "debug: true," to diagnose sentry issues 46 | transport: Some(Arc::new(curl_transport_factory)), 47 | release: sentry::release_name!(), 48 | ..sentry::ClientOptions::default() 49 | }); 50 | opts.integrations.retain(|i| i.name() != "debug-images"); 51 | opts.default_integrations = false; 52 | let _sentry = sentry::init(opts); 53 | 54 | // Setup and run the server 55 | let banner = settings.banner(); 56 | let server = if !settings.syncstorage.enabled { 57 | server::Server::tokenserver_only_with_settings(settings) 58 | .await 59 | .unwrap() 60 | } else { 61 | server::Server::with_settings(settings).await.unwrap() 62 | }; 63 | info!("Server running on {}", banner); 64 | server.await?; 65 | info!("Server closing"); 66 | logging::reset_logging(); 67 | 68 | Ok(()) 69 | } 70 | -------------------------------------------------------------------------------- /syncserver/src/tokenserver/README.md: -------------------------------------------------------------------------------- 1 | # Tokenserver 2 | 3 | Tokenserver is used to: 4 | 1. Authenticate Sync clients via an OAuth token that clients receive from FxA 5 | 1. Direct Sync clients to the appropriate Sync Storage node 6 | 1. Present Sync clients with the encryption key necessary to access their Sync Storage nodes 7 | 8 | This functionality was previously provided by a [Python service](https://github.com/mozilla-services/tokenserver/). Originally, the intent was to use Tokenserver as a standalone authentication service for use with various, independent microservices. In practice, it is only used for Firefox Sync, so it was rewritten in Rust to be part of the same code repository as the Sync Storage node. 9 | 10 | 11 | 12 | 13 | 14 | 15 | - [Configuration](#configuration) 16 | - [Disabling Syncstorage](#disabling-syncstorage) 17 | - [Test Mode](#test-mode) 18 | - [Connecting to Firefox](#connecting-to-firefox) 19 | - [Database](#database) 20 | - [Running](#running) 21 | - [Testing](#testing) 22 | 23 | 24 | 25 | ## Configuration 26 | 27 | You can find example settings for Tokenserver in [config/local.example.toml](../../config/local.example.toml). The available settings are described in doc comments [here](../../src/tokenserver/settings.rs). 28 | 29 | ### Disabling Syncstorage 30 | 31 | Tokenserver can be run as a standalone service by disabling the Sync Storage endpoints. This can be done simply by setting the `disable_syncstorage` setting to `true`. **Note that the Sync Storage settings must still be set even when those endpoints are disabled.** 32 | 33 | ### Connecting to Firefox 34 | 35 | 1. Visit `about:config` in Firefox 36 | 1. Set `identity.sync.tokenserver.uri` to `http://localhost:8000/1.0/sync/1.5` 37 | 38 | This will point Firefox to the Tokenserver running alongside your local instance of Sync Storage. 39 | 40 | ## Database 41 | 42 | Prior to using Tokenserver, the migrations must be run. First, install the [diesel](https://diesel.rs/guides/getting-started) CLI tool: 43 | ``` 44 | cargo install diesel_cli 45 | ``` 46 | Then, run the migrations: 47 | ``` 48 | diesel --database-url mysql://sample_user:sample_password@localhost/tokenserver_rs migration --migration-dir src/tokenserver/migrations run 49 | ``` 50 | You should replace the above database Data Source Name (DSN) with the DSN of the database you are using. 51 | 52 | ## Running 53 | 54 | Tokenserver is run alongside Sync Storage using `make run`. 55 | 56 | ## Testing 57 | Tokenserver includes unit tests and a comprehensive suite of integration tests. These tests are run alongside the Sync Storage tests and can be run by following the instructions [here](../../README.md#unit-tests) and [here](../../README.md#end-to-end-tests). 58 | -------------------------------------------------------------------------------- /syncserver/src/tokenserver/logging.rs: -------------------------------------------------------------------------------- 1 | use actix_web::{ 2 | dev::{Service, ServiceRequest, ServiceResponse}, 3 | HttpMessage, 4 | }; 5 | use futures::future::Future; 6 | 7 | use super::LogItems; 8 | 9 | pub fn handle_request_log_line( 10 | request: ServiceRequest, 11 | service: &impl Service, Error = actix_web::Error>, 12 | ) -> impl Future, actix_web::Error>> { 13 | let items = LogItems::from(request.head()); 14 | request.extensions_mut().insert(items); 15 | let fut = service.call(request); 16 | 17 | async move { 18 | let sresp = fut.await?; 19 | 20 | if let Some(items) = sresp.request().extensions().get::() { 21 | info!("{}", items); 22 | } 23 | 24 | Ok(sresp) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /syncserver/src/web/middleware/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod rejectua; 2 | pub mod weave; 3 | 4 | // # Web Middleware 5 | // 6 | // Matches the [Sync Storage middleware](https://github.com/mozilla-services/server-syncstorage/blob/master/syncstorage/tweens.py) (tweens). 7 | 8 | use std::collections::HashMap; 9 | use std::future::Future; 10 | 11 | use actix_web::{ 12 | dev::{Service, ServiceRequest, ServiceResponse}, 13 | web::Data, 14 | HttpMessage, 15 | }; 16 | use syncserver_common::Metrics; 17 | use tokenserver_auth::TokenserverOrigin; 18 | 19 | use crate::error::{ApiError, ApiErrorKind}; 20 | use crate::server::ServerState; 21 | 22 | pub fn emit_http_status_with_tokenserver_origin( 23 | req: ServiceRequest, 24 | srv: &impl Service, Error = actix_web::Error>, 25 | ) -> impl Future, actix_web::Error>> { 26 | let fut = srv.call(req); 27 | 28 | async move { 29 | let res = fut.await?; 30 | let req = res.request(); 31 | let metrics = { 32 | let statsd_client = req 33 | .app_data::>() 34 | .map(|state| state.metrics.clone()) 35 | .ok_or_else(|| ApiError::from(ApiErrorKind::NoServerState))?; 36 | 37 | Metrics::from(&statsd_client) 38 | }; 39 | 40 | let mut tags = HashMap::default(); 41 | if let Some(origin) = req.extensions().get::().copied() { 42 | tags.insert("tokenserver_origin".to_string(), origin.to_string()); 43 | }; 44 | 45 | if res.status().is_informational() { 46 | metrics.incr_with_tags("http_1XX", tags); 47 | } else if res.status().is_success() { 48 | metrics.incr_with_tags("http_2XX", tags); 49 | } else if res.status().is_redirection() { 50 | metrics.incr_with_tags("http_3XX", tags); 51 | } else if res.status().is_client_error() { 52 | metrics.incr_with_tags("http_4XX", tags); 53 | } else if res.status().is_server_error() { 54 | metrics.incr_with_tags("http_5XX", tags); 55 | } 56 | 57 | Ok(res) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /syncserver/src/web/middleware/rejectua.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::type_complexity)] 2 | 3 | use actix_web::{ 4 | body::EitherBody, 5 | dev::{Service, ServiceRequest, ServiceResponse}, 6 | http::header::USER_AGENT, 7 | FromRequest, HttpResponse, 8 | }; 9 | use futures::future::LocalBoxFuture; 10 | use lazy_static::lazy_static; 11 | use regex::Regex; 12 | 13 | use crate::server::MetricsWrapper; 14 | 15 | lazy_static! { 16 | // e.g. "Firefox-iOS-Sync/18.0b1 (iPhone; iPhone OS 13.2.2) (Fennec (synctesting))" 17 | // https://github.com/mozilla-mobile/firefox-ios/blob/v19.x/Shared/UserAgent.swift#L12 18 | static ref IOS_UA_REGEX: Regex = Regex::new( 19 | r"(?x) 20 | ^ 21 | Firefox-iOS-Sync/ 22 | (?P[0-9]+)\.[.0-9]+ # . 23 | b.* # b 24 | \s\(.+ # ( 25 | ;\siPhone\sOS # ; iPhone OS 26 | \s.+\) # ) 27 | \s\(.*\) # () 28 | $ 29 | " 30 | ) 31 | .unwrap(); 32 | } 33 | 34 | pub fn reject_user_agent( 35 | request: ServiceRequest, 36 | service: &(impl Service, Error = actix_web::Error> 37 | + 'static), 38 | ) -> LocalBoxFuture<'static, Result>, actix_web::Error>> { 39 | match request.headers().get(USER_AGENT).cloned() { 40 | Some(header) if header.to_str().is_ok_and(should_reject) => Box::pin(async move { 41 | trace!("Rejecting User-Agent: {:?}", header); 42 | let (req, payload) = request.into_parts(); 43 | MetricsWrapper::extract(&req) 44 | .await? 45 | .0 46 | .incr("error.rejectua"); 47 | let sreq = ServiceRequest::from_parts(req, payload); 48 | 49 | Ok(sreq.into_response( 50 | HttpResponse::ServiceUnavailable() 51 | .body("0") 52 | .map_into_right_body(), 53 | )) 54 | }), 55 | _ => { 56 | let fut = service.call(request); 57 | Box::pin(async move { fut.await.map(|resp| resp.map_into_left_body()) }) 58 | } 59 | } 60 | } 61 | 62 | /// Determine if a User-Agent should be rejected w/ an error response. 63 | /// 64 | /// firefox-ios < v20 suffers from a bug where our response headers 65 | /// can cause it to crash. They're sent an error response instead that 66 | /// avoids the crash. 67 | /// 68 | /// Dev builds were originally labeled as v0 (or now "Firefox-iOS-Sync/dev") so 69 | /// we don't reject those. 70 | /// 71 | /// https://github.com/mozilla-services/syncstorage-rs/issues/293 72 | fn should_reject(ua: &str) -> bool { 73 | let major = IOS_UA_REGEX 74 | .captures(ua) 75 | .and_then(|captures| captures.name("major")) 76 | .and_then(|major| major.as_str().parse::().ok()) 77 | .unwrap_or(20); 78 | 0 < major && major < 20 79 | } 80 | -------------------------------------------------------------------------------- /syncserver/src/web/mod.rs: -------------------------------------------------------------------------------- 1 | //! Web authentication, handlers, and middleware 2 | pub mod auth; 3 | pub mod error; 4 | pub mod extractors; 5 | pub mod handlers; 6 | pub mod middleware; 7 | mod transaction; 8 | 9 | // Known DockerFlow commands for Ops callbacks 10 | pub const DOCKER_FLOW_ENDPOINTS: [&str; 4] = [ 11 | "/__heartbeat__", 12 | "/__lbheartbeat__", 13 | "/__version__", 14 | "/__error__", 15 | ]; 16 | -------------------------------------------------------------------------------- /syncserver/version.json: -------------------------------------------------------------------------------- 1 | { 2 | "build": "TBD", 3 | "commit": "TBD", 4 | "source": "https://github.com/mozilla-services/syncstorage-rs", 5 | "version": "TBD" 6 | } 7 | -------------------------------------------------------------------------------- /syncstorage-db-common/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "syncstorage-db-common" 3 | version.workspace = true 4 | license.workspace = true 5 | authors.workspace = true 6 | edition.workspace = true 7 | 8 | [dependencies] 9 | backtrace.workspace = true 10 | chrono.workspace = true 11 | futures.workspace = true 12 | lazy_static.workspace = true 13 | http.workspace = true 14 | serde.workspace = true 15 | serde_json.workspace = true 16 | thiserror.workspace = true 17 | 18 | async-trait = "0.1.40" 19 | # diesel = 1.4 20 | diesel = { workspace = true, features = ["mysql", "r2d2"] } 21 | syncserver-common = { path = "../syncserver-common" } 22 | syncserver-db-common = { path = "../syncserver-db-common" } 23 | -------------------------------------------------------------------------------- /syncstorage-db-common/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use backtrace::Backtrace; 4 | use http::StatusCode; 5 | use syncserver_common::{impl_fmt_display, ReportableError}; 6 | use thiserror::Error; 7 | 8 | /// Errors common to all supported syncstorage database backends. These errors can be thought of 9 | /// as being related more to the syncstorage application logic as opposed to a particular 10 | /// database backend. 11 | #[derive(Debug)] 12 | pub struct SyncstorageDbError { 13 | kind: SyncstorageDbErrorKind, 14 | pub status: StatusCode, 15 | pub backtrace: Backtrace, 16 | } 17 | 18 | #[derive(Debug, Error)] 19 | enum SyncstorageDbErrorKind { 20 | #[error("Specified collection does not exist")] 21 | CollectionNotFound, 22 | 23 | #[error("Specified bso does not exist")] 24 | BsoNotFound, 25 | 26 | #[error("Specified batch does not exist")] 27 | BatchNotFound, 28 | 29 | #[error("An attempt at a conflicting write")] 30 | Conflict, 31 | 32 | #[error("Unexpected error: {}", _0)] 33 | Internal(String), 34 | 35 | #[error("User over quota")] 36 | Quota, 37 | } 38 | 39 | impl SyncstorageDbError { 40 | pub fn batch_not_found() -> Self { 41 | SyncstorageDbErrorKind::BatchNotFound.into() 42 | } 43 | 44 | pub fn bso_not_found() -> Self { 45 | SyncstorageDbErrorKind::BsoNotFound.into() 46 | } 47 | 48 | pub fn collection_not_found() -> Self { 49 | SyncstorageDbErrorKind::CollectionNotFound.into() 50 | } 51 | 52 | pub fn conflict() -> Self { 53 | SyncstorageDbErrorKind::Conflict.into() 54 | } 55 | 56 | pub fn internal(msg: String) -> Self { 57 | SyncstorageDbErrorKind::Internal(msg).into() 58 | } 59 | 60 | pub fn quota() -> Self { 61 | SyncstorageDbErrorKind::Quota.into() 62 | } 63 | } 64 | 65 | pub trait DbErrorIntrospect { 66 | fn is_collection_not_found(&self) -> bool; 67 | fn is_conflict(&self) -> bool; 68 | fn is_quota(&self) -> bool; 69 | fn is_bso_not_found(&self) -> bool; 70 | fn is_batch_not_found(&self) -> bool; 71 | } 72 | 73 | impl DbErrorIntrospect for SyncstorageDbError { 74 | fn is_collection_not_found(&self) -> bool { 75 | matches!(self.kind, SyncstorageDbErrorKind::CollectionNotFound) 76 | } 77 | 78 | fn is_conflict(&self) -> bool { 79 | matches!(self.kind, SyncstorageDbErrorKind::Conflict) 80 | } 81 | 82 | fn is_quota(&self) -> bool { 83 | matches!(self.kind, SyncstorageDbErrorKind::Quota) 84 | } 85 | 86 | fn is_bso_not_found(&self) -> bool { 87 | matches!(self.kind, SyncstorageDbErrorKind::BsoNotFound) 88 | } 89 | 90 | fn is_batch_not_found(&self) -> bool { 91 | matches!(self.kind, SyncstorageDbErrorKind::BatchNotFound) 92 | } 93 | } 94 | 95 | impl ReportableError for SyncstorageDbError { 96 | fn reportable_source(&self) -> Option<&(dyn ReportableError + 'static)> { 97 | None 98 | } 99 | 100 | fn is_sentry_event(&self) -> bool { 101 | !matches!(&self.kind, SyncstorageDbErrorKind::Conflict) 102 | } 103 | 104 | fn metric_label(&self) -> Option<&'static str> { 105 | match self.kind { 106 | SyncstorageDbErrorKind::Conflict => Some("storage.conflict"), 107 | _ => None, 108 | } 109 | } 110 | 111 | fn backtrace(&self) -> Option<&Backtrace> { 112 | Some(&self.backtrace) 113 | } 114 | } 115 | 116 | impl From for SyncstorageDbError { 117 | fn from(kind: SyncstorageDbErrorKind) -> Self { 118 | let status = match kind { 119 | SyncstorageDbErrorKind::CollectionNotFound | SyncstorageDbErrorKind::BsoNotFound => { 120 | StatusCode::NOT_FOUND 121 | } 122 | // Matching the Python code here (a 400 vs 404) 123 | SyncstorageDbErrorKind::BatchNotFound => StatusCode::BAD_REQUEST, 124 | // NOTE: the protocol specification states that we should return a 125 | // "409 Conflict" response here, but clients currently do not 126 | // handle these respones very well: 127 | // * desktop bug: https://bugzilla.mozilla.org/show_bug.cgi?id=959034 128 | // * android bug: https://bugzilla.mozilla.org/show_bug.cgi?id=959032 129 | SyncstorageDbErrorKind::Conflict => StatusCode::SERVICE_UNAVAILABLE, 130 | SyncstorageDbErrorKind::Quota => StatusCode::FORBIDDEN, 131 | _ => StatusCode::INTERNAL_SERVER_ERROR, 132 | }; 133 | 134 | Self { 135 | kind, 136 | status, 137 | backtrace: Backtrace::new(), 138 | } 139 | } 140 | } 141 | 142 | impl_fmt_display!(SyncstorageDbError, SyncstorageDbErrorKind); 143 | -------------------------------------------------------------------------------- /syncstorage-db-common/src/results.rs: -------------------------------------------------------------------------------- 1 | //! Result types for database methods. 2 | use std::collections::HashMap; 3 | 4 | use diesel::{ 5 | sql_types::{BigInt, Integer, Nullable, Text}, 6 | Queryable, QueryableByName, 7 | }; 8 | use serde::{Deserialize, Serialize}; 9 | 10 | use super::params; 11 | use crate::util::SyncTimestamp; 12 | 13 | pub type LockCollection = (); 14 | pub type GetBsoTimestamp = SyncTimestamp; 15 | pub type GetCollectionTimestamps = HashMap; 16 | pub type GetCollectionTimestamp = SyncTimestamp; 17 | pub type GetCollectionCounts = HashMap; 18 | pub type GetCollectionUsage = HashMap; 19 | pub type GetStorageTimestamp = SyncTimestamp; 20 | pub type GetStorageUsage = u64; 21 | pub type DeleteStorage = (); 22 | pub type DeleteCollection = SyncTimestamp; 23 | pub type DeleteBsos = SyncTimestamp; 24 | pub type DeleteBso = SyncTimestamp; 25 | pub type PutBso = SyncTimestamp; 26 | 27 | #[derive(Debug, Default, Clone)] 28 | pub struct CreateBatch { 29 | pub id: String, 30 | pub size: Option, 31 | } 32 | 33 | pub type ValidateBatch = bool; 34 | pub type AppendToBatch = (); 35 | pub type GetBatch = params::Batch; 36 | pub type DeleteBatch = (); 37 | pub type CommitBatch = SyncTimestamp; 38 | pub type ValidateBatchId = (); 39 | pub type Check = bool; 40 | 41 | #[derive(Debug, Default)] 42 | pub struct GetQuotaUsage { 43 | pub total_bytes: usize, 44 | pub count: i32, 45 | } 46 | 47 | #[derive(Debug, Default, Deserialize, Queryable, QueryableByName, Serialize)] 48 | pub struct GetBso { 49 | #[sql_type = "Text"] 50 | pub id: String, 51 | #[sql_type = "BigInt"] 52 | pub modified: SyncTimestamp, 53 | #[sql_type = "Text"] 54 | pub payload: String, 55 | #[serde(skip_serializing_if = "Option::is_none")] 56 | #[sql_type = "Nullable"] 57 | pub sortindex: Option, 58 | // NOTE: expiry (ttl) is never rendered to clients and only loaded for 59 | // tests: this and its associated queries/loading could be wrapped in 60 | // #[cfg(test)] 61 | #[serde(skip_serializing)] 62 | #[serde(skip_deserializing)] 63 | #[sql_type = "BigInt"] 64 | pub expiry: i64, 65 | } 66 | 67 | #[derive(Debug, Default)] 68 | pub struct Paginated 69 | where 70 | T: Serialize, 71 | { 72 | pub items: Vec, 73 | pub offset: Option, 74 | } 75 | 76 | pub type GetBsos = Paginated; 77 | pub type GetBsoIds = Paginated; 78 | 79 | #[derive(Debug, Default, Deserialize, Serialize)] 80 | pub struct PostBsos { 81 | pub modified: SyncTimestamp, 82 | pub success: Vec, 83 | pub failed: HashMap, 84 | } 85 | 86 | #[derive(Debug, Default)] 87 | pub struct ConnectionInfo { 88 | pub age: i64, 89 | pub spanner_age: i64, 90 | pub spanner_idle: i64, 91 | } 92 | 93 | pub type GetCollectionId = i32; 94 | 95 | pub type CreateCollection = i32; 96 | 97 | pub type UpdateCollection = SyncTimestamp; 98 | -------------------------------------------------------------------------------- /syncstorage-db/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "syncstorage-db" 3 | version.workspace = true 4 | license.workspace = true 5 | authors.workspace = true 6 | edition.workspace = true 7 | 8 | [dependencies] 9 | env_logger.workspace = true 10 | futures.workspace = true 11 | lazy_static.workspace = true 12 | rand.workspace = true 13 | slog-scope.workspace = true 14 | 15 | async-trait = "0.1.40" 16 | log = { version = "0.4", features = [ 17 | "max_level_debug", 18 | "release_max_level_info", 19 | ] } 20 | syncserver-common = { path = "../syncserver-common" } 21 | syncserver-db-common = { path = "../syncserver-db-common" } 22 | syncserver-settings = { path = "../syncserver-settings" } 23 | syncstorage-db-common = { path = "../syncstorage-db-common" } 24 | syncstorage-mysql = { path = "../syncstorage-mysql", optional = true } 25 | syncstorage-settings = { path = "../syncstorage-settings" } 26 | syncstorage-spanner = { path = "../syncstorage-spanner", optional = true } 27 | tokio = { workspace = true, features = ["macros", "sync"] } 28 | 29 | [features] 30 | mysql = ['syncstorage-mysql'] 31 | spanner = ['syncstorage-spanner'] 32 | -------------------------------------------------------------------------------- /syncstorage-db/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Generic db abstration. 2 | 3 | #[cfg(test)] 4 | #[macro_use] 5 | extern crate slog_scope; 6 | 7 | pub mod mock; 8 | #[cfg(test)] 9 | mod tests; 10 | 11 | #[cfg(feature = "mysql")] 12 | pub type DbPoolImpl = syncstorage_mysql::MysqlDbPool; 13 | #[cfg(feature = "mysql")] 14 | pub use syncstorage_mysql::DbError; 15 | #[cfg(feature = "mysql")] 16 | pub type DbImpl = syncstorage_mysql::MysqlDb; 17 | 18 | #[cfg(feature = "spanner")] 19 | pub type DbPoolImpl = syncstorage_spanner::SpannerDbPool; 20 | #[cfg(feature = "spanner")] 21 | pub use syncstorage_spanner::DbError; 22 | #[cfg(feature = "spanner")] 23 | pub type DbImpl = syncstorage_spanner::SpannerDb; 24 | 25 | pub use syncserver_db_common::{GetPoolState, PoolState}; 26 | pub use syncstorage_db_common::error::DbErrorIntrospect; 27 | 28 | pub use syncstorage_db_common::{ 29 | params, results, 30 | util::{to_rfc3339, SyncTimestamp}, 31 | Db, DbPool, Sorting, UserIdentifier, 32 | }; 33 | 34 | #[cfg(all(feature = "mysql", feature = "spanner"))] 35 | compile_error!("only one of the \"mysql\" and \"spanner\" features can be enabled at a time"); 36 | 37 | #[cfg(not(any(feature = "mysql", feature = "spanner")))] 38 | compile_error!("exactly one of the \"mysql\" and \"spanner\" features must be enabled"); 39 | -------------------------------------------------------------------------------- /syncstorage-db/src/mock.rs: -------------------------------------------------------------------------------- 1 | //! Mock db implementation with methods stubbed to return default values. 2 | #![allow(clippy::new_without_default)] 3 | use async_trait::async_trait; 4 | use futures::future; 5 | use syncserver_db_common::{GetPoolState, PoolState}; 6 | use syncstorage_db_common::{params, results, util::SyncTimestamp, Db, DbPool}; 7 | 8 | use crate::DbError; 9 | 10 | type DbFuture<'a, T> = syncserver_db_common::DbFuture<'a, T, DbError>; 11 | 12 | #[derive(Clone, Debug)] 13 | pub struct MockDbPool; 14 | 15 | impl MockDbPool { 16 | pub fn new() -> Self { 17 | MockDbPool 18 | } 19 | } 20 | 21 | #[async_trait] 22 | impl DbPool for MockDbPool { 23 | type Error = DbError; 24 | 25 | async fn get(&self) -> Result>, Self::Error> { 26 | Ok(Box::new(MockDb::new())) 27 | } 28 | 29 | fn validate_batch_id(&self, _: params::ValidateBatchId) -> Result<(), DbError> { 30 | Ok(()) 31 | } 32 | 33 | fn box_clone(&self) -> Box> { 34 | Box::new(self.clone()) 35 | } 36 | } 37 | 38 | impl GetPoolState for MockDbPool { 39 | fn state(&self) -> PoolState { 40 | PoolState::default() 41 | } 42 | } 43 | 44 | #[derive(Clone, Debug)] 45 | pub struct MockDb; 46 | 47 | impl MockDb { 48 | pub fn new() -> Self { 49 | MockDb 50 | } 51 | } 52 | 53 | macro_rules! mock_db_method { 54 | ($name:ident, $type:ident) => { 55 | mock_db_method!($name, $type, results::$type); 56 | }; 57 | ($name:ident, $type:ident, $result:ty) => { 58 | fn $name(&self, _params: params::$type) -> DbFuture<'_, $result> { 59 | let result: $result = Default::default(); 60 | Box::pin(future::ok(result)) 61 | } 62 | }; 63 | } 64 | 65 | impl Db for MockDb { 66 | type Error = DbError; 67 | 68 | fn commit(&self) -> DbFuture<'_, ()> { 69 | Box::pin(future::ok(())) 70 | } 71 | 72 | fn rollback(&self) -> DbFuture<'_, ()> { 73 | Box::pin(future::ok(())) 74 | } 75 | 76 | fn begin(&self, _for_write: bool) -> DbFuture<'_, ()> { 77 | Box::pin(future::ok(())) 78 | } 79 | 80 | fn box_clone(&self) -> Box> { 81 | Box::new(self.clone()) 82 | } 83 | 84 | fn check(&self) -> DbFuture<'_, results::Check> { 85 | Box::pin(future::ok(true)) 86 | } 87 | 88 | mock_db_method!(lock_for_read, LockCollection); 89 | mock_db_method!(lock_for_write, LockCollection); 90 | mock_db_method!(get_collection_timestamps, GetCollectionTimestamps); 91 | mock_db_method!(get_collection_timestamp, GetCollectionTimestamp); 92 | mock_db_method!(get_collection_counts, GetCollectionCounts); 93 | mock_db_method!(get_collection_usage, GetCollectionUsage); 94 | mock_db_method!(get_storage_timestamp, GetStorageTimestamp); 95 | mock_db_method!(get_storage_usage, GetStorageUsage); 96 | mock_db_method!(get_quota_usage, GetQuotaUsage); 97 | mock_db_method!(delete_storage, DeleteStorage); 98 | mock_db_method!(delete_collection, DeleteCollection); 99 | mock_db_method!(delete_bsos, DeleteBsos); 100 | mock_db_method!(get_bsos, GetBsos); 101 | mock_db_method!(get_bso_ids, GetBsoIds); 102 | mock_db_method!(post_bsos, PostBsos); 103 | mock_db_method!(delete_bso, DeleteBso); 104 | mock_db_method!(get_bso, GetBso, Option); 105 | mock_db_method!(get_bso_timestamp, GetBsoTimestamp); 106 | mock_db_method!(put_bso, PutBso); 107 | mock_db_method!(create_batch, CreateBatch); 108 | mock_db_method!(validate_batch, ValidateBatch); 109 | mock_db_method!(append_to_batch, AppendToBatch); 110 | mock_db_method!(get_batch, GetBatch, Option); 111 | mock_db_method!(commit_batch, CommitBatch); 112 | 113 | fn get_connection_info(&self) -> results::ConnectionInfo { 114 | results::ConnectionInfo::default() 115 | } 116 | 117 | mock_db_method!(get_collection_id, GetCollectionId); 118 | mock_db_method!(create_collection, CreateCollection); 119 | mock_db_method!(update_collection, UpdateCollection); 120 | 121 | fn timestamp(&self) -> SyncTimestamp { 122 | Default::default() 123 | } 124 | 125 | fn set_timestamp(&self, _: SyncTimestamp) {} 126 | 127 | mock_db_method!(delete_batch, DeleteBatch); 128 | 129 | fn clear_coll_cache(&self) -> DbFuture<'_, ()> { 130 | Box::pin(future::ok(())) 131 | } 132 | 133 | fn set_quota(&mut self, _: bool, _: usize, _: bool) {} 134 | } 135 | 136 | unsafe impl Send for MockDb {} 137 | -------------------------------------------------------------------------------- /syncstorage-db/src/tests/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | #[macro_use] 3 | pub mod support; 4 | 5 | #[cfg(test)] 6 | pub mod batch; 7 | #[cfg(test)] 8 | mod db; 9 | -------------------------------------------------------------------------------- /syncstorage-mysql/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "syncstorage-mysql" 3 | version.workspace = true 4 | license.workspace = true 5 | authors.workspace = true 6 | edition.workspace = true 7 | 8 | [dependencies] 9 | backtrace.workspace = true 10 | base64.workspace = true 11 | futures.workspace = true 12 | http.workspace = true 13 | slog-scope.workspace = true 14 | thiserror.workspace = true 15 | 16 | async-trait = "0.1.40" 17 | # There appears to be a compilation error with diesel 18 | diesel = { workspace = true, features = ["mysql", "r2d2"] } 19 | diesel_logger = { workspace = true } 20 | diesel_migrations = { workspace = true, features = ["mysql"] } 21 | syncserver-common = { path = "../syncserver-common" } 22 | syncserver-db-common = { path = "../syncserver-db-common" } 23 | syncstorage-db-common = { path = "../syncstorage-db-common" } 24 | syncstorage-settings = { path = "../syncstorage-settings" } 25 | url = "2.1" 26 | 27 | [dev-dependencies] 28 | env_logger.workspace = true 29 | syncserver-settings = { path = "../syncserver-settings" } 30 | -------------------------------------------------------------------------------- /syncstorage-mysql/migrations/2018-08-28-010336_init/down.sql: -------------------------------------------------------------------------------- 1 | -- DROP TABLE IF EXISTS `bso`; 2 | -- DROP TABLE IF EXISTS `collections`; 3 | -- DROP TABLE IF EXISTS `user_collections`; 4 | -- DROP TABLE IF EXISTS `batches`; -------------------------------------------------------------------------------- /syncstorage-mysql/migrations/2018-08-28-010336_init/up.sql: -------------------------------------------------------------------------------- 1 | -- /*!40100 DEFAULT CHARACTER SET latin1 */ 2 | 3 | -- DROP TABLE IF EXISTS `bso`; 4 | -- XXX: bsov1, etc 5 | CREATE TABLE IF NOT EXISTS `bso`( 6 | `user_id` INT NOT NULL, 7 | `collection_id` INT NOT NULL, 8 | `id` VARCHAR(64) NOT NULL, 9 | 10 | `sortindex` INT, 11 | 12 | `payload` MEDIUMTEXT NOT NULL, 13 | 14 | -- last modified time in milliseconds since epoch 15 | `modified` BIGINT NOT NULL, 16 | -- expiration in milliseconds since epoch 17 | `expiry` BIGINT DEFAULT '3153600000000' NOT NULL, 18 | 19 | PRIMARY KEY (`user_id`, `collection_id`, `id`), 20 | KEY `bso_expiry_idx` (`expiry`), 21 | KEY `bso_usr_col_mod_idx` (`user_id`, `collection_id`, `modified`) 22 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 23 | 24 | 25 | DROP TABLE IF EXISTS `collections`; 26 | CREATE TABLE IF NOT EXISTS `collections`( 27 | `id` INT PRIMARY KEY NOT NULL AUTO_INCREMENT, 28 | `name` VARCHAR(32) UNIQUE NOT NULL 29 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 30 | INSERT INTO collections (id, name) VALUES 31 | ( 1, "clients"), 32 | ( 2, "crypto"), 33 | ( 3, "forms"), 34 | ( 4, "history"), 35 | ( 5, "keys"), 36 | ( 6, "meta"), 37 | ( 7, "bookmarks"), 38 | ( 8, "prefs"), 39 | ( 9, "tabs"), 40 | (10, "passwords"), 41 | (11, "addons"), 42 | (12, "addresses"), 43 | (13, "creditcards"); 44 | 45 | 46 | -- DROP TABLE IF EXISTS `user_collections`; 47 | CREATE TABLE IF NOT EXISTS `user_collections`( 48 | `user_id` INT NOT NULL, 49 | `collection_id` INT NOT NULL, 50 | -- last modified time in milliseconds since epoch 51 | `modified` BIGINT NOT NULL, 52 | PRIMARY KEY (`user_id`, `collection_id`) 53 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 54 | 55 | 56 | -- XXX: based on the go version (bsos is a concatenated blob of BSO jsons separated by newlines) 57 | -- DROP TABLE IF EXISTS `batches`; 58 | CREATE TABLE IF NOT EXISTS `batches`( 59 | `user_id` INT NOT NULL, 60 | `collection_id` INT NOT NULL, 61 | `id` BIGINT NOT NULL, 62 | 63 | `bsos` LONGTEXT NOT NULL, 64 | 65 | -- expiration in milliseconds since epoch 66 | `expiry` BIGINT DEFAULT '3153600000000' NOT NULL, 67 | 68 | PRIMARY KEY (`user_id`, `collection_id`, `id`) 69 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 70 | -------------------------------------------------------------------------------- /syncstorage-mysql/migrations/2019-09-11-164500/down.sql: -------------------------------------------------------------------------------- 1 | -- At this point, it's sanest to just drop the tables rather than revert them 2 | -- there are a number of non-backwards compatible changes performed and data 3 | -- corruption is HIGHLY likely. 4 | -- Best just try and install the python version (probably in a docker), and 5 | -- let the client try and reconnect and restore. 6 | DROP TABLE IF EXISTS `bso`; 7 | DROP TABLE IF EXISTS `collections`; 8 | DROP TABLE IF EXISTS `user_collections`; 9 | DROP TABLE IF EXISTS `batches`; 10 | -------------------------------------------------------------------------------- /syncstorage-mysql/migrations/2019-09-11-164500/up.sql: -------------------------------------------------------------------------------- 1 | -- tests to see if columns need adjustment 2 | -- this is because the `init` may create the tables 3 | -- with column names already correct 4 | 5 | CREATE PROCEDURE UPDATE_165600() 6 | BEGIN 7 | IF EXISTS( SELECT column_name 8 | FROM INFORMATION_SCHEMA.COLUMNS 9 | WHERE table_schema=database() AND table_name='bso' AND column_name="user_id") 10 | THEN 11 | BEGIN 12 | alter table `bso` change column `user_id` `userid` int(11) not null; 13 | alter table `bso` change column `collection_id` `collection` int(11) not null; 14 | alter table `bso` change column `expiry` `ttl` bigint(20) not null; 15 | END; 16 | END IF; 17 | 18 | IF EXISTS( SELECT column_name 19 | FROM INFORMATION_SCHEMA.COLUMNS 20 | WHERE table_schema=database() AND table_name='batches' AND column_name="user_id") 21 | THEN 22 | BEGIN 23 | alter table `batches` change column `user_id` `userid` int(11) not null; 24 | alter table `batches` change column `collection_id` `collection` int(11) not null; 25 | END; 26 | END IF; 27 | 28 | IF EXISTS( SELECT column_name 29 | FROM INFORMATION_SCHEMA.COLUMNS 30 | WHERE table_schema=database() AND table_name='user_collections' AND column_name="user_id") 31 | THEN 32 | BEGIN 33 | alter table `user_collections` change column `user_id` `userid` int(11) not null; 34 | alter table `user_collections` change column `collection_id` `collection` int(11) not null; 35 | alter table `user_collections` change column `modified` `last_modified` bigint(20) not null; 36 | END; 37 | END IF; 38 | 39 | -- must be last in case of error 40 | -- the following column is not used, but preserved for legacy and stand alone systems. 41 | IF NOT EXISTS( SELECT column_name 42 | FROM INFORMATION_SCHEMA.COLUMNS 43 | where table_schema=database() AND table_name='bso' AND column_name="payload_size") 44 | THEN 45 | BEGIN 46 | alter table `bso` add column `payload_size` int(11) default 0; 47 | END; 48 | 49 | END IF; 50 | END; 51 | 52 | CALL UPDATE_165600(); 53 | 54 | DROP PROCEDURE UPDATE_165600; 55 | -------------------------------------------------------------------------------- /syncstorage-mysql/migrations/2019-09-25-174347_min_collection_id/down.sql: -------------------------------------------------------------------------------- 1 | DELETE FROM collections 2 | WHERE name = "" 3 | -------------------------------------------------------------------------------- /syncstorage-mysql/migrations/2019-09-25-174347_min_collection_id/up.sql: -------------------------------------------------------------------------------- 1 | -- Reserve space for additions to the standard collections 2 | INSERT INTO collections (id, name) 3 | VALUES (100, "") 4 | -------------------------------------------------------------------------------- /syncstorage-mysql/migrations/2020-04-03-102015_change_userid/down.sql: -------------------------------------------------------------------------------- 1 | -- This file should undo anything in `up.sql` -------------------------------------------------------------------------------- /syncstorage-mysql/migrations/2020-04-03-102015_change_userid/up.sql: -------------------------------------------------------------------------------- 1 | alter table `batches` change column `userid` `userid` bigint(20) not null; 2 | alter table `bso` change column `userid` `userid` bigint(20) not null; 3 | alter table `user_collections` change column `userid` `userid` bigint(20) not null; 4 | -------------------------------------------------------------------------------- /syncstorage-mysql/migrations/2020-06-12-231034_new_batch/down.sql: -------------------------------------------------------------------------------- 1 | -- This file should undo anything in `up.sql` -------------------------------------------------------------------------------- /syncstorage-mysql/migrations/2020-06-12-231034_new_batch/up.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE `batches`; 2 | 3 | CREATE TABLE `batch_uploads` ( 4 | `batch` bigint(20) NOT NULL, 5 | `userid` bigint(20) NOT NULL, 6 | `collection` int(11) NOT NULL, 7 | PRIMARY KEY (`batch`, `userid`) 8 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 9 | 10 | CREATE TABLE `batch_upload_items` ( 11 | `batch` bigint(20) NOT NULL, 12 | `userid` bigint(20) NOT NULL, 13 | `id` varchar(64) NOT NULL, 14 | `sortindex` int(11) DEFAULT NULL, 15 | `payload` mediumtext, 16 | `payload_size` int(11) DEFAULT NULL, 17 | `ttl_offset` int(11) DEFAULT NULL, 18 | PRIMARY KEY (`batch`, `userid`, `id`) 19 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 20 | -------------------------------------------------------------------------------- /syncstorage-mysql/migrations/2020-08-24-091401_add_quota/down.sql: -------------------------------------------------------------------------------- 1 | alter table user_collections drop column total_bytes; 2 | alter table user_collections drop column count; -------------------------------------------------------------------------------- /syncstorage-mysql/migrations/2020-08-24-091401_add_quota/up.sql: -------------------------------------------------------------------------------- 1 | alter table user_collections add column (total_bytes bigint, count int); -------------------------------------------------------------------------------- /syncstorage-mysql/src/batch_commit.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO bso (userid, collection, id, modified, sortindex, ttl, payload, payload_size) 2 | SELECT 3 | ?, 4 | ?, 5 | id, 6 | ?, 7 | sortindex, 8 | COALESCE((ttl_offset * 1000) + ?, ?), 9 | COALESCE(payload, ''), 10 | COALESCE(payload_size, 0) 11 | FROM batch_upload_items 12 | WHERE batch = ? 13 | AND userid = ? 14 | ON DUPLICATE KEY UPDATE 15 | modified = ?, 16 | sortindex = COALESCE(batch_upload_items.sortindex, bso.sortindex), 17 | ttl = COALESCE((batch_upload_items.ttl_offset * 1000) + ?, bso.ttl), 18 | payload = COALESCE(batch_upload_items.payload, bso.payload), 19 | payload_size = COALESCE(batch_upload_items.payload_size, bso.payload_size) 20 | -------------------------------------------------------------------------------- /syncstorage-mysql/src/diesel_ext.rs: -------------------------------------------------------------------------------- 1 | use diesel::{ 2 | backend::Backend, 3 | insertable::CanInsertInSingleQuery, 4 | mysql::Mysql, 5 | query_builder::{AstPass, InsertStatement, QueryFragment, QueryId}, 6 | query_dsl::methods::LockingDsl, 7 | result::QueryResult, 8 | Expression, RunQueryDsl, Table, 9 | }; 10 | 11 | /// Emit MySQL <= 5.7's `LOCK IN SHARE MODE` 12 | /// 13 | /// MySQL 8 supports `FOR SHARE` as an alias (which diesel natively supports) 14 | pub trait LockInShareModeDsl { 15 | type Output; 16 | 17 | fn lock_in_share_mode(self) -> Self::Output; 18 | } 19 | 20 | impl LockInShareModeDsl for T 21 | where 22 | T: LockingDsl, 23 | { 24 | type Output = >::Output; 25 | 26 | fn lock_in_share_mode(self) -> Self::Output { 27 | self.with_lock(LockInShareMode) 28 | } 29 | } 30 | 31 | #[derive(Debug, Clone, Copy, QueryId)] 32 | pub struct LockInShareMode; 33 | 34 | impl QueryFragment for LockInShareMode { 35 | fn walk_ast(&self, mut out: AstPass<'_, Mysql>) -> QueryResult<()> { 36 | out.push_sql(" LOCK IN SHARE MODE"); 37 | Ok(()) 38 | } 39 | } 40 | 41 | #[derive(Debug, Clone)] 42 | pub struct OnDuplicateKeyUpdate(Box>, X); 43 | 44 | impl QueryFragment for OnDuplicateKeyUpdate 45 | where 46 | DB: Backend, 47 | T: Table, 48 | T::FromClause: QueryFragment, 49 | U: QueryFragment + CanInsertInSingleQuery, 50 | Op: QueryFragment, 51 | Ret: QueryFragment, 52 | X: Expression, 53 | { 54 | fn walk_ast(&self, mut out: AstPass<'_, DB>) -> QueryResult<()> { 55 | self.0.walk_ast(out.reborrow())?; 56 | out.push_sql(" ON DUPLICATE KEY UPDATE "); 57 | //self.1.walk_ast(out.reborrow())?; 58 | Ok(()) 59 | } 60 | } 61 | 62 | impl RunQueryDsl for OnDuplicateKeyUpdate {} 63 | 64 | impl QueryId for OnDuplicateKeyUpdate { 65 | type QueryId = (); 66 | 67 | const HAS_STATIC_QUERY_ID: bool = false; 68 | } 69 | -------------------------------------------------------------------------------- /syncstorage-mysql/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_local_definitions)] 2 | #[macro_use] 3 | extern crate diesel; 4 | #[macro_use] 5 | extern crate diesel_migrations; 6 | #[macro_use] 7 | extern crate slog_scope; 8 | 9 | #[macro_use] 10 | mod batch; 11 | mod diesel_ext; 12 | mod error; 13 | mod models; 14 | mod pool; 15 | mod schema; 16 | #[cfg(test)] 17 | mod test; 18 | 19 | pub use error::DbError; 20 | pub use models::MysqlDb; 21 | pub use pool::MysqlDbPool; 22 | 23 | pub(crate) type DbResult = Result; 24 | -------------------------------------------------------------------------------- /syncstorage-mysql/src/schema.rs: -------------------------------------------------------------------------------- 1 | table! { 2 | batch_uploads (batch_id, user_id) { 3 | #[sql_name="batch"] 4 | batch_id -> Bigint, 5 | #[sql_name="userid"] 6 | user_id -> Bigint, 7 | #[sql_name="collection"] 8 | collection_id -> Integer, 9 | } 10 | } 11 | 12 | table! { 13 | batch_upload_items (batch_id, user_id, id) { 14 | #[sql_name="batch"] 15 | batch_id -> Bigint, 16 | #[sql_name="userid"] 17 | user_id -> Bigint, 18 | id -> Varchar, 19 | sortindex -> Nullable, 20 | payload -> Nullable, 21 | payload_size -> Nullable, 22 | ttl_offset -> Nullable, 23 | } 24 | } 25 | 26 | table! { 27 | bso (user_id, collection_id, id) { 28 | #[sql_name="userid"] 29 | user_id -> BigInt, 30 | #[sql_name="collection"] 31 | collection_id -> Integer, 32 | id -> Varchar, 33 | sortindex -> Nullable, 34 | payload -> Mediumtext, 35 | // not used, but legacy 36 | payload_size -> Bigint, 37 | modified -> Bigint, 38 | #[sql_name="ttl"] 39 | expiry -> Bigint, 40 | } 41 | } 42 | 43 | table! { 44 | collections (id) { 45 | id -> Integer, 46 | name -> Varchar, 47 | } 48 | } 49 | 50 | table! { 51 | user_collections (user_id, collection_id) { 52 | #[sql_name="userid"] 53 | user_id -> BigInt, 54 | #[sql_name="collection"] 55 | collection_id -> Integer, 56 | #[sql_name="last_modified"] 57 | modified -> Bigint, 58 | #[sql_name="count"] 59 | count -> Integer, 60 | #[sql_name="total_bytes"] 61 | total_bytes -> BigInt, 62 | } 63 | } 64 | 65 | allow_tables_to_appear_in_same_query!( 66 | batch_uploads, 67 | batch_upload_items, 68 | bso, 69 | collections, 70 | user_collections, 71 | ); 72 | -------------------------------------------------------------------------------- /syncstorage-mysql/src/test.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, sync::Arc}; 2 | 3 | use diesel::{ 4 | // expression_methods::TextExpressionMethods, // See note below about `not_like` becoming swedish 5 | ExpressionMethods, 6 | QueryDsl, 7 | RunQueryDsl, 8 | }; 9 | use syncserver_common::{BlockingThreadpool, Metrics}; 10 | use syncserver_settings::Settings as SyncserverSettings; 11 | use syncstorage_settings::Settings as SyncstorageSettings; 12 | use url::Url; 13 | 14 | use crate::{models::MysqlDb, pool::MysqlDbPool, schema::collections, DbResult}; 15 | 16 | pub fn db(settings: &SyncstorageSettings) -> DbResult { 17 | let _ = env_logger::try_init(); 18 | // inherit SYNC_SYNCSTORAGE__DATABASE_URL from the env 19 | 20 | let pool = MysqlDbPool::new( 21 | settings, 22 | &Metrics::noop(), 23 | Arc::new(BlockingThreadpool::new(512)), 24 | )?; 25 | pool.get_sync() 26 | } 27 | 28 | #[test] 29 | fn static_collection_id() -> DbResult<()> { 30 | let settings = SyncserverSettings::test_settings().syncstorage; 31 | if Url::parse(&settings.database_url).unwrap().scheme() != "mysql" { 32 | // Skip this test if we're not using mysql 33 | return Ok(()); 34 | } 35 | let db = db(&settings)?; 36 | 37 | // ensure DB actually has predefined common collections 38 | let cols: Vec<(i32, _)> = vec![ 39 | (1, "clients"), 40 | (2, "crypto"), 41 | (3, "forms"), 42 | (4, "history"), 43 | (5, "keys"), 44 | (6, "meta"), 45 | (7, "bookmarks"), 46 | (8, "prefs"), 47 | (9, "tabs"), 48 | (10, "passwords"), 49 | (11, "addons"), 50 | (12, "addresses"), 51 | (13, "creditcards"), 52 | ]; 53 | // The integration tests can create collections that start 54 | // with `xxx%`. We should not include those in our counts for local 55 | // unit tests. 56 | let results: HashMap = collections::table 57 | .select((collections::id, collections::name)) 58 | .filter(collections::name.ne("")) 59 | .filter(collections::name.ne("xxx_col2")) // from server::test 60 | .filter(collections::name.ne("col2")) // from older intergration tests 61 | .load(&db.inner.conn)? 62 | .into_iter() 63 | .collect(); 64 | assert_eq!(results.len(), cols.len(), "mismatched columns"); 65 | for (id, name) in &cols { 66 | assert_eq!(results.get(id).unwrap(), name); 67 | } 68 | 69 | for (id, name) in &cols { 70 | let result = db.get_collection_id(name)?; 71 | assert_eq!(result, *id); 72 | } 73 | 74 | let cid = db.get_or_create_collection_id("col1")?; 75 | assert!(cid >= 100); 76 | Ok(()) 77 | } 78 | -------------------------------------------------------------------------------- /syncstorage-settings/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "syncstorage-settings" 3 | version.workspace=true 4 | license.workspace=true 5 | authors.workspace=true 6 | edition.workspace=true 7 | 8 | [dependencies] 9 | rand.workspace=true 10 | serde.workspace=true 11 | 12 | syncserver-common = { path = "../syncserver-common" } 13 | time = "^0.3" 14 | -------------------------------------------------------------------------------- /syncstorage-spanner/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "syncstorage-spanner" 3 | version.workspace = true 4 | license.workspace = true 5 | authors.workspace = true 6 | edition.workspace = true 7 | 8 | [dependencies] 9 | actix-web.workspace = true 10 | backtrace.workspace = true 11 | cadence.workspace = true 12 | deadpool.workspace = true 13 | env_logger.workspace = true 14 | futures.workspace = true 15 | http.workspace = true 16 | slog-scope.workspace = true 17 | thiserror.workspace = true 18 | uuid.workspace = true 19 | 20 | async-trait = "0.1.40" 21 | google-cloud-rust-raw = { version = "0.16.1", features = ["spanner"] } 22 | form_urlencoded = "1.2" 23 | # Some versions of OpenSSL 1.1.1 conflict with grpcio's built-in boringssl which can cause 24 | # syncserver to either fail to either compile, or start. In those cases, try 25 | # `cargo build --features grpcio/openssl ...` 26 | grpcio = { version = "0.13.0", features = ["openssl"] } 27 | log = { version = "0.4", features = [ 28 | "max_level_debug", 29 | "release_max_level_info", 30 | ] } 31 | protobuf = { version = "2.28.0" } # must match what's used by googleapis-raw 32 | syncserver-common = { path = "../syncserver-common" } 33 | syncserver-db-common = { path = "../syncserver-db-common" } 34 | syncstorage-db-common = { path = "../syncstorage-db-common" } 35 | syncstorage-settings = { path = "../syncstorage-settings" } 36 | tokio = { workspace = true, features = [ 37 | "macros", 38 | "sync", 39 | ] } 40 | url = "2.1" 41 | 42 | [[bin]] 43 | name = "purge_ttl" 44 | path = "src/bin/purge_ttl.rs" 45 | -------------------------------------------------------------------------------- /syncstorage-spanner/src/BATCH_COMMIT.txt: -------------------------------------------------------------------------------- 1 | Batches serve as a temporary storage for POST'd bsos. Committing a batch 2 | entails moving the bsos from the batch table to their final destination in the 3 | `bsos` table. 4 | 5 | The move operation is like an UPSERT: the bsos in the batch will either be 6 | INSERT'd into the `bsos` table or UPDATE'd if they already exist. Only columns 7 | with NON NULL values in the batch table will be UPDATE'd in the `bsos` table: 8 | those with NULL values won't be touched. When INSERTing bsos, columns with NULL 9 | values recieve defaults. 10 | 11 | The entire move (batch commit) happens in one single transaction. Spanner 12 | limits the number of writes (or "mutations") within a transaction to 20000 13 | total mutations. A mutation is a write to an individual column. Any writes 14 | requiring modifications to secondary indices also incur additional mutations. 15 | DELETEs are generally cheaper (incurring one mutation per DELETE, not including 16 | secondary indices). 17 | 18 | The batch commit mutations are as follows: 19 | 20 | - Ensure a parent record exists in `user_collections` (due to bsos INTERLEAVE 21 | IN PARENT `user_collections`): 4 mutations (quota: False) or 6 mutations 22 | (quota: True) 23 | - INSERT or UPDATE: 24 | - 3 key columns 25 | - quota: False 26 | - 1 non key column 27 | - quota: True 28 | - 3 non key columns 29 | 30 | - Possibly direct inserts via post_bsos (but these only reduce total mutations 31 | by avoiding writing to the batch table, so not included here) 32 | 33 | - Write 1664 (MAX_TOTAL_RECORDS) to `bsos`: max 19968 (1664 * 12) mutations 34 | - INSERT takes 10 mutations: 35 | - 4 key columns 36 | - 4 non key columns 37 | - INSERT into 2 secondary indices: 2 mutations 38 | - UPDATE takes 12 mutations: 39 | - 4 key columns 40 | - 4 non key columns 41 | - UPDATE of 2 secondary indices: Each requires deleting + inserting a row: 42 | 2 mutations each. 4 total 43 | 44 | - Delete the batch 45 | - DELETE incurs 1 mutation 46 | 47 | - Update `user_collections` quota counts (only when quota: True): 6 mutations 48 | - UPDATE: 49 | - 3 key columns 50 | - quota: True 51 | - 3 non key columns 52 | 53 | Totals: 54 | - quota: False 55 | 4 + 19968 + 1 = 19973 56 | 57 | - quota: True 58 | 6 + 19968 + 1 + 6 = 19981 59 | -------------------------------------------------------------------------------- /syncstorage-spanner/src/batch_commit_insert.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO bsos (fxa_uid, fxa_kid, collection_id, bso_id, sortindex, payload, modified, expiry) 2 | SELECT 3 | batch_bsos.fxa_uid, 4 | batch_bsos.fxa_kid, 5 | batch_bsos.collection_id, 6 | batch_bsos.batch_bso_id, 7 | 8 | batch_bsos.sortindex, 9 | COALESCE(batch_bsos.payload, ''), 10 | @timestamp, 11 | COALESCE( 12 | TIMESTAMP_ADD(@timestamp, INTERVAL batch_bsos.ttl SECOND), 13 | TIMESTAMP_ADD(@timestamp, INTERVAL @default_bso_ttl SECOND) 14 | ) 15 | FROM batch_bsos 16 | WHERE fxa_uid = @fxa_uid 17 | AND fxa_kid = @fxa_kid 18 | AND collection_id = @collection_id 19 | AND batch_id = @batch_id 20 | AND batch_bso_id NOT in ( 21 | SELECT bso_id 22 | FROM bsos 23 | WHERE fxa_uid = @fxa_uid 24 | AND fxa_kid = @fxa_kid 25 | AND collection_id = @collection_id 26 | ) 27 | -------------------------------------------------------------------------------- /syncstorage-spanner/src/batch_commit_update.sql: -------------------------------------------------------------------------------- 1 | UPDATE bsos 2 | SET sortindex = COALESCE( 3 | (SELECT sortindex 4 | FROM batch_bsos 5 | WHERE fxa_uid = @fxa_uid 6 | AND fxa_kid = @fxa_kid 7 | AND collection_id = @collection_id 8 | AND batch_id = @batch_id 9 | AND batch_bso_id = bsos.bso_id 10 | ), 11 | bsos.sortindex 12 | ), 13 | 14 | payload = COALESCE( 15 | (SELECT payload 16 | FROM batch_bsos 17 | WHERE fxa_uid = @fxa_uid 18 | AND fxa_kid = @fxa_kid 19 | AND collection_id = @collection_id 20 | AND batch_id = @batch_id 21 | AND batch_bso_id = bsos.bso_id 22 | ), 23 | bsos.payload 24 | ), 25 | 26 | modified = @timestamp, 27 | 28 | expiry = COALESCE( 29 | -- TIMESTAMP_ADD returns NULL when ttl is null 30 | (SELECT TIMESTAMP_ADD(@timestamp, INTERVAL ttl SECOND) 31 | FROM batch_bsos 32 | WHERE fxa_uid = @fxa_uid 33 | AND fxa_kid = @fxa_kid 34 | AND collection_id = @collection_id 35 | AND batch_id = @batch_id 36 | AND batch_bso_id = bsos.bso_id 37 | ), 38 | bsos.expiry 39 | ) 40 | WHERE fxa_uid = @fxa_uid 41 | AND fxa_kid = @fxa_kid 42 | AND collection_id = @collection_id 43 | AND bso_id in ( 44 | SELECT batch_bso_id 45 | FROM batch_bsos 46 | WHERE fxa_uid = @fxa_uid 47 | AND fxa_kid = @fxa_kid 48 | AND collection_id = @collection_id 49 | AND batch_id = @batch_id 50 | ) 51 | -------------------------------------------------------------------------------- /syncstorage-spanner/src/batch_index.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX BatchExpireId 2 | ON batches ( 3 | fxa_uid, 4 | fxa_kid, 5 | collection_id, 6 | expiry 7 | ), INTERLEAVE IN user_collections 8 | 9 | -------------------------------------------------------------------------------- /syncstorage-spanner/src/insert_standard_collections.sql: -------------------------------------------------------------------------------- 1 | -- These are the 13 standard collections that are expected to exist by clients. 2 | -- The IDs are fixed. The below statement can be used to add these collections 3 | -- to a Spanner instance. 4 | INSERT INTO collections (collection_id, name) VALUES 5 | ( 1, "clients"), 6 | ( 2, "crypto"), 7 | ( 3, "forms"), 8 | ( 4, "history"), 9 | ( 5, "keys"), 10 | ( 6, "meta"), 11 | ( 7, "bookmarks"), 12 | ( 8, "prefs"), 13 | ( 9, "tabs"), 14 | (10, "passwords"), 15 | (11, "addons"), 16 | (12, "addresses"), 17 | (13, "creditcards"); 18 | -------------------------------------------------------------------------------- /syncstorage-spanner/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::time::SystemTime; 2 | 3 | #[macro_use] 4 | extern crate slog_scope; 5 | 6 | #[macro_use] 7 | mod macros; 8 | 9 | mod batch; 10 | mod error; 11 | mod manager; 12 | mod metadata; 13 | mod models; 14 | mod pool; 15 | mod support; 16 | 17 | pub use error::DbError; 18 | pub use models::SpannerDb; 19 | pub use pool::SpannerDbPool; 20 | 21 | type DbResult = Result; 22 | 23 | fn now() -> i64 { 24 | SystemTime::now() 25 | .duration_since(SystemTime::UNIX_EPOCH) 26 | .unwrap_or_default() 27 | .as_secs() as i64 28 | } 29 | -------------------------------------------------------------------------------- /syncstorage-spanner/src/manager/bb8.rs: -------------------------------------------------------------------------------- 1 | use std::marker::PhantomData; 2 | use std::{fmt, sync::Arc}; 3 | 4 | use async_trait::async_trait; 5 | use bb8::{ManageConnection, PooledConnection}; 6 | use grpcio::{EnvBuilder, Environment}; 7 | 8 | use crate::{ 9 | db::{ 10 | error::{DbError, DbErrorKind}, 11 | PoolState, 12 | }, 13 | server::Metrics, 14 | settings::Settings, 15 | }; 16 | 17 | use super::session::{create_spanner_session, recycle_spanner_session, SpannerSession}; 18 | 19 | #[allow(dead_code)] 20 | pub type Conn<'a> = PooledConnection<'a, SpannerSessionManager>; 21 | 22 | pub(super) struct SpannerSessionManager { 23 | database_name: String, 24 | /// The gRPC environment 25 | env: Arc, 26 | metrics: Metrics, 27 | test_transactions: bool, 28 | phantom: PhantomData, 29 | } 30 | 31 | impl fmt::Debug for SpannerSessionManager { 32 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 33 | fmt.debug_struct("bb8::SpannerSessionManager") 34 | .field("database_name", &self.database_name) 35 | .field("test_transactions", &self.test_transactions) 36 | .finish() 37 | } 38 | } 39 | 40 | #[async_trait] 41 | impl ManageConnection for SpannerSessionManager { 42 | type Connection = SpannerSession; 43 | type Error = DbError; 44 | 45 | async fn connect(&self) -> Result { 46 | create_spanner_session( 47 | Arc::clone(&self.env), 48 | self.metrics.clone(), 49 | &self.database_name, 50 | self.test_transactions, 51 | ) 52 | .await 53 | } 54 | 55 | async fn is_valid(&self, mut conn: Self::Connection) -> Result { 56 | recycle_spanner_session(&mut conn, &self.database_name).await?; 57 | Ok(conn) 58 | } 59 | 60 | fn has_broken(&self, _conn: &mut Self::Connection) -> bool { 61 | false 62 | } 63 | } 64 | 65 | impl From for PoolState { 66 | fn from(state: bb8::State) -> PoolState { 67 | PoolState { 68 | connections: state.connections, 69 | idle_connections: state.idle_connections, 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /syncstorage-spanner/src/manager/deadpool.rs: -------------------------------------------------------------------------------- 1 | use std::{fmt, sync::Arc}; 2 | 3 | use deadpool::managed::{Manager, RecycleError, RecycleResult}; 4 | use grpcio::{EnvBuilder, Environment}; 5 | use syncserver_common::{BlockingThreadpool, Metrics}; 6 | use syncstorage_settings::Settings; 7 | 8 | use super::session::{ 9 | create_spanner_session, recycle_spanner_session, SpannerSession, SpannerSessionSettings, 10 | }; 11 | use crate::error::DbError; 12 | 13 | pub(crate) type Conn = deadpool::managed::Object; 14 | 15 | pub(crate) struct SpannerSessionManager { 16 | pub settings: SpannerSessionSettings, 17 | /// The gRPC environment 18 | env: Arc, 19 | metrics: Metrics, 20 | blocking_threadpool: Arc, 21 | } 22 | 23 | impl fmt::Debug for SpannerSessionManager { 24 | fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { 25 | fmt.debug_struct("deadpool::SpannerSessionManager") 26 | .field("settings", &self.settings) 27 | .field("blocking_threadpool", &self.blocking_threadpool) 28 | .finish() 29 | } 30 | } 31 | 32 | impl SpannerSessionManager { 33 | pub fn new( 34 | settings: &Settings, 35 | metrics: &Metrics, 36 | blocking_threadpool: Arc, 37 | ) -> Result { 38 | Ok(Self { 39 | settings: SpannerSessionSettings::from_settings(settings)?, 40 | env: Arc::new(EnvBuilder::new().build()), 41 | metrics: metrics.clone(), 42 | blocking_threadpool, 43 | }) 44 | } 45 | } 46 | 47 | impl Manager for SpannerSessionManager { 48 | type Type = SpannerSession; 49 | type Error = DbError; 50 | 51 | async fn create(&self) -> Result { 52 | let session = create_spanner_session( 53 | &self.settings, 54 | Arc::clone(&self.env), 55 | self.metrics.clone(), 56 | self.blocking_threadpool.clone(), 57 | ) 58 | .await?; 59 | Ok(session) 60 | } 61 | 62 | async fn recycle( 63 | &self, 64 | conn: &mut SpannerSession, 65 | _: &deadpool::managed::Metrics, 66 | ) -> RecycleResult { 67 | recycle_spanner_session(conn, &self.metrics) 68 | .await 69 | .map_err(RecycleError::Backend) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /syncstorage-spanner/src/manager/mod.rs: -------------------------------------------------------------------------------- 1 | // mod bb8; 2 | mod deadpool; 3 | mod session; 4 | 5 | pub(super) use self::deadpool::{Conn, SpannerSessionManager}; 6 | -------------------------------------------------------------------------------- /syncstorage-spanner/src/schema.ddl: -------------------------------------------------------------------------------- 1 | -- fxa_uid: a 16 byte identifier, randomly generated by the fxa server 2 | -- usually a UUID, so presuming a formatted form. 3 | -- fxa_kid: <`mono_num`>-<`client_state`> 4 | -- 5 | -- - mono_num: a monotonically increasing timestamp or generation number 6 | -- in hex and padded to 13 digits, provided by the fxa server 7 | -- - client_state: the first 16 bytes of a SHA256 hash of the user's sync 8 | -- encryption key. 9 | -- 10 | -- NOTE: DO NOT INCLUDE COMMENTS IF PASTING INTO CONSOLE 11 | -- ALSO, CONSOLE WANTS ONE SPACE BETWEEN DDL COMMANDS 12 | 13 | CREATE TABLE user_collections ( 14 | fxa_uid STRING(MAX) NOT NULL, 15 | fxa_kid STRING(MAX) NOT NULL, 16 | collection_id INT64 NOT NULL, 17 | modified TIMESTAMP NOT NULL, 18 | 19 | count INT64, 20 | total_bytes INT64, 21 | ) PRIMARY KEY(fxa_uid, fxa_kid, collection_id); 22 | 23 | CREATE TABLE bsos ( 24 | fxa_uid STRING(MAX) NOT NULL, 25 | fxa_kid STRING(MAX) NOT NULL, 26 | collection_id INT64 NOT NULL, 27 | bso_id STRING(MAX) NOT NULL, 28 | 29 | sortindex INT64, 30 | 31 | payload STRING(MAX) NOT NULL, 32 | 33 | modified TIMESTAMP NOT NULL, 34 | expiry TIMESTAMP NOT NULL, 35 | ) PRIMARY KEY(fxa_uid, fxa_kid, collection_id, bso_id), 36 | INTERLEAVE IN PARENT user_collections ON DELETE CASCADE; 37 | 38 | CREATE INDEX BsoModified 39 | ON bsos(fxa_uid, fxa_kid, collection_id, modified DESC), 40 | INTERLEAVE IN user_collections; 41 | 42 | CREATE INDEX BsoExpiry 43 | ON bsos(fxa_uid, fxa_kid, collection_id, expiry), 44 | INTERLEAVE IN user_collections; 45 | 46 | CREATE TABLE collections ( 47 | collection_id INT64 NOT NULL, 48 | name STRING(32) NOT NULL, 49 | ) PRIMARY KEY(collection_id); 50 | 51 | CREATE UNIQUE INDEX CollectionName 52 | ON collections(name); 53 | 54 | CREATE TABLE batches ( 55 | fxa_uid STRING(MAX) NOT NULL, 56 | fxa_kid STRING(MAX) NOT NULL, 57 | collection_id INT64 NOT NULL, 58 | batch_id STRING(MAX) NOT NULL, 59 | expiry TIMESTAMP NOT NULL, 60 | ) PRIMARY KEY(fxa_uid, fxa_kid, collection_id, batch_id), 61 | INTERLEAVE IN PARENT user_collections ON DELETE CASCADE; 62 | 63 | CREATE INDEX BatchExpireId 64 | ON batches(fxa_uid, fxa_kid, collection_id, expiry), 65 | INTERLEAVE IN user_collections; 66 | 67 | CREATE TABLE batch_bsos ( 68 | fxa_uid STRING(MAX) NOT NULL, 69 | fxa_kid STRING(MAX) NOT NULL, 70 | collection_id INT64 NOT NULL, 71 | batch_id STRING(MAX) NOT NULL, 72 | batch_bso_id STRING(MAX) NOT NULL, 73 | 74 | sortindex INT64, 75 | payload STRING(MAX), 76 | ttl INT64, 77 | ) PRIMARY KEY(fxa_uid, fxa_kid, collection_id, batch_id, batch_bso_id), 78 | INTERLEAVE IN PARENT batches ON DELETE CASCADE; 79 | 80 | -- batch_bsos' bso fields are nullable as the batch upload may or may 81 | -- not set each individual field of each item. Also note that there's 82 | -- no "modified" column because the modification timestamp gets set on 83 | -- batch commit. 84 | 85 | -- *NOTE*: 86 | -- Newly created Spanner instances should pre-populate the `collections` table by 87 | -- running the content of `insert_standard_collections.sql ` 88 | -------------------------------------------------------------------------------- /tests.ini: -------------------------------------------------------------------------------- 1 | [server:main] 2 | use = egg:Paste#http 3 | host = 0.0.0.0 4 | port = 5000 5 | 6 | [app:main] 7 | use = egg:SyncStorage 8 | 9 | [storage] 10 | backend = syncstorage.storage.sql.SQLStorage 11 | sqluri = ${MOZSVC_SQLURI} 12 | standard_collections = true 13 | quota_size = 5242880 14 | pool_size = 100 15 | pool_recycle = 3600 16 | reset_on_return = true 17 | create_tables = true 18 | max_post_records = 4000 19 | batch_upload_enabled = true 20 | force_consistent_sort_order = true 21 | 22 | [hawkauth] 23 | secret = "secret0" 24 | -------------------------------------------------------------------------------- /tokenserver-auth/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tokenserver-auth" 3 | version.workspace = true 4 | license.workspace = true 5 | authors.workspace = true 6 | edition.workspace = true 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | serde.workspace = true 12 | serde_json.workspace = true 13 | hex.workspace = true 14 | hkdf.workspace = true 15 | hmac.workspace = true 16 | jsonwebtoken.workspace = true 17 | base64.workspace = true 18 | sha2.workspace = true 19 | thiserror.workspace = true 20 | slog-scope.workspace = true 21 | 22 | async-trait = "0.1.40" 23 | dyn-clone = "1.0.4" 24 | reqwest = { workspace = true, features = ["json"] } 25 | ring = "0.17" 26 | syncserver-common = { path = "../syncserver-common" } 27 | tokenserver-common = { path = "../tokenserver-common" } 28 | tokenserver-settings = { path = "../tokenserver-settings" } 29 | tokio = { workspace = true } 30 | pyo3 = { version = "0.24", features = ["auto-initialize"], optional = true } 31 | 32 | 33 | [dev-dependencies] 34 | # mockito = "0.30" 35 | mockito = "1.4.0" 36 | tokio = { workspace = true, features = ["macros"] } 37 | 38 | [features] 39 | default = ["py"] 40 | py = ["pyo3"] 41 | -------------------------------------------------------------------------------- /tokenserver-auth/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[cfg(not(feature = "py"))] 2 | mod crypto; 3 | 4 | #[cfg(not(feature = "py"))] 5 | pub use crypto::{JWTVerifier, JWTVerifierImpl}; 6 | #[allow(clippy::result_large_err)] 7 | pub mod oauth; 8 | #[allow(clippy::result_large_err)] 9 | mod token; 10 | use syncserver_common::Metrics; 11 | pub use token::Tokenlib; 12 | 13 | use std::fmt; 14 | 15 | use async_trait::async_trait; 16 | use dyn_clone::{self, DynClone}; 17 | use serde::{Deserialize, Serialize}; 18 | use tokenserver_common::TokenserverError; 19 | /// Represents the origin of the token used by Sync clients to access their data. 20 | #[derive(Clone, Copy, Default, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] 21 | #[serde(rename_all = "lowercase")] 22 | pub enum TokenserverOrigin { 23 | /// The Python Tokenserver. 24 | #[default] 25 | Python, 26 | /// The Rust Tokenserver. 27 | Rust, 28 | } 29 | 30 | impl fmt::Display for TokenserverOrigin { 31 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 32 | match self { 33 | TokenserverOrigin::Python => write!(f, "python"), 34 | TokenserverOrigin::Rust => write!(f, "rust"), 35 | } 36 | } 37 | } 38 | 39 | /// The plaintext needed to build a token. 40 | #[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq, Eq)] 41 | pub struct MakeTokenPlaintext { 42 | pub node: String, 43 | pub fxa_kid: String, 44 | pub fxa_uid: String, 45 | pub hashed_device_id: String, 46 | pub hashed_fxa_uid: String, 47 | pub expires: u64, 48 | pub uid: i64, 49 | pub tokenserver_origin: TokenserverOrigin, 50 | } 51 | 52 | /// Implementers of this trait can be used to verify tokens for Tokenserver. 53 | #[async_trait] 54 | pub trait VerifyToken: DynClone + Sync + Send { 55 | type Output: Clone; 56 | 57 | /// Verifies the given token. This function is async because token verification often involves 58 | /// making a request to a remote server. 59 | async fn verify( 60 | &self, 61 | token: String, 62 | metrics: &Metrics, 63 | ) -> Result; 64 | } 65 | 66 | dyn_clone::clone_trait_object!( VerifyToken); 67 | 68 | /// A mock verifier to be used for testing purposes. 69 | #[derive(Clone, Default)] 70 | pub struct MockVerifier { 71 | pub valid: bool, 72 | pub verify_output: T, 73 | } 74 | 75 | #[async_trait] 76 | impl VerifyToken for MockVerifier { 77 | type Output = T; 78 | 79 | async fn verify(&self, _token: String, _metrics: &Metrics) -> Result { 80 | self.valid 81 | .then(|| self.verify_output.clone()) 82 | .ok_or_else(|| TokenserverError::invalid_credentials("Unauthorized".to_owned())) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /tokenserver-auth/src/oauth.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | #[cfg(not(feature = "py"))] 4 | mod native; 5 | #[cfg(feature = "py")] 6 | mod py; 7 | 8 | #[cfg(feature = "py")] 9 | pub type Verifier = py::Verifier; 10 | 11 | #[cfg(not(feature = "py"))] 12 | pub type Verifier = native::Verifier; 13 | 14 | /// The information extracted from a valid OAuth token. 15 | #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] 16 | pub struct VerifyOutput { 17 | #[serde(rename = "user")] 18 | pub fxa_uid: String, 19 | pub generation: Option, 20 | } 21 | -------------------------------------------------------------------------------- /tokenserver-auth/src/oauth/verify.py: -------------------------------------------------------------------------------- 1 | from fxa.oauth import Client 2 | from fxa.errors import ClientError, TrustError 3 | import json 4 | 5 | DEFAULT_OAUTH_SCOPE = 'https://identity.mozilla.com/apps/oldsync' 6 | 7 | 8 | class FxaOAuthClient: 9 | def __init__(self, server_url=None, jwks=None): 10 | self._client = Client(server_url=server_url, jwks=jwks) 11 | 12 | def verify_token(self, token): 13 | try: 14 | token_data = self._client.verify_token(token, DEFAULT_OAUTH_SCOPE) 15 | 16 | # Serialize the data to make it easier to parse in Rust 17 | return json.dumps(token_data) 18 | except (ClientError, TrustError): 19 | return None 20 | -------------------------------------------------------------------------------- /tokenserver-auth/src/token.rs: -------------------------------------------------------------------------------- 1 | #[cfg(not(feature = "py"))] 2 | mod native; 3 | 4 | #[cfg(feature = "py")] 5 | mod py; 6 | 7 | #[cfg(feature = "py")] 8 | pub type Tokenlib = py::PyTokenlib; 9 | 10 | #[cfg(not(feature = "py"))] 11 | pub type Tokenlib = native::Tokenlib; 12 | -------------------------------------------------------------------------------- /tokenserver-auth/src/token/py.rs: -------------------------------------------------------------------------------- 1 | use crate::{MakeTokenPlaintext, TokenserverError}; 2 | use pyo3::{ 3 | prelude::{IntoPyObject, PyErr, PyModule, Python}, 4 | types::{IntoPyDict, PyAnyMethods, PyDict}, 5 | Bound, 6 | }; 7 | 8 | pub struct PyTokenlib {} 9 | impl<'py> IntoPyObject<'py> for MakeTokenPlaintext { 10 | type Target = PyDict; 11 | type Output = Bound<'py, Self::Target>; 12 | type Error = PyErr; 13 | 14 | fn into_pyobject(self, py: Python<'py>) -> Result { 15 | let dict = [ 16 | ("node", self.node), 17 | ("fxa_kid", self.fxa_kid), 18 | ("fxa_uid", self.fxa_uid), 19 | ("hashed_device_id", self.hashed_device_id), 20 | ("hashed_fxa_uid", self.hashed_fxa_uid), 21 | ("tokenserver_origin", self.tokenserver_origin.to_string()), 22 | ] 23 | .into_py_dict(py)?; 24 | 25 | // These need to be set separately since they aren't strings, and 26 | // Rust doesn't support heterogeneous arrays 27 | dict.set_item("expires", self.expires)?; 28 | dict.set_item("uid", self.uid)?; 29 | 30 | Ok(dict) 31 | } 32 | } 33 | impl PyTokenlib { 34 | pub fn get_token_and_derived_secret( 35 | plaintext: MakeTokenPlaintext, 36 | shared_secret: &str, 37 | ) -> Result<(String, String), TokenserverError> { 38 | Python::with_gil(|py| { 39 | // `import tokenlib` 40 | let module = PyModule::import(py, "tokenlib") 41 | .inspect_err(|e| e.print_and_set_sys_last_vars(py))?; 42 | // `kwargs = { 'secret': shared_secret }` 43 | let kwargs = [("secret", shared_secret)].into_py_dict(py)?; 44 | // `token = tokenlib.make_token(plaintext, **kwargs)` 45 | // Adding a note, since not having explicit string type resulted in a very pesky and hard to find 46 | // error, described https://github.com/PyO3/pyo3/issues/4702. To reproduce, remove type annotation 47 | // from token. 48 | let token: String = module 49 | .getattr("make_token")? 50 | .call((plaintext,), Some(&kwargs)) 51 | .inspect_err(|e| e.print_and_set_sys_last_vars(py)) 52 | .and_then(|x| x.extract())?; 53 | // `derived_secret = tokenlib.get_derived_secret(token, **kwargs)` 54 | let derived_secret = module 55 | .getattr("get_derived_secret")? 56 | .call((&token,), Some(&kwargs)) 57 | .inspect_err(|e| e.print_and_set_sys_last_vars(py)) 58 | .and_then(|x| x.extract())?; 59 | // `return (token, derived_secret)` 60 | Ok((token, derived_secret)) 61 | }) 62 | .map_err(pyerr_to_tokenserver_error) 63 | } 64 | } 65 | 66 | fn pyerr_to_tokenserver_error(e: PyErr) -> TokenserverError { 67 | TokenserverError { 68 | context: e.to_string(), 69 | ..TokenserverError::internal_error() 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /tokenserver-common/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tokenserver-common" 3 | version.workspace = true 4 | license.workspace = true 5 | authors.workspace = true 6 | edition.workspace = true 7 | 8 | [dependencies] 9 | actix-web.workspace = true 10 | backtrace.workspace = true 11 | http.workspace = true 12 | serde.workspace = true 13 | 14 | pyo3 = { version = "0.24", features = ["auto-initialize"], optional = true } 15 | syncserver-common = { path = "../syncserver-common" } 16 | 17 | [features] 18 | default = ["py"] 19 | py = ["pyo3"] -------------------------------------------------------------------------------- /tokenserver-common/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod error; 2 | 3 | use serde::{Deserialize, Serialize}; 4 | 5 | pub use error::{ErrorLocation, TokenserverError}; 6 | 7 | #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] 8 | pub enum NodeType { 9 | #[serde(rename = "mysql")] 10 | MySql, 11 | #[serde(rename = "spanner")] 12 | Spanner, 13 | } 14 | 15 | impl NodeType { 16 | pub fn spanner() -> Self { 17 | Self::Spanner 18 | } 19 | } 20 | 21 | impl Default for NodeType { 22 | fn default() -> Self { 23 | Self::Spanner 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tokenserver-db/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tokenserver-db" 3 | version.workspace = true 4 | license.workspace = true 5 | authors.workspace = true 6 | edition.workspace = true 7 | 8 | [dependencies] 9 | backtrace.workspace = true 10 | futures.workspace = true 11 | http.workspace = true 12 | serde.workspace = true 13 | slog-scope.workspace = true 14 | thiserror.workspace = true 15 | 16 | async-trait = "0.1.40" 17 | # diesel 1.4 18 | diesel = { workspace = true, features = ["mysql", "r2d2"] } 19 | diesel_logger = { workspace = true } 20 | diesel_migrations = { workspace = true, features = ["mysql"] } 21 | syncserver-common = { path = "../syncserver-common" } 22 | syncserver-db-common = { path = "../syncserver-db-common" } 23 | tokenserver-common = { path = "../tokenserver-common" } 24 | tokenserver-settings = { path = "../tokenserver-settings" } 25 | tokio = { workspace = true, features = ["macros", "sync"] } 26 | 27 | [dev-dependencies] 28 | env_logger.workspace = true 29 | 30 | syncserver-settings = { path = "../syncserver-settings" } 31 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-07-16-001122_init/down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS `users`; 2 | DROP TABLE IF EXISTS `nodes`; 3 | DROP TABLE IF EXISTS `services`; 4 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-07-16-001122_init/up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS `services` ( 2 | `id` int NOT NULL AUTO_INCREMENT, 3 | `service` varchar(30) DEFAULT NULL, 4 | `pattern` varchar(128) DEFAULT NULL, 5 | PRIMARY KEY (`id`), 6 | UNIQUE KEY `service` (`service`) 7 | ); 8 | 9 | CREATE TABLE IF NOT EXISTS `nodes` ( 10 | `id` bigint NOT NULL AUTO_INCREMENT, 11 | `service` int NOT NULL, 12 | `node` varchar(64) NOT NULL, 13 | `available` int NOT NULL DEFAULT '0', 14 | `current_load` int NOT NULL DEFAULT '0', 15 | `capacity` int NOT NULL DEFAULT '0', 16 | `downed` int NOT NULL DEFAULT '0', 17 | `backoff` int NOT NULL DEFAULT '0', 18 | PRIMARY KEY (`id`), 19 | KEY `service` (`service`), 20 | CONSTRAINT `nodes_ibfk_1` FOREIGN KEY (`service`) REFERENCES `services` (`id`) 21 | ); 22 | 23 | CREATE TABLE IF NOT EXISTS `users` ( 24 | `uid` bigint NOT NULL AUTO_INCREMENT, 25 | `service` int NOT NULL, 26 | `email` varchar(255) NOT NULL, 27 | `generation` bigint NOT NULL, 28 | `client_state` varchar(32) NOT NULL, 29 | `created_at` bigint NOT NULL, 30 | `replaced_at` bigint DEFAULT NULL, 31 | `nodeid` bigint NOT NULL, 32 | `keys_changed_at` bigint DEFAULT NULL, 33 | PRIMARY KEY (`uid`), 34 | KEY `nodeid` (`nodeid`), 35 | CONSTRAINT `users_ibfk_1` FOREIGN KEY (`nodeid`) REFERENCES `nodes` (`id`) 36 | ) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8 37 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-08-03-234845_populate_services/down.sql: -------------------------------------------------------------------------------- 1 | DELETE FROM services 2 | WHERE id=1 3 | OR id=2; 4 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-08-03-234845_populate_services/up.sql: -------------------------------------------------------------------------------- 1 | INSERT IGNORE INTO services (id, service, pattern) VALUES 2 | (1, "sync-1.1", "{node}/1.1/{uid}"), 3 | (2, "sync-1.5", "{node}/1.5/{uid}"); 4 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-09-30-142643_remove_foreign_key_constraints/down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE `nodes` ADD CONSTRAINT `nodes_ibfk_1` FOREIGN KEY (`service`) REFERENCES `services` (`id`); 2 | ALTER TABLE `users` ADD CONSTRAINT `users_ibfk_1` FOREIGN KEY (`nodeid`) REFERENCES `nodes` (`id`); 3 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-09-30-142643_remove_foreign_key_constraints/up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE `nodes` DROP FOREIGN KEY `nodes_ibfk_1`; 2 | ALTER TABLE `users` DROP FOREIGN KEY `users_ibfk_1`; 3 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-09-30-142654_remove_node_defaults/down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE `nodes` ALTER `available` SET DEFAULT 0; 2 | ALTER TABLE `nodes` ALTER `current_load` SET DEFAULT 0; 3 | ALTER TABLE `nodes` ALTER `capacity` SET DEFAULT 0; 4 | ALTER TABLE `nodes` ALTER `downed` SET DEFAULT 0; 5 | ALTER TABLE `nodes` ALTER `backoff` SET DEFAULT 0; 6 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-09-30-142654_remove_node_defaults/up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE `nodes` ALTER `available` DROP DEFAULT; 2 | ALTER TABLE `nodes` ALTER `current_load` DROP DEFAULT; 3 | ALTER TABLE `nodes` ALTER `capacity` DROP DEFAULT; 4 | ALTER TABLE `nodes` ALTER `downed` DROP DEFAULT; 5 | ALTER TABLE `nodes` ALTER `backoff` DROP DEFAULT; 6 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-09-30-142746_add_indexes/down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE `nodes` DROP INDEX `unique_idx`; 2 | ALTER TABLE `users` DROP INDEX `lookup_idx`; 3 | ALTER TABLE `users` DROP INDEX `replaced_at_idx`; 4 | ALTER TABLE `users` DROP INDEX `node_idx`; 5 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-09-30-142746_add_indexes/up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE `nodes` ADD UNIQUE KEY `unique_idx` (`service`, `node`); 2 | ALTER TABLE `users` ADD INDEX `lookup_idx` (`email`, `service`, `created_at`); 3 | ALTER TABLE `users` ADD INDEX `replaced_at_idx` (`service`, `replaced_at`); 4 | ALTER TABLE `users` ADD INDEX `node_idx` (`nodeid`); 5 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-09-30-144043_remove_nodes_service_key/down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE `nodes` ADD KEY `service` (`service`); 2 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-09-30-144043_remove_nodes_service_key/up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE `nodes` DROP KEY `service`; 2 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-09-30-144225_remove_users_nodeid_key/down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE `users` ADD KEY `nodeid` (`nodeid`); 2 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-09-30-144225_remove_users_nodeid_key/up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE `users` DROP KEY `nodeid`; 2 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-12-22-160451_remove_services/down.sql: -------------------------------------------------------------------------------- 1 | INSERT IGNORE INTO services (id, service, pattern) VALUES 2 | (1, "sync-1.1", "{node}/1.1/{uid}"), 3 | (2, "sync-1.5", "{node}/1.5/{uid}"); 4 | -------------------------------------------------------------------------------- /tokenserver-db/migrations/2021-12-22-160451_remove_services/up.sql: -------------------------------------------------------------------------------- 1 | DELETE FROM services 2 | WHERE id=1 3 | OR id=2; 4 | -------------------------------------------------------------------------------- /tokenserver-db/src/error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use backtrace::Backtrace; 4 | use http::StatusCode; 5 | use syncserver_common::{from_error, impl_fmt_display, InternalError, ReportableError}; 6 | use syncserver_db_common::error::SqlError; 7 | use thiserror::Error; 8 | use tokenserver_common::TokenserverError; 9 | 10 | pub(crate) type DbFuture<'a, T> = syncserver_db_common::DbFuture<'a, T, DbError>; 11 | pub(crate) type DbResult = Result; 12 | 13 | /// An error type that represents any database-related errors that may occur while processing a 14 | /// tokenserver request. 15 | #[derive(Debug)] 16 | pub struct DbError { 17 | kind: DbErrorKind, 18 | pub status: StatusCode, 19 | pub backtrace: Box, 20 | } 21 | 22 | impl DbError { 23 | pub(crate) fn internal(msg: String) -> Self { 24 | DbErrorKind::Internal(msg).into() 25 | } 26 | } 27 | 28 | impl ReportableError for DbError { 29 | fn backtrace(&self) -> Option<&Backtrace> { 30 | match &self.kind { 31 | DbErrorKind::Sql(e) => e.backtrace(), 32 | _ => Some(&self.backtrace), 33 | } 34 | } 35 | 36 | fn is_sentry_event(&self) -> bool { 37 | match &self.kind { 38 | DbErrorKind::Sql(e) => e.is_sentry_event(), 39 | _ => true, 40 | } 41 | } 42 | 43 | fn metric_label(&self) -> Option<&str> { 44 | match &self.kind { 45 | DbErrorKind::Sql(e) => e.metric_label(), 46 | _ => None, 47 | } 48 | } 49 | } 50 | 51 | #[derive(Debug, Error)] 52 | enum DbErrorKind { 53 | #[error("{}", _0)] 54 | Sql(SqlError), 55 | 56 | #[error("Unexpected error: {}", _0)] 57 | Internal(String), 58 | } 59 | 60 | impl From for DbError { 61 | fn from(kind: DbErrorKind) -> Self { 62 | match kind { 63 | DbErrorKind::Sql(ref mysql_error) => Self { 64 | status: mysql_error.status, 65 | backtrace: Box::new(mysql_error.backtrace.clone()), 66 | kind, 67 | }, 68 | DbErrorKind::Internal(_) => Self { 69 | kind, 70 | status: StatusCode::INTERNAL_SERVER_ERROR, 71 | backtrace: Box::new(Backtrace::new()), 72 | }, 73 | } 74 | } 75 | } 76 | 77 | impl From for TokenserverError { 78 | fn from(db_error: DbError) -> Self { 79 | TokenserverError { 80 | description: db_error.to_string(), 81 | context: db_error.to_string(), 82 | backtrace: db_error.backtrace.clone(), 83 | http_status: if db_error.status.is_server_error() { 84 | // Use the status code from the DbError if it already suggests an internal error; 85 | // it might be more specific than `StatusCode::SERVICE_UNAVAILABLE` 86 | db_error.status 87 | } else { 88 | StatusCode::SERVICE_UNAVAILABLE 89 | }, 90 | source: Some(Box::new(db_error)), 91 | // An unhandled DbError in the Tokenserver code is an internal error 92 | ..TokenserverError::internal_error() 93 | } 94 | } 95 | } 96 | 97 | impl InternalError for DbError { 98 | fn internal_error(message: String) -> Self { 99 | DbErrorKind::Internal(message).into() 100 | } 101 | } 102 | 103 | impl_fmt_display!(DbError, DbErrorKind); 104 | 105 | from_error!( 106 | diesel::result::Error, 107 | DbError, 108 | |error: diesel::result::Error| DbError::from(DbErrorKind::Sql(SqlError::from(error))) 109 | ); 110 | from_error!( 111 | diesel::result::ConnectionError, 112 | DbError, 113 | |error: diesel::result::ConnectionError| DbError::from(DbErrorKind::Sql(SqlError::from(error))) 114 | ); 115 | from_error!( 116 | diesel::r2d2::PoolError, 117 | DbError, 118 | |error: diesel::r2d2::PoolError| DbError::from(DbErrorKind::Sql(SqlError::from(error))) 119 | ); 120 | from_error!( 121 | diesel_migrations::RunMigrationsError, 122 | DbError, 123 | |error: diesel_migrations::RunMigrationsError| DbError::from(DbErrorKind::Sql(SqlError::from( 124 | error 125 | ))) 126 | ); 127 | -------------------------------------------------------------------------------- /tokenserver-db/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_local_definitions)] 2 | extern crate diesel; 3 | #[macro_use] 4 | extern crate diesel_migrations; 5 | #[macro_use] 6 | extern crate slog_scope; 7 | 8 | mod error; 9 | pub mod mock; 10 | mod models; 11 | pub mod params; 12 | mod pool; 13 | pub mod results; 14 | 15 | pub use models::{Db, TokenserverDb}; 16 | pub use pool::{DbPool, TokenserverPool}; 17 | -------------------------------------------------------------------------------- /tokenserver-db/src/mock.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::new_without_default)] 2 | 3 | use async_trait::async_trait; 4 | use futures::future; 5 | use syncserver_db_common::{GetPoolState, PoolState}; 6 | 7 | use super::error::{DbError, DbFuture}; 8 | use super::models::Db; 9 | use super::params; 10 | use super::pool::DbPool; 11 | use super::results; 12 | 13 | #[derive(Clone, Debug)] 14 | pub struct MockDbPool; 15 | 16 | impl MockDbPool { 17 | pub fn new() -> Self { 18 | MockDbPool 19 | } 20 | } 21 | 22 | #[async_trait] 23 | impl DbPool for MockDbPool { 24 | async fn get(&self) -> Result, DbError> { 25 | Ok(Box::new(MockDb::new())) 26 | } 27 | 28 | fn box_clone(&self) -> Box { 29 | Box::new(self.clone()) 30 | } 31 | } 32 | 33 | impl GetPoolState for MockDbPool { 34 | fn state(&self) -> PoolState { 35 | PoolState::default() 36 | } 37 | } 38 | 39 | #[derive(Clone, Debug)] 40 | pub struct MockDb; 41 | 42 | impl MockDb { 43 | pub fn new() -> Self { 44 | MockDb 45 | } 46 | } 47 | 48 | impl Db for MockDb { 49 | fn replace_user(&self, _params: params::ReplaceUser) -> DbFuture<'_, results::ReplaceUser> { 50 | Box::pin(future::ok(())) 51 | } 52 | 53 | fn replace_users(&self, _params: params::ReplaceUsers) -> DbFuture<'_, results::ReplaceUsers> { 54 | Box::pin(future::ok(())) 55 | } 56 | 57 | fn post_user(&self, _params: params::PostUser) -> DbFuture<'_, results::PostUser> { 58 | Box::pin(future::ok(results::PostUser::default())) 59 | } 60 | 61 | fn put_user(&self, _params: params::PutUser) -> DbFuture<'_, results::PutUser> { 62 | Box::pin(future::ok(())) 63 | } 64 | 65 | fn check(&self) -> DbFuture<'_, results::Check> { 66 | Box::pin(future::ok(true)) 67 | } 68 | 69 | fn get_node_id(&self, _params: params::GetNodeId) -> DbFuture<'_, results::GetNodeId> { 70 | Box::pin(future::ok(results::GetNodeId::default())) 71 | } 72 | 73 | fn get_best_node(&self, _params: params::GetBestNode) -> DbFuture<'_, results::GetBestNode> { 74 | Box::pin(future::ok(results::GetBestNode::default())) 75 | } 76 | 77 | fn add_user_to_node( 78 | &self, 79 | _params: params::AddUserToNode, 80 | ) -> DbFuture<'_, results::AddUserToNode> { 81 | Box::pin(future::ok(())) 82 | } 83 | 84 | fn get_users(&self, _params: params::GetUsers) -> DbFuture<'_, results::GetUsers> { 85 | Box::pin(future::ok(results::GetUsers::default())) 86 | } 87 | 88 | fn get_or_create_user( 89 | &self, 90 | _params: params::GetOrCreateUser, 91 | ) -> DbFuture<'_, results::GetOrCreateUser> { 92 | Box::pin(future::ok(results::GetOrCreateUser::default())) 93 | } 94 | 95 | fn get_service_id(&self, _params: params::GetServiceId) -> DbFuture<'_, results::GetServiceId> { 96 | Box::pin(future::ok(results::GetServiceId::default())) 97 | } 98 | 99 | #[cfg(test)] 100 | fn set_user_created_at( 101 | &self, 102 | _params: params::SetUserCreatedAt, 103 | ) -> DbFuture<'_, results::SetUserCreatedAt> { 104 | Box::pin(future::ok(())) 105 | } 106 | 107 | #[cfg(test)] 108 | fn set_user_replaced_at( 109 | &self, 110 | _params: params::SetUserReplacedAt, 111 | ) -> DbFuture<'_, results::SetUserReplacedAt> { 112 | Box::pin(future::ok(())) 113 | } 114 | 115 | #[cfg(test)] 116 | fn get_user(&self, _params: params::GetUser) -> DbFuture<'_, results::GetUser> { 117 | Box::pin(future::ok(results::GetUser::default())) 118 | } 119 | 120 | #[cfg(test)] 121 | fn post_node(&self, _params: params::PostNode) -> DbFuture<'_, results::PostNode> { 122 | Box::pin(future::ok(results::PostNode::default())) 123 | } 124 | 125 | #[cfg(test)] 126 | fn get_node(&self, _params: params::GetNode) -> DbFuture<'_, results::GetNode> { 127 | Box::pin(future::ok(results::GetNode::default())) 128 | } 129 | 130 | #[cfg(test)] 131 | fn unassign_node(&self, _params: params::UnassignNode) -> DbFuture<'_, results::UnassignNode> { 132 | Box::pin(future::ok(())) 133 | } 134 | 135 | #[cfg(test)] 136 | fn remove_node(&self, _params: params::RemoveNode) -> DbFuture<'_, results::RemoveNode> { 137 | Box::pin(future::ok(())) 138 | } 139 | 140 | #[cfg(test)] 141 | fn post_service(&self, _params: params::PostService) -> DbFuture<'_, results::PostService> { 142 | Box::pin(future::ok(results::PostService::default())) 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /tokenserver-db/src/params.rs: -------------------------------------------------------------------------------- 1 | //! Parameter types for database methods. 2 | 3 | #[derive(Clone, Default)] 4 | pub struct PostNode { 5 | pub service_id: i32, 6 | pub node: String, 7 | pub available: i32, 8 | pub current_load: i32, 9 | pub capacity: i32, 10 | pub downed: i32, 11 | pub backoff: i32, 12 | } 13 | 14 | #[derive(Clone, Default)] 15 | pub struct GetNode { 16 | pub id: i64, 17 | } 18 | 19 | #[derive(Default)] 20 | pub struct PostService { 21 | pub service: String, 22 | pub pattern: String, 23 | } 24 | 25 | pub struct GetUsers { 26 | pub service_id: i32, 27 | pub email: String, 28 | } 29 | 30 | #[derive(Clone, Default)] 31 | pub struct GetOrCreateUser { 32 | pub service_id: i32, 33 | pub email: String, 34 | pub generation: i64, 35 | pub client_state: String, 36 | pub keys_changed_at: Option, 37 | pub capacity_release_rate: Option, 38 | } 39 | 40 | pub type AllocateUser = GetOrCreateUser; 41 | 42 | #[derive(Clone, Default)] 43 | pub struct PostUser { 44 | pub service_id: i32, 45 | pub email: String, 46 | pub generation: i64, 47 | pub client_state: String, 48 | pub created_at: i64, 49 | pub node_id: i64, 50 | pub keys_changed_at: Option, 51 | } 52 | 53 | /// The parameters used to update a user record. `generation` and `keys_changed_at` are applied to 54 | /// the user record matching the given `service_id` and `email`. 55 | #[derive(Default)] 56 | pub struct PutUser { 57 | pub service_id: i32, 58 | pub email: String, 59 | pub generation: i64, 60 | pub keys_changed_at: Option, 61 | } 62 | 63 | #[derive(Default)] 64 | pub struct ReplaceUsers { 65 | pub email: String, 66 | pub service_id: i32, 67 | pub replaced_at: i64, 68 | } 69 | 70 | #[derive(Default)] 71 | pub struct ReplaceUser { 72 | pub uid: i64, 73 | pub service_id: i32, 74 | pub replaced_at: i64, 75 | } 76 | 77 | #[derive(Debug, Default)] 78 | pub struct GetNodeId { 79 | pub service_id: i32, 80 | pub node: String, 81 | } 82 | 83 | #[derive(Default)] 84 | pub struct GetBestNode { 85 | pub service_id: i32, 86 | pub capacity_release_rate: Option, 87 | } 88 | 89 | #[derive(Default)] 90 | pub struct AddUserToNode { 91 | pub service_id: i32, 92 | pub node: String, 93 | } 94 | 95 | pub struct GetServiceId { 96 | pub service: String, 97 | } 98 | 99 | #[cfg(test)] 100 | pub struct SetUserCreatedAt { 101 | pub uid: i64, 102 | pub created_at: i64, 103 | } 104 | 105 | #[cfg(test)] 106 | pub struct SetUserReplacedAt { 107 | pub uid: i64, 108 | pub replaced_at: i64, 109 | } 110 | 111 | #[cfg(test)] 112 | #[derive(Default)] 113 | pub struct GetUser { 114 | pub id: i64, 115 | } 116 | 117 | #[cfg(test)] 118 | pub struct UnassignNode { 119 | pub node_id: i64, 120 | } 121 | 122 | #[cfg(test)] 123 | pub struct RemoveNode { 124 | pub node_id: i64, 125 | } 126 | -------------------------------------------------------------------------------- /tokenserver-db/src/results.rs: -------------------------------------------------------------------------------- 1 | use diesel::{ 2 | sql_types::{Bigint, Integer, Nullable, Text}, 3 | QueryableByName, 4 | }; 5 | use serde::{Deserialize, Serialize}; 6 | 7 | /// Represents a user record as it is stored in the database. 8 | #[derive(Clone, Debug, Default, Deserialize, QueryableByName, Serialize)] 9 | pub struct GetRawUser { 10 | #[sql_type = "Bigint"] 11 | pub uid: i64, 12 | #[sql_type = "Text"] 13 | pub client_state: String, 14 | #[sql_type = "Bigint"] 15 | pub generation: i64, 16 | #[sql_type = "Nullable"] 17 | pub node: Option, 18 | #[sql_type = "Nullable"] 19 | pub keys_changed_at: Option, 20 | #[sql_type = "Bigint"] 21 | pub created_at: i64, 22 | #[sql_type = "Nullable"] 23 | pub replaced_at: Option, 24 | } 25 | 26 | pub type GetUsers = Vec; 27 | 28 | #[derive(Debug, Default, Eq, PartialEq)] 29 | pub struct AllocateUser { 30 | pub uid: i64, 31 | pub node: String, 32 | pub created_at: i64, 33 | } 34 | 35 | /// Represents the relevant information from the most recently-created user record in the database 36 | /// for a given email and service ID, along with any previously-seen client states seen for the 37 | /// user. 38 | #[derive(Debug, Default, Eq, PartialEq)] 39 | pub struct GetOrCreateUser { 40 | pub uid: i64, 41 | pub email: String, 42 | pub client_state: String, 43 | pub generation: i64, 44 | pub node: String, 45 | pub keys_changed_at: Option, 46 | pub created_at: i64, 47 | pub replaced_at: Option, 48 | pub first_seen_at: i64, 49 | pub old_client_states: Vec, 50 | } 51 | 52 | #[derive(Default, QueryableByName)] 53 | pub struct LastInsertId { 54 | #[sql_type = "Bigint"] 55 | pub id: i64, 56 | } 57 | 58 | pub type PostUser = LastInsertId; 59 | pub type ReplaceUsers = (); 60 | pub type ReplaceUser = (); 61 | pub type PutUser = (); 62 | 63 | #[derive(Default, QueryableByName)] 64 | pub struct GetNodeId { 65 | #[sql_type = "Bigint"] 66 | pub id: i64, 67 | } 68 | 69 | #[derive(Default, QueryableByName)] 70 | pub struct GetBestNode { 71 | #[sql_type = "Bigint"] 72 | pub id: i64, 73 | #[sql_type = "Text"] 74 | pub node: String, 75 | } 76 | 77 | pub type AddUserToNode = (); 78 | 79 | #[derive(Default, QueryableByName)] 80 | pub struct GetServiceId { 81 | #[sql_type = "Integer"] 82 | pub id: i32, 83 | } 84 | 85 | #[cfg(test)] 86 | #[derive(Debug, Default, Eq, PartialEq, QueryableByName)] 87 | pub struct GetUser { 88 | #[sql_type = "Integer"] 89 | #[column_name = "service"] 90 | pub service_id: i32, 91 | #[sql_type = "Text"] 92 | pub email: String, 93 | #[sql_type = "Bigint"] 94 | pub generation: i64, 95 | #[sql_type = "Text"] 96 | pub client_state: String, 97 | #[sql_type = "Nullable"] 98 | pub replaced_at: Option, 99 | #[sql_type = "Bigint"] 100 | #[column_name = "nodeid"] 101 | pub node_id: i64, 102 | #[sql_type = "Nullable"] 103 | pub keys_changed_at: Option, 104 | } 105 | 106 | #[cfg(test)] 107 | pub type PostNode = LastInsertId; 108 | 109 | #[cfg(test)] 110 | #[derive(Default, QueryableByName)] 111 | pub struct GetNode { 112 | #[sql_type = "Bigint"] 113 | pub id: i64, 114 | #[sql_type = "Integer"] 115 | #[column_name = "service"] 116 | pub service_id: i32, 117 | #[sql_type = "Text"] 118 | pub node: String, 119 | #[sql_type = "Integer"] 120 | pub available: i32, 121 | #[sql_type = "Integer"] 122 | pub current_load: i32, 123 | #[sql_type = "Integer"] 124 | pub capacity: i32, 125 | #[sql_type = "Integer"] 126 | pub downed: i32, 127 | #[sql_type = "Integer"] 128 | pub backoff: i32, 129 | } 130 | 131 | #[cfg(test)] 132 | #[derive(Default, QueryableByName)] 133 | pub struct PostService { 134 | #[sql_type = "Integer"] 135 | pub id: i32, 136 | } 137 | 138 | #[cfg(test)] 139 | pub type SetUserCreatedAt = (); 140 | 141 | #[cfg(test)] 142 | pub type SetUserReplacedAt = (); 143 | 144 | pub type Check = bool; 145 | 146 | #[cfg(test)] 147 | pub type UnassignNode = (); 148 | 149 | #[cfg(test)] 150 | pub type RemoveNode = (); 151 | -------------------------------------------------------------------------------- /tokenserver-settings/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tokenserver-settings" 3 | version.workspace=true 4 | license.workspace=true 5 | authors.workspace=true 6 | edition.workspace=true 7 | 8 | [dependencies] 9 | serde.workspace=true 10 | jsonwebtoken.workspace=true 11 | 12 | tokenserver-common = { path = "../tokenserver-common" } 13 | -------------------------------------------------------------------------------- /tokenserver-settings/src/lib.rs: -------------------------------------------------------------------------------- 1 | use jsonwebtoken::jwk::Jwk; 2 | use serde::Deserialize; 3 | use tokenserver_common::NodeType; 4 | 5 | #[derive(Clone, Debug, Deserialize)] 6 | #[serde(default)] 7 | pub struct Settings { 8 | /// The URL of the Tokenserver MySQL database. 9 | pub database_url: String, 10 | /// The max size of the database connection pool. 11 | pub database_pool_max_size: u32, 12 | // NOTE: Not supported by deadpool! 13 | /// The minimum number of database connections to be maintained at any given time. 14 | pub database_pool_min_idle: Option, 15 | /// Pool timeout when waiting for a slot to become available, in seconds 16 | pub database_pool_connection_timeout: Option, 17 | /// Database request timeout, in seconds 18 | pub database_request_timeout: Option, 19 | // XXX: This is a temporary setting used to enable Tokenserver-related features. In 20 | // the future, Tokenserver will always be enabled, and this setting will be 21 | // removed. 22 | /// Whether or not to enable Tokenserver. 23 | pub enabled: bool, 24 | /// The secret to be used when computing the hash for a Tokenserver user's metrics UID. 25 | pub fxa_metrics_hash_secret: String, 26 | /// The email domain for users' FxA accounts. This should be set according to the 27 | /// desired FxA environment (production or stage). 28 | pub fxa_email_domain: String, 29 | /// The URL of the FxA server used for verifying OAuth tokens. 30 | pub fxa_oauth_server_url: String, 31 | /// The timeout to be used when making requests to the FxA OAuth verification server. 32 | pub fxa_oauth_request_timeout: u64, 33 | /// The JWK to be used to verify OAuth tokens. Passing a JWK to the PyFxA Python library 34 | /// prevents it from making an external API call to FxA to get the JWK, yielding substantial 35 | /// performance benefits. This value should match that on the `/v1/jwks` endpoint on the FxA 36 | /// Auth Server. 37 | pub fxa_oauth_primary_jwk: Option, 38 | /// A secondary JWK to be used to verify OAuth tokens. This is intended to be used to enable 39 | /// seamless key rotations on FxA. 40 | pub fxa_oauth_secondary_jwk: Option, 41 | /// The rate at which capacity should be released from nodes that are at capacity. 42 | pub node_capacity_release_rate: Option, 43 | /// The type of the storage nodes used by this instance of Tokenserver. 44 | #[serde(default = "NodeType::spanner")] 45 | pub node_type: NodeType, 46 | /// The label to be used when reporting Metrics. 47 | pub statsd_label: String, 48 | /// Whether or not to run the Tokenserver migrations upon startup. 49 | pub run_migrations: bool, 50 | /// The database ID of the Spanner node. 51 | pub spanner_node_id: Option, 52 | /// The number of additional blocking threads to add to the blocking threadpool to handle 53 | /// OAuth verification requests to FxA. This value is added to the worker_max_blocking_threads 54 | /// config var. 55 | /// Note that this setting only applies if the OAuth public JWK is not cached, since OAuth 56 | /// verifications do not require requests to FXA if the JWK is set on Tokenserver. The server 57 | /// will return an error at startup if the JWK is not cached and this setting is `None`. 58 | pub additional_blocking_threads_for_fxa_requests: Option, 59 | /// The amount of time in seconds before a token provided by Tokenserver expires. 60 | pub token_duration: u64, 61 | } 62 | 63 | impl Default for Settings { 64 | fn default() -> Settings { 65 | Settings { 66 | database_url: "mysql://root@127.0.0.1/tokenserver".to_owned(), 67 | database_pool_max_size: 10, 68 | database_pool_min_idle: None, 69 | database_pool_connection_timeout: Some(30), 70 | database_request_timeout: None, 71 | enabled: false, 72 | fxa_email_domain: "api-accounts.stage.mozaws.net".to_owned(), 73 | fxa_metrics_hash_secret: "secret".to_owned(), 74 | fxa_oauth_server_url: "https://oauth.stage.mozaws.net".to_owned(), 75 | fxa_oauth_request_timeout: 10, 76 | fxa_oauth_primary_jwk: None, 77 | fxa_oauth_secondary_jwk: None, 78 | node_capacity_release_rate: None, 79 | node_type: NodeType::Spanner, 80 | statsd_label: "syncstorage.tokenserver".to_owned(), 81 | run_migrations: cfg!(test), 82 | spanner_node_id: None, 83 | additional_blocking_threads_for_fxa_requests: Some(1), 84 | token_duration: 3600, 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /tools/README.md: -------------------------------------------------------------------------------- 1 | # A collection of Sync Tools and utilities 2 | 3 | See each directory for details: 4 | 5 | * [hawk](hawk) - a tool for generating test HAWK authorization headers 6 | * [spanner](spanner) - Google Cloud Platform Spanner tools for maintenance and testing 7 | * [user_migration](user_migration) - scripts for dumping and moving user data from SQL to Spanner 8 | 9 | ## Installation 10 | 11 | These tools are mostly written in python. It is recommended that you create a commonly shared virtual environment using something like: 12 | 13 | `python3 -m venv venv` 14 | 15 | to create a `/venv` directory. To activate this, call `sh /venv/bin/activate`. 16 | 17 | Script dependencies can be installed via `pip install -r requirements.txt` for each tool. 18 | -------------------------------------------------------------------------------- /tools/examples/README.md: -------------------------------------------------------------------------------- 1 | # Example scripts for working with syncstorage-rs 2 | 3 | * `put.bash` - First `chmod +x put.bash` then `./put.bash` to simulate a PUT. 4 | -------------------------------------------------------------------------------- /tools/examples/put.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | NODE="http://localhost:8000" 3 | URI="/1.5/1/storage/col2/DEADBEEF" 4 | METHOD="PUT" 5 | SYNC_MASTER_SECRET="INSERT_SECRET_KEY_HERE" 6 | AUTH=`../hawk/venv/bin/python ../hawk/make_hawk_token.py --node $NODE --uri $URI --method $METHOD --secret=$SYNC_MASTER_SECRET --as_header` 7 | curl -vv -X PUT "$NODE$URI" \ 8 | -H "$AUTH" \ 9 | -H 'Content-Type: application/json' \ 10 | -H 'Accept: application/json' \ 11 | -d '{"id": "womble", "payload": "mary had a little lamb with a nice mint jelly", "sortindex": 0, "ttl": 86400}' 12 | -------------------------------------------------------------------------------- /tools/hawk/README.md: -------------------------------------------------------------------------------- 1 | # Make a Hawk compatible Auth header 2 | 3 | 1) The best way to install this is probably to set up a python virtual 4 | env. 5 | 6 | `python3 -m venv venv && venv/bin/pip install -r requirements.txt` 7 | 8 | this will create a python virtual environment in the `/venv` directory. 9 | 10 | *Note* You may need to install `python3-venv` for the above to work. 11 | 12 | Once the virtual env is installed, run `. venv/bin/activate`. This 13 | will ensure that calls to python and python tools happen within this 14 | virtual environment. 15 | 16 | 2) install the requirements using: 17 | 18 | `venv/bin/pip install -r requirements.txt` 19 | 20 | 3) To create a Token Header: 21 | 22 | You'll need to pass along your `SYNC_MASTER_SECRET` and the uri you'll be testing in order to generate a valid Hawk Id: 23 | 24 | `venv/bin/python make_hawk_token.py --uri /1.5/1/storage/meta/global --secret=$SYNC_MASTER_SECRET --as_header` 25 | 26 | ** For testing against uri's using methods other than GET, you'll need to pass along the `--method` flag to generate your token. Ie, `venv/bin/python make_hawk_token.py --method PUT --uri /1.5/1/storage/meta/global --secret=$SYNC_MASTER_SECRET --as_header`. See [examples/put.bash](https://github.com/mozilla-services/syncstorage-rs/blob/master/tools/examples/put.bash) for an example of this. 27 | 28 | 29 | Use `-h` for help. 30 | -------------------------------------------------------------------------------- /tools/hawk/make_hawk_token.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Create a Hawk token for tests 3 | 4 | requires hawkauthlib, tokenlib, webob 5 | 6 | Creates the hawk headers for auth::tests, in particular valid_header and 7 | valid_header_with_querystring. 8 | 9 | The latter modifies the query string which changes the mac/nonce and 10 | potentially ts values (in the Hawk header). 11 | 12 | """ 13 | import argparse 14 | import hmac 15 | import os 16 | import time 17 | from binascii import hexlify 18 | from datetime import timedelta 19 | from hashlib import sha256 20 | 21 | import hawkauthlib 22 | import tokenlib 23 | from webob.request import Request 24 | 25 | LEGACY_UID = 1 26 | COL = "col2" 27 | URI = "/1.5/{uid}/storage/{col}/".format(uid=LEGACY_UID, col=COL) 28 | METHOD = "GET" 29 | FXA_UID = "DEADBEEF00004be4ae957006c0ceb620" 30 | FXA_KID = "DEADBEEF00004be4ae957006c0ceb620" 31 | DEVICE_ID = "device1" 32 | NODE = "http://localhost:8000" 33 | SECRET = os.envrion.get("SYNC_MASTER_SECRET", "Ted_Koppel_is_a_robot") 34 | HMAC_KEY = b"foo" 35 | 36 | # 10 years 37 | DURATION = timedelta(days=10 * 365).total_seconds() 38 | 39 | SALT = hexlify(os.urandom(3)).decode('ascii') 40 | 41 | 42 | def get_args(): 43 | parser = argparse.ArgumentParser( 44 | description="Create a hawk header for use in testing" 45 | ) 46 | parser.add_argument( 47 | '--uid', type=int, default=LEGACY_UID, 48 | help="Legacy UID ({})".format(LEGACY_UID)) 49 | parser.add_argument( 50 | '--uri', default=URI, 51 | help="URI path ({})".format(URI)) 52 | parser.add_argument( 53 | '--method', default=METHOD, 54 | help="The HTTP Method ({})".format(METHOD)) 55 | parser.add_argument( 56 | '--fxa_uid', default=FXA_UID, 57 | help="FxA User ID ({})".format(FXA_UID)) 58 | parser.add_argument( 59 | '--fxa_kid', default=FXA_KID, 60 | help="FxA K ID ({})".format(FXA_KID)) 61 | parser.add_argument( 62 | '--device_id', default=DEVICE_ID, 63 | help="FxA Device ID ({})".format(DEVICE_ID)) 64 | parser.add_argument( 65 | '--node', default=NODE, 66 | help="HTTP Host URI for node ({})".format(NODE)) 67 | parser.add_argument( 68 | '--duration', type=int, default=DURATION, 69 | help="Hawk TTL ({})".format(DURATION)) 70 | parser.add_argument( 71 | '--secret', default=SECRET, 72 | help="Shared HAWK secret ({})".format(SECRET)) 73 | parser.add_argument( 74 | '--hmac_key', default=HMAC_KEY, 75 | help="HAWK HMAC key ({})".format(HMAC_KEY)) 76 | parser.add_argument( 77 | '--as_header', action="store_true", default=False, 78 | help="return only header (False)") 79 | return parser.parse_args() 80 | 81 | 82 | def create_token(args): 83 | expires = int(time.time()) + args.duration 84 | token_data = { 85 | 'uid': args.uid, 86 | 'node': args.node, 87 | 'expires': expires, 88 | 'fxa_uid': args.fxa_uid, 89 | 'fxa_kid': args.fxa_kid, 90 | 'hashed_fxa_uid': metrics_hash(args, args.fxa_uid), 91 | 'hashed_device_id': metrics_hash(args, args.device_id), 92 | 'salt': SALT, 93 | } 94 | token = tokenlib.make_token(token_data, secret=args.secret) 95 | key = tokenlib.get_derived_secret(token, secret=args.secret) 96 | return token, key, expires, SALT 97 | 98 | 99 | def metrics_hash(args, value): 100 | if isinstance(args.hmac_key, str): 101 | args.hmac_key = args.hmac_key.encode() 102 | hasher = hmac.new(args.hmac_key, b'', sha256) 103 | # value may be an email address, in which case we only want the first part 104 | hasher.update(value.encode('utf-8').split(b"@", 1)[0]) 105 | return hasher.hexdigest() 106 | 107 | 108 | def main(): 109 | args = get_args() 110 | token, key, expires, salt = create_token(args) 111 | path = "{node}{uri}".format( 112 | node=args.node, 113 | uri=args.uri) 114 | req = Request.blank(path) 115 | req.method = args.method 116 | header = hawkauthlib.sign_request(req, token, key) 117 | if not args.as_header: 118 | print("Expires: ", expires) 119 | print("Salt: ", salt) 120 | print("\nPath: ", path) 121 | print("Hawk Authorization Header: ", header) 122 | else: 123 | print("Authorization:", header) 124 | 125 | 126 | if __name__ == '__main__': 127 | main() 128 | -------------------------------------------------------------------------------- /tools/hawk/requirements.txt: -------------------------------------------------------------------------------- 1 | hawkauthlib 2 | tokenlib 3 | webob 4 | -------------------------------------------------------------------------------- /tools/integration_tests/requirements.txt: -------------------------------------------------------------------------------- 1 | cryptography==44.0.2 2 | hawkauthlib 3 | konfig 4 | mysqlclient 5 | psutil 6 | pyjwt 7 | pyramid 8 | pyramid_hawkauth 9 | pyfxa==0.8.1 10 | pytest 11 | requests 12 | simplejson 13 | sqlalchemy==1.4.46 14 | tokenlib 15 | webtest 16 | wsgiproxy2 17 | -------------------------------------------------------------------------------- /tools/integration_tests/tests-no-batch.ini: -------------------------------------------------------------------------------- 1 | [server:main] 2 | use = egg:Paste#http 3 | host = 0.0.0.0 4 | port = 5000 5 | 6 | [app:main] 7 | use = egg:SyncStorage 8 | 9 | [storage] 10 | backend = syncstorage.storage.sql.SQLStorage 11 | sqluri = ${MOZSVC_SQLURI} 12 | standard_collections = true 13 | quota_size = 5242880 14 | pool_size = 100 15 | pool_recycle = 3600 16 | reset_on_return = true 17 | create_tables = true 18 | max_post_records = 4000 19 | 20 | [hawkauth] 21 | secret = "TED KOPPEL IS A ROBOT" 22 | -------------------------------------------------------------------------------- /tools/integration_tests/tests-paginated.ini: -------------------------------------------------------------------------------- 1 | [server:main] 2 | use = egg:Paste#http 3 | host = 0.0.0.0 4 | port = 5000 5 | 6 | [app:main] 7 | use = egg:SyncStorage 8 | 9 | [storage] 10 | backend = syncstorage.storage.sql.SQLStorage 11 | sqluri = ${MOZSVC_SQLURI} 12 | standard_collections = true 13 | quota_size = 5242880 14 | pool_size = 100 15 | pool_recycle = 3600 16 | reset_on_return = true 17 | create_tables = true 18 | batch_upload_enabled = true 19 | # Use a small batch-size to help test internal pagination usage. 20 | pagination_batch_size = 4 21 | 22 | [hawkauth] 23 | secret = "TED KOPPEL IS A ROBOT" 24 | -------------------------------------------------------------------------------- /tools/integration_tests/tests.ini: -------------------------------------------------------------------------------- 1 | [server:main] 2 | use = egg:Paste#http 3 | host = 0.0.0.0 4 | port = 5000 5 | 6 | [app:main] 7 | use = egg:SyncStorage 8 | 9 | [storage] 10 | backend = syncstorage.storage.sql.SQLStorage 11 | sqluri = ${MOZSVC_SQLURI} 12 | standard_collections = true 13 | quota_size = 5242880 14 | pool_size = 100 15 | pool_recycle = 3600 16 | reset_on_return = true 17 | create_tables = true 18 | max_post_records = 4000 19 | batch_upload_enabled = true 20 | force_consistent_sort_order = true 21 | 22 | [hawkauth] 23 | secret = "TED KOPPEL IS A ROBOT" 24 | 25 | [endpoints] 26 | syncstorage-rs = http://localhost:8000/1.5/1 27 | -------------------------------------------------------------------------------- /tools/integration_tests/tokenserver/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/syncstorage-rs/0768d4975de555b2c7af64dfef1ba40e5e2f99df/tools/integration_tests/tokenserver/__init__.py -------------------------------------------------------------------------------- /tools/integration_tests/tokenserver/mock_fxa_server.py: -------------------------------------------------------------------------------- 1 | from wsgiref.simple_server import make_server as _make_server 2 | from pyramid.config import Configurator 3 | from pyramid.response import Response 4 | from pyramid.view import view_config 5 | import json 6 | import os 7 | 8 | 9 | @view_config(route_name='mock_oauth_verify', renderer='json') 10 | def _mock_oauth_verify(request): 11 | body = json.loads(request.json_body['token']) 12 | 13 | return Response(json=body['body'], content_type='application/json', 14 | status=body['status']) 15 | 16 | 17 | # The PyFxA OAuth client makes a request to the FxA OAuth server for its 18 | # current public RSA key. While the client allows us to pass in a JWK to 19 | # prevent this request from happening, mocking the endpoint is simpler. 20 | @view_config(route_name='mock_oauth_jwk', renderer='json') 21 | def _mock_oauth_jwk(request): 22 | return {'keys': [{'fake': 'RSA key'}]} 23 | 24 | 25 | def make_server(host, port): 26 | with Configurator() as config: 27 | config.add_route('mock_oauth_verify', '/v1/verify') 28 | config.add_view(_mock_oauth_verify, route_name='mock_oauth_verify', 29 | renderer='json') 30 | 31 | config.add_route('mock_oauth_jwk', '/v1/jwks') 32 | config.add_view(_mock_oauth_jwk, route_name='mock_oauth_jwk', 33 | renderer='json') 34 | app = config.make_wsgi_app() 35 | 36 | return _make_server(host, port, app) 37 | 38 | 39 | if __name__ == '__main__': 40 | host = os.environ.get('MOCK_FXA_SERVER_HOST', 'localhost') 41 | port = os.environ.get('MOCK_FXA_SERVER_PORT', 6000) 42 | 43 | with make_server(host, int(port)) as httpd: 44 | print("Running mock FxA server on %s:%s" % (host, port)) 45 | httpd.serve_forever() 46 | -------------------------------------------------------------------------------- /tools/spanner/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12-bookworm 2 | 3 | COPY purge_ttl.py count_expired_rows.py count_users.py requirements.txt /app/ 4 | 5 | RUN pip install -r /app/requirements.txt 6 | 7 | USER nobody 8 | 9 | ENTRYPOINT ["/usr/local/bin/python"] 10 | CMD ["/app/purge_ttl.py"] 11 | -------------------------------------------------------------------------------- /tools/spanner/README.md: -------------------------------------------------------------------------------- 1 | # Spanner Tools and Scripts 2 | 3 | These tools are supplemental scripts for working with the Google Cloud Platform. Follow [the general installation instructions](https://cloud.google.com/spanner/docs/getting-started/python/), as well as fetch the proper service account credentials file. 4 | 5 | Remember, the `GOOGLE_APPLICATION_CREDENTIALS` environment variable should point to the absolute path location of your service account credential file. 6 | 7 | e.g. 8 | ```bash 9 | GOOGLE_APPLICATION_CREDENTIALS=`pwd`/keys/project-id-service-cred.json venv/bin/python purge_ttl.py 10 | ``` 11 | See each script for details about function and use. 12 | -------------------------------------------------------------------------------- /tools/spanner/count_expired_rows.py: -------------------------------------------------------------------------------- 1 | # Count the number of users in the spanner database 2 | # Specifically, the number of unique fxa_uid found in the user_collections table 3 | # 4 | # This Source Code Form is subject to the terms of the Mozilla Public 5 | # License, v. 2.0. If a copy of the MPL was not distributed with this 6 | # file, You can obtain one at https://mozilla.org/MPL/2.0/. 7 | 8 | import os 9 | import sys 10 | import logging 11 | from datetime import datetime 12 | from statsd.defaults.env import statsd 13 | from urllib import parse 14 | 15 | from google.cloud import spanner 16 | 17 | # set up logger 18 | logging.basicConfig( 19 | format='{"datetime": "%(asctime)s", "message": "%(message)s"}', 20 | stream=sys.stdout, 21 | level=logging.INFO) 22 | 23 | # Change these to match your install. 24 | client = spanner.Client() 25 | 26 | 27 | def from_env(): 28 | try: 29 | url = os.environ.get("SYNC_SYNCSTORAGE__DATABASE_URL") 30 | if not url: 31 | raise Exception("no url") 32 | purl = parse.urlparse(url) 33 | if purl.scheme == "spanner": 34 | path = purl.path.split("/") 35 | instance_id = path[-3] 36 | database_id = path[-1] 37 | except Exception as e: 38 | # Change these to reflect your Spanner instance install 39 | print("Exception {}".format(e)) 40 | instance_id = os.environ.get("INSTANCE_ID", "spanner-test") 41 | database_id = os.environ.get("DATABASE_ID", "sync_stage") 42 | return (instance_id, database_id) 43 | 44 | 45 | def spanner_read_data(query, table): 46 | (instance_id, database_id) = from_env() 47 | instance = client.instance(instance_id) 48 | database = instance.database(database_id) 49 | 50 | logging.info("For {}:{}".format(instance_id, database_id)) 51 | 52 | # Count bsos expired rows 53 | with statsd.timer(f"syncstorage.count_expired_{table}_rows.duration"): 54 | with database.snapshot() as snapshot: 55 | result = snapshot.execute_sql(query) 56 | row_count = result.one()[0] 57 | statsd.gauge(f"syncstorage.expired_{table}_rows", row_count) 58 | logging.info(f"Found {row_count} expired rows in {table}") 59 | 60 | 61 | if __name__ == "__main__": 62 | logging.info('Starting count_expired_rows.py') 63 | 64 | for table in ['batches', 'bsos']: 65 | query = f'SELECT COUNT(*) FROM {table} WHERE expiry < CURRENT_TIMESTAMP()' 66 | spanner_read_data(query, table) 67 | 68 | logging.info('Completed count_expired_rows.py') 69 | -------------------------------------------------------------------------------- /tools/spanner/count_users.py: -------------------------------------------------------------------------------- 1 | # Count the number of users in the spanner database 2 | # Specifically, the number of unique fxa_uid found in the user_collections table 3 | # 4 | # This Source Code Form is subject to the terms of the Mozilla Public 5 | # License, v. 2.0. If a copy of the MPL was not distributed with this 6 | # file, You can obtain one at https://mozilla.org/MPL/2.0/. 7 | 8 | import os 9 | import sys 10 | import logging 11 | from datetime import datetime 12 | from statsd.defaults.env import statsd 13 | from urllib import parse 14 | 15 | from google.cloud import spanner 16 | 17 | # set up logger 18 | logging.basicConfig( 19 | format='{"datetime": "%(asctime)s", "message": "%(message)s"}', 20 | stream=sys.stdout, 21 | level=logging.INFO) 22 | 23 | # Change these to match your install. 24 | client = spanner.Client() 25 | 26 | 27 | def from_env(): 28 | try: 29 | url = os.environ.get("SYNC_SYNCSTORAGE__DATABASE_URL") 30 | if not url: 31 | raise Exception("no url") 32 | purl = parse.urlparse(url) 33 | if purl.scheme == "spanner": 34 | path = purl.path.split("/") 35 | instance_id = path[-3] 36 | database_id = path[-1] 37 | except Exception as e: 38 | # Change these to reflect your Spanner instance install 39 | print("Exception {}".format(e)) 40 | instance_id = os.environ.get("INSTANCE_ID", "spanner-test") 41 | database_id = os.environ.get("DATABASE_ID", "sync_stage") 42 | return (instance_id, database_id) 43 | 44 | 45 | def spanner_read_data(request=None): 46 | (instance_id, database_id) = from_env() 47 | instance = client.instance(instance_id) 48 | database = instance.database(database_id) 49 | 50 | logging.info("For {}:{}".format(instance_id, database_id)) 51 | 52 | # Count users 53 | with statsd.timer("syncstorage.count_users.duration"): 54 | with database.snapshot() as snapshot: 55 | query = 'SELECT COUNT (DISTINCT fxa_uid) FROM user_collections' 56 | result = snapshot.execute_sql(query) 57 | user_count = result.one()[0] 58 | statsd.gauge("syncstorage.distinct_fxa_uid", user_count) 59 | logging.info("Count found {} distinct users".format(user_count)) 60 | 61 | 62 | if __name__ == "__main__": 63 | logging.info('Starting count_users.py') 64 | 65 | spanner_read_data() 66 | 67 | logging.info('Completed count_users.py') 68 | -------------------------------------------------------------------------------- /tools/spanner/requirements.txt: -------------------------------------------------------------------------------- 1 | google-cloud-spanner >= 1.16.0 2 | statsd 3 | -------------------------------------------------------------------------------- /tools/tokenserver/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla-services/syncstorage-rs/0768d4975de555b2c7af64dfef1ba40e5e2f99df/tools/tokenserver/__init__.py -------------------------------------------------------------------------------- /tools/tokenserver/add_node.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | """ 6 | 7 | Script to add a new node to the system. 8 | 9 | """ 10 | 11 | import logging 12 | import optparse 13 | 14 | from database import Database, SERVICE_NAME 15 | import util 16 | 17 | 18 | logger = logging.getLogger("tokenserver.scripts.add_node") 19 | 20 | 21 | def add_node(node, capacity, **kwds): 22 | """Add the specific node to the system.""" 23 | logger.info("Adding node %s to service %s", node, SERVICE_NAME) 24 | try: 25 | database = Database() 26 | database.add_node(node, capacity, **kwds) 27 | except Exception: 28 | logger.exception("Error while adding node") 29 | return False 30 | else: 31 | logger.info("Finished adding node %s", node) 32 | return True 33 | 34 | 35 | def main(args=None): 36 | """Main entry-point for running this script. 37 | 38 | This function parses command-line arguments and passes them on 39 | to the add_node() function. 40 | """ 41 | usage = "usage: %prog [options] node_name capacity" 42 | descr = "Add a new node to the tokenserver database" 43 | parser = optparse.OptionParser(usage=usage, description=descr) 44 | parser.add_option("", "--available", type="int", 45 | help="How many user slots the node has available") 46 | parser.add_option("", "--current-load", type="int", 47 | help="How many user slots the node has occupied") 48 | parser.add_option("", "--downed", action="store_true", 49 | help="Mark the node as down in the db") 50 | parser.add_option("", "--backoff", action="store_true", 51 | help="Mark the node as backed-off in the db") 52 | parser.add_option("-v", "--verbose", action="count", dest="verbosity", 53 | help="Control verbosity of log messages") 54 | 55 | opts, args = parser.parse_args(args) 56 | if len(args) != 2: 57 | parser.print_usage() 58 | return 1 59 | 60 | util.configure_script_logging(opts) 61 | 62 | node_name = args[0] 63 | capacity = int(args[1]) 64 | 65 | kwds = {} 66 | if opts.available is not None: 67 | kwds["available"] = opts.available 68 | if opts.current_load is not None: 69 | kwds["current_load"] = opts.current_load 70 | if opts.backoff is not None: 71 | kwds["backoff"] = opts.backoff 72 | if opts.downed is not None: 73 | kwds["downed"] = opts.downed 74 | 75 | add_node(node_name, capacity, **kwds) 76 | return 0 77 | 78 | 79 | if __name__ == "__main__": 80 | util.run_script(main) 81 | -------------------------------------------------------------------------------- /tools/tokenserver/allocate_user.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | """ 5 | 6 | Script to allocate a specific user to a node. 7 | 8 | This script allocates the specified user to a node. A particular node 9 | may be specified, or the best available node used by default. 10 | 11 | The allocated node is printed to stdout. 12 | 13 | """ 14 | 15 | import logging 16 | import optparse 17 | 18 | from database import Database 19 | import util 20 | 21 | 22 | logger = logging.getLogger("tokenserver.scripts.allocate_user") 23 | 24 | 25 | def allocate_user(email, node=None): 26 | logger.info("Allocating node for user %s", email) 27 | try: 28 | database = Database() 29 | user = database.get_user(email) 30 | if user is None: 31 | user = database.allocate_user(email, node=node) 32 | else: 33 | database.update_user(user, node=node) 34 | except Exception: 35 | logger.exception("Error while updating node") 36 | return False 37 | else: 38 | logger.info("Finished updating node %s", node) 39 | return True 40 | 41 | 42 | def main(args=None): 43 | """Main entry-point for running this script. 44 | 45 | This function parses command-line arguments and passes them on 46 | to the allocate_user() function. 47 | """ 48 | usage = "usage: %prog [options] email [node_name]" 49 | descr = "Allocate a user to a node. You may specify a particular node, "\ 50 | "or omit to use the best available node." 51 | parser = optparse.OptionParser(usage=usage, description=descr) 52 | parser.add_option("-v", "--verbose", action="count", dest="verbosity", 53 | help="Control verbosity of log messages") 54 | 55 | opts, args = parser.parse_args(args) 56 | if not 1 <= len(args) <= 2: 57 | parser.print_usage() 58 | return 1 59 | 60 | util.configure_script_logging(opts) 61 | 62 | email = args[0] 63 | if len(args) == 1: 64 | node_name = None 65 | else: 66 | node_name = args[1] 67 | 68 | allocate_user(email, node_name) 69 | return 0 70 | 71 | 72 | if __name__ == "__main__": 73 | util.run_script(main) 74 | -------------------------------------------------------------------------------- /tools/tokenserver/conftest.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 5 | -------------------------------------------------------------------------------- /tools/tokenserver/count_users.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | """ 5 | 6 | Script to emit total-user-count metrics for exec dashboard. 7 | 8 | """ 9 | 10 | import json 11 | import logging 12 | import optparse 13 | import os 14 | import socket 15 | import sys 16 | import time 17 | from datetime import datetime, timedelta, tzinfo 18 | 19 | from database import Database 20 | import util 21 | 22 | logger = logging.getLogger("tokenserver.scripts.count_users") 23 | 24 | ZERO = timedelta(0) 25 | 26 | 27 | class UTC(tzinfo): 28 | 29 | def utcoffset(self, dt): 30 | return ZERO 31 | 32 | def tzname(self, dt): 33 | return "UTC" 34 | 35 | def dst(self, dt): 36 | return ZERO 37 | 38 | 39 | utc = UTC() 40 | 41 | 42 | def count_users(outfile, timestamp=None): 43 | if timestamp is None: 44 | ts = time.gmtime() 45 | midnight = (ts[0], ts[1], ts[2], 0, 0, 0, ts[6], ts[7], ts[8]) 46 | timestamp = int(time.mktime(midnight)) * 1000 47 | database = Database() 48 | logger.debug("Counting users created before %i", timestamp) 49 | count = database.count_users(timestamp) 50 | logger.debug("Found %d users", count) 51 | # Output has heka-filter-compatible JSON object. 52 | ts_sec = timestamp / 1000 53 | output = { 54 | "hostname": socket.gethostname(), 55 | "pid": os.getpid(), 56 | "op": "sync_count_users", 57 | "total_users": count, 58 | "time": datetime.fromtimestamp(ts_sec, utc).isoformat(), 59 | "v": 0 60 | } 61 | json.dump(output, outfile) 62 | outfile.write("\n") 63 | 64 | 65 | def main(args=None): 66 | """Main entry-point for running this script. 67 | 68 | This function parses command-line arguments and passes them on 69 | to the add_node() function. 70 | """ 71 | usage = "usage: %prog [options]" 72 | descr = "Count total users in the tokenserver database" 73 | parser = optparse.OptionParser(usage=usage, description=descr) 74 | parser.add_option("-t", "--timestamp", type="int", 75 | help="Max creation timestamp; default previous midnight") 76 | parser.add_option("-o", "--output", 77 | help="Output file; default stderr") 78 | parser.add_option("-v", "--verbose", action="count", dest="verbosity", 79 | help="Control verbosity of log messages") 80 | 81 | opts, args = parser.parse_args(args) 82 | if len(args) != 0: 83 | parser.print_usage() 84 | return 1 85 | 86 | util.configure_script_logging(opts) 87 | 88 | if opts.output in (None, "-"): 89 | count_users(sys.stdout, opts.timestamp) 90 | else: 91 | with open(opts.output, "a") as outfile: 92 | count_users(outfile, opts.timestamp) 93 | 94 | return 0 95 | 96 | 97 | if __name__ == "__main__": 98 | util.run_script(main) 99 | -------------------------------------------------------------------------------- /tools/tokenserver/loadtests/README.md: -------------------------------------------------------------------------------- 1 | # Tokenserver Load Tests 2 | 3 | This directory contains everything needed to run the suite of load tests for Tokenserver. 4 | 5 | ## Building and Running 6 | 7 | 1. Install the load testing dependencies: 8 | 9 | ```sh 10 | pip3 install -r requirements.txt 11 | ``` 12 | 13 | 1. Run the `generate-keys.sh` script to generate an RSA keypair and derive the public JWK: 14 | 15 | ```sh 16 | ./generate-keys.sh 17 | ``` 18 | 19 | This script will output two files: 20 | 21 | - `load_test.pem`: The private key to be used by the load tests to create OAuth tokens 22 | - `jwk.json`: The public JWK associated with the private key. This is a key of the form 23 | 24 | ```json 25 | { 26 | "n": ..., 27 | "e": ..., 28 | "kty": "RSA" 29 | } 30 | ``` 31 | 32 | 1. Set the following environment variables/settings on Tokenserver: 33 | 34 | ```sh 35 | # Should be set to the "n" component of the JWK 36 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK_N 37 | # Should be set to the "e" component of the JWK (this value should almost always be "AQAB") 38 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK_E 39 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK_KTY=RSA 40 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK_USE=sig 41 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK_ALG=RS256 42 | 43 | # These two environment variables don't affect the load tests, but they need to be set: 44 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK_KID="" 45 | SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK_FXA_CREATED_AT=0 46 | ``` 47 | 48 | 1. Configure Tokenserver to verify BrowserID assertions through FxA stage. This is done by setting the following environment variables: 49 | 50 | ```sh 51 | # The exact value of this environment variable is not important as long as it matches the `BROWSERID_AUDIENCE` environment variable set on the machine running the load tests, as described below 52 | SYNC_TOKENSERVER__FXA_BROWSERID_SERVER_URL=https://verifier.stage.mozaws.net/v2 53 | 54 | SYNC_TOKENSERVER__FXA_BROWSERID_AUDIENCE=https://token.stage.mozaws.net 55 | SYNC_TOKENSERVER__FXA_BROWSERID_ISSUER=mockmyid.s3-us-west-2.amazonaws.com 56 | ``` 57 | 58 | Note that, because we have cached the JWK used to verify OAuth tokens, no verification requests will be made to FxA, so the value of `SYNC_TOKENSERVER__FXA_OAUTH_VERIFIER_URL` does not matter; however, Tokenserver expects it to be set, so setting it to something like `http://localhost` will suffice. 59 | 60 | 1. Set the following environment variables on the machine that will be running the load tests: 61 | 62 | - `OAUTH_PEM_FILE` should be set to the location of the private RSA key generated in a previous step 63 | - `BROWSERID_AUDIENCE` should be set to match the `SYNC_TOKENSERVER__FXA_BROWSERID_AUDIENCE` environment variable on Tokenserver 64 | 65 | 1. Tokenserver uses [locust](https://locust.io/) for load testing. To run the load tests, simply run the following command in this directory: 66 | 67 | ```sh 68 | locust 69 | ``` 70 | 71 | 1. Navigate your browser to , where you'll find the locust GUI. Enter the following information: 72 | 73 | - Number of users: The peak number of Tokenserver users to be used during the load tests 74 | - Spawn rate: The rate at which new users are spawned 75 | - Host: The URL of the server to be load tested. Note that this URL must include the protocol (e.g. "http://") 76 | 77 | 1. Click the "Start swarming" button to begin the load tests. 78 | 79 | ## Populating the Database 80 | 81 | This directory includes an optional `populate_db.py` script that can be used to add test users to the database en masse. The script can be run like so: 82 | 83 | ```sh 84 | python3 populate_db.py 85 | ``` 86 | 87 | where `sqluri` is the URL of the Tokenserver database, `nodes` is a comma-separated list of nodes **that are already present in the database** to which the users will be randomly assigned, and `number of users` is the number of users to be created. 88 | -------------------------------------------------------------------------------- /tools/tokenserver/loadtests/generate-keys.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Generate a private RSA key 4 | openssl genrsa -out load_test.pem 2048 5 | 6 | # Derive the public key from the private key 7 | openssl rsa -in load_test.pem -pubout > load_test.pub 8 | 9 | # Derive and print the JWK from the public key 10 | python3 get_jwk.py load_test.pub > jwk.json 11 | rm load_test.pub 12 | -------------------------------------------------------------------------------- /tools/tokenserver/loadtests/get_jwk.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from authlib.jose import JsonWebKey 3 | 4 | raw_public_key = open(sys.argv[1], "rb").read() 5 | public_key = JsonWebKey.import_key(raw_public_key, {"kty": "RSA"}) 6 | print(public_key.as_json()) 7 | -------------------------------------------------------------------------------- /tools/tokenserver/loadtests/populate_db.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # script to populate the database with records 3 | import time 4 | import random 5 | from sqlalchemy import create_engine 6 | from sqlalchemy.sql import text as sqltext 7 | 8 | _CREATE_USER_RECORD = sqltext("""\ 9 | insert into 10 | users 11 | (service, email, nodeid, generation, client_state, 12 | created_at, replaced_at) 13 | values 14 | (:service, :email, :nodeid, 0, "", :timestamp, NULL) 15 | """) 16 | 17 | _GET_SERVICE_ID = sqltext("""\ 18 | select 19 | id 20 | from 21 | services 22 | where 23 | service = :service 24 | """) 25 | 26 | _GET_NODE_ID = sqltext("""\ 27 | select 28 | id 29 | from 30 | nodes 31 | where 32 | service=:service and node=:node 33 | """) 34 | 35 | _SERVICE_NAME = 'sync-1.5' 36 | 37 | 38 | # This class creates a bunch of users associated with the sync-1.5 service. 39 | # 40 | # The resulting users will have an address in the form of @ where 41 | # uid is an int from 0 to :param user_range:. 42 | # 43 | # This class is useful to populate the database during the load tests. It 44 | # allows us to test a specific behaviour: making sure that we are not reading 45 | # the values from memory when retrieving the node information. 46 | # 47 | # :param sqluri: the sqluri string used to connect to the database 48 | # :param nodes: the list of available nodes for this service 49 | # :param user_range: the number of users to create 50 | # :param host: the hostname to use when generating users 51 | class PopulateDatabase: 52 | def __init__(self, sqluri, nodes, user_range, host="loadtest.local"): 53 | engine = create_engine(sqluri) 54 | self.database = engine. \ 55 | execution_options(isolation_level="AUTOCOMMIT"). \ 56 | connect() 57 | 58 | self.service_id = self._get_service_id() 59 | self.node_ids = [self._get_node_id(node) for node in nodes] 60 | self.user_range = user_range 61 | self.host = host 62 | 63 | def _get_node_id(self, node_name): 64 | """Get numeric id for a node.""" 65 | res = self.database.execute(_GET_NODE_ID, 66 | service=self.service_id, 67 | node=node_name) 68 | row = res.fetchone() 69 | res.close() 70 | if row is None: 71 | raise ValueError("unknown node: " + node_name) 72 | return row[0] 73 | 74 | def _get_service_id(self): 75 | res = self.database.execute(_GET_SERVICE_ID, service=_SERVICE_NAME) 76 | row = res.fetchone() 77 | res.close() 78 | return row.id 79 | 80 | def run(self): 81 | params = { 82 | 'service': self.service_id, 83 | 'timestamp': int(time.time() * 1000), 84 | } 85 | 86 | # for each user in the range, assign them to a node 87 | for idx in range(0, self.user_range): 88 | email = "%s@%s" % (idx, self.host) 89 | nodeid = random.choice(self.node_ids) 90 | self.database.execute(_CREATE_USER_RECORD, 91 | email=email, 92 | nodeid=nodeid, 93 | **params) 94 | 95 | 96 | def main(): 97 | # Read the arguments from the command line and pass them to the 98 | # PopulateDb class. 99 | # 100 | # Example use: 101 | # 102 | # python3 populate-db.py sqlite:////tmp/tokenserver\ 103 | # node1,node2,node3,node4,node5,node6 100 104 | import sys 105 | if len(sys.argv) < 4: 106 | raise ValueError('You need to specify (in this order) sqluri, ' 107 | 'nodes (comma separated), and user_range') 108 | # transform the values from the cli to python objects 109 | sys.argv[2] = sys.argv[2].split(',') # comma separated => list 110 | sys.argv[3] = int(sys.argv[3]) 111 | 112 | PopulateDatabase(*sys.argv[1:]).run() 113 | print("created {nb_users} users".format(nb_users=sys.argv[3])) 114 | 115 | 116 | if __name__ == '__main__': 117 | main() 118 | -------------------------------------------------------------------------------- /tools/tokenserver/loadtests/requirements.txt: -------------------------------------------------------------------------------- 1 | authlib 2 | cryptography 3 | locust 4 | pybrowserid 5 | pyjwt 6 | sqlalchemy 7 | -------------------------------------------------------------------------------- /tools/tokenserver/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | markers = 3 | migration_records: mark a test as a migration records test 4 | 5 | addopts = 6 | -m 'not migration_records' -------------------------------------------------------------------------------- /tools/tokenserver/remove_node.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | """ 5 | 6 | Script to remove a node from the system. 7 | 8 | This script nukes any references to the named node - it is removed from 9 | the "nodes" table and any users currently assigned to that node have their 10 | assignments cleared. 11 | 12 | """ 13 | 14 | import logging 15 | import optparse 16 | 17 | import util 18 | from database import Database 19 | 20 | logger = logging.getLogger("tokenserver.scripts.remove_node") 21 | 22 | 23 | def remove_node(node): 24 | """Remove the named node from the system.""" 25 | logger.info("Removing node %s", node) 26 | try: 27 | database = Database() 28 | found = False 29 | try: 30 | database.remove_node(node) 31 | except ValueError: 32 | logger.debug(" not found") 33 | else: 34 | found = True 35 | logger.debug(" removed") 36 | except Exception: 37 | logger.exception("Error while removing node") 38 | return False 39 | else: 40 | if not found: 41 | logger.info("Node %s was not found", node) 42 | else: 43 | logger.info("Finished removing node %s", node) 44 | return True 45 | 46 | 47 | def main(args=None): 48 | """Main entry-point for running this script. 49 | 50 | This function parses command-line arguments and passes them on 51 | to the remove_node() function. 52 | """ 53 | usage = "usage: %prog [options] node_name" 54 | descr = "Remove a node from the tokenserver database" 55 | parser = optparse.OptionParser(usage=usage, description=descr) 56 | parser.add_option("-v", "--verbose", action="count", dest="verbosity", 57 | help="Control verbosity of log messages") 58 | 59 | opts, args = parser.parse_args(args) 60 | if len(args) != 1: 61 | parser.print_usage() 62 | return 1 63 | 64 | util.configure_script_logging(opts) 65 | 66 | node_name = args[0] 67 | 68 | remove_node(node_name) 69 | return 0 70 | 71 | 72 | if __name__ == "__main__": 73 | util.run_script(main) 74 | -------------------------------------------------------------------------------- /tools/tokenserver/requirements.txt: -------------------------------------------------------------------------------- 1 | boto==2.49.0 2 | hawkauthlib==2.0.0 3 | mysqlclient==2.1.1 4 | pyramid==2.0.2 5 | sqlalchemy==1.4.46 6 | testfixtures 7 | tokenlib==2.0.0 8 | PyBrowserID==0.14.0 9 | pytest==8.3.5 10 | datadog 11 | backoff 12 | 13 | -------------------------------------------------------------------------------- /tools/tokenserver/unassign_node.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | """ 5 | 6 | Script to remove a node from the system. 7 | 8 | This script clears any assignments to the named node. 9 | 10 | """ 11 | 12 | import logging 13 | import optparse 14 | 15 | from database import Database 16 | import util 17 | 18 | 19 | logger = logging.getLogger("tokenserver.scripts.unassign_node") 20 | 21 | 22 | def unassign_node(node): 23 | """Clear any assignments to the named node.""" 24 | logger.info("Unassignment node %s", node) 25 | try: 26 | database = Database() 27 | found = False 28 | try: 29 | database.unassign_node(node) 30 | except ValueError: 31 | logger.debug(" not found") 32 | else: 33 | found = True 34 | logger.debug(" unassigned") 35 | except Exception: 36 | logger.exception("Error while unassigning node") 37 | return False 38 | else: 39 | if not found: 40 | logger.info("Node %s was not found", node) 41 | else: 42 | logger.info("Finished unassigning node %s", node) 43 | return True 44 | 45 | 46 | def main(args=None): 47 | """Main entry-point for running this script. 48 | 49 | This function parses command-line arguments and passes them on 50 | to the unassign_node() function. 51 | """ 52 | usage = "usage: %prog [options] node_name" 53 | descr = "Clear all assignments to node in the tokenserver database" 54 | parser = optparse.OptionParser(usage=usage, description=descr) 55 | parser.add_option("-v", "--verbose", action="count", dest="verbosity", 56 | help="Control verbosity of log messages") 57 | 58 | opts, args = parser.parse_args(args) 59 | if len(args) != 1: 60 | parser.print_usage() 61 | return 1 62 | 63 | util.configure_script_logging(opts) 64 | 65 | node_name = args[0] 66 | 67 | unassign_node(node_name) 68 | return 0 69 | 70 | 71 | if __name__ == "__main__": 72 | util.run_script(main) 73 | -------------------------------------------------------------------------------- /tools/tokenserver/update_node.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | """ 5 | 6 | Script to update node status in the db. 7 | 8 | """ 9 | 10 | import logging 11 | import optparse 12 | 13 | from database import Database 14 | import util 15 | 16 | 17 | logger = logging.getLogger("tokenserver.scripts.update_node") 18 | 19 | 20 | def update_node(node, **kwds): 21 | """Update details of a node.""" 22 | logger.info("Updating node %s for service %s", node) 23 | logger.debug("Value: %r", kwds) 24 | try: 25 | database = Database() 26 | database.update_node(node, **kwds) 27 | except Exception: 28 | logger.exception("Error while updating node") 29 | return False 30 | else: 31 | logger.info("Finished updating node %s", node) 32 | return True 33 | 34 | 35 | def main(args=None): 36 | """Main entry-point for running this script. 37 | 38 | This function parses command-line arguments and passes them on 39 | to the update_node() function. 40 | """ 41 | usage = "usage: %prog [options] node_name" 42 | descr = "Update node details in the tokenserver database" 43 | parser = optparse.OptionParser(usage=usage, description=descr) 44 | parser.add_option("", "--capacity", type="int", 45 | help="How many user slots the node has overall") 46 | parser.add_option("", "--available", type="int", 47 | help="How many user slots the node has available") 48 | parser.add_option("", "--current-load", type="int", 49 | help="How many user slots the node has occupied") 50 | parser.add_option("", "--downed", action="store_true", 51 | help="Mark the node as down in the db") 52 | parser.add_option("", "--backoff", action="store_true", 53 | help="Mark the node as backed-off in the db") 54 | parser.add_option("-v", "--verbose", action="count", dest="verbosity", 55 | help="Control verbosity of log messages") 56 | 57 | opts, args = parser.parse_args(args) 58 | if len(args) != 1: 59 | parser.print_usage() 60 | return 1 61 | 62 | util.configure_script_logging(opts) 63 | 64 | node_name = args[0] 65 | 66 | kwds = {} 67 | if opts.capacity is not None: 68 | kwds["capacity"] = opts.capacity 69 | if opts.available is not None: 70 | kwds["available"] = opts.available 71 | if opts.current_load is not None: 72 | kwds["current_load"] = opts.current_load 73 | if opts.backoff is not None: 74 | kwds["backoff"] = opts.backoff 75 | if opts.downed is not None: 76 | kwds["downed"] = opts.downed 77 | 78 | update_node(node_name, **kwds) 79 | return 0 80 | 81 | 82 | if __name__ == "__main__": 83 | util.run_script(main) 84 | -------------------------------------------------------------------------------- /tools/tokenserver/util.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | """ 5 | 6 | Admin/managment scripts for TokenServer. 7 | 8 | """ 9 | 10 | import sys 11 | import time 12 | import logging 13 | import base64 14 | import optparse 15 | import os 16 | import json 17 | from datetime import datetime 18 | 19 | from datadog import initialize, statsd 20 | 21 | 22 | def encode_bytes_b64(value): 23 | return base64.urlsafe_b64encode(value).rstrip(b'=').decode('ascii') 24 | 25 | 26 | def run_script(main): 27 | """Simple wrapper for running scripts in __main__ section.""" 28 | try: 29 | exitcode = main() 30 | except KeyboardInterrupt: 31 | exitcode = 1 32 | sys.exit(exitcode) 33 | 34 | 35 | def configure_script_logging(opts=None, logger_name=""): 36 | """Configure stdlib logging to produce output from the script. 37 | 38 | This basically configures logging to send messages to stderr, with 39 | formatting that's more for human readability than machine parsing. 40 | It also takes care of the --verbosity command-line option. 41 | """ 42 | 43 | verbosity = ( 44 | opts and getattr( 45 | opts, "verbosity", logging.NOTSET)) or logging.NOTSET 46 | logger = logging.getLogger(logger_name) 47 | level = os.environ.get("PYTHON_LOG", "").upper() or \ 48 | max(logging.DEBUG, logging.WARNING - (verbosity * 10)) or \ 49 | logger.getEffectiveLevel() 50 | 51 | # if we've previously setup a handler, adjust it instead 52 | if logger.hasHandlers(): 53 | handler = logger.handlers[0] 54 | else: 55 | handler = logging.StreamHandler() 56 | 57 | formatter = GCP_JSON_Formatter() 58 | # if we've opted for "human_logs", specify a simpler message. 59 | if opts: 60 | if getattr(opts, "human_logs", None): 61 | formatter = logging.Formatter( 62 | "{levelname:<8s}: {message}", 63 | style="{") 64 | 65 | handler.setFormatter(formatter) 66 | handler.setLevel(level) 67 | logger = logging.getLogger("") 68 | logger.addHandler(handler) 69 | logger.setLevel(level) 70 | return logger 71 | 72 | 73 | # We need to reformat a few things to get the record to display correctly 74 | # This includes "escaping" the message as well as converting the timestamp 75 | # into a parsable format. 76 | class GCP_JSON_Formatter(logging.Formatter): 77 | 78 | def format(self, record): 79 | return json.dumps({ 80 | "severity": record.levelname, 81 | "message": record.getMessage(), 82 | "timestamp": datetime.fromtimestamp( 83 | record.created).strftime( 84 | "%Y-%m-%dT%H:%M:%SZ" # RFC3339 85 | ), 86 | }) 87 | 88 | 89 | def format_key_id(keys_changed_at, key_hash): 90 | """Format an FxA key ID from a timestamp and key hash.""" 91 | return "{:013d}-{}".format( 92 | keys_changed_at, 93 | encode_bytes_b64(key_hash), 94 | ) 95 | 96 | 97 | def get_timestamp(): 98 | """Get current timestamp in milliseconds.""" 99 | return int(time.time() * 1000) 100 | 101 | 102 | class Metrics(): 103 | 104 | def __init__(self, opts, namespace=""): 105 | options = dict( 106 | namespace=namespace, 107 | statsd_namespace=namespace, 108 | statsd_host=getattr(opts, "metric_host"), 109 | statsd_port=getattr(opts, "metric_port"), 110 | ) 111 | self.prefix = options.get("namespace") 112 | initialize(**options) 113 | 114 | def incr(self, label, tags=None): 115 | statsd.increment(label, tags=tags) 116 | 117 | 118 | def add_metric_options(parser: optparse.OptionParser): 119 | """Add generic metric related options to an OptionParser""" 120 | parser.add_option( 121 | "", 122 | "--metric_host", 123 | default=os.environ.get("SYNC_STATSD_HOST"), 124 | help="Metric host name" 125 | ) 126 | parser.add_option( 127 | "", 128 | "--metric_port", 129 | default=os.environ.get("SYNC_STATSD_PORT"), 130 | help="Metric host port" 131 | ) 132 | -------------------------------------------------------------------------------- /tools/user_migration/README.md: -------------------------------------------------------------------------------- 1 | # User Migration Script 2 | 3 | This is a workspace for testing user migration from the old databases 4 | to the new durable one. 5 | 6 | There are several candidate scripts that you can use. 7 | 8 | These progress off of each other in order to provide cached results. 9 | 10 | There are a few base files you'll want to declare: 11 | 12 | * *dsns* - a file containing the mysql and spanner DSNs for the users. 13 | Each DSN should be on a single line. Currently only one DSN of a 14 | given type is permitted. 15 | 16 | (e.g.) 17 | 18 | ```text 19 | mysql://test:test@localhost/syncstorage 20 | spanner://projects/sync-spanner-dev-225401/instances/spanner-test/databases/sync_schema3 21 | ``` 22 | 23 | * *users.csv* - a mysql dump of the token database. This file is only needed if the `--deanon` de-anonymization flag is set. By default, data is anononymized to prevent accidental movement. 24 | You can produce this file from the following: 25 | ```bash 26 | mysql -e "select uid, email, generation, keys_changed_at, \ 27 | client_state from users;" > users.csv` 28 | ``` 29 | The script will automatically skip the title row, and presumes that fields are tab separated. 30 | 31 | 32 | With those files you can now run: 33 | 34 | ```bash 35 | gen_fxa_users.py 36 | ``` 37 | which will take the `users.csv` raw data and generate a 38 | `fxa_users_{date}.lst` file. 39 | 40 | ```bash 41 | gen_bso_users.py --bso_num # 42 | ``` 43 | which will automatically read in the `fxa_users_{date}.lst` file, 44 | connect to the mysql database, and geneate a list of sorted users 45 | taken from the `bso#` table. This will create the 46 | `bso_users_{bso_num}_{date}.lst` file 47 | 48 | and finally: 49 | 50 | ```bash 51 | GOOGLE_APPLICATION_CREDENTIALS=credentials.json migrate_node.py \ 52 | [--start_bso=0] \ 53 | [--end_bso=19] \ 54 | [--user_percent 1:100] 55 | ``` 56 | 57 | Which will read the `bso_users_#_{date}.lst` files and move the users 58 | based on `--user_percent` 59 | 60 | More importantly `--help` is your friend. feel free to use liberally. 61 | 62 | ## installation 63 | 64 | ```bash 65 | virtualenv venv && venv/bin/pip install -r requirements.txt 66 | ``` 67 | 68 | ## running 69 | 70 | Since you will be connecting to the GCP Spanner API, you will need to have set the `GOOGLE_APPLICATION_CREDENTIALS` env var before running these scripts. This environment variable should point to the exported Google Credentials acquired from the GCP console. 71 | 72 | The scripts will take the following actions: 73 | 74 | 1. fetch all users from a given node. 75 | 1. compare and port all user_collections over (NOTE: this may involve remapping collecitonid values.) 76 | 1. begin copying over user information from mysql to spanner. 77 | 78 | Overall performance may be improved by "batching" BSOs to different 79 | processes using: 80 | 81 | `--start_bso` the BSO database (defaults to 0, inclusive) to begin 82 | copying from 83 | 84 | `--end_bso` the final BSO database (defaults to 19, inclusive) to copy 85 | from. 86 | 87 | Note that these are inclusive values. So to split between two 88 | processes, you would want to use 89 | 90 | ```bash 91 | migrate_node.py --start_bso=0 --end_bso=9 & 92 | migrate_node.py --start_bso=10 --end_bso=19 & 93 | ``` 94 | 95 | (As short hand for this case, you could also do: 96 | ``` 97 | migrate_node.py --end_bso=9 & 98 | migrate_node.py --start_bso=10 & 99 | ``` 100 | and let the defaults handle the rest.) 101 | -------------------------------------------------------------------------------- /tools/user_migration/fix_collections.sql: -------------------------------------------------------------------------------- 1 | INSERT IGNORE INTO weave0.collections (name, collectionid) VALUES 2 | ("clients", 1), 3 | ("crypto", 2), 4 | ("forms", 3), 5 | ("history", 4), 6 | ("keys", 5), 7 | ("meta", 6), 8 | ("bookmarks", 7), 9 | ("prefs", 8), 10 | ("tabs", 9), 11 | ("passwords", 10), 12 | ("addons", 11), 13 | ("addresses", 12), 14 | ("creditcards", 13), 15 | ("reserved", 99); 16 | -------------------------------------------------------------------------------- /tools/user_migration/old/dump_avro.py: -------------------------------------------------------------------------------- 1 | #! venv/bin/python 2 | 3 | # painfully stupid script to check out dumping a spanner database to avro. 4 | # Avro is basically "JSON" for databases. It's not super complicated & it has 5 | # issues (one of which is that it requires Python2). 6 | # test run Dumped 2770783 rows in 457.566066027 seconds and produced a 7 | # roughly 6.5GB file. 8 | # 9 | # Spanner also has a Deadline issue where it will kill a db connection after 10 | # so many minutes (5?). Might be better to just divvy things up into clusters 11 | # and have threads handle transporting records over. 12 | # 13 | 14 | import avro.schema 15 | import argparse 16 | import time 17 | 18 | from avro.datafile import DataFileWriter 19 | from avro.io import DatumWriter 20 | from google.cloud import spanner 21 | 22 | 23 | def get_args(): 24 | parser = argparse.ArgumentParser(description="dump spanner to arvo files") 25 | parser.add_argument( 26 | '--instance_id', default="spanner-test", 27 | help="Spanner instance name") 28 | parser.add_argument( 29 | '--database_id', default="sync_schema3", 30 | help="Spanner database name") 31 | parser.add_argument( 32 | '--schema', default="sync.avsc", 33 | help="Database schema description") 34 | parser.add_argument( 35 | '--output', default="output.avso", 36 | help="Output file") 37 | parser.add_argument( 38 | '--limit', type=int, default=1500000, 39 | help="Limit to n rows") 40 | return parser.parse_args() 41 | 42 | 43 | def conf_spanner(args): 44 | spanner_client = spanner.Client() 45 | instance = spanner_client.instance(args.instance_id) 46 | database = instance.database(args.database_id) 47 | return database 48 | 49 | 50 | def dump_rows(offset, db, writer, args): 51 | print("Querying.... @{}".format(offset)) 52 | sql = """ 53 | SELECT collection_id, fxa_kid, fxa_uid, bso_id, 54 | UNIX_MICROS(expiry), UNIX_MICROS(modified), payload, 55 | sortindex from bsos LIMIT {} OFFSET {}""".format(args.limit, offset) 56 | try: 57 | with db.snapshot() as snapshot: 58 | result = snapshot.execute_sql(sql) 59 | print("Dumping...") 60 | for row in result: 61 | writer.append({ 62 | "collection_id": row[0], 63 | "fxa_kid": row[1], 64 | "fxa_uid": row[2], 65 | "bso_id": row[3], 66 | "expiry": row[4], 67 | "modified": row[5], 68 | "payload": row[6], 69 | "sortindex": row[7]}) 70 | offset += 1 71 | if offset % 1000 == 0: 72 | print("Row: {}".format(offset)) 73 | return offset 74 | except Exception as ex: 75 | print("Deadline hit at: {} ({})".format(offset, ex)) 76 | return offset 77 | 78 | 79 | def count_rows(db): 80 | with db.snapshot() as snapshot: 81 | result = snapshot.execute_sql("SELECT Count(*) from bsos") 82 | return result.one()[0] 83 | 84 | 85 | def dump_data(args, schema): 86 | offset = 0 87 | # things time out around 1_500_000 rows. 88 | db = conf_spanner(args) 89 | writer = DataFileWriter( 90 | open(args.output, "wb"), DatumWriter(), schema) 91 | row_count = count_rows(db) 92 | print("Dumping {} rows".format(row_count)) 93 | while offset < row_count: 94 | old_offset = offset 95 | offset = dump_rows(offset=offset, db=db, writer=writer, args=args) 96 | if offset == old_offset: 97 | break 98 | writer.close() 99 | return row_count 100 | 101 | 102 | def main(): 103 | start = time.time() 104 | args = get_args() 105 | schema = avro.schema.parse(open(args.schema, "rb").read()) 106 | rows = dump_data(args, schema) 107 | print("Dumped: {} rows in {} seconds".format(rows, time.time() - start)) 108 | 109 | 110 | if __name__ == "__main__": 111 | main() 112 | -------------------------------------------------------------------------------- /tools/user_migration/old/requirements.txt: -------------------------------------------------------------------------------- 1 | wheel 2 | avro-python3 3 | google-cloud-spanner 4 | mysql-connector 5 | -------------------------------------------------------------------------------- /tools/user_migration/old/sync.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "bso.avro", 2 | "type": "record", 3 | "name": "bso", 4 | "fields": [ 5 | {"name": "fxa_uid", "type": ["null", "string"]}, 6 | {"name": "fxa_kid", "type": ["null", "string"]}, 7 | {"name": "collection_id", "type": ["null", "long"]}, 8 | {"name": "bso_id", "type": "string"}, 9 | {"name": "expiry", "type": "long"}, 10 | {"name": "modified", "type": "long"}, 11 | {"name": "payload", "type": "string"}, 12 | {"name": "sortindex", "type": ["null", "long"]} 13 | ]} -------------------------------------------------------------------------------- /tools/user_migration/requirements.txt: -------------------------------------------------------------------------------- 1 | wheel 2 | google-cloud-spanner 3 | mysql-connector 4 | --------------------------------------------------------------------------------