├── .dockerignore ├── .github └── workflows │ ├── check.yml │ └── fly.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── DEVELOPMENT.md ├── Dockerfile ├── LICENSE ├── README.md ├── Rocket.toml ├── examples ├── crowdin_download_english_file.rs ├── crowdin_download_translations.rs ├── github_clone_repository.rs ├── github_download_all_repositories.rs ├── github_fork_all_not_forked_repositories.rs ├── github_get_all_installations.rs ├── github_get_all_repositories.rs ├── github_is_default_branch_protected.rs ├── github_star_all_not_starred_repositories.rs ├── github_star_repository.rs └── on_repository_added.rs ├── fly.toml ├── rust-toolchain ├── src ├── crowdin │ ├── http.rs │ └── mod.rs ├── documentation.md ├── git_util.rs ├── github.rs ├── github_config.rs ├── github_repo_info.rs ├── lib.rs ├── main.rs ├── mod_directory.rs ├── myenv.rs ├── sentry.rs ├── server │ ├── debug_routes.rs │ ├── example_error_routes.rs │ ├── mod.rs │ ├── trigger_update.rs │ ├── trigger_update_public.rs │ └── webhook_util.rs ├── util │ ├── case.rs │ ├── escape.rs │ └── mod.rs └── webhooks.rs ├── tests ├── all_github_repositories_are_forked.rs ├── all_github_repositories_are_starred.rs ├── crowdin_github_matches_english_files.rs ├── crowdin_github_matches_english_files_content.rs ├── crowdin_github_matches_mods.rs ├── crowdin_no_translations_has_newline.rs ├── github_check_config_file.rs └── github_translated_files_matches_locale_en.rs └── todo.md /.dockerignore: -------------------------------------------------------------------------------- 1 | .git/ 2 | target/ 3 | temp/ 4 | .env 5 | -------------------------------------------------------------------------------- /.github/workflows/check.yml: -------------------------------------------------------------------------------- 1 | name: check 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | # every day at 03:45 UTC 7 | - cron: "45 03 * * *" 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | - name: Frees disk space 18 | run: | 19 | sudo rm -rf /usr/local/.ghcup 20 | sudo rm -rf /usr/share/dotnet 21 | sudo rm -rf /usr/local/lib/android 22 | sudo rm -rf /opt/hostedtoolcache/CodeQL 23 | sudo rm -rf "$AGENT_TOOLSDIRECTORY" 24 | - name: Set up cargo cache 25 | uses: actions/cache@v4 26 | with: 27 | path: | 28 | ~/.cargo/bin/ 29 | ~/.cargo/registry/index/ 30 | ~/.cargo/registry/cache/ 31 | ~/.cargo/git/db/ 32 | target/ 33 | key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} 34 | restore-keys: ${{ runner.os }}-cargo- 35 | - name: Build 36 | run: cargo build 37 | - name: Run tests 38 | run: cargo test 39 | env: 40 | CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} 41 | CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }} 42 | # Note that secrets are not allowed to start with GITHUB_, so added MY_ prefix 43 | GITHUB_APP_ID: ${{ secrets.MY_GITHUB_APP_ID }} 44 | GITHUB_APP_PRIVATE_KEY: ${{ secrets.MY_GITHUB_APP_PRIVATE_KEY }} 45 | GITHUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.MY_GITHUB_PERSONAL_ACCESS_TOKEN }} 46 | RUST_BACKTRACE: 1 47 | -------------------------------------------------------------------------------- /.github/workflows/fly.yml: -------------------------------------------------------------------------------- 1 | # https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/ 2 | name: Fly Deploy 3 | 4 | on: 5 | push: 6 | branches: 7 | - master 8 | 9 | jobs: 10 | deploy: 11 | name: Deploy app 12 | runs-on: ubuntu-latest 13 | concurrency: deploy-group # ensure only one action runs at a time 14 | steps: 15 | - uses: actions/checkout@v4 16 | - uses: superfly/flyctl-actions/setup-flyctl@master 17 | - run: flyctl deploy --remote-only --image-label $(git rev-parse HEAD) 18 | env: 19 | FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | # fml — acronym of "factorio-mods-localization" 3 | name = "fml" 4 | version = "1.0.2" 5 | authors = ["Dmitry Murzin "] 6 | edition = "2021" 7 | default-run = "fml" 8 | 9 | [profile.dev] 10 | debug = true 11 | [profile.release] 12 | debug = true 13 | [profile.test] 14 | debug = true 15 | 16 | [profile.dev.package."*"] 17 | opt-level = 3 18 | debug = true 19 | [profile.release.package."*"] 20 | opt-level = 3 21 | debug = true 22 | [profile.test.package."*"] 23 | opt-level = 3 24 | debug = true 25 | 26 | [dependencies] 27 | async-trait = "0.1.74" 28 | dotenv = "0.15.0" 29 | hex = "0.4.3" 30 | hmac = { version = "0.12.1", features = ["std"] } 31 | http = "0.2.9" 32 | hyper = "0.14.27" 33 | jsonwebtoken = "8.3.0" 34 | log = "0.4.20" 35 | octocrab = { version = "0.31.2", features = ["stream"] } 36 | pretty_env_logger = "0.5.0" 37 | regex = { version = "1.10.2", features = ["pattern"] } 38 | reqwest = { version = "0.11.22", features = ["json"] } 39 | rocket = { version = "0.5.0", default-features = false, features = ["json"] } 40 | secrecy = "0.8.0" 41 | # Disable debug-images feature - https://github.com/getsentry/sentry-rust/issues/574 42 | sentry = { version = "0.31.8", default-features = false, features = ["backtrace", "contexts", "panic", "transport", "log"] } 43 | sentry-log = "0.31.8" 44 | serde = "1.0.190" 45 | serde_json = "1.0.108" 46 | sha2 = "0.10.8" 47 | tempfile = "3.8.1" 48 | tokio = { version = "1.33.0" } 49 | url = "2.5.0" 50 | zip = { version = "0.6.0", default-features = false, features = ["deflate"] } 51 | -------------------------------------------------------------------------------- /DEVELOPMENT.md: -------------------------------------------------------------------------------- 1 | ## Webserver routes 2 | * `/` - Main page with link to GitHub repository 3 | * `/webhook` - Github app webhooks handler 4 | * `/triggerUpdate?secret=X` - Update all repositories 5 | * `/triggerUpdate?secret=X&repo=REPO` - Update specific repository 6 | * `/api/triggerUpdate?repo=REPO` - Public API for updating specific repository with GitHub OAuth 7 | * `/importRepository?secret=X&repo=REPO` - Readd repository to Crowdin (both english files and translations) 8 | * `/importEnglish?secret=X&repo=REPO` - Overwrites english files on Crowdin based on GitHub 9 | 10 | 11 | ## fly.io configuration 12 | * First time 13 | ```sh 14 | yay -S flyctl-bin 15 | fly secrets set KEY=VALUE 16 | fly launch 17 | ``` 18 | 19 | * Deploy from local 20 | ```sh 21 | fly deploy --image-label $(git rev-parse HEAD) 22 | ``` 23 | 24 | * Deploy from github 25 | https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/ 26 | See .github/workflows/fly.yml 27 | `FLY_API_TOKEN` - get using `fly tokens create deploy -x 999999h` 28 | 29 | * Get git commit hash for current release 30 | ```sh 31 | fly image show # column TAG 32 | ``` 33 | 34 | 35 | ## Needed environment variables 36 | From https://crowdin.com/project/factorio-mods-localization/tools/api: 37 | * `CROWDIN_PROJECT_ID` 38 | * `CROWDIN_API_KEY` 39 | 40 | From https://github.com/settings/apps/factorio-mods-localization-helper: 41 | * `GITHUB_APP_ID` - App ID 42 | * `GITHUB_APP_PRIVATE_KEY` - Private keys (convert pem file content to one line by replacing newlines with \n) 43 | * `GITHUB_APP_WEBHOOKS_SECRET` - Webhook secret 44 | 45 | From https://github.com/settings/tokens for https://github.com/factorio-mods-helper: 46 | * `GITHUB_PERSONAL_ACCESS_TOKEN` - classic token with `repo` and `workflow` scopes 47 | * Without `workflow` scope we can't change `.github/workflows` in the fork 48 | 49 | From https://github.com/settings/developers: 50 | * `GITHUB_OAUTH_CLIENT_ID` 51 | * `GITHUB_OAUTH_CLIENT_SECRET` 52 | 53 | From https://diralik.sentry.io/settings/projects/factorio-mods-localization/keys/ 54 | * SENTRY_DSN - `https://...@....ingest.sentry.io/...` 55 | 56 | * `GIT_COMMIT_USER_NAME` - "Factorio Mods Helper" 57 | * `GIT_COMMIT_USER_EMAIL` - Email of https://github.com/factorio-mods-helper 58 | * `GIT_COMMIT_MESSAGE` - "Update translations from Crowdin" 59 | 60 | * `RUST_LOG` - "fml=info" 61 | * `WEBSERVER_SECRET` - any string for `/triggerUpdate` route 62 | 63 | 64 | ## GitHub Apps 65 | Main 66 | id: 13052 67 | private key: MIIEog... 68 | 69 | Fml-test 70 | id: 97456 71 | private key: MIIEow... 72 | 73 | ## Crowdin projects 74 | Main id: 307377 75 | Fml-test id: 613717 76 | Api key for both: 2ad62d... 77 | 78 | 79 | ## Weekly updates from crowdin to github 80 | https://cron-job.org/ 81 | 82 | 83 | ## .cfg/.ini files format 84 | Factorio uses .cfg extension, Crowdin uses .ini extension, so we just change the extension when needed 85 | 86 | General format is (https://wiki.factorio.com/Tutorial:Localisation): 87 | ```ini 88 | [section] 89 | key=value 90 | ``` 91 | 92 | Notes: 93 | * `key=foo # bar` - Crowdin will export it as `key="foo # bar"`, I wrote to support to disable quoting 94 | * `key=foo ; bar` - Crowdin will threat `; bar` as comment, we need to wrap value in quotes `"` before uploading english file to Crowdin 95 | * Multiline english strings are not allowed 96 | * If translated string has new lines, e.g.: 97 | ``` 98 | line1 99 | line2 100 | line3 101 | ``` 102 | then Crowdin will export it in format 103 | ``` 104 | key=line1 105 | line2= 106 | line3= 107 | ``` 108 | Probably our helper should replace it to `key=line1\nline2\nline3` 109 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust AS builder 2 | COPY . . 3 | RUN cargo build --release 4 | 5 | FROM archlinux 6 | RUN pacman -Sy --noconfirm git && rm -rf /var/cache/pacman/pkg/* 7 | COPY --from=builder ./target/release/fml ./target/release/fml 8 | COPY --from=builder Rocket.toml Rocket.toml 9 | ENV RUST_BACKTRACE 1 10 | CMD ["/target/release/fml"] 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Dmitry Murzin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Translate your Factorio mod easily with the power of Crowdin 2 | 3 | [![Crowdin](https://badges.crowdin.net/factorio-mods-localization/localized.svg)](https://crowdin.com/project/factorio-mods-localization) 4 | [![Website factorio-mods-localization.fly.dev](https://img.shields.io/website-up-down-green-red/https/factorio-mods-localization.fly.dev.svg)](https://factorio-mods-localization.fly.dev/) 5 | [![GitHub Actions Status](https://img.shields.io/github/actions/workflow/status/dima74/factorio-mods-localization/check.yml)](https://github.com/dima74/factorio-mods-localization/actions/workflows/check.yml) 6 | [![GitHub license](https://img.shields.io/github/license/dima74/factorio-mods-localization.svg)](https://github.com/dima74/factorio-mods-localization/blob/master/LICENSE) 7 | [![GitHub issues](https://img.shields.io/github/issues/dima74/factorio-mods-localization.svg)](https://GitHub.com/dima74/factorio-mods-localization/issues/) 8 | 9 | ## Description 10 | We provide service for simplifying [Factorio](https://www.factorio.com/) mods translation. You only need to install [our GitHub app][1]. After this, the following actions will be performed automatically: 11 | 12 | * All existing English strings of your mod will be uploaded to [Crowdin](https://crowdin.com/) 13 | * All existing translations will be uploaded too 14 | * Every week our [FactorioBot](https://github.com/factorio-mods-helper) will fetch translation updates from Crowdin and commit them to your repository 15 | 16 | ## Motivation 17 | There are a lot of Factorio mods hosted on GitHub. Most of them are translated using pull requests. It is not very convenient (because it is unclear which strings are untranslated yet and translators have to know how to use git). So, I created a helper tool for configuring the translation process on Crowdin, a powerful localization platform. 18 | 19 | ## Installation 20 | 1. Go to our [GitHub app page][1] 21 | 2. Click the install button 22 | 3. By default, the app will be installed for all repositories, and the app will automatically find repositories that have `locale` folder. Alternatively, you can manually select repositories for which app will be installed 23 | 4. Click the install button 24 | 25 | You are done! Now share the link to [this Crowdin project][2] with translators. 26 | 27 | Please note that **only Crowdin should be used for translation**. GitHub pull requests must not be used for translation, otherwise translations will be lost after the next synchronization from Crowdin! Consider adding link to [Crowdin][2] to your repository Readme ([example](https://github.com/softmix/AutoDeconstruct/pull/6/files)). 28 | 29 | ## How to translate using Crowdin 30 | We have a single Crowdin project. It consists of several folders, each folder corresponds to one mod. So, here are instructions on how to translate specific mod: 31 | 32 | 1. Go to [Crowdin project page][2] 33 | 2. Select language 34 | 3. Find the folder with your mod 35 | 4. Open the menu (click on three points) right of the folder name 36 | 5. Click "Open in Editor": ![menu](https://user-images.githubusercontent.com/6505554/85887708-bdfa5880-b801-11ea-99c1-766ad92ae4af.png) 37 | 38 | Then Crowdin translation interface will be opened where you can translate strings. 39 | 40 | ## Notes 41 | 42 | * To correctly upload your existing translations to Crowdin, files in any localization folder (such as `/locale/de`) **must have the same names as files in `/locale/en` folder**. 43 | * If a repository has branch protection rules, our helper will create a pull request (instead of pushing to the main branch directly). 44 | * Please ask any questions or report bugs by creating a new [issue](https://github.com/dima74/factorio-mods-localization/issues). 45 | 46 | ## Configuration 47 | There are options which can be added to `factorio-mods-localization.json` config file located in the root of your repository. 48 | 49 | List of currently supported options (see corresponding section for description of each option): 50 | ```json 51 | { 52 | "mods": ["mod1", "mod2"], 53 | "weekly_update_from_crowdin": false, 54 | "branch": "dev" 55 | } 56 | ``` 57 | 58 | ### Configuration: Multimods 59 | It is possible to have multiple Factorio mods in a single GitHub repository. Add `"mods"` option with a list of submods to the [config](#configuration): 60 | ``` 61 | ├── factorio-mods-localization.json // {"mods": ["Mod1", "Mod2"]} 62 | ├── Mod1 63 | │ ├── locale/en 64 | ├── Mod2 65 | │ ├── locale/en 66 | ``` 67 | 68 | ### Configuration: Disable weekly updates from Crowdin 69 | It is possible to disable automatic weekly updates from Crowdin and perform them manually when needed. To do so, add `"weekly_update_from_crowdin": false` option to the [config](#configuration). 70 | 71 | Now you can perform an update manually using the following URL: 72 | ``` 73 | https://factorio-mods-localization.fly.dev/api/triggerUpdate?repo=OWNER/REPO 74 | ``` 75 | 76 | ### Configuration: Specify branch 77 | It is possible to use some other branch instead of the default branch. To do so, add `"branch"` option to the [config](#configuration). 78 | 79 | Note that the `factorio-mods-localization.json` config file should still be in the **default** branch. 80 | 81 | ## Detailed description of how it works 82 | 0. Mod author has a mod repository on GitHub 83 | 1. Mod author installs GitHub app (for mod repository) 84 | 2. Our service creates a subdirectory in our Crowdin project and uploads original English strings and existing translations into it 85 | 3. Every week our service takes translated strings from Crowdin and makes a commit to the GitHub repository (if there are any changes) 86 | 4. Every time original (locale/en) strings are changed, our service changes appropriate strings on Crowdin 87 | 88 | 89 | [1]: https://github.com/apps/factorio-mods-localization-helper 90 | [2]: https://crowdin.com/project/factorio-mods-localization 91 | -------------------------------------------------------------------------------- /Rocket.toml: -------------------------------------------------------------------------------- 1 | [release] 2 | address = "0.0.0.0" 3 | cli_colors = false 4 | port = 8080 5 | -------------------------------------------------------------------------------- /examples/crowdin_download_english_file.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io; 3 | use std::io::{Seek, SeekFrom}; 4 | use std::path::Path; 5 | 6 | use fml::crowdin; 7 | 8 | #[tokio::main] 9 | async fn main() { 10 | fml::init_with_crowdin().await; 11 | 12 | let directory_id = crowdin::find_directory_id("Factorio Mod Example (dima74)").await.unwrap(); 13 | let mut files = crowdin::list_files(directory_id).await; 14 | let (_, file_id) = files.next().unwrap(); 15 | let mut file = crowdin::download_file(file_id).await; 16 | file.seek(SeekFrom::Start(0)).unwrap(); 17 | let target = Path::new("temp/download.ini"); 18 | let mut target = File::create(target).unwrap(); 19 | io::copy(&mut file, &mut target).unwrap(); 20 | } 21 | -------------------------------------------------------------------------------- /examples/crowdin_download_translations.rs: -------------------------------------------------------------------------------- 1 | #[tokio::main] 2 | async fn main() { 3 | fml::init_with_crowdin().await; 4 | let path = fml::crowdin::download_all_translations().await; 5 | dbg!(&path); 6 | #[allow(clippy::empty_loop)] 7 | loop {} 8 | } 9 | -------------------------------------------------------------------------------- /examples/github_clone_repository.rs: -------------------------------------------------------------------------------- 1 | use fml::github_repo_info::GithubRepoInfo; 2 | 3 | #[tokio::main] 4 | async fn main() { 5 | fml::init(); 6 | let api = fml::github::as_app(); 7 | let owner = "dima74"; 8 | let repo = "factorio-mod-example"; 9 | let installation = api 10 | .apps().get_repository_installation(owner, repo) 11 | .await.unwrap(); 12 | 13 | let repo_info = GithubRepoInfo::new_single_mod(&format!("{}/{}", owner, repo)); 14 | fml::github::clone_repository(&repo_info, installation.id).await; 15 | } 16 | -------------------------------------------------------------------------------- /examples/github_download_all_repositories.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | use fml::github::get_all_repositories; 4 | use fml::github_repo_info::GithubRepoInfo; 5 | 6 | const USE_CACHED: bool = false; 7 | 8 | #[tokio::main] 9 | async fn main() { 10 | fml::init(); 11 | let cache_path = Path::new("temp/repositories.json"); 12 | let repositories: Vec = if USE_CACHED { 13 | let json = std::fs::read_to_string(cache_path).unwrap(); 14 | serde_json::from_str(&json).unwrap() 15 | } else { 16 | let api = fml::github::as_app(); 17 | let repositories = get_all_repositories(&api).await 18 | .into_iter() 19 | .map(|(repo_info, _id)| repo_info) 20 | .collect::>(); 21 | 22 | let json = serde_json::to_string_pretty(&repositories).unwrap(); 23 | std::fs::write(cache_path, json).unwrap(); 24 | 25 | repositories 26 | }; 27 | 28 | for repo_info in repositories { 29 | let branch = match repo_info.branch { 30 | Some(branch) => format!("--branch {}", branch), 31 | None => "".to_owned(), 32 | }; 33 | println!("git clone --depth 1 {} git@github.com:{}.git &", branch, repo_info.full_name); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /examples/github_fork_all_not_forked_repositories.rs: -------------------------------------------------------------------------------- 1 | use fml::github; 2 | 3 | #[tokio::main] 4 | async fn main() { 5 | fml::init(); 6 | 7 | let not_forked = github::get_not_forked_repositories().await.not_forked; 8 | 9 | let api_personal = github::as_personal_account(); 10 | for full_name in not_forked { 11 | println!("Forking {}", full_name); 12 | github::fork_repository_without_check(&api_personal, &full_name).await; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/github_get_all_installations.rs: -------------------------------------------------------------------------------- 1 | // https://github.com/XAMPPRocky/octocrab/blob/9f8a94e70e4707fd240c65dd96f934d1dd46938c/examples/github_app_authentication.rs 2 | 3 | use fml::github::get_all_installations; 4 | 5 | #[tokio::main] 6 | async fn main() { 7 | fml::init(); 8 | let api = fml::github::as_app(); 9 | let installations = get_all_installations(&api).await; 10 | dbg!(installations.len()); 11 | 12 | let installations = installations.iter() 13 | .map(|it| (&it.account.login, it.id.0)) 14 | .collect::>(); 15 | dbg!(installations); 16 | } 17 | -------------------------------------------------------------------------------- /examples/github_get_all_repositories.rs: -------------------------------------------------------------------------------- 1 | use fml::github::get_all_repositories; 2 | 3 | #[tokio::main] 4 | async fn main() { 5 | fml::init(); 6 | let api = fml::github::as_app(); 7 | let repositories = get_all_repositories(&api).await; 8 | dbg!(repositories); 9 | } 10 | -------------------------------------------------------------------------------- /examples/github_is_default_branch_protected.rs: -------------------------------------------------------------------------------- 1 | use fml::github; 2 | 3 | #[tokio::main] 4 | async fn main() { 5 | fml::init(); 6 | let full_name = "jingleheimer-schmidt/factorio-trainsaver"; // true 7 | // let full_name = "jingleheimer-schmidt/cutscene-creator"; // false 8 | let api = github::as_installation_for_user("jingleheimer-schmidt").await.unwrap(); 9 | let default_branch = github::get_default_branch(&api, full_name).await; 10 | let is_protected = github::is_branch_protected(&api, full_name, &default_branch).await; 11 | dbg!(is_protected); 12 | } 13 | -------------------------------------------------------------------------------- /examples/github_star_all_not_starred_repositories.rs: -------------------------------------------------------------------------------- 1 | use fml::github; 2 | 3 | #[tokio::main] 4 | async fn main() { 5 | fml::init(); 6 | 7 | let not_starred = github::get_not_starred_repositories().await; 8 | 9 | let api_personal = github::as_personal_account(); 10 | for full_name in not_starred { 11 | println!("Starring {}", full_name); 12 | github::star_repository(&api_personal, &full_name).await; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/github_star_repository.rs: -------------------------------------------------------------------------------- 1 | use fml::github; 2 | 3 | #[tokio::main] 4 | async fn main() { 5 | fml::init(); 6 | let api_personal = github::as_personal_account(); 7 | github::star_repository(&api_personal, "dima74/factorio-mod-example").await; 8 | } 9 | -------------------------------------------------------------------------------- /examples/on_repository_added.rs: -------------------------------------------------------------------------------- 1 | use fml::{github, webhooks}; 2 | use fml::github_repo_info::GithubRepoInfo; 3 | 4 | #[tokio::main] 5 | async fn main() { 6 | fml::init_with_crowdin().await; 7 | let installation_id = github::get_installation_id_for_user("dima74").await; 8 | let repo_info = GithubRepoInfo::new_single_mod("dima74/factorio-mod-example"); 9 | webhooks::on_repository_added(repo_info, installation_id).await; 10 | } 11 | -------------------------------------------------------------------------------- /fly.toml: -------------------------------------------------------------------------------- 1 | # fly.toml app configuration file generated for factorio-mods-localization on 2024-03-25T23:19:01+03:00 2 | # 3 | # See https://fly.io/docs/reference/configuration/ for information about how to use this file. 4 | # 5 | 6 | app = "factorio-mods-localization" 7 | primary_region = "waw" 8 | 9 | [http_service] 10 | internal_port = 8080 11 | force_https = true 12 | auto_stop_machines = false 13 | auto_start_machines = true 14 | min_machines_running = 1 15 | processes = ["app"] 16 | 17 | [[vm]] 18 | cpu_kind = "shared" 19 | cpus = 1 20 | memory_mb = 512 21 | swap_size_mb = 512 22 | -------------------------------------------------------------------------------- /rust-toolchain: -------------------------------------------------------------------------------- 1 | nightly-2025-01-01 2 | -------------------------------------------------------------------------------- /src/crowdin/http.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Deref; 2 | 3 | use reqwest::{Method, RequestBuilder}; 4 | use serde::{Deserialize, Serialize}; 5 | use serde::de::DeserializeOwned; 6 | 7 | use crate::crowdin::StorageId; 8 | use crate::myenv::{CROWDIN_API_KEY, CROWDIN_PROJECT_ID}; 9 | 10 | const BASE_URL: &str = "https://api.crowdin.com/api/v2"; 11 | 12 | #[derive(Deserialize)] 13 | pub struct DataWrapper { pub data: T } 14 | 15 | #[derive(Deserialize)] 16 | pub struct IdResponse { pub id: i64 } 17 | 18 | #[derive(Deserialize)] 19 | pub struct UnitResponse {} 20 | 21 | async fn send_request( 22 | crowdin_path: &str, 23 | http_method: Method, 24 | before_send: impl FnOnce(RequestBuilder) -> RequestBuilder 25 | ) -> T { 26 | let url = format!("{}/projects/{}{}", BASE_URL, CROWDIN_PROJECT_ID.deref(), crowdin_path); 27 | let request = reqwest::Client::new() 28 | .request(http_method, &url) 29 | .bearer_auth(CROWDIN_API_KEY.deref()); 30 | let request = before_send(request); 31 | 32 | let response = request.send().await.unwrap(); 33 | let status = response.status(); 34 | if status.is_client_error() || status.is_server_error() { 35 | let response_text = response.text().await.unwrap(); 36 | panic!("Request to {} failed with code {}, response: `{}`", url, status.as_u16(), response_text); 37 | } 38 | 39 | response.json::>().await.unwrap().data 40 | } 41 | 42 | async fn crowdin_get( 43 | path: &str, 44 | before_send: impl FnOnce(RequestBuilder) -> RequestBuilder 45 | ) -> Res { 46 | send_request(path, Method::GET, before_send).await 47 | } 48 | 49 | pub async fn crowdin_get_empty_query(path: &str) -> Res { 50 | crowdin_get(path, |request| request).await 51 | } 52 | 53 | pub async fn crowdin_get_pagination(path: &str, before_send: impl Fn(RequestBuilder) -> RequestBuilder) -> Vec { 54 | const LIMIT: usize = 500; 55 | let mut result = Vec::new(); 56 | for i in (0..).step_by(LIMIT) { 57 | let items: Vec = crowdin_get(path, |request| { 58 | let request = before_send(request); 59 | let request = request.query(&[("offset", i)]); 60 | let request = request.query(&[("limit", LIMIT)]); 61 | #[allow(clippy::let_and_return)] 62 | request 63 | }).await; 64 | if items.is_empty() { break; } 65 | result.extend(items); 66 | } 67 | result 68 | } 69 | 70 | pub async fn crowdin_get_pagination_empty_query(path: &str) -> Vec { 71 | crowdin_get_pagination(path, |request| request).await 72 | } 73 | 74 | pub async fn crowdin_post(path: &str, data: Req) -> Res { 75 | send_request(path, Method::POST, |request| request.json(&data)).await 76 | } 77 | 78 | pub async fn crowdin_post_empty_body(path: &str) -> Res { 79 | send_request(path, Method::POST, |request| request).await 80 | } 81 | 82 | pub async fn crowdin_put(method: &str, data: Req) -> Res { 83 | send_request(method, Method::PUT, |request| request.json(&data)).await 84 | } 85 | 86 | // https://developer.crowdin.com/api/v2/#operation/api.storages.post 87 | pub async fn upload_file_to_storage(file_content: String, file_name: &str) -> StorageId { 88 | let url = format!("{}/storages", BASE_URL); 89 | reqwest::Client::new() 90 | .post(&url) 91 | .body(file_content) 92 | .header("Crowdin-API-FileName", file_name) 93 | .bearer_auth(CROWDIN_API_KEY.deref()) 94 | .send().await.unwrap() 95 | .json::>().await.unwrap() 96 | .data.id 97 | } 98 | -------------------------------------------------------------------------------- /src/crowdin/mod.rs: -------------------------------------------------------------------------------- 1 | use std::collections::{HashMap, HashSet}; 2 | use std::fs; 3 | use std::ops::Deref; 4 | use std::path::Path; 5 | use std::sync::{LazyLock, OnceLock}; 6 | use std::time::Duration; 7 | 8 | use log::info; 9 | use octocrab::models::InstallationId; 10 | use regex::Regex; 11 | use serde::{Deserialize, Serialize}; 12 | use tempfile::TempDir; 13 | 14 | use crate::crowdin::http::{crowdin_get_empty_query, crowdin_get_pagination, crowdin_get_pagination_empty_query, crowdin_post, crowdin_put, DataWrapper, IdResponse, UnitResponse, upload_file_to_storage}; 15 | use crate::github_repo_info::{GithubModInfo, GithubRepoInfo}; 16 | use crate::mod_directory::{LanguageCode, ModDirectory}; 17 | use crate::myenv::is_development; 18 | use crate::util; 19 | 20 | pub mod http; 21 | 22 | pub static PROJECT_LANGUAGE_CODES: OnceLock> = OnceLock::new(); 23 | 24 | pub async fn init() { 25 | let info = get_project_info().await; 26 | if !is_development() { 27 | assert_eq!(info.name, "Factorio mods localization"); 28 | } 29 | 30 | let codes = info.target_language_ids; 31 | assert!(codes.len() > 20); 32 | PROJECT_LANGUAGE_CODES.set(codes).unwrap(); 33 | } 34 | 35 | type DirectoryId = i64; 36 | /// id of english file 37 | type FileId = i64; 38 | /// id of english/localized file in storage 39 | type StorageId = i64; 40 | 41 | #[derive(Deserialize)] 42 | struct ProjectInfo { 43 | #[serde(rename = "targetLanguageIds")] 44 | target_language_ids: Vec, 45 | name: String, 46 | } 47 | 48 | // https://developer.crowdin.com/api/v2/#operation/api.projects.get 49 | async fn get_project_info() -> ProjectInfo { 50 | crowdin_get_empty_query("").await 51 | } 52 | 53 | pub async fn find_directory_id(crowdin_name: &str) -> Option { 54 | list_directories().await 55 | .find(|(name, _id)| name == crowdin_name) 56 | .map(|(_name, id)| id) 57 | } 58 | 59 | // https://developer.crowdin.com/api/v2/#operation/api.projects.directories.getMany 60 | pub async fn list_directories() -> impl Iterator { 61 | #[derive(Deserialize)] 62 | struct Directory { id: DirectoryId, name: String } 63 | let directories: Vec> = crowdin_get_pagination_empty_query("/directories").await; 64 | directories.into_iter().map(|d| (d.data.name, d.data.id)) 65 | } 66 | 67 | // https://developer.crowdin.com/api/v2/#operation/api.projects.files.getMany 68 | pub async fn list_files(directory_id: DirectoryId) -> impl Iterator { 69 | #[derive(Deserialize)] 70 | struct File { id: FileId, name: String } 71 | let files: Vec> = crowdin_get_pagination("/files", |request| { 72 | request.query(&[("directoryId", directory_id)]) 73 | }).await; 74 | files.into_iter().map(|d| (d.data.name, d.data.id)) 75 | } 76 | 77 | // https://developer.crowdin.com/api/v2/#operation/api.projects.directories.post 78 | pub async fn create_directory(name: &str) -> DirectoryId { 79 | #[derive(Serialize)] 80 | struct Request<'a> { name: &'a str } 81 | let request = Request { name }; 82 | crowdin_post::<_, IdResponse>("/directories", request).await.id 83 | } 84 | 85 | pub async fn filter_repositories( 86 | repositories: Vec<(GithubRepoInfo, InstallationId)> 87 | ) -> Vec<(GithubRepoInfo, InstallationId)> { 88 | let directories = list_directories().await 89 | .map(|(name, _id)| name) 90 | .collect::>(); 91 | repositories 92 | .into_iter() 93 | .filter_map(|(repo_info, api)| { 94 | let repo_info = repo_info.filter_mods_present_on_crowdin(&directories)?; 95 | Some((repo_info, api)) 96 | }) 97 | .collect() 98 | } 99 | 100 | // https://developer.crowdin.com/api/v2/#operation/api.projects.files.post 101 | async fn add_english_file(directory_id: DirectoryId, storage_id: StorageId, file_name: &str) -> FileId { 102 | #[derive(Serialize)] 103 | struct Request<'a> { 104 | #[serde(rename = "directoryId")] 105 | directory_id: DirectoryId, 106 | #[serde(rename = "storageId")] 107 | storage_id: StorageId, 108 | #[serde(rename = "name")] 109 | file_name: &'a str, 110 | r#type: &'static str, 111 | } 112 | let request = Request { directory_id, storage_id, file_name, r#type: "ini" }; 113 | crowdin_post::<_, IdResponse>("/files", request).await.id 114 | } 115 | 116 | // https://developer.crowdin.com/api/v2/#operation/api.projects.files.put 117 | async fn update_english_file(file_id: FileId, storage_id: StorageId) { 118 | #[derive(Serialize)] 119 | struct Request { 120 | #[serde(rename = "storageId")] 121 | storage_id: StorageId, 122 | } 123 | let request = Request { storage_id }; 124 | let method = format!("/files/{}", file_id); 125 | crowdin_put::<_, UnitResponse>(&method, request).await; 126 | } 127 | 128 | // https://developer.crowdin.com/api/v2/#operation/api.projects.translations.postOnLanguage 129 | pub async fn add_localization_file( 130 | // id of source english file 131 | english_file_id: FileId, 132 | // id of localized file 133 | storage_id: StorageId, 134 | language_code: &LanguageCode, 135 | ) { 136 | #[derive(Serialize)] 137 | struct Request { 138 | #[serde(rename = "fileId")] 139 | english_file_id: FileId, 140 | #[serde(rename = "storageId")] 141 | storage_id: StorageId, 142 | /// Defines whether to add translation if it's the same as the source string 143 | #[serde(rename = "importEqSuggestions")] 144 | import_eq_suggestions: bool, 145 | /// Mark uploaded translations as approved 146 | #[serde(rename = "autoApproveImported")] 147 | auto_approve_imported: bool, 148 | } 149 | let request = Request { 150 | english_file_id, 151 | storage_id, 152 | import_eq_suggestions: false, 153 | auto_approve_imported: false, 154 | }; 155 | let path = format!("/translations/{}", language_code); 156 | crowdin_post::<_, UnitResponse>(&path, request).await; 157 | } 158 | 159 | pub async fn download_all_translations() -> TempDir { 160 | // https://developer.crowdin.com/api/v2/#operation/api.projects.translations.builds.post 161 | async fn build_translations() -> i64 { 162 | #[derive(Serialize)] 163 | struct Request { 164 | #[serde(rename = "skipUntranslatedStrings")] 165 | skip_untranslated_strings: bool, 166 | } 167 | let request = Request { skip_untranslated_strings: true }; 168 | crowdin_post::<_, IdResponse>("/translations/builds", request).await.id 169 | } 170 | // https://developer.crowdin.com/api/v2/#operation/api.projects.translations.builds.get 171 | async fn is_build_finished(build_id: i64) -> bool { 172 | #[derive(Deserialize)] 173 | struct Response { status: String } 174 | let url = format!("/translations/builds/{}", build_id); 175 | let response = crowdin_get_empty_query::(&url).await; 176 | response.status != "inProgress" 177 | } 178 | // https://developer.crowdin.com/api/v2/#operation/api.projects.translations.builds.download.download 179 | async fn get_build_download_url(build_id: i64) -> String { 180 | #[derive(Deserialize)] 181 | struct Response { url: String } 182 | let url = format!("/translations/builds/{}/download", build_id); 183 | crowdin_get_empty_query::(&url).await.url 184 | } 185 | 186 | let build_id = build_translations().await; 187 | while !is_build_finished(build_id).await { 188 | tokio::time::sleep(Duration::from_secs(1)).await; 189 | } 190 | let url = get_build_download_url(build_id).await; 191 | let result = util::download_and_extract_zip_file(&url).await; 192 | util::remove_empty_ini_files(result.path()); 193 | result 194 | } 195 | 196 | // https://developer.crowdin.com/api/v2/#operation/api.projects.files.get 197 | pub async fn download_file(file_id: FileId) -> fs::File { 198 | #[derive(Deserialize)] 199 | struct Response { url: String } 200 | let url = format!("/files/{}/download", file_id); 201 | let response: Response = crowdin_get_empty_query(&url).await; 202 | util::download_file(&response.url).await 203 | } 204 | 205 | pub struct CrowdinDirectory { 206 | crowdin_id: DirectoryId, 207 | #[allow(unused)] 208 | crowdin_name: String, 209 | pub mod_directory: ModDirectory, 210 | } 211 | 212 | impl CrowdinDirectory { 213 | pub async fn get_or_create(mod_directory: ModDirectory) -> (CrowdinDirectory, bool) { 214 | let crowdin_name = get_crowdin_directory_name(&mod_directory.mod_info); 215 | let (crowdin_id, created) = match find_directory_id(&crowdin_name).await { 216 | Some(crowdin_id) => (crowdin_id, false), 217 | None => (create_directory(&crowdin_name).await, true), 218 | }; 219 | (Self { crowdin_id, crowdin_name, mod_directory }, created) 220 | } 221 | 222 | pub async fn has_existing(mod_directory: &ModDirectory) -> bool { 223 | let crowdin_name = get_crowdin_directory_name(&mod_directory.mod_info); 224 | find_directory_id(&crowdin_name).await.is_some() 225 | } 226 | 227 | pub async fn add_english_and_localization_files(&self) { 228 | let english_file_ids = self.add_english_files().await; 229 | self.add_localization_files(english_file_ids).await; 230 | } 231 | 232 | pub async fn add_english_files(&self) -> HashMap { 233 | let existing_crowdin_files: HashMap = list_files(self.crowdin_id).await.collect(); 234 | let mut result = HashMap::new(); 235 | for file_path in self.mod_directory.get_english_files() { 236 | let file_name_ini = replace_cfg_to_ini(util::file_name(&file_path)); 237 | let file_id = self.add_or_update_english_file(&existing_crowdin_files, &file_path, &file_name_ini).await; 238 | result.insert(file_name_ini, file_id); 239 | } 240 | result 241 | } 242 | 243 | async fn add_or_update_english_file( 244 | &self, 245 | existing_crowdin_files: &HashMap, 246 | file_path: &Path, 247 | file_name_ini: &str, 248 | ) -> FileId { 249 | match existing_crowdin_files.get(file_name_ini) { 250 | Some(&file_id) => { 251 | self.update_english_file(file_id, file_path, file_name_ini).await; 252 | file_id 253 | } 254 | None => { 255 | self.add_english_file(file_path, file_name_ini).await 256 | } 257 | } 258 | } 259 | 260 | async fn add_english_file(&self, file: &Path, file_name: &str) -> FileId { 261 | let storage_id = self.upload_file_to_storage(file, file_name).await; 262 | add_english_file(self.crowdin_id, storage_id, file_name).await 263 | } 264 | 265 | async fn update_english_file(&self, file_id: FileId, file: &Path, file_name: &str) { 266 | let storage_id = self.upload_file_to_storage(file, file_name).await; 267 | update_english_file(file_id, storage_id).await; 268 | } 269 | 270 | async fn add_localization_files(&self, english_file_ids: HashMap) { 271 | for (language_code, files) in self.mod_directory.get_localizations() { 272 | for file in files { 273 | let file_name = replace_cfg_to_ini(util::file_name(&file)); 274 | let english_file_id = english_file_ids[&file_name]; 275 | self.add_localization_file(&file, &file_name, english_file_id, &language_code).await; 276 | } 277 | } 278 | } 279 | 280 | async fn add_localization_file( 281 | &self, 282 | file: &Path, 283 | file_name: &str, 284 | english_file_id: FileId, 285 | language_code: &LanguageCode, 286 | ) { 287 | let storage_id = self.upload_file_to_storage(file, file_name).await; 288 | add_localization_file(english_file_id, storage_id, language_code).await; 289 | } 290 | 291 | async fn upload_file_to_storage(&self, file: &Path, file_name: &str) -> StorageId { 292 | info!("[{}] upload file to storage: {}/{}", self.mod_directory.mod_info, util::file_name(file.parent().unwrap()), file_name); 293 | let file_content = fs::read_to_string(file).unwrap(); 294 | let mut file_content = util::escape::escape_strings_in_ini_file(&file_content); 295 | if file_content.is_empty() { 296 | file_content = "; empty".to_owned(); 297 | } 298 | upload_file_to_storage(file_content, file_name).await 299 | } 300 | } 301 | 302 | /// crowdin expects codes in format 'pt-BR' 303 | /// however some mods use 'pt-br' as language code 304 | /// (e.g. https://github.com/JonasJurczok/factorio-todo-list/tree/master/locale/pt-br) 305 | /// this function converts 'pt-br' to 'pt-BR' 306 | pub fn normalize_language_code(code: &str) -> String { 307 | match code.split_once('-') { 308 | None => code.to_ascii_lowercase(), 309 | Some((part1, part2)) => { 310 | format!("{}-{}", part1.to_ascii_lowercase(), part2.to_ascii_uppercase()) 311 | } 312 | } 313 | } 314 | 315 | pub fn is_correct_language_code(code: &str) -> bool { 316 | let codes = PROJECT_LANGUAGE_CODES.get().unwrap(); 317 | codes.iter().any(|it| it == code) 318 | } 319 | 320 | pub fn get_crowdin_directory_name(mod_info: &GithubModInfo) -> String { 321 | use util::case::to_title_case; 322 | let repo = to_title_case(&mod_info.repo); 323 | match &mod_info.crowdin_name { 324 | None => { 325 | format!("{} ({})", repo, mod_info.owner) 326 | } 327 | Some(crowdin_name) => { 328 | format!("{} - {} ({})", repo, to_title_case(crowdin_name), mod_info.owner) 329 | } 330 | } 331 | } 332 | 333 | pub fn replace_cfg_to_ini(name: &str) -> String { 334 | static DOT_CFG_REGEX: LazyLock = LazyLock::new(|| Regex::new(".cfg$").unwrap()); 335 | name.replace(DOT_CFG_REGEX.deref(), ".ini") 336 | } 337 | 338 | pub fn replace_ini_to_cfg(name: &str) -> String { 339 | static DOT_INI_REGEX: LazyLock = LazyLock::new(|| Regex::new(".ini$").unwrap()); 340 | name.replace(DOT_INI_REGEX.deref(), ".cfg") 341 | } 342 | -------------------------------------------------------------------------------- /src/documentation.md: -------------------------------------------------------------------------------- 1 | ## App responsibilities 2 | 1. Webhook for app installation: create subfolder in crowdin 3 | 1. Every week: update github from crowdin 4 | 1. Webhook for every push to update crowdin from github 5 | 6 | ## Installing github app 7 | 1. Find repositories with factorio mod (which has `locale/en` folder) 8 | 1. Clone github repository 9 | 1. Crowdin: 10 | * check that for each localization file (that is file in directory like `/locale/ru`) there is corresponding english file (in `/locale/en`) 11 | * crowdin-api/add-directory 12 | * for each file in `/locale/en`: crowdin-api/add-file 13 | * for each other `/locale` subfolder 14 | * for each subfolder file: crowdin-api/upload-translation 15 | 16 | ## Update github from crowdin (general overview) 17 | 1. Get list of all repositories for which github app is installed 18 | 1. Filter list to contain only repositories which also are presented on crowdin 19 | 1. Download all translations from crowdin 20 | 1. Update github from crowdin (for each repository) 21 | 22 | ## Update github from crowdin (for each repository) 23 | 1. Create installation token 24 | 25 | https://octokit.github.io/rest.js/#api-Apps-createInstallationToken 26 | 27 | 1. Clone repository `https://x-access-token:TOKEN@github.com/OWNER/REPO.git` 28 | 29 | https://developer.github.com/apps/building-github-apps/authenticating-with-github-apps/#http-based-git-access-by-an-installation 30 | 31 | 1. For each language: 32 | * locate language translations in downloaded files (see general overview for details) 33 | * copy them to cloned directory 34 | 1. Make git commit (or break if there are no changes) 35 | 1. Git push 36 | 37 | ### Branch protection rules 38 | If push to master is not allowed: 39 | 1. Create fork 40 | 1. Force-push to `crowdin-fml` branch in fork 41 | 1. Create pull request if not yet exists 42 | 43 | ## Webhook for every push to update crowdin from github 44 | 1. Check if pushed commits change `/locale/en` (Note that payload for push webhook contains added/modified/removed files) 45 | 1. For every modified/added english file: 46 | * check if we have same file on crowdin: 47 | * yes: crowdin-api/update-file 48 | * no: crowdin-api/add-file 49 | 1. (Note that for now we will not handle file removing) 50 | 51 | ## Public API to trigger update with GitHub OAuth authorization 52 | * Request `/api/triggerUpdate//` 53 | * Redirect to GitHub OAuth 54 | https://github.com/login/oauth/authorize 55 | https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps#web-application-flow 56 | * Redirect to `/api/triggerUpdate2` 57 | * Obtain access token using https://github.com/login/oauth/access_token 58 | * Get authorized user using https://api.github.com/user 59 | * Compare with , if matches trigger update 60 | 61 | ## Notes 62 | * Any localization folder (such as `/locale/en`, `/locale/ru`) may contain subfolder, and we should ignore subfolders, because factorio ignores them too. Here is [example](https://github.com/Karosieben/boblocale/tree/master/locale/en/old). 63 | 64 | * In some mods files names doesn't match across localization folders 65 | 66 | Examples: 67 | 68 | * `/locale/en/en.cfg` and `/locale/ru/ru.cfg` 69 | * `/locale/en/Angel.cfg` and `/locale/ru/Angel_ru.cfg` 70 | * `/locale/en/bobenemies_0.16.0.cfg` and `/locale/ru/bobenemies.cfg` 71 | 72 | We researched >1000 mods and it turns out that only 8% of them has unmatched files names in different languages directories 73 | 74 | So for now we decided to support only matched files names in different languages (mod author has to rename languages files if their names don't match) 75 | 76 | * Logs format: 77 | 78 | [action-name] [repository-name] comment 79 | -------------------------------------------------------------------------------- /src/git_util.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Deref; 2 | use std::path::Path; 3 | use std::process::{Command, Output}; 4 | 5 | use log::{error, warn}; 6 | 7 | use crate::github::{GITHUB_BRANCH_NAME, GITHUB_USER_NAME}; 8 | use crate::myenv::{GIT_COMMIT_MESSAGE, GIT_COMMIT_USER_EMAIL, GIT_COMMIT_USER_NAME, GITHUB_PERSONAL_ACCESS_TOKEN}; 9 | 10 | pub fn clone(url: &str, path: &Path, branch: Option<&str>) { 11 | let mut args = vec![ 12 | "clone", 13 | "--depth", "1", 14 | ]; 15 | if let Some(branch) = branch { 16 | args.extend(&[ 17 | "--branch", 18 | branch 19 | ]); 20 | } 21 | args.push(url); 22 | args.push("."); // clone to current directory 23 | execute_git_command(path, &args, true) 24 | } 25 | 26 | pub fn add_all_and_check_has_changes(path: &Path) -> bool { 27 | add_all(path); 28 | git_status_has_changes(path) 29 | } 30 | 31 | fn add_all(path: &Path) { 32 | execute_git_command(path, &["add", ".", "--all"], true); 33 | } 34 | 35 | pub fn commit(path: &Path) { 36 | let name = GIT_COMMIT_USER_NAME.deref(); 37 | let email = GIT_COMMIT_USER_EMAIL.deref(); 38 | let message = GIT_COMMIT_MESSAGE.deref(); 39 | let args = &[ 40 | "-c", &format!("user.name='{}'", name), 41 | "-c", &format!("user.email='{}'", email), 42 | "-c", "commit.gpgsign=false", 43 | "commit", 44 | "-m", message 45 | ]; 46 | execute_git_command(path, args, true); 47 | } 48 | 49 | pub fn push(path: &Path) { 50 | execute_git_command(path, &["push"], false); 51 | } 52 | 53 | pub fn push_to_my_fork(path: &Path, repo: &str) -> bool { 54 | let personal_token = GITHUB_PERSONAL_ACCESS_TOKEN.deref(); 55 | let url = format!("https://x-access-token:{}@github.com/{}/{}.git", personal_token, GITHUB_USER_NAME, repo); 56 | execute_git_command(path, &["remote", "add", "my", &url], true); 57 | 58 | execute_git_command(path, &["fetch", "my"], true); 59 | let diff_refspec = format!("HEAD..my/{}", GITHUB_BRANCH_NAME); 60 | if !git_diff_has_changes(path, &diff_refspec) { 61 | return false; 62 | } 63 | 64 | let push_refspec = format!("HEAD:{}", GITHUB_BRANCH_NAME); 65 | execute_git_command(path, &["push", "my", &push_refspec, "--force"], true); 66 | true 67 | } 68 | 69 | fn git_diff_has_changes(path: &Path, diff_refspec: &str) -> bool { 70 | let output = execute_git_command_unchecked( 71 | path, 72 | &["diff", "--exit-code", "--name-only", diff_refspec], 73 | ); 74 | !output.status.success() 75 | } 76 | 77 | fn git_status_has_changes(path: &Path) -> bool { 78 | let output = execute_git_command_unchecked( 79 | path, 80 | &["status", "--porcelain"], 81 | ); 82 | !output.stdout.is_empty() 83 | } 84 | 85 | fn execute_git_command_unchecked(path: &Path, args: &[&str]) -> Output { 86 | Command::new("git") 87 | .current_dir(path) 88 | .args(args) 89 | .output() 90 | .expect("Failed to execute git command") 91 | } 92 | 93 | fn execute_git_command(path: &Path, args: &[&str], panic_if_fail: bool) { 94 | let output = execute_git_command_unchecked(path, args); 95 | if !output.status.success() { 96 | let stderr = String::from_utf8_lossy(&output.stderr); 97 | let message = format!("Failed to execute `git {}`", args.join(" ")); 98 | if panic_if_fail { 99 | error!("{}", stderr); 100 | panic!("{}", message); 101 | } else { 102 | warn!("{}", stderr); 103 | warn!("{}", message); 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/github.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Deref; 2 | use std::path::Path; 3 | use std::time::Duration; 4 | 5 | use jsonwebtoken::EncodingKey; 6 | use log::info; 7 | use octocrab::{Error, Octocrab, Page}; 8 | use octocrab::models::{AppId, Installation, InstallationId, Repository}; 9 | use octocrab::models::pulls::PullRequest; 10 | use octocrab::models::repos::ContentItems; 11 | use rocket::serde::Deserialize; 12 | use serde::de::DeserializeOwned; 13 | use tokio::time::sleep; 14 | 15 | use crate::git_util; 16 | use crate::github_config::parse_github_repo_info_json; 17 | use crate::github_repo_info::{GithubRepoInfo}; 18 | use crate::mod_directory::RepositoryDirectory; 19 | use crate::myenv::{GITHUB_APP_ID, GITHUB_APP_PRIVATE_KEY, GITHUB_PERSONAL_ACCESS_TOKEN}; 20 | use crate::sentry::sentry_report_error; 21 | use crate::util::{create_temporary_directory, EmptyBody}; 22 | 23 | pub const GITHUB_USER_NAME: &str = "factorio-mods-helper"; 24 | pub const GITHUB_BRANCH_NAME: &str = "crowdin-fml"; 25 | pub const GITHUB_CONFIG_FILE_NAME: &str = "factorio-mods-localization.json"; 26 | const MAX_PER_PAGE: u8 = 100; 27 | 28 | fn get_credentials() -> (AppId, EncodingKey) { 29 | let github_app_id: u64 = GITHUB_APP_ID.deref().parse().unwrap(); 30 | let github_app_key = GITHUB_APP_PRIVATE_KEY.deref().replace("\\n", "\n"); 31 | let github_app_key = EncodingKey::from_rsa_pem(github_app_key.as_bytes()).unwrap(); 32 | (AppId(github_app_id), github_app_key) 33 | } 34 | 35 | pub fn as_app() -> Octocrab { 36 | let (app_id, key) = get_credentials(); 37 | Octocrab::builder().app(app_id, key).build().unwrap() 38 | } 39 | 40 | pub fn as_installation(installation_id: InstallationId) -> Octocrab { 41 | as_app().installation(installation_id) 42 | } 43 | 44 | // for tests/examples 45 | pub async fn as_installation_for_user(login: &str) -> Option { 46 | let api = as_app(); 47 | let installation = get_all_installations(&api).await 48 | .into_iter() 49 | .find(|it| it.account.login == login)?; 50 | Some(api.installation(installation.id)) 51 | } 52 | 53 | pub async fn get_installation_id_for_user(login: &str) -> InstallationId { 54 | let api = as_app(); 55 | get_all_installations(&api).await 56 | .iter().find(|it| it.account.login == login).unwrap() 57 | .id 58 | } 59 | 60 | pub async fn get_installation_id_for_repo(full_name: &str) -> Option { 61 | let (owner, repo) = full_name.split_once('/').unwrap(); 62 | as_app() 63 | .apps() 64 | .get_repository_installation(owner, repo).await 65 | .map(|it| it.id) 66 | .ok() 67 | } 68 | 69 | #[derive(Debug, Eq, PartialEq)] 70 | pub enum GetRepoInfoError { 71 | InvalidConfig, 72 | LocaleDirectoryMissing, 73 | LocaleEnDirectoryMissingOrEmpty, 74 | } 75 | 76 | pub async fn get_repo_info( 77 | installation_api: &Octocrab, 78 | full_name: &str, 79 | ) -> Result { 80 | let root_items = list_files_in_directory(installation_api, full_name, "").await.unwrap(); 81 | if root_items.iter().any(|it| it == GITHUB_CONFIG_FILE_NAME) { 82 | let mods_file = get_content(installation_api, full_name, GITHUB_CONFIG_FILE_NAME).await.unwrap(); 83 | let json = mods_file.items[0].decoded_content().unwrap(); 84 | parse_github_repo_info_json(full_name, &json) 85 | .ok_or(GetRepoInfoError::InvalidConfig) 86 | } else { 87 | if !root_items.iter().any(|it| it == "locale") { 88 | return Err(GetRepoInfoError::LocaleDirectoryMissing); 89 | } 90 | let locale_en_items = list_files_in_directory(installation_api, full_name, "locale/en").await; 91 | match locale_en_items { 92 | Ok(locale_en_items) if !locale_en_items.is_empty() => { 93 | Ok(GithubRepoInfo::new_single_mod(full_name)) 94 | } 95 | _ => { 96 | Err(GetRepoInfoError::LocaleEnDirectoryMissingOrEmpty) 97 | } 98 | } 99 | } 100 | } 101 | 102 | async fn get_content(installation_api: &Octocrab, full_name: &str, path: &str) -> octocrab::Result { 103 | let (owner, repo) = full_name.split_once('/').unwrap(); 104 | let result = installation_api 105 | .repos(owner, repo) 106 | .get_content() 107 | .path(path) 108 | .send() 109 | .await; 110 | if let Err(Error::GitHub { source, .. }) = &result { 111 | if path.is_empty() && source.errors.is_none() && source.message == "This repository is empty." { 112 | return Ok(ContentItems { items: vec![] }); 113 | } 114 | } 115 | result 116 | } 117 | 118 | pub async fn list_files_in_directory(installation_api: &Octocrab, full_name: &str, path: &str) -> octocrab::Result> { 119 | get_content(installation_api, full_name, path).await 120 | .map(|it| { 121 | it.items 122 | .into_iter() 123 | .map(|file| file.name) 124 | .collect() 125 | }) 126 | } 127 | 128 | trait PageExt { 129 | async fn all_pages(self, api: &Octocrab) -> octocrab::Result>; 130 | } 131 | 132 | impl PageExt for Page { 133 | async fn all_pages(self, api: &Octocrab) -> octocrab::Result> { 134 | api.all_pages(self).await 135 | } 136 | } 137 | 138 | pub async fn get_all_installations(api: &Octocrab) -> Vec { 139 | api 140 | .apps().installations().per_page(MAX_PER_PAGE) 141 | .send().await.unwrap() 142 | .all_pages(api).await.unwrap() 143 | } 144 | 145 | pub async fn get_all_repositories(api: &Octocrab) -> Vec<(GithubRepoInfo, InstallationId)> { 146 | let mut result = Vec::new(); 147 | let installations = get_all_installations(api).await; 148 | for installation in installations { 149 | let installation_api = api.installation(installation.id); 150 | let repositories = get_all_repositories_of_installation(&installation_api).await; 151 | for repository in repositories { 152 | let repo_info = get_repo_info(&installation_api, &repository).await; 153 | if let Ok(repo_info) = repo_info { 154 | result.push((repo_info, installation.id)); 155 | } 156 | } 157 | } 158 | result 159 | } 160 | 161 | pub async fn get_all_repositories_of_installation(installation_api: &Octocrab) -> Vec { 162 | let parameters = serde_json::json!({"per_page": MAX_PER_PAGE}); 163 | let repositories: Page = installation_api 164 | .get("/installation/repositories", Some(¶meters)).await.unwrap(); 165 | let repositories = repositories.all_pages(installation_api).await.unwrap(); 166 | repositories 167 | .into_iter() 168 | .filter(|it| !it.private.unwrap()) 169 | .map(|it| it.full_name.unwrap()) 170 | .collect() 171 | } 172 | 173 | pub async fn clone_repository( 174 | repo_info: &GithubRepoInfo, 175 | installation_id: InstallationId, 176 | ) -> RepositoryDirectory { 177 | info!("[{}] clone repository", repo_info.full_name); 178 | let directory = create_temporary_directory(); 179 | clone_repository_to(repo_info, installation_id, directory.path()).await; 180 | RepositoryDirectory::new(&repo_info.full_name, directory) 181 | } 182 | 183 | async fn clone_repository_to( 184 | repo_info: &GithubRepoInfo, 185 | installation_id: InstallationId, 186 | path: &Path, 187 | ) { 188 | use secrecy::ExposeSecret; 189 | let api = as_app(); 190 | let (_, installation_token) = api.installation_and_token(installation_id).await.unwrap(); 191 | let installation_token = installation_token.expose_secret(); 192 | let url = format!("https://x-access-token:{}@github.com/{}.git", installation_token, repo_info.full_name); 193 | git_util::clone(&url, path, repo_info.branch.as_deref()); 194 | } 195 | 196 | pub async fn create_pull_request(personal_api: &Octocrab, full_name: &str, base_branch: &str) { 197 | let (owner, repo) = full_name.split_once('/').unwrap(); 198 | let title = "Update translations from Crowdin"; 199 | let body = "See https://github.com/dima74/factorio-mods-localization for details"; 200 | let head_branch = format!("{}:{}", GITHUB_USER_NAME, GITHUB_BRANCH_NAME); 201 | let result = personal_api 202 | .pulls(owner, repo) 203 | .create(title, head_branch, base_branch) 204 | .body(body) 205 | .maintainer_can_modify(true) 206 | .send().await; 207 | check_create_pull_request_response(result, full_name); 208 | } 209 | 210 | fn check_create_pull_request_response(result: octocrab::Result, full_name: &str) { 211 | let Err(err) = result else { return; }; 212 | if is_error_pull_request_already_exists(&err) { 213 | // PR exists - no need to reopen, force push is enough 214 | return; 215 | } 216 | if is_error_repository_archived(&err) { 217 | // Ignore archived repositories, can't create PRs for them 218 | return; 219 | } 220 | panic!("[{}] Can't create pull request: {}", full_name, err); 221 | } 222 | 223 | fn is_error_pull_request_already_exists(error: &Error) -> bool { 224 | let Error::GitHub { source, .. } = &error else { return false; }; 225 | if source.message != "Validation Failed" { return false; }; 226 | let Some([error, ..]) = source.errors.as_deref() else { return false; }; 227 | let serde_json::Value::Object(error) = error else { return false; }; 228 | let Some(serde_json::Value::String(message)) = error.get("message") else { return false; }; 229 | message.starts_with("A pull request already exists for") 230 | } 231 | 232 | fn is_error_repository_archived(error: &Error) -> bool { 233 | let error_str = format!("{}", error); 234 | error_str.contains("Repository was archived so is read-only") 235 | } 236 | 237 | pub async fn get_default_branch(installation_api: &Octocrab, full_name: &str) -> String { 238 | #[derive(Deserialize)] 239 | struct Response { default_branch: String } 240 | let url = format!("/repos/{}", full_name); 241 | let response: Response = installation_api.get(&url, None::<&()>).await.unwrap(); 242 | response.default_branch 243 | } 244 | 245 | pub async fn is_branch_protected(installation_api: &Octocrab, full_name: &str, branch: &str) -> bool { 246 | #[derive(Deserialize)] 247 | struct Response { protected: bool } 248 | let url = format!("/repos/{}/branches/{}", full_name, branch); 249 | let result: Response = installation_api.get(&url, None::<&()>).await.unwrap(); 250 | result.protected 251 | } 252 | 253 | pub fn as_personal_account() -> Octocrab { 254 | let personal_token = GITHUB_PERSONAL_ACCESS_TOKEN.to_owned(); 255 | Octocrab::builder() 256 | .personal_token(personal_token) 257 | .build() 258 | .unwrap() 259 | } 260 | 261 | pub async fn fork_repository(personal_api: &Octocrab, full_name: &str) -> bool { 262 | if let Some(is_fork_name_correct) = check_fork_exists(personal_api, full_name).await { 263 | return is_fork_name_correct; 264 | } 265 | fork_repository_without_check(personal_api, full_name).await; 266 | true 267 | } 268 | 269 | pub async fn fork_repository_without_check(personal_api: &Octocrab, full_name: &str) { 270 | let (owner, repo) = full_name.split_once('/').unwrap(); 271 | info!("[{}] forking repository...", full_name); 272 | personal_api 273 | .repos(owner, repo) 274 | .create_fork() 275 | .send().await.unwrap(); 276 | sleep(Duration::from_secs(120)).await; 277 | } 278 | 279 | // None => no fork 280 | // Some(false) => fork with different name 281 | // Some(true) => fork exists and can be used 282 | async fn check_fork_exists(api: &Octocrab, full_name: &str) -> Option { 283 | let (owner, repo) = full_name.split_once('/').unwrap(); 284 | let forks = api 285 | .repos(owner, repo) 286 | .list_forks() 287 | .send().await.unwrap() 288 | .all_pages(api).await.unwrap(); 289 | for fork in forks { 290 | let fork_full_name = fork.full_name.unwrap(); 291 | let (fork_owner, fork_repo) = fork_full_name.split_once('/').unwrap(); 292 | if fork_owner == GITHUB_USER_NAME { 293 | return if fork_repo == repo { 294 | Some(true) // fork already exists 295 | } else { 296 | let message = format!("Fork name {} doesn't match repository {}/{}", fork_repo, owner, repo); 297 | sentry_report_error(&message); 298 | Some(false) 299 | }; 300 | } 301 | } 302 | None 303 | } 304 | 305 | #[derive(Default)] 306 | pub struct GetNotForkedResult { 307 | pub not_forked: Vec, 308 | pub forked_with_diferrent_name: Vec, 309 | } 310 | 311 | pub async fn get_not_forked_repositories() -> GetNotForkedResult { 312 | let api_app = as_app(); 313 | let repositories = get_all_repositories(&api_app).await; 314 | 315 | let api_personal = as_personal_account(); 316 | let mut result = GetNotForkedResult::default(); 317 | for (repo_info, _id) in repositories { 318 | let full_name = repo_info.full_name; 319 | match check_fork_exists(&api_personal, &full_name).await { 320 | None => result.not_forked.push(full_name), 321 | Some(false) => result.forked_with_diferrent_name.push(full_name), 322 | Some(true) => continue, 323 | } 324 | } 325 | result 326 | } 327 | 328 | pub async fn star_repository(api: &Octocrab, full_name: &str) { 329 | let _response: octocrab::Result = api 330 | .put(format!("/user/starred/{}", full_name), None::<&()>) 331 | .await; 332 | } 333 | 334 | pub async fn is_repository_starred(api: &Octocrab, full_name: &str) -> bool { 335 | let response: octocrab::Result = api 336 | .get(format!("/user/starred/{}", full_name), None::<&()>) 337 | .await; 338 | response.is_ok() 339 | } 340 | 341 | pub async fn get_not_starred_repositories() -> Vec { 342 | let api_app = as_app(); 343 | let repositories = get_all_repositories(&api_app).await; 344 | 345 | let api_personal = as_personal_account(); 346 | let mut not_starred = Vec::new(); 347 | for (repo_info, _id) in repositories { 348 | let full_name = repo_info.full_name; 349 | if !is_repository_starred(&api_personal, &full_name).await { 350 | not_starred.push(full_name); 351 | } 352 | } 353 | not_starred 354 | } 355 | 356 | pub async fn get_current_user(api_oauth: &Octocrab) -> String { 357 | let response: octocrab::models::Author = api_oauth 358 | .get("/user", None::<&()>) 359 | .await.unwrap(); 360 | response.login 361 | } 362 | 363 | #[cfg(test)] 364 | mod tests { 365 | use crate::github_repo_info::GithubModInfo; 366 | 367 | use super::*; 368 | 369 | #[tokio::test] 370 | async fn test_has_locale_en() { 371 | let api = as_installation_for_user("dima74").await.unwrap(); 372 | assert_eq!( 373 | get_repo_info(&api, "dima74/factorio-mod-example").await, 374 | Ok(GithubRepoInfo { 375 | full_name: "dima74/factorio-mod-example".to_owned(), 376 | mods: vec![GithubModInfo::new_root("dima74/factorio-mod-example")], 377 | weekly_update_from_crowdin: true, 378 | branch: None, 379 | }), 380 | ); 381 | assert_eq!( 382 | get_repo_info(&api, "dima74/factorio-multimod-example").await, 383 | Ok(GithubRepoInfo { 384 | full_name: "dima74/factorio-multimod-example".to_owned(), 385 | mods: vec![ 386 | GithubModInfo { 387 | owner: "dima74".to_owned(), 388 | repo: "factorio-multimod-example".to_owned(), 389 | locale_path: "Mod1/locale".to_owned(), 390 | crowdin_name: Some("Name1".to_owned()), 391 | }, 392 | GithubModInfo { 393 | owner: "dima74".to_owned(), 394 | repo: "factorio-multimod-example".to_owned(), 395 | locale_path: "Mod3/Data/locale".to_owned(), 396 | crowdin_name: Some("Name3".to_owned()), 397 | }, 398 | ], 399 | weekly_update_from_crowdin: true, 400 | branch: None, 401 | }), 402 | ); 403 | assert_eq!( 404 | get_repo_info(&api, "dima74/factorio-mods-localization").await, 405 | Err(GetRepoInfoError::LocaleDirectoryMissing), 406 | ); 407 | } 408 | } 409 | -------------------------------------------------------------------------------- /src/github_config.rs: -------------------------------------------------------------------------------- 1 | //! `factorio-mods-localization.json` - config file in root of the repository. 2 | //! It should be in the *default* branch, even if some other "branch" is specified in config. 3 | //! 4 | //! # Format of `factorio-mods-localization.json` 5 | //! Old format (deprecated): 6 | //! ```json 7 | //! ["mod1", "mod2"] 8 | //! ``` 9 | //! 10 | //! New format: 11 | //! ```json 12 | //! { 13 | //! "mods": ["mod1", "mod2"], 14 | //! "weekly_update_from_crowdin": false, 15 | //! "branch": "dev" 16 | //! } 17 | //! ``` 18 | //! 19 | //! Alternative format for "mods": 20 | //! ```json 21 | //! { 22 | //! "mods": [{"localePath": "custom/path", "crowdinName": "Foo"}] 23 | //! ... 24 | //! } 25 | //! ``` 26 | //! 27 | //! # Examples 28 | //! 29 | //! ## Single mod in github repository (no `factorio-mods-localization.json`) 30 | //! . 31 | //! ├── locale/en 32 | //! 33 | //! ## Multiple mods in github repository 34 | //! . 35 | //! ├── factorio-mods-localization.json // `{"mods": ["Mod1", "Mod2"]}` 36 | //! ├── Mod1 37 | //! │ ├── locale/en 38 | //! ├── Mod2 39 | //! │ ├── locale/en 40 | 41 | use crate::github_repo_info::{GithubModInfo, GithubRepoInfo}; 42 | use serde::Deserialize; 43 | use std::collections::HashSet; 44 | 45 | #[derive(Deserialize)] 46 | struct Config { 47 | mods: Option, 48 | weekly_update_from_crowdin: Option, 49 | branch: Option, 50 | } 51 | 52 | #[derive(Deserialize)] 53 | #[serde(untagged)] 54 | enum ConfigMods { 55 | Short(Vec), 56 | Full(Vec), 57 | } 58 | 59 | #[derive(Deserialize)] 60 | #[serde(rename_all = "camelCase")] 61 | struct ConfigMod { 62 | locale_path: String, 63 | crowdin_name: String, 64 | } 65 | 66 | #[derive(Deserialize)] 67 | struct ConfigOld(Vec); 68 | 69 | impl From for Config { 70 | fn from(config: ConfigOld) -> Self { 71 | Config { 72 | mods: Some(ConfigMods::Short(config.0)), 73 | weekly_update_from_crowdin: None, 74 | branch: None, 75 | } 76 | } 77 | } 78 | 79 | pub fn parse_github_repo_info_json(full_name: &str, json: &str) -> Option { 80 | let config: Config = parse_config(json)?; 81 | let mods = convert_mods(full_name, config.mods)?; 82 | if !check_no_duplicates(&mods) { return None; } 83 | GithubRepoInfo::new_from_config(full_name, mods, config.weekly_update_from_crowdin, config.branch) 84 | } 85 | 86 | fn parse_config(json: &str) -> Option { 87 | if let Ok(config) = serde_json::from_str::(json) { 88 | return Some(config.into()); 89 | } 90 | serde_json::from_str(json).ok() 91 | } 92 | 93 | fn convert_mods( 94 | full_name: &str, 95 | mods: Option, 96 | ) -> Option> { 97 | let Some(mods) = mods else { 98 | // { "weekly_update_from_crowdin": false } 99 | return Some(vec![GithubModInfo::new_root(full_name)]); 100 | }; 101 | 102 | match mods { 103 | ConfigMods::Short(mods) => { 104 | mods 105 | .into_iter() 106 | .map(|name| GithubModInfo::new_custom(full_name, None, name)) 107 | .collect() 108 | } 109 | ConfigMods::Full(mods) => { 110 | mods 111 | .into_iter() 112 | .map(|mod_| { 113 | GithubModInfo::new_custom(full_name, Some(mod_.locale_path), mod_.crowdin_name) 114 | }) 115 | .collect() 116 | } 117 | } 118 | } 119 | 120 | fn check_no_duplicates(mods: &[GithubModInfo]) -> bool { 121 | let mods_set = mods 122 | .iter() 123 | .map(|mod_| mod_.crowdin_name.as_ref()) 124 | .collect::>(); 125 | mods.len() == mods_set.len() 126 | } 127 | 128 | #[cfg(test)] 129 | mod tests { 130 | use super::*; 131 | 132 | #[test] 133 | fn test_parse_mods_old_version() { 134 | assert_eq!( 135 | parse_github_repo_info_json("owner/repo", r#"["mod1", "mod2"]"#), 136 | Some(GithubRepoInfo { 137 | full_name: "owner/repo".to_owned(), 138 | mods: vec![ 139 | GithubModInfo { 140 | owner: "owner".to_owned(), 141 | repo: "repo".to_owned(), 142 | locale_path: "mod1/locale".to_owned(), 143 | crowdin_name: Some("mod1".to_owned()), 144 | }, 145 | GithubModInfo { 146 | owner: "owner".to_owned(), 147 | repo: "repo".to_owned(), 148 | locale_path: "mod2/locale".to_owned(), 149 | crowdin_name: Some("mod2".to_owned()), 150 | }, 151 | ], 152 | weekly_update_from_crowdin: true, 153 | branch: None, 154 | }) 155 | ); 156 | } 157 | 158 | #[test] 159 | fn test_parse_mods_short_version() { 160 | assert_eq!( 161 | parse_github_repo_info_json("owner/repo", r#"{"mods": ["mod1", "mod2"]}"#), 162 | Some(GithubRepoInfo { 163 | full_name: "owner/repo".to_owned(), 164 | mods: 165 | vec![ 166 | GithubModInfo { 167 | owner: "owner".to_owned(), 168 | repo: "repo".to_owned(), 169 | locale_path: "mod1/locale".to_owned(), 170 | crowdin_name: Some("mod1".to_owned()), 171 | }, 172 | GithubModInfo { 173 | owner: "owner".to_owned(), 174 | repo: "repo".to_owned(), 175 | locale_path: "mod2/locale".to_owned(), 176 | crowdin_name: Some("mod2".to_owned()), 177 | }, 178 | ], 179 | weekly_update_from_crowdin: true, 180 | branch: None, 181 | }) 182 | ); 183 | } 184 | 185 | #[test] 186 | fn test_parse_mods_long_version() { 187 | assert_eq!( 188 | parse_github_repo_info_json("owner/repo", r#"{"mods": [{"localePath": "custom/path", "crowdinName": "Foo"}]}"#), 189 | Some(GithubRepoInfo { 190 | full_name: "owner/repo".to_owned(), 191 | mods: 192 | vec![ 193 | GithubModInfo { 194 | owner: "owner".to_owned(), 195 | repo: "repo".to_owned(), 196 | locale_path: "custom/path".to_owned(), 197 | crowdin_name: Some("Foo".to_owned()), 198 | }, 199 | ], 200 | weekly_update_from_crowdin: true, 201 | branch: None, 202 | }) 203 | ) 204 | } 205 | 206 | #[test] 207 | fn test_parse_weekly_update_from_crowdin() { 208 | assert_eq!( 209 | parse_github_repo_info_json("owner/repo", r#"{"weekly_update_from_crowdin": false}"#), 210 | Some(GithubRepoInfo { 211 | full_name: "owner/repo".to_owned(), 212 | mods: vec![ 213 | GithubModInfo { 214 | owner: "owner".to_owned(), 215 | repo: "repo".to_owned(), 216 | locale_path: "locale".to_owned(), 217 | crowdin_name: None, 218 | }, 219 | ], 220 | weekly_update_from_crowdin: false, 221 | branch: None, 222 | }) 223 | ); 224 | assert_eq!( 225 | parse_github_repo_info_json("owner/repo", r#"{"weekly_update_from_crowdin": true}"#), 226 | Some(GithubRepoInfo { 227 | full_name: "owner/repo".to_owned(), 228 | mods: vec![ 229 | GithubModInfo { 230 | owner: "owner".to_owned(), 231 | repo: "repo".to_owned(), 232 | locale_path: "locale".to_owned(), 233 | crowdin_name: None, 234 | }, 235 | ], 236 | weekly_update_from_crowdin: true, 237 | branch: None, 238 | }) 239 | ); 240 | } 241 | 242 | #[test] 243 | fn test_parse_branch() { 244 | assert_eq!( 245 | parse_github_repo_info_json("owner/repo", r#"{"branch": "dev"}"#), 246 | Some(GithubRepoInfo { 247 | full_name: "owner/repo".to_owned(), 248 | mods: vec![ 249 | GithubModInfo { 250 | owner: "owner".to_owned(), 251 | repo: "repo".to_owned(), 252 | locale_path: "locale".to_owned(), 253 | crowdin_name: None, 254 | }, 255 | ], 256 | weekly_update_from_crowdin: true, 257 | branch: Some("dev".to_owned()), 258 | }) 259 | ); 260 | } 261 | 262 | #[test] 263 | fn test_parse_all() { 264 | let json = r#" 265 | { 266 | "mods": ["mod1", "mod2"], 267 | "weekly_update_from_crowdin": false, 268 | "branch": "dev" 269 | } 270 | "#; 271 | assert_eq!( 272 | parse_github_repo_info_json("owner/repo", json), 273 | Some(GithubRepoInfo { 274 | full_name: "owner/repo".to_owned(), 275 | mods: vec![ 276 | GithubModInfo { 277 | owner: "owner".to_owned(), 278 | repo: "repo".to_owned(), 279 | locale_path: "mod1/locale".to_owned(), 280 | crowdin_name: Some("mod1".to_owned()), 281 | }, 282 | GithubModInfo { 283 | owner: "owner".to_owned(), 284 | repo: "repo".to_owned(), 285 | locale_path: "mod2/locale".to_owned(), 286 | crowdin_name: Some("mod2".to_owned()), 287 | }, 288 | ], 289 | weekly_update_from_crowdin: false, 290 | branch: Some("dev".to_owned()), 291 | }) 292 | ); 293 | } 294 | } 295 | -------------------------------------------------------------------------------- /src/github_repo_info.rs: -------------------------------------------------------------------------------- 1 | use regex::Regex; 2 | use serde::{Deserialize, Serialize}; 3 | use std::collections::HashSet; 4 | use std::fmt; 5 | use std::sync::LazyLock; 6 | 7 | use crate::crowdin::get_crowdin_directory_name; 8 | 9 | /// One [`GithubRepoInfo`] can contain multiple [`GithubModInfo`]. 10 | /// [`GithubRepoInfo`] corresponds 1-1 to github repository. 11 | /// [`GithubModInfo`] corresponds 1-1 to directory on crowdin. 12 | #[derive(Debug, Eq, PartialEq, Serialize, Deserialize)] 13 | pub struct GithubRepoInfo { 14 | pub full_name: String, 15 | pub mods: Vec, 16 | pub weekly_update_from_crowdin: bool, 17 | /// Branch from which english files will be tracked and to which translations will be pushed 18 | pub branch: Option, 19 | } 20 | 21 | impl GithubRepoInfo { 22 | fn new( 23 | full_name: &str, 24 | mods: Vec, 25 | weekly_update_from_crowdin: Option, 26 | branch: Option, 27 | ) -> Self { 28 | Self { 29 | full_name: full_name.to_owned(), 30 | mods, 31 | weekly_update_from_crowdin: weekly_update_from_crowdin.unwrap_or(true), 32 | branch, 33 | } 34 | } 35 | 36 | pub fn new_from_config( 37 | full_name: &str, 38 | mods: Vec, 39 | weekly_update_from_crowdin: Option, 40 | branch: Option, 41 | ) -> Option { 42 | if mods.is_empty() { return None; } 43 | Some(Self::new(full_name, mods, weekly_update_from_crowdin, branch)) 44 | } 45 | 46 | pub fn new_single_mod(full_name: &str) -> Self { 47 | let mods = vec![GithubModInfo::new_root(full_name)]; 48 | Self::new(full_name, mods, None, None) 49 | } 50 | 51 | // for debug routes 52 | pub fn keep_single_mod_with_crowdin_name(&mut self, crowdin_name: &str) -> bool { 53 | self.mods.retain(|mod_| { 54 | mod_.crowdin_name.as_deref() == Some(crowdin_name) 55 | }); 56 | !self.mods.is_empty() 57 | } 58 | 59 | pub fn filter_mods_present_on_crowdin( 60 | mut self, 61 | directories_crowdin: &HashSet, 62 | ) -> Option { 63 | self.mods.retain(|it| directories_crowdin.contains(&get_crowdin_directory_name(it))); 64 | if self.mods.is_empty() { return None; } 65 | Some(self) 66 | } 67 | } 68 | 69 | /// Depends on `factorio-mods-localization.json`: 70 | /// - No: 71 | /// locale_path = "locale" 72 | /// crowdin_name = None 73 | /// - `["Mod1", "Mod2"]` or `{"mods": ["Mod1", "Mod2"]}`: 74 | /// locale_path = "Mod1/locale" 75 | /// crowdin_name = Some("Mod1") 76 | /// - `{"mods": [{"localePath": "custom/path", "crowdinName": "Foo"}]}` 77 | /// locale_path = "custom/path" 78 | /// crowdin_name = Some("Foo") 79 | #[derive(Debug, Eq, PartialEq, Serialize, Deserialize)] 80 | pub struct GithubModInfo { 81 | pub owner: String, 82 | pub repo: String, 83 | pub locale_path: String, 84 | pub crowdin_name: Option, 85 | } 86 | 87 | // Used only for logging 88 | impl fmt::Display for GithubModInfo { 89 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 90 | match &self.crowdin_name { 91 | None => { 92 | write!(f, "{}/{}", self.owner, self.repo) 93 | } 94 | Some(crowdin_name) => { 95 | write!(f, "{}/{}/{}", self.owner, self.repo, crowdin_name) 96 | } 97 | } 98 | } 99 | } 100 | 101 | impl GithubModInfo { 102 | pub fn new_root(full_name: &str) -> Self { 103 | Self::new(full_name, "locale".to_owned(), None) 104 | } 105 | 106 | pub fn new_custom( 107 | full_name: &str, 108 | locale_path: Option, 109 | crowdin_name: String, 110 | ) -> Option { 111 | let locale_path = locale_path 112 | .unwrap_or_else(|| format!("{crowdin_name}/locale")); 113 | 114 | if !Self::check_locale_path(&locale_path) { return None; } 115 | if !Self::check_crowdin_name(&crowdin_name) { return None; } 116 | 117 | Some(Self::new(full_name, locale_path, Some(crowdin_name))) 118 | } 119 | 120 | fn new( 121 | full_name: &str, 122 | locale_path: String, 123 | crowdin_name: Option, 124 | ) -> Self { 125 | let (owner, repo) = full_name.split_once('/').unwrap(); 126 | Self { 127 | owner: owner.to_owned(), 128 | repo: repo.to_owned(), 129 | locale_path, 130 | crowdin_name, 131 | } 132 | } 133 | 134 | fn check_locale_path(locale_path: &str) -> bool { 135 | !locale_path.is_empty() 136 | && !locale_path.contains(['.', ' ', '<', '>', ':', '"', '\\', '|', '?', '*']) 137 | && !locale_path.split('/').any(|s| s.is_empty()) 138 | } 139 | 140 | fn check_crowdin_name(crowdin_name: &str) -> bool { 141 | static CROWDIN_NAME_REGEX: LazyLock = LazyLock::new(|| { 142 | Regex::new(r"^[a-zA-Z0-9._-]+$").unwrap() 143 | }); 144 | CROWDIN_NAME_REGEX.is_match(crowdin_name) 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! General overview of the process: 2 | //! 1. GitHub app installed - [webhooks::on_repositories_added] 3 | //! 2. English files updated on GitHub - [webhooks::on_push_event] 4 | //! 3. Weekly update from Crowdin to GitHub - [server::trigger_update::push_all_crowdin_changes_to_github] 5 | 6 | pub mod crowdin; 7 | pub mod git_util; 8 | pub mod github; 9 | pub mod mod_directory; 10 | pub mod myenv; 11 | pub mod sentry; 12 | pub mod server; 13 | pub mod util; 14 | pub mod webhooks; 15 | pub mod github_repo_info; 16 | pub mod github_config; 17 | 18 | pub fn init() { 19 | dotenv::dotenv().ok(); 20 | myenv::init(); 21 | sentry::init_logging(); 22 | } 23 | 24 | pub async fn init_with_crowdin() { 25 | init(); 26 | crowdin::init().await; 27 | } 28 | 29 | pub async fn main() { 30 | init_with_crowdin().await; 31 | server::main().await; 32 | } 33 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | dotenv::dotenv().ok(); 3 | let _sentry = fml::sentry::init_sentry(); 4 | rocket::async_main(fml::main()); 5 | } 6 | -------------------------------------------------------------------------------- /src/mod_directory.rs: -------------------------------------------------------------------------------- 1 | use std::path::{Path, PathBuf}; 2 | 3 | use log::{error, warn}; 4 | use tempfile::TempDir; 5 | 6 | use crate::{crowdin, util}; 7 | use crate::github_repo_info::GithubModInfo; 8 | use crate::sentry::sentry_report_error; 9 | 10 | pub type LanguageCode = String; 11 | 12 | /// Represents local directory containing cloned github repository 13 | pub struct RepositoryDirectory { 14 | /// full name of github repository in format "owner/repo" 15 | pub full_name: String, 16 | pub root: TempDir, 17 | } 18 | 19 | impl RepositoryDirectory { 20 | pub fn new(full_name: &str, root: TempDir) -> Self { 21 | Self { 22 | full_name: full_name.to_owned(), 23 | root, 24 | } 25 | } 26 | } 27 | 28 | /// Represents local directory containing factorio mod 29 | pub struct ModDirectory { 30 | pub locale_path: PathBuf, 31 | pub mod_info: GithubModInfo, 32 | } 33 | 34 | impl ModDirectory { 35 | pub fn new(repository_directory: &RepositoryDirectory, mod_info: GithubModInfo) -> Self { 36 | let repository_root = repository_directory.root.path(); 37 | let locale_path = repository_root.join(&mod_info.locale_path).to_owned(); 38 | Self { locale_path, mod_info } 39 | } 40 | 41 | pub fn locale_path(&self) -> &Path { 42 | &self.locale_path 43 | } 44 | 45 | pub fn locale_en_path(&self) -> PathBuf { 46 | self.locale_path.join("en") 47 | } 48 | 49 | pub fn check_structure(&self) -> bool { 50 | if !self.check_for_locale_folder() { 51 | warn!("[add-repository] [{}] Missing `locale/en`", &self.mod_info); 52 | return false; 53 | } 54 | 55 | self.check_translation_files_match_english_files(true) 56 | } 57 | 58 | pub fn check_for_locale_folder(&self) -> bool { 59 | self.locale_en_path().exists() 60 | } 61 | 62 | pub fn check_translation_files_match_english_files(&self, report_sentry: bool) -> bool { 63 | let localizations = self.get_localizations(); 64 | for (language_code, localized_files) in localizations { 65 | for localized_file in localized_files { 66 | let file_name = util::file_name(&localized_file); 67 | let english_file = self.locale_en_path().join(file_name); 68 | if !english_file.exists() { 69 | let message = format!( 70 | "[add-repository] [{}] matched english file not found for '{}/{}'", 71 | self.mod_info, 72 | language_code, 73 | file_name 74 | ); 75 | error!("{}", &message); 76 | if report_sentry { 77 | sentry_report_error(&message); 78 | } 79 | return false; 80 | } 81 | } 82 | } 83 | true 84 | } 85 | 86 | pub fn get_english_files(&self) -> Vec { 87 | util::get_directory_cfg_files_paths(&self.locale_en_path()) 88 | } 89 | 90 | pub fn get_localizations(&self) -> Vec<(LanguageCode, Vec)> { 91 | self.get_language_directories() 92 | .into_iter() 93 | .filter(|(code, _path)| code != "en") 94 | .map(|(code, path)| { 95 | let files = util::get_directory_cfg_files_paths(&path); 96 | (code, files) 97 | }) 98 | .collect() 99 | } 100 | 101 | fn get_language_directories(&self) -> Vec<(LanguageCode, PathBuf)> { 102 | util::read_dir(self.locale_path()) 103 | .filter(|(path, _name)| path.is_dir()) 104 | .map(|(path, name)| (crowdin::normalize_language_code(&name), path)) 105 | .filter(|(code, _path)| crowdin::is_correct_language_code(code)) 106 | .collect() 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/myenv.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Deref; 2 | use std::sync::LazyLock; 3 | 4 | #[derive(Debug, Eq, PartialEq)] 5 | enum Environment { 6 | /// Env not checked 7 | Development, 8 | /// Some env are required 9 | CI, 10 | /// All env are required 11 | Production, 12 | } 13 | 14 | fn current_environment() -> Environment { 15 | if is_development() { 16 | return Environment::Development; 17 | } 18 | if dotenv::var("CI").is_ok() { 19 | return Environment::CI; 20 | } 21 | Environment::Production 22 | } 23 | 24 | pub fn is_development() -> bool { 25 | dotenv::var("IS_DEVELOPMENT").ok() == Some("true".to_owned()) 26 | } 27 | 28 | macro_rules! gen { 29 | ($($ci:literal $name:ident),* $(,)?) => { 30 | $( 31 | pub static $name: LazyLock = LazyLock::new(|| dotenv::var(stringify!($name)).unwrap()); 32 | )* 33 | pub fn init() { 34 | let environment = current_environment(); 35 | if environment == Environment::Development { return; } 36 | $( 37 | if $ci == 1 || environment == Environment::Production { 38 | let _ = $name.deref(); 39 | } 40 | )* 41 | } 42 | } 43 | } 44 | 45 | // 1 if env is needed both for CI and production 46 | // 0 if env is needed only for production 47 | gen!( 48 | 1 CROWDIN_PROJECT_ID, 49 | 1 CROWDIN_API_KEY, 50 | 1 GITHUB_APP_ID, 51 | 1 GITHUB_APP_PRIVATE_KEY, 52 | 0 GITHUB_APP_WEBHOOKS_SECRET, 53 | 1 GITHUB_PERSONAL_ACCESS_TOKEN, 54 | 0 GITHUB_OAUTH_CLIENT_ID, 55 | 0 GITHUB_OAUTH_CLIENT_SECRET, 56 | 0 SENTRY_DSN, 57 | 0 GIT_COMMIT_USER_NAME, 58 | 0 GIT_COMMIT_USER_EMAIL, 59 | 0 GIT_COMMIT_MESSAGE, 60 | 0 WEBSERVER_SECRET, 61 | ); 62 | -------------------------------------------------------------------------------- /src/sentry.rs: -------------------------------------------------------------------------------- 1 | use log::error; 2 | use sentry::Level; 3 | use sentry_log::LogFilter; 4 | 5 | use crate::myenv::is_development; 6 | 7 | // https://docs.sentry.io/platforms/rust/ 8 | pub fn init_sentry() -> Option { 9 | if is_development() { return None; } 10 | let sentry = sentry::init(sentry::ClientOptions { 11 | release: sentry::release_name!(), 12 | ..Default::default() 13 | }); 14 | Some(sentry) 15 | } 16 | 17 | // Analogue of `pretty_env_logger::init();` but with sentry middleware 18 | // Note that it overrides rocket logging 19 | pub fn init_logging() { 20 | let mut builder = pretty_env_logger::formatted_builder(); 21 | builder.parse_default_env(); 22 | let logger = builder.build(); 23 | let max_level = logger.filter(); 24 | 25 | let sentry_logger = sentry_log::SentryLogger::with_dest(logger) 26 | .filter(|data| match data.level() { 27 | // Ignore Level::Error because rocket prints error when route handler panics, 28 | // so we end up with duplicated events on sentry (one for panic, one for rocket error log). 29 | log::Level::Warn | log::Level::Info => LogFilter::Breadcrumb, 30 | _ => LogFilter::Ignore, 31 | }); 32 | log::set_boxed_logger(Box::new(sentry_logger)).unwrap(); 33 | log::set_max_level(max_level); 34 | } 35 | 36 | pub fn sentry_report_error(message: &str) { 37 | error!("{}", message); 38 | sentry::capture_message(message, Level::Error); 39 | } 40 | -------------------------------------------------------------------------------- /src/server/debug_routes.rs: -------------------------------------------------------------------------------- 1 | use rocket::get; 2 | use serde::Serialize; 3 | 4 | use crate::{github, webhooks}; 5 | use crate::server::check_secret; 6 | use crate::server::trigger_update::{get_installation_id_and_repo_info, get_trigger_update_mutex}; 7 | 8 | #[get("/listRepos?")] 9 | pub async fn list_repositories(secret: Option) -> String { 10 | if !check_secret(secret) { return "Missing secret".to_owned(); } 11 | let api = github::as_app(); 12 | let repositories = github::get_all_repositories(&api).await 13 | .into_iter() 14 | .map(|(repo_info, _)| repo_info.full_name) 15 | .collect::>(); 16 | serde_json::to_string_pretty(&repositories).unwrap() 17 | } 18 | 19 | #[get("/listReposForUser?&")] 20 | pub async fn list_repositories_for_user(user: String, secret: Option) -> String { 21 | if !check_secret(secret) { return "Missing secret".to_owned(); } 22 | let Some(api) = github::as_installation_for_user(&user).await else { 23 | return format!("App is not installed for {}", user); 24 | }; 25 | let repositories = github::get_all_repositories_of_installation(&api).await; 26 | serde_json::to_string_pretty(&repositories).unwrap() 27 | } 28 | 29 | #[get("/listUsers?")] 30 | pub async fn list_users(secret: Option) -> String { 31 | #[derive(Serialize)] 32 | struct User { 33 | login: String, 34 | repository_selection: Option, 35 | } 36 | 37 | if !check_secret(secret) { return "Missing secret".to_owned(); } 38 | let api = github::as_app(); 39 | let repositories = github::get_all_installations(&api).await 40 | .into_iter() 41 | .map(|installation| User { 42 | login: installation.account.login, 43 | repository_selection: installation.repository_selection, 44 | }) 45 | .collect::>(); 46 | serde_json::to_string_pretty(&repositories).unwrap() 47 | } 48 | 49 | /// For cases when repository was not imported correctly for some reason and manual intervention is needed 50 | #[get("/importRepository?&&")] 51 | pub async fn import_repository( 52 | repo: String, 53 | subpath: Option, 54 | secret: Option, 55 | ) -> &'static str { 56 | if !check_secret(secret) { return "Missing secret"; } 57 | let _lock = get_trigger_update_mutex().await; 58 | let (installation_id, repo_info) = match get_installation_id_and_repo_info(&repo, subpath).await { 59 | Ok(value) => value, 60 | Err(value) => return value, 61 | }; 62 | webhooks::on_repository_added(repo_info, installation_id).await; 63 | "Ok." 64 | } 65 | 66 | /// Overwrites all english file on crowdin based on github 67 | #[get("/importEnglish?&&")] 68 | pub async fn import_english( 69 | repo: String, 70 | subpath: Option, 71 | secret: Option, 72 | ) -> &'static str { 73 | if !check_secret(secret) { return "Missing secret"; } 74 | let _lock = get_trigger_update_mutex().await; 75 | let (installation_id, repo_info) = match get_installation_id_and_repo_info(&repo, subpath).await { 76 | Ok(value) => value, 77 | Err(value) => return value, 78 | }; 79 | webhooks::import_english(repo_info, installation_id).await; 80 | "Ok." 81 | } 82 | 83 | #[get("/triggerOOM?")] 84 | pub async fn trigger_oom(secret: Option) -> &'static str { 85 | if !check_secret(secret) { return "Missing secret"; } 86 | let _lock = get_trigger_update_mutex().await; 87 | 88 | eprintln!("\nTrying to trigger OOM..."); 89 | let mut v = Vec::new(); 90 | loop { 91 | v.push(v.len()); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/server/example_error_routes.rs: -------------------------------------------------------------------------------- 1 | /// For testing Sentry 2 | 3 | use std::time::Duration; 4 | 5 | use log::info; 6 | use rocket::get; 7 | 8 | #[get("/error1")] 9 | pub fn error1() -> String { 10 | info!("info message"); 11 | panic!("Example error 1") 12 | } 13 | 14 | #[get("/error2")] 15 | pub async fn error2() -> String { 16 | tokio::time::sleep(Duration::from_secs(1)).await; 17 | panic!("Example error 2"); 18 | } 19 | -------------------------------------------------------------------------------- /src/server/mod.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Deref; 2 | 3 | use log::info; 4 | use rocket::{get, post, routes}; 5 | use rocket::response::content::RawHtml; 6 | 7 | use crate::myenv::WEBSERVER_SECRET; 8 | use crate::server::webhook_util::GithubEvent; 9 | use crate::webhooks; 10 | 11 | mod debug_routes; 12 | mod example_error_routes; 13 | mod trigger_update; 14 | mod trigger_update_public; 15 | pub mod webhook_util; 16 | 17 | #[get("/")] 18 | fn index() -> RawHtml<&'static str> { 19 | RawHtml("

Factorio mods localization

See GitHub repository for documentation

") 20 | } 21 | 22 | #[post("/webhook", format = "json", data = "")] 23 | fn webhook(event: GithubEvent) { 24 | let task = webhooks::webhook_impl(event.0); 25 | // execute task in another thread, because it may be long 26 | tokio::spawn(task); 27 | } 28 | 29 | #[get("/version")] 30 | fn version() -> &'static str { 31 | env!("CARGO_PKG_VERSION") 32 | } 33 | 34 | fn check_secret(secret: Option) -> bool { 35 | secret.as_ref() == Some(WEBSERVER_SECRET.deref()) 36 | } 37 | 38 | pub async fn main() { 39 | info!("launching Rocket..."); 40 | let routes = routes![ 41 | index, 42 | webhook, 43 | trigger_update::trigger_update, 44 | trigger_update_public::trigger_update, 45 | trigger_update_public::trigger_update2, 46 | version, 47 | debug_routes::import_repository, 48 | debug_routes::import_english, 49 | debug_routes::list_repositories, 50 | debug_routes::list_repositories_for_user, 51 | debug_routes::list_users, 52 | debug_routes::trigger_oom, 53 | example_error_routes::error1, 54 | example_error_routes::error2, 55 | ]; 56 | rocket::build() 57 | .mount("/", routes) 58 | .launch().await.unwrap(); 59 | } 60 | -------------------------------------------------------------------------------- /src/server/trigger_update.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashSet; 2 | use std::fs; 3 | use std::ops::Deref; 4 | use std::path::Path; 5 | use std::sync::LazyLock; 6 | use std::time::Duration; 7 | 8 | use log::info; 9 | use octocrab::models::InstallationId; 10 | use rocket::get; 11 | use tempfile::TempDir; 12 | use tokio::time::sleep; 13 | 14 | use crate::{crowdin, git_util, github, util}; 15 | use crate::crowdin::{get_crowdin_directory_name, normalize_language_code, replace_ini_to_cfg}; 16 | use crate::github::{as_personal_account, get_repo_info}; 17 | use crate::github_repo_info::{GithubModInfo, GithubRepoInfo}; 18 | use crate::mod_directory::ModDirectory; 19 | use crate::server::check_secret; 20 | 21 | const TRIGGER_UPDATE_IGNORED_REPOSITORIES: &[&str] = &[ 22 | // TODO: 23 | // Large repository, causes OOM for some reason 24 | // https://github.com/dima74/factorio-mods-localization/issues/25 25 | "robot256/cargo_ships", 26 | "jingleheimer-schmidt/status_bars", 27 | ]; 28 | 29 | #[get("/triggerUpdate?&&")] 30 | pub async fn trigger_update( 31 | repo: Option, 32 | subpath: Option, 33 | secret: Option, 34 | ) -> &'static str { 35 | if !check_secret(secret) { return "Missing secret"; } 36 | match repo { 37 | Some(repo) => { 38 | trigger_update_single_repository_part1(repo, subpath).await 39 | .unwrap_or_else(|it| it) 40 | } 41 | None => { 42 | let task = trigger_update_all_repositories(); 43 | tokio::spawn(task); 44 | // TODO link to logs 45 | "Triggered. See logs for details." 46 | } 47 | } 48 | } 49 | 50 | pub async fn get_trigger_update_mutex() -> impl Drop { 51 | // Note that tokio Mutex doesn't poisoning in contrast to stdlib Mutex. 52 | // This means that it will work correctly if thread panicked. 53 | use tokio::sync::Mutex; 54 | static MUTEX: LazyLock> = LazyLock::new(|| Mutex::new(())); 55 | MUTEX.lock().await 56 | } 57 | 58 | pub async fn trigger_update_single_repository_part1( 59 | full_name: String, 60 | subpath: Option, 61 | ) -> Result<&'static str, &'static str> { 62 | info!("\n[update-github-from-crowdin] [{}] starting...", full_name); 63 | let (installation_id, repo_info) = match get_installation_id_and_repo_info(&full_name, subpath).await { 64 | Ok(value) => value, 65 | Err(message) => return Err(message), 66 | }; 67 | let repositories = vec![(repo_info, installation_id)]; 68 | let task = trigger_update_single_repository_part2(repositories, full_name); 69 | tokio::spawn(task); 70 | Ok("Triggered. See logs for details.") 71 | } 72 | 73 | async fn trigger_update_single_repository_part2( 74 | repositories: Vec<(GithubRepoInfo, InstallationId)>, 75 | full_name: String, 76 | ) { 77 | let _lock = get_trigger_update_mutex().await; 78 | push_crowdin_changes_to_repositories(repositories).await; 79 | info!("[update-github-from-crowdin] [{}] success", full_name); 80 | } 81 | 82 | pub async fn get_installation_id_and_repo_info( 83 | full_name: &str, 84 | subpath: Option, 85 | ) -> Result<(InstallationId, GithubRepoInfo), &'static str> { 86 | let installation_id = match github::get_installation_id_for_repo(full_name).await { 87 | Some(id) => id, 88 | None => return Err("Can't find installation for repository"), 89 | }; 90 | 91 | let api = github::as_installation(installation_id); 92 | let mut repo_info = match get_repo_info(&api, full_name).await { 93 | Err(_) => return Err("No mods."), 94 | Ok(repo_info) => repo_info, 95 | }; 96 | if let Some(subpath) = subpath { 97 | if !repo_info.keep_single_mod_with_crowdin_name(&subpath) { 98 | return Err("Can't find mod with provided subpath"); 99 | } 100 | } 101 | 102 | Ok((installation_id, repo_info)) 103 | } 104 | 105 | async fn trigger_update_all_repositories() { 106 | let _lock = get_trigger_update_mutex().await; 107 | info!("\n[update-github-from-crowdin] [*] starting..."); 108 | let api = github::as_app(); 109 | let repositories = github::get_all_repositories(&api).await; 110 | let repositories = filter_repositories_for_update_all(repositories); 111 | push_crowdin_changes_to_repositories(repositories).await; 112 | info!("[update-github-from-crowdin] [*] success"); 113 | } 114 | 115 | fn filter_repositories_for_update_all( 116 | mut repositories: Vec<(GithubRepoInfo, InstallationId)> 117 | ) -> Vec<(GithubRepoInfo, InstallationId)> { 118 | repositories 119 | .retain(|(repo_info, _)| { 120 | if TRIGGER_UPDATE_IGNORED_REPOSITORIES.contains(&repo_info.full_name.deref()) { 121 | info!( 122 | "[update-github-from-crowdin] [{}] skipping update (ignored)", 123 | repo_info.full_name 124 | ); 125 | return false; 126 | }; 127 | 128 | let weekly_update_from_crowdin = repo_info.weekly_update_from_crowdin; 129 | if !weekly_update_from_crowdin { 130 | info!( 131 | "[update-github-from-crowdin] [{}] skipping update because weekly_update_from_crowdin=false", 132 | repo_info.full_name 133 | ); 134 | } 135 | weekly_update_from_crowdin 136 | }); 137 | repositories 138 | } 139 | 140 | async fn push_crowdin_changes_to_repositories(repositories: Vec<(GithubRepoInfo, InstallationId)>) { 141 | let repositories = crowdin::filter_repositories(repositories).await; 142 | if repositories.is_empty() { return; } 143 | let translations_directory = crowdin::download_all_translations().await; 144 | for (repo_info, installation_id) in repositories { 145 | push_crowdin_changes_to_repository(repo_info, installation_id, &translations_directory).await; 146 | // TODO: https://github.com/dima74/factorio-mods-localization/issues/25 147 | sleep(Duration::from_secs(30)).await; 148 | } 149 | } 150 | 151 | async fn push_crowdin_changes_to_repository( 152 | repo_info: GithubRepoInfo, 153 | installation_id: InstallationId, 154 | translations_directory: &TempDir, 155 | ) { 156 | let full_name = &repo_info.full_name; 157 | let repository_directory = github::clone_repository(&repo_info, installation_id).await; 158 | for mod_ in repo_info.mods { 159 | let mod_directory = ModDirectory::new(&repository_directory, mod_); 160 | move_translated_files_to_mod_directory(&mod_directory, translations_directory.path()).await; 161 | } 162 | let path = repository_directory.root.path(); 163 | let are_changes_exists = git_util::add_all_and_check_has_changes(path); 164 | if are_changes_exists { 165 | info!("[update-github-from-crowdin] [{}] found changes", full_name); 166 | git_util::commit(path); 167 | let installation_api = github::as_installation(installation_id); 168 | let default_branch = github::get_default_branch(&installation_api, full_name).await; 169 | let base_branch = repo_info.branch.unwrap_or(default_branch); 170 | let is_protected = github::is_branch_protected(&installation_api, full_name, &base_branch).await; 171 | if is_protected { 172 | push_changes_using_pull_request(path, full_name, &base_branch).await; 173 | } else { 174 | git_util::push(path); 175 | info!("[update-github-from-crowdin] [{}] pushed", full_name); 176 | } 177 | } else { 178 | info!("[update-github-from-crowdin] [{}] no changes found", full_name); 179 | } 180 | } 181 | 182 | async fn push_changes_using_pull_request(path: &Path, full_name: &str, base_branch: &str) { 183 | let personal_api = as_personal_account(); 184 | if !github::fork_repository(&personal_api, full_name).await { 185 | return; 186 | } 187 | let (_owner, repo) = full_name.split_once('/').unwrap(); 188 | let pushed = git_util::push_to_my_fork(path, repo); 189 | if pushed { 190 | sleep(Duration::from_secs(30)).await; 191 | github::create_pull_request(&personal_api, full_name, base_branch).await; 192 | info!("[update-github-from-crowdin] [{}] pushed to crowdin-fml branch and created PR", full_name); 193 | } else { 194 | info!("[update-github-from-crowdin] [{}] existing crowdin-fml branch has same content", full_name); 195 | } 196 | } 197 | 198 | async fn move_translated_files_to_mod_directory(mod_directory: &ModDirectory, translation_directory: &Path) { 199 | delete_unmatched_localization_files(mod_directory); 200 | for (language_path, language) in util::read_dir(translation_directory) { 201 | if !language_is_enabled_for_mod_on_crowdin(&mod_directory.mod_info, &language) { continue; } 202 | 203 | let language_path_crowdin = language_path.join(get_crowdin_directory_name(&mod_directory.mod_info)); 204 | assert!(language_path_crowdin.exists()); 205 | let files = util::read_dir(&language_path_crowdin).collect::>(); 206 | if files.is_empty() { continue; } 207 | 208 | let language_original = util::read_dir(mod_directory.locale_path()) 209 | .map(|(_path, name)| name) 210 | .find(|it| normalize_language_code(it) == language) 211 | .unwrap_or(language); 212 | let language_path_repository = mod_directory.locale_path().join(language_original); 213 | fs::create_dir(&language_path_repository).ok(); 214 | for (old_path, name) in files { 215 | assert!(name.ends_with(".ini"), "file {} from crowdin must ends with .ini`", name); 216 | let file_renamed = replace_ini_to_cfg(&name); 217 | let new_path = language_path_repository.join(&file_renamed); 218 | fs::rename(old_path, new_path).unwrap(); 219 | } 220 | } 221 | } 222 | 223 | /// Consider: 224 | /// locale/en: ["locale1.cfg"] 225 | /// locale/ru: ["locale1.cfg", "locale2.cfg"] 226 | /// 227 | /// Result: 228 | /// locale/en: ["locale1.cfg"] 229 | /// locale/ru: ["locale1.cfg"] 230 | fn delete_unmatched_localization_files(mod_directory: &ModDirectory) { 231 | let english_files = mod_directory.get_english_files() 232 | .into_iter() 233 | .map(|it| util::file_name(&it).to_owned()) 234 | .collect::>(); 235 | for (_, localized_files) in mod_directory.get_localizations() { 236 | for localized_file in localized_files { 237 | let name = util::file_name(&localized_file); 238 | if !english_files.contains(name) { 239 | fs::remove_file(&localized_file).unwrap(); 240 | } 241 | } 242 | } 243 | } 244 | 245 | fn language_is_enabled_for_mod_on_crowdin(mod_info: &GithubModInfo, language: &str) -> bool { 246 | // This is a hack for this specific use case. See #19 247 | if mod_info.owner == "PennyJim" && mod_info.repo == "pirate-locale" { 248 | return language == "fr"; 249 | } 250 | 251 | true 252 | } 253 | -------------------------------------------------------------------------------- /src/server/trigger_update_public.rs: -------------------------------------------------------------------------------- 1 | //! Comparison with [super::trigger_update]: 2 | //! `/triggerUpdate?secret=X&repo=REPO` - for private use, secret based authorization 3 | //! `/api/triggerUpdate?repo=REPO` - for public use, GitHub OAuth based authorization 4 | 5 | use std::ops::Deref; 6 | 7 | use rocket::get; 8 | use rocket::response::Redirect; 9 | use url::Url; 10 | 11 | use crate::github; 12 | use crate::myenv::GITHUB_OAUTH_CLIENT_ID; 13 | use crate::server::trigger_update::trigger_update_single_repository_part1; 14 | 15 | #[get("/api/triggerUpdate?")] 16 | pub async fn trigger_update(repo: Option) -> Result { 17 | let Some(repo) = repo else { 18 | return Err("Missing `repo` query parameter"); 19 | }; 20 | if repo.chars().filter(|it| *it == '/').count() != 1 21 | || repo.starts_with('/') || repo.ends_with('/') { 22 | return Err("`repo` parameter should be in format `owner/repo`"); 23 | } 24 | let owner = repo.split_once('/').unwrap().0; 25 | 26 | let mut redirect_url = Url::parse("https://factorio-mods-localization.fly.dev/api/triggerUpdate2").unwrap(); 27 | redirect_url.query_pairs_mut() 28 | .append_pair("repo", &repo); 29 | 30 | let mut url = Url::parse("https://github.com/login/oauth/authorize").unwrap(); 31 | url.query_pairs_mut() 32 | .append_pair("client_id", GITHUB_OAUTH_CLIENT_ID.deref()) 33 | .append_pair("redirect_uri", redirect_url.as_ref()) 34 | .append_pair("login", owner) 35 | .append_pair("allow_signup", "false"); 36 | Ok(Redirect::to(url.to_string())) 37 | } 38 | 39 | #[get("/api/triggerUpdate2?&")] 40 | pub async fn trigger_update2(repo: String, code: String) -> String { 41 | let owner = repo.split_once('/').unwrap().0; 42 | 43 | let api = github_oauth::authenticate(&code).await; 44 | let authenticated_user = github::get_current_user(&api).await; 45 | 46 | if owner != authenticated_user { 47 | return format!("Authentication failed, expected `{}` user, found `{}`", owner, authenticated_user); 48 | } 49 | 50 | match trigger_update_single_repository_part1(repo, None).await { 51 | Ok(_) => "Triggered. Repository will be updated if there are changes on Crowdin.".to_owned(), 52 | Err(e) => e.to_owned(), 53 | } 54 | } 55 | 56 | // this should be in octocrab library 57 | mod github_oauth { 58 | use std::collections::HashMap; 59 | use std::ops::Deref; 60 | 61 | use http::header::ACCEPT; 62 | use octocrab::auth::OAuth; 63 | use octocrab::Octocrab; 64 | use rocket::serde::Deserialize; 65 | use secrecy::SecretString; 66 | 67 | use crate::myenv::{GITHUB_OAUTH_CLIENT_ID, GITHUB_OAUTH_CLIENT_SECRET}; 68 | 69 | pub async fn authenticate(code: &str) -> Octocrab { 70 | let oauth = get_access_token(code).await; 71 | Octocrab::builder().oauth(oauth).build().unwrap() 72 | } 73 | 74 | async fn get_access_token(code: &str) -> OAuth { 75 | let mut params = HashMap::<&str, &str>::new(); 76 | params.insert("client_id", GITHUB_OAUTH_CLIENT_ID.deref()); 77 | params.insert("client_secret", GITHUB_OAUTH_CLIENT_SECRET.deref()); 78 | params.insert("code", code); 79 | let api = Octocrab::builder() 80 | .add_header(ACCEPT, "application/json".to_string()) 81 | .build() 82 | .unwrap(); 83 | let response: OAuthWire = api 84 | .post("https://github.com/login/oauth/access_token", Some(¶ms)) 85 | .await 86 | .unwrap(); 87 | response.into() 88 | } 89 | 90 | #[derive(Deserialize)] 91 | struct OAuthWire { 92 | access_token: String, 93 | token_type: String, 94 | scope: String, 95 | } 96 | 97 | impl From for OAuth { 98 | fn from(value: OAuthWire) -> Self { 99 | OAuth { 100 | access_token: SecretString::from(value.access_token), 101 | token_type: value.token_type, 102 | scope: value.scope.split(',').map(ToString::to_string).collect(), 103 | } 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/server/webhook_util.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | use std::ops::Deref; 3 | 4 | use hmac::{Hmac, Mac}; 5 | use octocrab::models::webhook_events::WebhookEvent; 6 | use rocket::{Data, Request}; 7 | use rocket::data::{FromData, Outcome, ToByteUnit}; 8 | use rocket::http::Status; 9 | use sha2::Sha256; 10 | 11 | use crate::myenv::GITHUB_APP_WEBHOOKS_SECRET; 12 | 13 | pub struct GithubEvent(pub WebhookEvent); 14 | 15 | #[rocket::async_trait] 16 | impl<'r> FromData<'r> for GithubEvent { 17 | type Error = String; 18 | 19 | async fn from_data(request: &'r Request<'_>, data: Data<'r>) -> Outcome<'r, Self> { 20 | match GithubEvent::from_data_impl(request, data).await { 21 | Ok(result) => Outcome::Success(result), 22 | Err(err) => { 23 | let message = format!("{}", err); 24 | Outcome::Error((Status::BadRequest, message)) 25 | } 26 | } 27 | } 28 | } 29 | 30 | impl GithubEvent { 31 | // https://docs.github.com/en/webhooks/webhook-events-and-payloads#delivery-headers 32 | async fn from_data_impl(request: &Request<'_>, data: Data<'_>) -> Result> { 33 | // Parse the event type 34 | let event_type = request.headers() 35 | .get_one("X-Github-Event") 36 | .ok_or("Missing X-Github-Event header")?; 37 | 38 | // Parse the signature 39 | let signature = request 40 | .headers() 41 | .get_one("X-Hub-Signature-256") 42 | .and_then(parse_signature) 43 | .ok_or("Invalid signature")?; 44 | 45 | // Read the data into a String 46 | let limit = request.limits().get("json").unwrap_or(1.mebibytes()); 47 | let mut content = Vec::new(); 48 | data.open(limit).stream_to(&mut content).await?; 49 | 50 | // Validate signature 51 | verify_signature(&signature, &content)?; 52 | 53 | let event = WebhookEvent::try_from_header_and_body(event_type, &content)?; 54 | Ok(GithubEvent(event)) 55 | } 56 | } 57 | 58 | fn verify_signature(signature: &[u8], content: &[u8]) -> Result<(), impl Error> { 59 | let secret = GITHUB_APP_WEBHOOKS_SECRET.deref(); 60 | let mut mac = Hmac::::new_from_slice(secret.as_bytes()) 61 | .expect("HMAC can take key of any size"); 62 | mac.update(content); 63 | mac.verify_slice(signature) 64 | } 65 | 66 | fn parse_signature(header: &str) -> Option> { 67 | let header = header.trim(); 68 | let digest = header.strip_prefix("sha256=")?; 69 | hex::decode(digest).ok() 70 | } 71 | -------------------------------------------------------------------------------- /src/util/case.rs: -------------------------------------------------------------------------------- 1 | pub fn to_title_case(s: &str) -> String { 2 | let mut prev_char = ' '; 3 | let mut is_first_char = true; 4 | let mut result = String::new(); 5 | for char in s.chars() { 6 | if char.is_alphanumeric() { 7 | if prev_char.is_lowercase() && char.is_uppercase() { 8 | prev_char = ' '; 9 | } 10 | if prev_char.is_alphanumeric() { 11 | result.push(char); 12 | } else { 13 | if !is_first_char { result.push(' '); } 14 | result.push(char.to_ascii_uppercase()); 15 | } 16 | is_first_char = false; 17 | } 18 | prev_char = char; 19 | } 20 | result 21 | } 22 | 23 | #[cfg(test)] 24 | mod tests { 25 | use super::to_title_case; 26 | 27 | fn test(source: &str, expected: &str) { 28 | assert_eq!(expected, to_title_case(source)); 29 | } 30 | 31 | #[test] 32 | fn simple() { 33 | test("fooBar", "Foo Bar"); 34 | test("FooBar", "Foo Bar"); 35 | test("foo-bar", "Foo Bar"); 36 | test("Foo bar", "Foo Bar"); 37 | test("Foo Bar", "Foo Bar"); 38 | test("Foo-Bar", "Foo Bar"); 39 | test("FOO-BAR", "FOO BAR"); 40 | test("foo_bar", "Foo Bar"); 41 | } 42 | 43 | #[test] 44 | fn complex() { 45 | test("foo bar", "Foo Bar"); 46 | test("-_.foo-_.bar-_.", "Foo Bar"); 47 | test("factorio-mod-example", "Factorio Mod Example"); 48 | test("LTN-Language-Pack", "LTN Language Pack"); 49 | test("Noxys StackSizeMultiplier", "Noxys Stack Size Multiplier"); 50 | } 51 | 52 | #[test] 53 | fn all() { 54 | test("arachnophobia", "Arachnophobia"); 55 | test("Atomic_Overhaul", "Atomic Overhaul"); 56 | test("AutoDeconstruct", "Auto Deconstruct"); 57 | test("automatic-discharge-defense", "Automatic Discharge Defense"); 58 | test("biters_drop_money", "Biters Drop Money"); 59 | test("biter-trails", "Biter Trails"); 60 | test("brush-tools", "Brush Tools"); 61 | test("commands", "Commands"); 62 | test("cutscene-creator", "Cutscene Creator"); 63 | test("dim_lamps", "Dim Lamps"); 64 | test("diplomacy", "Diplomacy"); 65 | test("eco-friendly-electric-machines", "Eco Friendly Electric Machines"); 66 | test("enemy_race_manager", "Enemy Race Manager"); 67 | test("erm_marspeople", "Erm Marspeople"); 68 | test("erm_redarmy", "Erm Redarmy"); 69 | test("erm_terran", "Erm Terran"); 70 | test("erm_toss", "Erm Toss"); 71 | test("erm_zerg", "Erm Zerg"); 72 | test("ExtendedAngels", "Extended Angels"); 73 | test("extended-factorio", "Extended Factorio"); 74 | test("Factorio.AdvancedAirPurification", "Factorio Advanced Air Purification"); 75 | test("factorio-autobuild", "Factorio Autobuild"); 76 | test("factorio-AutoPauseForAFKplayers", "Factorio Auto Pause For AFKplayers"); 77 | test("factorio-beltlayer", "Factorio Beltlayer"); 78 | test("factorio-cooked-fish", "Factorio Cooked Fish"); 79 | test("factorio-ender-pearl", "Factorio Ender Pearl"); 80 | test("factorio-free-market", "Factorio Free Market"); 81 | test("factorio.InserterCranes", "Factorio Inserter Cranes"); 82 | test("factorio-kill_nest_get_money", "Factorio Kill Nest Get Money"); 83 | test("Factorio.LongStorageTanks", "Factorio Long Storage Tanks"); 84 | test("Factorio.LongWarehouses", "Factorio Long Warehouses"); 85 | test("factorio-lua-compiler", "Factorio Lua Compiler"); 86 | test("FactorioMilestones", "Factorio Milestones"); 87 | test("factorio-minable_tiles", "Factorio Minable Tiles"); 88 | test("factorio-miniloader", "Factorio Miniloader"); 89 | test("factorio-mod-example", "Factorio Mod Example"); 90 | test("Factorio-Modules-T4", "Factorio Modules T4"); 91 | test("factorio-money-UI", "Factorio Money UI"); 92 | test("Factorio-Non-Colliding-Rails", "Factorio Non Colliding Rails"); 93 | test("factorio-ODAD", "Factorio ODAD"); 94 | test("factorio-passive_player_income", "Factorio Passive Player Income"); 95 | test("factorio-passive_team_income", "Factorio Passive Team Income"); 96 | test("factorio-pipelayer", "Factorio Pipelayer"); 97 | test("factorio-Players_info", "Factorio Players Info"); 98 | test("factorio-PrivateElectricity", "Factorio Private Electricity"); 99 | test("factorio-railloader", "Factorio Railloader"); 100 | test("factorio-restrict_building", "Factorio Restrict Building"); 101 | test("factorio-shifted-worlds", "Factorio Shifted Worlds"); 102 | test("factorio-show_my_damage", "Factorio Show My Damage"); 103 | test("factorio-skip-hours", "Factorio Skip Hours"); 104 | test("Factorio.SmallTank", "Factorio Small Tank"); 105 | test("Factorio-Sniper-Rifle", "Factorio Sniper Rifle"); 106 | test("Factorio.SpaceScienceDelivery", "Factorio Space Science Delivery"); 107 | test("Factorio.SpaceShuttle", "Factorio Space Shuttle"); 108 | test("Factorio-Start-With-Nanobots", "Factorio Start With Nanobots"); 109 | test("factorio-surface_floors", "Factorio Surface Floors"); 110 | test("factorio-switchable_mods", "Factorio Switchable Mods"); 111 | test("factorio-tank-pvp", "Factorio Tank Pvp"); 112 | test("factorio-techs_for_science", "Factorio Techs For Science"); 113 | test("factorio-todo-list", "Factorio Todo List"); 114 | test("factorio-trainsaver", "Factorio Trainsaver"); 115 | test("factorio-useful_book", "Factorio Useful Book"); 116 | test("factorio-WhereIsMyBody", "Factorio Where Is My Body"); 117 | test("Factorissimo2", "Factorissimo2"); 118 | test("FactorySearch", "Factory Search"); 119 | test("firework_rockets", "Firework Rockets"); 120 | test("fish-farm", "Fish Farm"); 121 | test("flow-control", "Flow Control"); 122 | test("FreightForwarding", "Freight Forwarding"); 123 | test("glowing_trees", "Glowing Trees"); 124 | test("hiladdars-robots", "Hiladdars Robots"); 125 | test("ick-automatic-train-repair", "Ick Automatic Train Repair"); 126 | test("ickputzdirwech-vanilla-tweaks", "Ickputzdirwech Vanilla Tweaks"); 127 | test("Industrial-Revolution-Language-Pack", "Industrial Revolution Language Pack"); 128 | test("inserter-visualizer", "Inserter Visualizer"); 129 | test("IntermodalContainers", "Intermodal Containers"); 130 | test("LTN-Language-Pack", "LTN Language Pack"); 131 | test("M-Dirigible", "M Dirigible"); 132 | test("misery", "Misery"); 133 | test("m-lawful-evil", "M Lawful Evil"); 134 | test("m-microcontroller", "M Microcontroller"); 135 | test("m-multiplayertrading", "M Multiplayertrading"); 136 | test("ModuleInserterSimplified", "Module Inserter Simplified"); 137 | test("More_Ammo", "More Ammo"); 138 | test("more-fish", "More Fish"); 139 | test("More_Repair_Packs", "More Repair Packs"); 140 | test("Noxys_Achievement_Helper", "Noxys Achievement Helper"); 141 | test("Noxys_Deep_Core_Mining_Tweak", "Noxys Deep Core Mining Tweak"); 142 | test("Noxys_Extra_Settings_Info", "Noxys Extra Settings Info"); 143 | test("Noxys_Fading", "Noxys Fading"); 144 | test("Noxys_Multidirectional_Trains", "Noxys Multidirectional Trains"); 145 | test("Noxys_Robot_Battery_Tweak", "Noxys Robot Battery Tweak"); 146 | test("Noxys_StackSizeMultiplier", "Noxys Stack Size Multiplier"); 147 | test("Noxys_Swimming", "Noxys Swimming"); 148 | test("Noxys_Trees", "Noxys Trees"); 149 | test("Noxys_Waterfill", "Noxys Waterfill"); 150 | test("OmniLocales", "Omni Locales"); 151 | test("OmniSea", "Omni Sea"); 152 | test("PowerOverload", "Power Overload"); 153 | test("prismatic-belts", "Prismatic Belts"); 154 | test("RemoteConfiguration", "Remote Configuration"); 155 | test("reskins-angels", "Reskins Angels"); 156 | test("reskins-bobs", "Reskins Bobs"); 157 | test("reskins-compatibility", "Reskins Compatibility"); 158 | test("reskins-library", "Reskins Library"); 159 | test("rpg_items", "Rpg Items"); 160 | test("SeaBlockCustomPack", "Sea Block Custom Pack"); 161 | test("secondary-chat", "Secondary Chat"); 162 | test("sentient_spiders", "Sentient Spiders"); 163 | test("Shortcuts-ick", "Shortcuts Ick"); 164 | test("show-health-and-shield", "Show Health And Shield"); 165 | test("slashgamble", "Slashgamble"); 166 | test("spell-pack", "Spell Pack"); 167 | test("SpidertronEngineer", "Spidertron Engineer"); 168 | test("SpidertronEnhancements", "Spidertron Enhancements"); 169 | test("SpidertronPatrols", "Spidertron Patrols"); 170 | test("SpidertronWeaponSwitcher", "Spidertron Weapon Switcher"); 171 | test("stationary_chat", "Stationary Chat"); 172 | test("status_bars", "Status Bars"); 173 | test("teams-zo", "Teams Zo"); 174 | test("train-trails", "Train Trails"); 175 | test("vanilla-loaders-hd", "Vanilla Loaders Hd"); 176 | test("vanilla-loaders-hd-krastorio", "Vanilla Loaders Hd Krastorio"); 177 | test("VehicleSnap", "Vehicle Snap"); 178 | test("Warptorio2-Language-Pack", "Warptorio2 Language Pack"); 179 | test("zk-lib", "Zk Lib"); 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /src/util/escape.rs: -------------------------------------------------------------------------------- 1 | use std::sync::LazyLock; 2 | 3 | use regex::{Captures, Regex}; 4 | 5 | /// .ini file looks like this: 6 | /// ```ini 7 | /// [section1] 8 | /// key1=value1 9 | /// key2=value2 10 | /// ``` 11 | /// We need to escape (wrap in quotes) values which contains quotes or semicolon, because: 12 | /// - semicolon is used for commenting, so by default everything after semicolon is ignored 13 | /// - quotes are used for escaping, so by default something strange happens 14 | pub fn escape_strings_in_ini_file(ini_content: &str) -> String { 15 | static REGEX: LazyLock = LazyLock::new(|| Regex::new(r"(?mR)^([^\[=\n]*)=([^\n]*)$").unwrap()); 16 | REGEX.replace_all(ini_content, |captures: &Captures| { 17 | let key = &captures[1]; 18 | let value = &captures[2]; 19 | let should_escape = value.contains('"') || value.contains(';'); 20 | let already_escaped = value.starts_with('"') && value.ends_with('"'); 21 | if should_escape && !already_escaped { 22 | format!("{}=\"{}\"", key, value) 23 | } else { 24 | captures[0].to_owned() 25 | } 26 | }).into_owned() 27 | } 28 | 29 | #[cfg(test)] 30 | mod tests { 31 | use super::escape_strings_in_ini_file; 32 | 33 | fn test(source: &str, expected: &str) { 34 | assert_eq!(expected, escape_strings_in_ini_file(source)); 35 | } 36 | 37 | #[test] 38 | fn test_no_escape() { 39 | test("[section]", "[section]"); 40 | test("key=value", "key=value"); 41 | test(r#"key="value""#, r#"key="value""#); 42 | test(r" 43 | [section] 44 | key1=value1 45 | key2=value2 46 | ", r" 47 | [section] 48 | key1=value1 49 | key2=value2 50 | "); 51 | } 52 | 53 | #[test] 54 | fn test_escape() { 55 | test(r#"key=foo"bar"#, r#"key="foo"bar""#); 56 | test(r#"key=foo;bar"#, r#"key="foo;bar""#); 57 | test(r#" 58 | key1=value;1 59 | key2=value;2 60 | "#, r#" 61 | key1="value;1" 62 | key2="value;2" 63 | "#); 64 | test(r#" 65 | [se"ct;ion] 66 | key1="value1 67 | key2=value2" 68 | key2=v;a"l;u"e;3 69 | "#, r#" 70 | [se"ct;ion] 71 | key1=""value1" 72 | key2="value2"" 73 | key2="v;a"l;u"e;3" 74 | "#); 75 | } 76 | 77 | #[test] 78 | fn test_escape_crlf() { 79 | test( 80 | "key1=foo\"bar\r\nkey2=value2", 81 | "key1=\"foo\"bar\"\r\nkey2=value2" 82 | ); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/util/mod.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | use std::fs::File; 3 | use std::path::{Path, PathBuf}; 4 | 5 | use tempfile::TempDir; 6 | 7 | pub mod case; 8 | pub mod escape; 9 | 10 | pub fn read_dir(path: &Path) -> impl Iterator { 11 | fs::read_dir(path).unwrap() 12 | .map(|entry| { 13 | let path = entry.unwrap().path(); 14 | let name = file_name(&path).to_owned(); 15 | (path, name) 16 | }) 17 | } 18 | 19 | pub fn file_name(path: &Path) -> &str { 20 | path.file_name().unwrap().to_str().unwrap() 21 | } 22 | 23 | pub fn get_directory_cfg_files_paths(path: &Path) -> Vec { 24 | read_dir(path) 25 | .filter(|(path, name)| path.is_file() && name.ends_with(".cfg")) 26 | .map(|(path, _name)| path) 27 | .collect() 28 | } 29 | 30 | pub async fn download_and_extract_zip_file(url: &str) -> TempDir { 31 | use zip::ZipArchive; 32 | 33 | let file = download_file(url).await; 34 | let mut zip = ZipArchive::new(file).unwrap(); 35 | let directory = create_temporary_directory(); 36 | zip.extract(&directory).unwrap(); 37 | directory 38 | } 39 | 40 | pub async fn download_file(url: &str) -> File { 41 | let file = create_temporary_file(); 42 | let mut file = tokio::fs::File::from_std(file); 43 | let response = reqwest::get(url) 44 | .await.unwrap() 45 | .bytes().await.unwrap(); 46 | tokio::io::copy(&mut &*response, &mut file).await.unwrap(); 47 | file.into_std().await 48 | } 49 | 50 | // See [create_temporary_directory] 51 | fn create_temporary_file() -> File { 52 | tempfile::tempfile_in("./").unwrap() 53 | } 54 | 55 | // Temporary files/directories should NOT be created inside /tmp, 56 | // because /tmp is mounted inside RAM (tmpfs) 57 | pub fn create_temporary_directory() -> TempDir { 58 | TempDir::with_prefix_in("FML.", "./").unwrap() 59 | } 60 | 61 | pub fn remove_empty_ini_files(root: &Path) { 62 | // `ru/Factorio Mod Example (dima74)/test.ini` 63 | for (language_path, _) in read_dir(root) { 64 | for (repository_path, _) in read_dir(&language_path) { 65 | for (file_path, _) in read_dir(&repository_path) { 66 | let content = fs::read_to_string(&file_path).unwrap(); 67 | if is_empty_ini_file(content) { 68 | fs::remove_file(file_path).unwrap(); 69 | } 70 | } 71 | } 72 | } 73 | } 74 | 75 | fn is_empty_ini_file(content: String) -> bool { 76 | content.lines().all(|line| { 77 | let line = line.trim(); 78 | line.is_empty() || line.starts_with('[') || line.starts_with(';') 79 | }) 80 | } 81 | 82 | #[derive(Debug)] 83 | pub struct EmptyBody; 84 | 85 | #[async_trait::async_trait] 86 | impl ::octocrab::FromResponse for EmptyBody { 87 | async fn from_response(_: ::http::Response<::hyper::Body>) -> ::octocrab::Result { 88 | Ok(Self) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/webhooks.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Deref; 2 | 3 | use InstallationWebhookEventAction::{Created, Deleted}; 4 | use log::info; 5 | use octocrab::models::InstallationId; 6 | use octocrab::models::webhook_events::{EventInstallation, InstallationEventRepository, WebhookEvent, WebhookEventPayload}; 7 | use octocrab::models::webhook_events::payload::{InstallationWebhookEventAction, PushWebhookEventPayload}; 8 | use WebhookEventPayload::{Installation, InstallationRepositories, Push}; 9 | 10 | use crate::crowdin::CrowdinDirectory; 11 | use crate::github; 12 | use crate::github::GITHUB_CONFIG_FILE_NAME; 13 | use crate::github_repo_info::GithubRepoInfo; 14 | use crate::mod_directory::ModDirectory; 15 | 16 | pub async fn webhook_impl(event: WebhookEvent) { 17 | match event.specific { 18 | Installation(_) | InstallationRepositories(_) => { 19 | handle_installation_event(event).await; 20 | } 21 | Push(payload) => { 22 | let EventInstallation::Minimal(installation) = event.installation.as_ref().unwrap() else { 23 | panic!("Unexpected installation data"); 24 | }; 25 | let full_name = event.repository.unwrap().full_name.unwrap(); 26 | on_push_event(&payload, installation.id, full_name).await; 27 | } 28 | // TODO InstallationTarget event (user changed login) 29 | _ => info!("[webhook] unknown event: {:?}", event.kind), 30 | }; 31 | } 32 | 33 | async fn handle_installation_event(event: WebhookEvent) { 34 | let user = event.sender.unwrap().login; 35 | let EventInstallation::Full(installation) = event.installation.as_ref().unwrap() else { 36 | panic!("Unexpected installation data"); 37 | }; 38 | let repositories = match event.specific { 39 | Installation(payload) => { 40 | match payload.action { 41 | Created => { 42 | payload.repositories.unwrap() 43 | } 44 | Deleted => { 45 | info!("[email] app uninstalled for user {}", user); 46 | return; 47 | } 48 | _ => { 49 | info!("[installation-webhook] [{}] unknown action: {:?}", user, payload.action); 50 | return; 51 | } 52 | } 53 | } 54 | InstallationRepositories(payload) => { 55 | for repository_removed in payload.repositories_removed { 56 | info!("[email] app uninstalled for repository {}", repository_removed.full_name); 57 | } 58 | payload.repositories_added 59 | } 60 | _ => panic!("Unexpected event type"), 61 | }; 62 | 63 | info!("\n[installation-webhook] [{}] starting for {} repositories...", user, repositories.len()); 64 | let installation_id = installation.id; 65 | on_repositories_added(repositories, installation_id).await; 66 | info!("[installation-webhook] [{}] success", user); 67 | } 68 | 69 | async fn on_repositories_added(repositories: Vec, installation_id: InstallationId) { 70 | let repositories = repositories 71 | .into_iter() 72 | .filter(|it| !it.private) 73 | .map(|it| it.full_name) 74 | .collect::>(); 75 | let api = github::as_installation(installation_id); 76 | for repository in repositories { 77 | let Ok(repo_info) = github::get_repo_info(&api, &repository).await else { 78 | continue; 79 | }; 80 | on_repository_added(repo_info, installation_id).await; 81 | star_and_fork_repository(&repository).await; 82 | } 83 | } 84 | 85 | pub async fn on_repository_added(repo_info: GithubRepoInfo, installation_id: InstallationId) { 86 | info!("[email] app installed for repository {}", repo_info.full_name); 87 | let repository_directory = github::clone_repository(&repo_info, installation_id).await; 88 | for mod_ in repo_info.mods { 89 | let mod_directory = ModDirectory::new(&repository_directory, mod_); 90 | if !mod_directory.check_structure() { continue; } 91 | 92 | let (crowdin_directory, _) = CrowdinDirectory::get_or_create(mod_directory).await; 93 | crowdin_directory.add_english_and_localization_files().await; 94 | } 95 | info!("[add-repository] [{}] success", repo_info.full_name); 96 | } 97 | 98 | pub async fn import_english(repo_info: GithubRepoInfo, installation_id: InstallationId) { 99 | let repository_directory = github::clone_repository(&repo_info, installation_id).await; 100 | for mod_ in repo_info.mods { 101 | let mod_directory = ModDirectory::new(&repository_directory, mod_); 102 | if !mod_directory.check_for_locale_folder() { continue; } 103 | 104 | if !CrowdinDirectory::has_existing(&mod_directory).await { continue; } 105 | let (crowdin_directory, _) = CrowdinDirectory::get_or_create(mod_directory).await; 106 | crowdin_directory.add_english_files().await; 107 | } 108 | } 109 | 110 | pub async fn on_push_event( 111 | event: &PushWebhookEventPayload, 112 | installation_id: InstallationId, 113 | full_name: String, 114 | ) { 115 | info!("\n[push-webhook] [{}] starting...", full_name); 116 | 117 | if !has_interesting_changes(event) { 118 | info!("[push-webhook] [{}] no modified/added english files found", full_name); 119 | return; 120 | }; 121 | 122 | let api = github::as_installation(installation_id); 123 | let Ok(repo_info) = github::get_repo_info(&api, &full_name).await else { 124 | info!("[push-webhook] [{}] no mods found", full_name); 125 | return; 126 | }; 127 | 128 | let repository_directory = github::clone_repository(&repo_info, installation_id).await; 129 | let mut created = false; 130 | for mod_ in repo_info.mods { 131 | let mod_directory = ModDirectory::new(&repository_directory, mod_); 132 | if !mod_directory.check_for_locale_folder() { continue; } 133 | created |= handle_push_event_for_mod(mod_directory).await; 134 | } 135 | info!("[push-webhook] [{}] success", full_name); 136 | 137 | if created { 138 | star_and_fork_repository(&full_name).await; 139 | } 140 | } 141 | 142 | async fn handle_push_event_for_mod(mod_directory: ModDirectory) -> bool { 143 | let exists = CrowdinDirectory::has_existing(&mod_directory).await; 144 | if !exists && !mod_directory.check_translation_files_match_english_files(true) { 145 | return false; 146 | } 147 | 148 | let (crowdin_directory, created) = CrowdinDirectory::get_or_create(mod_directory).await; 149 | if created { 150 | info!("[push-webhook] [{}] created directory on crowdin - performing full import", crowdin_directory.mod_directory.mod_info); 151 | crowdin_directory.add_english_and_localization_files().await; 152 | } else { 153 | crowdin_directory.add_english_files().await; 154 | } 155 | created 156 | } 157 | 158 | fn has_interesting_changes(event: &PushWebhookEventPayload) -> bool { 159 | let mut changed_files = get_all_changed_files(event); 160 | changed_files.any(|file| { 161 | file == GITHUB_CONFIG_FILE_NAME || file.contains("locale/en/") 162 | }) 163 | } 164 | 165 | fn get_all_changed_files(event: &PushWebhookEventPayload) -> impl Iterator { 166 | event.commits.iter() 167 | .flat_map(|commit| { 168 | let added = commit.added.iter(); 169 | let modified = commit.modified.iter(); 170 | let removed = commit.removed.iter(); 171 | added.chain(modified).chain(removed).map(Deref::deref) 172 | }) 173 | } 174 | 175 | // This is needed for correct counting of contributions, 176 | // so they will be displayed at https://github.com/factorio-mods-helper. 177 | // Previously it was enough to star repository, but it was changed somewhere in 2023-2024. 178 | // https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-github-profile/managing-contribution-settings-on-your-profile/why-are-my-contributions-not-showing-up-on-my-profile 179 | // 180 | // Note that it will not work (commits will not be shown), if mod repository is a fork. 181 | async fn star_and_fork_repository(repository: &str) { 182 | let api_personal = github::as_personal_account(); 183 | github::star_repository(&api_personal, repository).await; 184 | github::fork_repository(&api_personal, repository).await; 185 | } 186 | -------------------------------------------------------------------------------- /tests/all_github_repositories_are_forked.rs: -------------------------------------------------------------------------------- 1 | //! Without forking contributions will not be displayed for 2 | //! https://github.com/factorio-mods-helper 3 | 4 | use fml::github; 5 | 6 | #[tokio::test] 7 | async fn main() { 8 | fml::init(); 9 | let result = github::get_not_forked_repositories().await; 10 | 11 | let forked_with_diferrent_name = result.forked_with_diferrent_name; 12 | if !forked_with_diferrent_name.is_empty() { 13 | for full_name in &forked_with_diferrent_name { 14 | println!("{}", full_name); 15 | } 16 | panic!("{} repositories have forks with different name", forked_with_diferrent_name.len()); 17 | } 18 | 19 | let not_forked = result.not_forked; 20 | if !not_forked.is_empty() { 21 | for full_name in ¬_forked { 22 | println!("{}", full_name); 23 | } 24 | panic!("{} repositories not forked", not_forked.len()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /tests/all_github_repositories_are_starred.rs: -------------------------------------------------------------------------------- 1 | //! Starring previously was needed for displaying contributions. 2 | //! However it is not working anymore (forking is needed instead). 3 | //! So we keep star check just so. 4 | 5 | use fml::github; 6 | 7 | #[tokio::test] 8 | async fn main() { 9 | fml::init(); 10 | let not_starred = github::get_not_starred_repositories().await; 11 | if !not_starred.is_empty() { 12 | for full_name in ¬_starred { 13 | println!("{}", full_name); 14 | } 15 | panic!("{} repositories not starred", not_starred.len()); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/crowdin_github_matches_english_files.rs: -------------------------------------------------------------------------------- 1 | //! Checks that all GitHub english files are present on Crowdin, 2 | //! And that there are no extra english files on Crowdin. 3 | 4 | use std::collections::{HashMap, HashSet}; 5 | 6 | use octocrab::Octocrab; 7 | 8 | use fml::{crowdin, github}; 9 | use fml::crowdin::get_crowdin_directory_name; 10 | use fml::github_repo_info::GithubModInfo; 11 | 12 | #[tokio::test] 13 | async fn main() { 14 | fml::init_with_crowdin().await; 15 | let github_data = get_github_data().await; 16 | let crowdin_data = get_crowdin_data().await; 17 | 18 | let mut matches = true; 19 | for (crowdin_name, ref crowdin_files) in crowdin_data { 20 | let Some(github_files) = github_data.get(&crowdin_name) else { continue; }; 21 | 22 | for file in github_files { 23 | if !crowdin_files.contains(file) { 24 | println!("[{}] Missing on crowdin: '{}'", crowdin_name, file) 25 | } 26 | } 27 | for files in crowdin_files { 28 | if !github_files.contains(files) { 29 | println!("[{}] Extra on crowdin: '{}'", crowdin_name, files) 30 | } 31 | } 32 | matches &= crowdin_files == github_files; 33 | } 34 | assert!(matches, "Crowdin and GitHub names doesn't match"); 35 | } 36 | 37 | async fn get_crowdin_data() -> HashMap> { 38 | let mut result = HashMap::new(); 39 | let directories = crowdin::list_directories().await; 40 | for (crowdin_name, directory_id) in directories { 41 | let files = crowdin::list_files(directory_id).await; 42 | let files = files 43 | .map(|(name, _)| crowdin::replace_ini_to_cfg(&name)) 44 | .collect(); 45 | result.insert(crowdin_name, files); 46 | } 47 | result 48 | } 49 | 50 | async fn get_github_data() -> HashMap> { 51 | let api = github::as_app(); 52 | let repositories = github::get_all_repositories(&api).await; 53 | let mut result = HashMap::new(); 54 | for (repo_info, installation_id) in repositories { 55 | for mod_ in repo_info.mods { 56 | let installation_api = api.installation(installation_id); 57 | let files = list_locale_en_files_for_mod(&repo_info.full_name, &mod_, &installation_api).await; 58 | let files = match files { 59 | Some(value) => value, 60 | None => continue, 61 | }; 62 | let crowdin_name = get_crowdin_directory_name(&mod_); 63 | result.insert(crowdin_name, files); 64 | } 65 | } 66 | result 67 | } 68 | 69 | async fn list_locale_en_files_for_mod( 70 | full_name: &str, 71 | mod_info: &GithubModInfo, 72 | installation_api: &Octocrab, 73 | ) -> Option> { 74 | let path = format!("{}/en", mod_info.locale_path); 75 | let files = github::list_files_in_directory(installation_api, full_name, &path).await.ok()?; 76 | let files = files 77 | .into_iter() 78 | .filter(|name| name.ends_with(".cfg")) 79 | .collect(); 80 | Some(files) 81 | } 82 | -------------------------------------------------------------------------------- /tests/crowdin_github_matches_english_files_content.rs: -------------------------------------------------------------------------------- 1 | //! Creates two directories: 2 | //! * `temp/compare/english-crowdin` 3 | //! * `temp/compare/english-github` 4 | //! 5 | //! After you should manually execute `diff -r` on them. 6 | 7 | use std::{fs, io}; 8 | use std::fs::File; 9 | use std::io::{Seek, SeekFrom}; 10 | use std::path::Path; 11 | 12 | use fml::{crowdin, util}; 13 | use fml::crowdin::{get_crowdin_directory_name, replace_cfg_to_ini}; 14 | use fml::github_repo_info::GithubRepoInfo; 15 | use fml::util::escape::escape_strings_in_ini_file; 16 | 17 | #[ignore] // Ignore on CI 18 | #[tokio::test] 19 | async fn main() { 20 | fml::init_with_crowdin().await; 21 | download_crowdin_files().await; 22 | download_github_files().await; 23 | } 24 | 25 | async fn download_crowdin_files() { 26 | let root = Path::new("temp/compare/english-crowdin"); 27 | if root.exists() { 28 | panic!("Delete existing {:?}", root); 29 | } 30 | fs::create_dir_all(root).unwrap(); 31 | 32 | let directories = crowdin::list_directories().await; 33 | for (directory_name, directory_id) in directories { 34 | let directory_path = root.join(&directory_name); 35 | fs::create_dir(&directory_path).unwrap(); 36 | for (file_name, file_id) in crowdin::list_files(directory_id).await { 37 | let mut file = crowdin::download_file(file_id).await; 38 | file.seek(SeekFrom::Start(0)).unwrap(); 39 | let target = directory_path.join(file_name); 40 | let mut target = File::create(target).unwrap(); 41 | io::copy(&mut file, &mut target).unwrap(); 42 | } 43 | } 44 | } 45 | 46 | async fn download_github_files() { 47 | let target_root = Path::new("temp/compare/english-github"); 48 | if target_root.exists() { 49 | panic!("Delete existing {:?}", target_root); 50 | } 51 | fs::create_dir_all(target_root).unwrap(); 52 | 53 | let source_root = Path::new("temp/repositories"); 54 | if !source_root.exists() { 55 | panic!("Run `examples/github_download_all_repositories.rs`") 56 | } 57 | 58 | let json = fs::read_to_string(Path::new("temp/repositories.json")).unwrap(); 59 | let repositories: Vec = serde_json::from_str(&json).unwrap(); 60 | for repo_info in repositories { 61 | let (_owner, repo) = repo_info.full_name.split_once('/').unwrap(); 62 | let repository_directory = source_root.join(repo); 63 | for mod_ in repo_info.mods { 64 | let target_directory_name = get_crowdin_directory_name(&mod_); 65 | let target_directory = target_root.join(target_directory_name); 66 | fs::create_dir(&target_directory).unwrap(); 67 | 68 | let source_directory = repository_directory.join("locale/en"); 69 | for source_path in util::get_directory_cfg_files_paths(&source_directory) { 70 | let source_file_name = util::file_name(&source_path); 71 | let target_file_name = replace_cfg_to_ini(source_file_name); 72 | let target_path = target_directory.join(target_file_name); 73 | 74 | let content = fs::read_to_string(source_path).unwrap(); 75 | let content = escape_strings_in_ini_file(&content); 76 | fs::write(target_path, content).unwrap(); 77 | } 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /tests/crowdin_github_matches_mods.rs: -------------------------------------------------------------------------------- 1 | //! Checks that all GitHub repositories are present on Crowdin, 2 | //! And that there are no extra directories on Crowdin. 3 | 4 | use std::collections::HashSet; 5 | 6 | use fml::{crowdin, github}; 7 | use fml::crowdin::get_crowdin_directory_name; 8 | 9 | const IGNORED_GITHUB: &[&str] = &[]; 10 | const IGNORED_CROWDIN: &[&str] = &[ 11 | // Used for testing 12 | "Factorio Mod Example (dima74)", 13 | // github repository deleted or hidden, but mod page still has link to crowdin, so keep for now 14 | "Factorio Ntech Chemistry (NathaU)", 15 | ]; 16 | 17 | #[tokio::test] 18 | async fn main() { 19 | fml::init_with_crowdin().await; 20 | 21 | let crowdin_names = crowdin::list_directories().await 22 | .map(|(name, _id)| name) 23 | .filter(|name| !IGNORED_CROWDIN.contains(&name.as_str())) 24 | .collect::>(); 25 | 26 | let api = github::as_app(); 27 | let github_names = github::get_all_repositories(&api).await 28 | .into_iter() 29 | .flat_map(|(repo_info, _id)| repo_info.mods) 30 | .map(|it| get_crowdin_directory_name(&it)) 31 | .filter(|name| !IGNORED_GITHUB.contains(&name.as_str())) 32 | .collect::>(); 33 | 34 | for name in &github_names { 35 | if !crowdin_names.contains(name) { 36 | println!("Missing on crowdin: '{}'", name) 37 | } 38 | } 39 | for name in &crowdin_names { 40 | if !github_names.contains(name) { 41 | println!("Extra on crowdin: '{}'", name) 42 | } 43 | } 44 | if crowdin_names != github_names { 45 | panic!("Crowdin and GitHub names doesn't match"); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tests/crowdin_no_translations_has_newline.rs: -------------------------------------------------------------------------------- 1 | //! Newlines in translations will break .cfg file format 2 | 3 | use std::fs; 4 | 5 | use fml::crowdin; 6 | use fml::util::read_dir; 7 | 8 | const IGNORED: &[(&str, &str, &str, &str)] = &[ 9 | // Empty translation for unknown reason, Crowdin support doesn't help 10 | ("Factorio Ultracube (grandseiken)", "de", "tips-and-tricks.ini", "cube-boiler"), 11 | ]; 12 | 13 | #[tokio::test] 14 | async fn main() { 15 | fml::init_with_crowdin().await; 16 | 17 | let translations_directory = crowdin::download_all_translations().await; 18 | let mut has_newlines = false; 19 | // `ru/Factorio Mod Example (dima74)/locale.ini` 20 | for (language_path, language) in read_dir(translations_directory.path()) { 21 | for (repository_path, crowdin_name) in read_dir(&language_path) { 22 | for (file_path, file_name) in read_dir(&repository_path) { 23 | let content = fs::read_to_string(&file_path).unwrap(); 24 | for line in content.lines() { 25 | if let Some(key) = line.strip_suffix('=') { 26 | if IGNORED.contains(&(crowdin_name.as_str(), language.as_str(), file_name.as_str(), key)) { 27 | continue; 28 | } 29 | has_newlines = true; 30 | eprintln!("[{}] {}/{}: incorrect translation for key {}", crowdin_name, language, file_name, key) 31 | } 32 | } 33 | } 34 | } 35 | } 36 | assert!(!has_newlines); 37 | } 38 | -------------------------------------------------------------------------------- /tests/github_check_config_file.rs: -------------------------------------------------------------------------------- 1 | //! Checks `factorio-mods-localization.json` config file. 2 | //! Repository either should not have it, or have it correct. 3 | 4 | use fml::github; 5 | use fml::github::{get_all_installations, get_all_repositories_of_installation, get_repo_info, GetRepoInfoError}; 6 | 7 | #[tokio::test] 8 | async fn main() { 9 | fml::init(); 10 | let api = github::as_app(); 11 | 12 | let mut repos_with_invalid_config = Vec::new(); 13 | let installations = get_all_installations(&api).await; 14 | for installation in installations { 15 | let installation_api = api.installation(installation.id); 16 | let repositories = get_all_repositories_of_installation(&installation_api).await; 17 | for repository in repositories { 18 | let repo_info = get_repo_info(&installation_api, &repository).await; 19 | if let Err(GetRepoInfoError::InvalidConfig) = repo_info { 20 | repos_with_invalid_config.push(repository); 21 | } 22 | } 23 | } 24 | 25 | if !repos_with_invalid_config.is_empty() { 26 | eprintln!("\n\nFound {} repositories with invalid config:", repos_with_invalid_config.len()); 27 | for repo in repos_with_invalid_config { 28 | eprintln!("{repo}"); 29 | } 30 | eprintln!("\n"); 31 | 32 | panic!("There are repositories with invalid config."); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /tests/github_translated_files_matches_locale_en.rs: -------------------------------------------------------------------------------- 1 | //! Checks that on GitHub files in translation directory (e.g. "locale/ru") 2 | //! matches file in english directory ("locale/en"). 3 | //! Expected flow: 4 | //! * Mod author deletes english file on GitHub 5 | //! * Our helper deletes corresponding english file on Crowdin 6 | //! * Our helper deletes corresponding translated files on GitHub 7 | //! 8 | //! Last step happens with delays (one-week sync delay and delay for merging PR), 9 | //! therefor this test is ignored on CI. 10 | 11 | use fml::github; 12 | use fml::mod_directory::ModDirectory; 13 | 14 | #[ignore] 15 | #[tokio::test] 16 | async fn main() { 17 | // Disable info logging in `github::clone_repository` 18 | std::env::set_var("RUST_LOG", "fml=warn"); 19 | fml::init(); 20 | 21 | let api = github::as_app(); 22 | let repositories = github::get_all_repositories(&api).await; 23 | let mut all_matches = true; 24 | for (repo_info, installation_id) in repositories { 25 | let repository_directory = github::clone_repository(&repo_info, installation_id).await; 26 | for mod_ in repo_info.mods { 27 | let mod_directory = ModDirectory::new(&repository_directory, mod_); 28 | if !mod_directory.check_for_locale_folder() { continue; } 29 | if !mod_directory.check_translation_files_match_english_files(false) { 30 | all_matches = false; 31 | continue; 32 | } 33 | } 34 | } 35 | assert!(all_matches); 36 | } 37 | -------------------------------------------------------------------------------- /todo.md: -------------------------------------------------------------------------------- 1 | * После установки github app - страничка на vue со статусом и логом 2 | --------------------------------------------------------------------------------