├── .eslintrc.js ├── .github ├── actions │ ├── pnpm-workspace │ │ └── action.yaml │ ├── rustup │ │ └── action.yaml │ └── setup-build-env │ │ └── action.yaml ├── docker │ ├── README.md │ └── linux.Dockerfile └── workflows │ ├── build-napi-platform-package.yaml │ ├── build-swc-intl-message-transformer.yaml │ ├── publish-canary.yaml │ └── release.yaml ├── .gitignore ├── .idea ├── .gitignore ├── codeStyles │ ├── Project.xml │ └── codeStyleConfig.xml ├── inspectionProfiles │ └── Project_Default.xml ├── intl.iml ├── jsLibraryMappings.xml ├── misc.xml ├── modules.xml ├── prettier.xml └── vcs.xml ├── .prettierrc ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── crates ├── intl_allocator │ ├── Cargo.toml │ └── src │ │ └── lib.rs ├── intl_database_core │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ └── src │ │ ├── database │ │ ├── message.rs │ │ ├── mod.rs │ │ ├── source.rs │ │ └── symbol.rs │ │ ├── error.rs │ │ ├── lib.rs │ │ └── message │ │ ├── meta.rs │ │ ├── mod.rs │ │ ├── source_file.rs │ │ ├── value.rs │ │ └── variables │ │ ├── mod.rs │ │ └── visitor.rs ├── intl_database_exporter │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ └── src │ │ ├── bundle.rs │ │ ├── export.rs │ │ └── lib.rs ├── intl_database_js_source │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ └── src │ │ ├── extractor.rs │ │ └── lib.rs ├── intl_database_json_source │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ └── src │ │ └── lib.rs ├── intl_database_service │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ └── src │ │ └── lib.rs ├── intl_database_types_generator │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ └── src │ │ ├── comment.rs │ │ ├── lib.rs │ │ ├── type_def.rs │ │ └── writer.rs ├── intl_flat_json_parser │ ├── .cargo │ │ └── config.toml │ ├── Cargo.toml │ ├── README.md │ ├── bench │ │ └── index.js │ ├── build.rs │ ├── index.d.ts │ ├── index.js │ ├── npm │ │ ├── darwin-arm64 │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── darwin-x64 │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── linux-arm64-gnu │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── linux-x64-gnu │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── win32-arm64-msvc │ │ │ ├── README.md │ │ │ └── package.json │ │ └── win32-x64-msvc │ │ │ ├── README.md │ │ │ └── package.json │ ├── package.json │ └── src │ │ ├── lib.rs │ │ ├── napi.rs │ │ ├── parser.rs │ │ └── util.rs ├── intl_markdown │ ├── .cargo │ │ └── config.toml │ ├── .gitignore │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ ├── benches │ │ ├── icu_messageformat_parser_bench.rs │ │ ├── long_documents.rs │ │ └── spec.md │ ├── examples │ │ ├── gen-html-entities.rs │ │ ├── gen-spec-tests.rs │ │ ├── html-entities.json │ │ └── spec_tests.json │ ├── src │ │ ├── ast │ │ │ ├── format.rs │ │ │ ├── mod.rs │ │ │ ├── process.rs │ │ │ └── util.rs │ │ ├── block_parser.rs │ │ ├── byte_lookup.rs │ │ ├── delimiter.rs │ │ ├── event.rs │ │ ├── html_entities.rs │ │ ├── icu │ │ │ ├── compile.rs │ │ │ ├── format.rs │ │ │ ├── mod.rs │ │ │ ├── serialize.rs │ │ │ └── tags.rs │ │ ├── lexer.rs │ │ ├── lib.rs │ │ ├── parser │ │ │ ├── block.rs │ │ │ ├── code_span.rs │ │ │ ├── delimiter.rs │ │ │ ├── emphasis.rs │ │ │ ├── icu.rs │ │ │ ├── inline.rs │ │ │ ├── link.rs │ │ │ ├── mod.rs │ │ │ ├── strikethrough.rs │ │ │ └── text.rs │ │ ├── syntax.rs │ │ ├── token.rs │ │ └── tree_builder │ │ │ ├── cst.rs │ │ │ └── mod.rs │ └── tests │ │ ├── harness.rs │ │ ├── icu.rs │ │ ├── icu_ast.rs │ │ ├── md_extensions.rs │ │ ├── mod.rs │ │ └── test_ast.rs ├── intl_markdown_macros │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ └── src │ │ └── lib.rs ├── intl_markdown_visitor │ ├── Cargo.toml │ ├── README.md │ └── src │ │ ├── lib.rs │ │ ├── visit_with.rs │ │ └── visitor.rs ├── intl_message_database │ ├── .cargo │ │ └── config.toml │ ├── .gitignore │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ ├── bench │ │ ├── .gitignore │ │ ├── native.js │ │ └── util.js │ ├── build.rs │ ├── examples │ │ └── public-test.rs │ ├── index.d.ts │ ├── index.js │ ├── npm │ │ ├── darwin-arm64 │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── darwin-x64 │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── linux-arm64-gnu │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── linux-arm64-musl │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── linux-x64-gnu │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── linux-x64-musl │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── win32-arm64-msvc │ │ │ ├── README.md │ │ │ └── package.json │ │ ├── win32-ia32-msvc │ │ │ ├── README.md │ │ │ └── package.json │ │ └── win32-x64-msvc │ │ │ ├── README.md │ │ │ └── package.json │ ├── package.json │ └── src │ │ ├── lib.rs │ │ ├── napi │ │ ├── mod.rs │ │ └── types.rs │ │ ├── public.rs │ │ ├── sources │ │ └── mod.rs │ │ └── threading.rs ├── intl_message_utils │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ └── src │ │ └── lib.rs ├── intl_validator │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ └── src │ │ ├── content.rs │ │ ├── diagnostic.rs │ │ ├── lib.rs │ │ ├── severity.rs │ │ └── validators │ │ ├── mod.rs │ │ ├── no_repeated_plural_names.rs │ │ ├── no_repeated_plural_options.rs │ │ ├── no_trimmable_whitespace.rs │ │ ├── no_unicode_variable_names.rs │ │ └── validator.rs └── keyless_json │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ └── src │ ├── error.rs │ ├── lib.rs │ ├── serializer.rs │ └── string.rs ├── package.json ├── packages ├── babel-plugin-transform-discord-intl │ ├── LICENSE │ ├── README.md │ ├── index.js │ ├── package.json │ └── traverse.js ├── eslint-plugin-discord-intl │ ├── LICENSE │ ├── README.md │ ├── index.js │ ├── lib │ │ ├── is-typescript.js │ │ ├── native-validation.js │ │ └── traverse.js │ ├── package.json │ └── rules │ │ ├── native │ │ ├── no-duplicate-message-keys.js │ │ ├── no-repeated-plural-names.js │ │ ├── no-repeated-plural-options.js │ │ ├── no-trimmable-whitespace.js │ │ ├── no-trimmable-whitespace.test.js │ │ └── no-unicode-variable-names.js │ │ ├── no-duplicate-message-keys.test.js │ │ ├── no-opaque-messages-objects.js │ │ ├── no-opaque-messages-objects.test.js │ │ ├── use-static-access.js │ │ └── use-static-access.test.js ├── intl-ast │ ├── .gitignore │ ├── README.md │ ├── index.ts │ ├── package.json │ └── tsconfig.json ├── intl-loader-core │ ├── .gitignore │ ├── README.md │ ├── index.js │ ├── package.json │ ├── src │ │ ├── database.js │ │ ├── processing.js │ │ ├── transformer.js │ │ ├── util.js │ │ └── watcher.js │ ├── tsconfig.json │ └── types.d.ts ├── intl │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ ├── package.json │ ├── src │ │ ├── data-formatters │ │ │ ├── cache.ts │ │ │ ├── config.ts │ │ │ └── index.ts │ │ ├── format.ts │ │ ├── formatters │ │ │ ├── ast.ts │ │ │ ├── index.ts │ │ │ ├── markdown.ts │ │ │ ├── react.ts │ │ │ └── string.ts │ │ ├── hash.ts │ │ ├── index.ts │ │ ├── intl-manager.ts │ │ ├── message-loader.ts │ │ ├── message.ts │ │ ├── runtime-utils.ts │ │ └── types.d.ts │ └── tsconfig.json ├── jest-processor-discord-intl │ ├── LICENSE │ ├── README.md │ ├── index.js │ └── package.json ├── metro-intl-transformer │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ ├── asset-plugin.js │ ├── index.js │ └── package.json ├── rspack-intl-loader │ ├── LICENSE │ ├── README.md │ ├── package.json │ ├── rspack-loader.js │ └── types-plugin.js └── swc-intl-message-transformer │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ ├── package.json │ └── src │ ├── config.rs │ ├── lib.rs │ └── transformer.rs ├── pnpm-lock.yaml ├── pnpm-workspace.yaml ├── rust-toolchain.toml ├── shell.nix ├── tools ├── index.js ├── package.json ├── src │ ├── ci │ │ └── commands.js │ ├── constants.js │ ├── db │ │ └── commands.js │ ├── ecosystem │ │ └── commands.js │ ├── js-package.js │ ├── json │ │ └── commands.js │ ├── napi.js │ ├── npm.js │ ├── pnpm.js │ ├── util-commands.js │ ├── util │ │ ├── gh.js │ │ ├── git.js │ │ ├── platform.js │ │ └── rustup.js │ └── versioning.js └── tsconfig.json ├── tsconfig.base.json ├── tsconfig.client.json ├── tsconfig.json └── tsconfig.node.json /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | rules: { 3 | 'no-console': 'off', 4 | }, 5 | }; 6 | -------------------------------------------------------------------------------- /.github/actions/pnpm-workspace/action.yaml: -------------------------------------------------------------------------------- 1 | name: Node/pnpm workspace setup 2 | 3 | description: | 4 | Install NodeJS and pnpm on the system, then install dependencies for the whole workspace. Any pnpm 5 | command in the repository can be run after this action has completed. 6 | 7 | inputs: 8 | node-version: 9 | default: "20" 10 | required: false 11 | type: string 12 | frozen-lockfile: 13 | default: false 14 | required: false 15 | type: boolean 16 | save-if: 17 | default: false 18 | required: false 19 | type: boolean 20 | 21 | runs: 22 | using: composite 23 | steps: 24 | - name: Install Node.js 25 | uses: actions/setup-node@v4 26 | with: 27 | node-version: ${{ inputs.node-version }} 28 | registry-url: 'https://registry.npmjs.org/' 29 | 30 | - name: Install pnpm 31 | uses: pnpm/action-setup@v4 32 | with: 33 | version: 9 34 | standalone: ${{ startsWith(inputs.node-version, '16') }} 35 | 36 | 37 | - name: Get pnpm store directory 38 | id: pnpm-cache 39 | shell: bash 40 | run: | 41 | # set store-dir to $(pnpm config get store-dir)/$(pnpm -v) 42 | global_store_path=$(pnpm config get store-dir) 43 | if [ -z "${global_store_path}" ] || [ "${global_store_path}" = "undefined" ]; then 44 | global_store_path=~/.cache/pnpm 45 | fi 46 | pnpm config set store-dir $global_store_path/$(pnpm -v) --location project 47 | echo "STORE_PATH is $(pnpm store path)" 48 | echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT 49 | 50 | - name: Install dependencies 51 | shell: bash 52 | run: | 53 | if [[ "${{ inputs.frozen-lockfile}}" == 'true' ]]; then 54 | pnpm install --frozen-lockfile --prefer-offline 55 | else 56 | pnpm install --no-frozen-lockfile --prefer-offline 57 | fi -------------------------------------------------------------------------------- /.github/actions/rustup/action.yaml: -------------------------------------------------------------------------------- 1 | name: rustup 2 | 3 | description: Install Rust toolchains 4 | 5 | inputs: 6 | save-cache: 7 | default: false 8 | required: false 9 | type: boolean 10 | shared-key: 11 | default: 'check' 12 | required: false 13 | type: string 14 | 15 | runs: 16 | using: composite 17 | steps: 18 | - name: Print Inputs 19 | shell: bash 20 | run: | 21 | echo 'save-cache: ${{ inputs.save-cache }}' 22 | echo 'shared-key: ${{ inputs.shared-key }}' 23 | 24 | - name: Get toolchain from rust-toolchain.toml 25 | id: rust-toolchain-toml 26 | shell: bash 27 | run: | 28 | CHANNEL=$(grep "channel" rust-toolchain.toml | cut -f 3 -d' ' | sed s/\"//g) 29 | echo "channel=$CHANNEL" >> "$GITHUB_OUTPUT" 30 | 31 | - name: Install Rust toolchain 32 | uses: dtolnay/rust-toolchain@master 33 | with: 34 | toolchain: ${{steps.rust-toolchain-toml.outputs.channel}} 35 | 36 | - name: Cache on ${{ github.ref_name }} 37 | uses: Swatinem/rust-cache@v2 38 | if: ${{ startsWith(runner.name, 'GitHub Actions') }} 39 | with: 40 | shared-key: ${{ inputs.shared-key }} 41 | save-if: ${{ inputs.save-cache == 'true' }} -------------------------------------------------------------------------------- /.github/docker/README.md: -------------------------------------------------------------------------------- 1 | # Dockerfiles for building in CI 2 | 3 | These dockerfiles help speed up CI by pre-installing all of the necessary build environment tools, including: 4 | 5 | - node 6 | - pnpm 7 | - rust 8 | - zig 9 | - cargo-zigbuild 10 | - cargo-xwin 11 | 12 | It also includes a pre-downloaded cache of most dependencies (cargo and npm) to speed up installation times. 13 | -------------------------------------------------------------------------------- /.github/docker/linux.Dockerfile: -------------------------------------------------------------------------------- 1 | # Start from the cargo-zigbuild base image to automatically get Rust and Zig 2 | FROM messense/cargo-zigbuild 3 | # Add Node for package development. 4 | FROM node:20.6.1-bookworm-slim 5 | 6 | # Make the working directory the repo root to reference pnpm-lock.yaml and 7 | # Cargo.lock as expected. 8 | WORKDIR ../../ 9 | 10 | # Install pnpm. Keep this version in sync with the monorepo 11 | RUN corepack prepare pnpm@9.0.6 --activate 12 | RUN corepack enable 13 | 14 | # Do a pre-installation to get most dependencies cached in a local store that \ 15 | # gets copied into the contain 16 | RUN pnpm config set store-dir ./.cache/pnpm-store 17 | RUN pnpm install 18 | 19 | # Do the same for Cargo dependencies 20 | ENV CARGO_HOME = "./.cache/cargo-home" 21 | RUN cargo fetch 22 | 23 | COPY ./.cache ./.cache -------------------------------------------------------------------------------- /.github/workflows/build-napi-platform-package.yaml: -------------------------------------------------------------------------------- 1 | name: Build NAPI platform package 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | package: 7 | description: "Base name of the NPM package being built (no @discord, no platform name included)" 8 | required: true 9 | type: string 10 | crate: 11 | description: "Path name of the crate being built (e.g., some_crate instead of some-crate)" 12 | required: true 13 | type: string 14 | cli-name: 15 | description: "Name of the project in the intl-cli tool (e.g., rt, db, swc)" 16 | required: true 17 | type: string 18 | target: 19 | description: "Platform package name that maps to a Rust target triple." 20 | required: true 21 | type: string 22 | runner: 23 | description: "Which os/version to run this workflow on." 24 | required: true 25 | type: string 26 | 27 | env: 28 | # This should ensure that the mac packages are allowed to run on all 29 | # supported macOS versions. 30 | MACOSX_DEPLOYMENT_TARGET: "10.13" 31 | DEBUG: "napi:*" 32 | XWIN_CACHE_DIR: ${{ github.workspace }}/.xwin 33 | 34 | jobs: 35 | build: 36 | runs-on: ${{ inputs.runner }} 37 | defaults: 38 | run: 39 | shell: bash 40 | outputs: 41 | runner-labels: ${{ steps.upload-artifact.outputs.runner-labels || inputs.runner }} 42 | steps: 43 | - uses: actions/checkout@v4 44 | 45 | - name: Setup Build Env 46 | uses: ./.github/actions/setup-build-env 47 | with: 48 | runner: ${{ inputs.runner }} 49 | target: ${{ inputs.target }} 50 | 51 | - name: Build @discord/${{inputs.package}}-${{inputs.target}} 52 | run: | 53 | pnpm intl-cli ${{inputs.cli-name}} build --target ${{inputs.target}} 54 | 55 | - name: Upload artifact 56 | id: upload-artifact 57 | uses: actions/upload-artifact@v4 58 | with: 59 | name: ${{inputs.package}}.${{inputs.target}}.node 60 | path: crates/${{inputs.crate}}/npm/${{inputs.target}}/${{inputs.package}}.${{inputs.target}}.node 61 | if-no-files-found: error 62 | -------------------------------------------------------------------------------- /.github/workflows/build-swc-intl-message-transformer.yaml: -------------------------------------------------------------------------------- 1 | name: Build swc-intl-message-transformer 2 | 3 | on: 4 | workflow_call: 5 | 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | defaults: 10 | run: 11 | shell: bash 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - name: Setup Build Env 16 | uses: ./.github/actions/setup-build-env 17 | with: 18 | target: wasm 19 | 20 | - name: Build @discord/swc-intl-message-transformer 21 | run: | 22 | pnpm intl-cli swc build 23 | 24 | - name: Upload artifact 25 | id: upload-artifact 26 | uses: actions/upload-artifact@v4 27 | with: 28 | name: swc_intl_message_transformer.wasm 29 | path: packages/swc-intl-message-transformer/swc_intl_message_transformer.wasm 30 | if-no-files-found: error 31 | -------------------------------------------------------------------------------- /.github/workflows/publish-canary.yaml: -------------------------------------------------------------------------------- 1 | name: Publish canary packages 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | packages: 7 | required: true 8 | type: string 9 | description: "Space-separated list of packages to publish" 10 | strict: 11 | required: true 12 | type: boolean 13 | description: "Strict Mode (only bump explicitly given packages)" 14 | 15 | permissions: 16 | # To publish packages with provenance 17 | contents: write 18 | id-token: write 19 | 20 | jobs: 21 | release: 22 | name: Release and Publish 23 | runs-on: ubuntu-latest 24 | permissions: 25 | contents: write 26 | id-token: write 27 | steps: 28 | - uses: actions/checkout@v4 29 | 30 | - name: Setup Build Env 31 | uses: ./.github/actions/setup-build-env 32 | with: 33 | native: false 34 | target: wasm 35 | 36 | - name: Set canary versions 37 | run: | 38 | pnpm intl-cli ecosystem version bump canary 39 | 40 | - name: Publish all @discord/intl-message-database* packages 41 | run: | 42 | pnpm intl-cli ecosystem publish-only ${{inputs.packages}} \ 43 | --tag canary ${{inputs.strict && '--strict' || ''}} \ 44 | --yes --skip-existing --no-git-checks \ 45 | --provenance --access public 46 | env: 47 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | target/ 3 | 4 | *.tsbuildinfo 5 | 6 | *.DS_Store 7 | 8 | # Build artifacts 9 | *.wasm 10 | *.node 11 | .local-packs/ -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | -------------------------------------------------------------------------------- /.idea/codeStyles/Project.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 15 | 16 | 24 | 25 | 32 | 33 | 40 | 41 | 48 | 49 | -------------------------------------------------------------------------------- /.idea/codeStyles/codeStyleConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 10 | -------------------------------------------------------------------------------- /.idea/intl.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /.idea/jsLibraryMappings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/prettier.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": true, 3 | "printWidth": 100, 4 | "singleQuote": true, 5 | "bracketSameLine": true 6 | } -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "crates/intl_allocator", 4 | "crates/intl_database_core", 5 | "crates/intl_database_exporter", 6 | "crates/intl_database_js_source", 7 | "crates/intl_database_json_source", 8 | "crates/intl_database_service", 9 | "crates/intl_database_types_generator", 10 | "crates/intl_flat_json_parser", 11 | "crates/intl_markdown", 12 | "crates/intl_markdown_macros", 13 | "crates/intl_markdown_visitor", 14 | "crates/intl_message_database", 15 | "crates/intl_message_utils", 16 | "crates/intl_validator", 17 | "crates/keyless_json", 18 | "packages/swc-intl-message-transformer", 19 | ] 20 | resolver = "2" 21 | 22 | [workspace.dependencies] 23 | intl_allocator = { path = "./crates/intl_allocator" } 24 | intl_database_core = { path = "./crates/intl_database_core" } 25 | intl_database_exporter = { path = "./crates/intl_database_exporter" } 26 | intl_database_js_source = { path = "./crates/intl_database_js_source" } 27 | intl_database_json_source = { path = "./crates/intl_database_json_source" } 28 | intl_database_service = { path = "./crates/intl_database_service" } 29 | intl_database_types_generator = { path = "./crates/intl_database_types_generator" } 30 | intl_flat_json_parser = { path = "./crates/intl_flat_json_parser" } 31 | intl_markdown = { path = "./crates/intl_markdown" } 32 | intl_markdown_macros = { path = "./crates/intl_markdown_macros" } 33 | intl_markdown_visitor = { path = "./crates/intl_markdown_visitor" } 34 | intl_message_database = { path = "./crates/intl_message_database" } 35 | intl_message_utils = { path = "./crates/intl_message_utils" } 36 | intl_validator = { path = "./crates/intl_validator" } 37 | keyless_json = { path = "./crates/keyless_json" } 38 | 39 | anyhow = "1" 40 | ignore = "0.4.19" 41 | mimalloc = { version = "0.1", features = ["local_dynamic_tls"] } 42 | napi = { version = "3.0.0-alpha.8", features = ["error_anyhow", "serde-json"] } 43 | napi-derive = "3.0.0-alpha.7" 44 | rustc-hash = "2" 45 | serde = { version = "1", features = ["derive"] } 46 | serde_json = "1" 47 | swc_common = "5.0.0" 48 | swc_core = "10.6.1" 49 | thiserror = "1" 50 | threadpool = "1.8.1" 51 | unescape_zero_copy = "2.1.1" 52 | unicode-xid = "0.2.6" 53 | xxhash-rust = { version = "0.8.10", features = ["xxh64"] } 54 | ustr = { version = "1.0.0", features = ["serde"] } 55 | memchr = "2.7.4" 56 | once_cell = "1.19.0" 57 | 58 | # Build dependencies 59 | napi-build = "2" 60 | 61 | [profile.release-profiling] 62 | inherits = "release" 63 | codegen-units = 1 64 | lto = true 65 | # Optimize for performance 66 | opt-level = 3 67 | debug = true 68 | strip = false 69 | panic = "abort" 70 | 71 | 72 | [profile.release] 73 | codegen-units = 1 74 | lto = true 75 | # Optimize for performance 76 | opt-level = 3 77 | # Strip debug symbols 78 | strip = "symbols" 79 | panic = "abort" 80 | 81 | [workspace.metadata.cross.build] 82 | zig = "2.17" 83 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/intl_allocator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_allocator" 3 | description = "Global allocator configuration for Discord's intl* crates" 4 | version = "0.1.0" 5 | edition = "2021" 6 | 7 | [dependencies] 8 | mimalloc = { workspace = true } -------------------------------------------------------------------------------- /crates/intl_allocator/src/lib.rs: -------------------------------------------------------------------------------- 1 | #[global_allocator] 2 | #[cfg(not(miri))] 3 | static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; 4 | -------------------------------------------------------------------------------- /crates/intl_database_core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_database_core" 3 | description = "A core database for managing intl messages, supporting CRUD for messages with attribution to their source files." 4 | version = "0.1.0" 5 | edition = "2021" 6 | publish = false 7 | 8 | [dependencies] 9 | intl_markdown = { workspace = true } 10 | intl_markdown_visitor = { workspace = true } 11 | intl_message_utils = { workspace = true } 12 | path-absolutize = "3.1.1" 13 | rustc-hash = { workspace = true } 14 | serde = { workspace = true } 15 | thiserror = { workspace = true } 16 | ustr = { workspace = true } 17 | -------------------------------------------------------------------------------- /crates/intl_database_core/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/intl_database_core/README.md: -------------------------------------------------------------------------------- 1 | # intl_database_core 2 | 3 | The core logic and types for implementing a Messages database. This crate contains definitions for `Message`, `SourceFile`, `Source`, `KeySymbol`, `MessageDatabase`, and more, which are the fundamental building blocks for constructing and interacting with messages inside a database. 4 | 5 | This is a library crate that is only built as part of another crate. 6 | -------------------------------------------------------------------------------- /crates/intl_database_core/src/database/symbol.rs: -------------------------------------------------------------------------------- 1 | //! Small module for creating and working with Symbols (aka Atoms), which are 2 | //! internal handles to commonly-shared values like message keys, file names 3 | //! locale ids, or anything else that needs to be shared. 4 | use ustr::{existing_ustr, ustr, Ustr, UstrMap, UstrSet}; 5 | 6 | /// A symbol representing a message key, file name, or any other frequently- 7 | /// copied value. This is not the same as hashing a message key, which creates 8 | /// a new, shorter representation of a message key for identification, 9 | /// obfuscation, and minification. 10 | pub type KeySymbol = Ustr; 11 | 12 | pub type KeySymbolMap = UstrMap; 13 | pub type KeySymbolSet = UstrSet; 14 | 15 | /// Return the KeySymbol that represents the given value. If the requested 16 | /// value is not currently known in the store, or if the store is poisoned, 17 | /// a DatabaseError is returned instead. 18 | pub fn get_key_symbol(value: &str) -> Option { 19 | existing_ustr(value) 20 | } 21 | 22 | /// Intern a new value into the global symbol store. This is thread-safe, but 23 | /// will lock any reads from the store that are happening concurrently. 24 | /// 25 | /// If the store can't be acquired or the write otherwise fails, this returns 26 | /// a DatabaseError. 27 | pub fn key_symbol(value: &str) -> KeySymbol { 28 | ustr(value) 29 | } 30 | -------------------------------------------------------------------------------- /crates/intl_database_core/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub use database::message::Message; 2 | pub use database::source::{ 3 | MessageDefinitionSource, MessageSourceError, MessageSourceResult, MessageTranslationSource, 4 | RawMessage, RawMessageDefinition, RawMessageTranslation, 5 | }; 6 | pub use database::symbol::{get_key_symbol, key_symbol, KeySymbol, KeySymbolMap, KeySymbolSet}; 7 | pub use database::{DatabaseInsertStrategy, MessagesDatabase}; 8 | pub use error::{DatabaseError, DatabaseResult}; 9 | pub use message::meta::{MessageMeta, SourceFileMeta}; 10 | pub use message::source_file::{ 11 | DefinitionFile, FilePosition, SourceFile, SourceFileKind, TranslationFile, 12 | }; 13 | pub use message::value::MessageValue; 14 | pub use message::variables::{ 15 | collect_message_variables, MessageVariableInstance, MessageVariableType, MessageVariables, 16 | }; 17 | 18 | mod database; 19 | mod error; 20 | mod message; 21 | 22 | // TODO: Allow this to be configurable, or determined by source files themselves through `meta`. 23 | pub static DEFAULT_LOCALE: &str = "en-US"; 24 | -------------------------------------------------------------------------------- /crates/intl_database_core/src/message/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod meta; 2 | pub mod source_file; 3 | pub mod value; 4 | pub mod variables; 5 | -------------------------------------------------------------------------------- /crates/intl_database_core/src/message/value.rs: -------------------------------------------------------------------------------- 1 | use serde::Serialize; 2 | 3 | use intl_markdown::{parse_intl_message, Document}; 4 | use intl_message_utils::message_may_have_blocks; 5 | 6 | use super::source_file::FilePosition; 7 | use super::variables::{collect_message_variables, MessageVariables}; 8 | 9 | #[derive(Clone, Debug, Serialize)] 10 | #[serde(rename_all = "camelCase")] 11 | pub struct MessageValue { 12 | pub raw: String, 13 | pub parsed: Document, 14 | pub variables: Option, 15 | pub file_position: FilePosition, 16 | } 17 | 18 | impl MessageValue { 19 | /// Creates a new value including the original raw content as given and 20 | /// parsing the content to a compiled AST. 21 | pub fn from_raw(content: &str, file_position: FilePosition) -> Self { 22 | let document = parse_intl_message(&content, message_may_have_blocks(content)); 23 | 24 | let variables = match collect_message_variables(&document) { 25 | Ok(variables) => Some(variables), 26 | _ => None, 27 | }; 28 | 29 | Self { 30 | raw: content.into(), 31 | parsed: document, 32 | variables, 33 | file_position, 34 | } 35 | } 36 | } 37 | 38 | // Messages are equal if they have the same starting raw content. Everything 39 | // else about a message is derived from that original string. 40 | impl PartialEq for MessageValue { 41 | fn eq(&self, other: &Self) -> bool { 42 | self.raw == other.raw 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /crates/intl_database_exporter/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_database_exporter" 3 | description = "Serialization utilities for exporting message database contents to other systems and general formats" 4 | version = "0.1.0" 5 | edition = "2021" 6 | 7 | [dependencies] 8 | intl_database_core = { workspace = true } 9 | intl_database_service = { workspace = true } 10 | intl_markdown = { workspace = true } 11 | keyless_json = { workspace = true } 12 | rustc-hash = { workspace = true } 13 | anyhow = { workspace = true } 14 | serde_json = { workspace = true } 15 | thiserror = { workspace = true } 16 | -------------------------------------------------------------------------------- /crates/intl_database_exporter/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/intl_database_exporter/README.md: -------------------------------------------------------------------------------- 1 | # intl_database_exporter 2 | 3 | Database services for serializing message contents into various formats. 4 | 5 | `export` is a generic service for persisting the entire _translation_ contents of a database to their appropriate files on the host system, according to the meta information for each message. 6 | 7 | `bundle` is a specialized service for serializing a set of messages into a single file for a given locale, both for definitions and translations. These services are utilized by bundlers to compile source files into content that can be consistently included in bundled applications. 8 | 9 | This is a library crate that is only built as part of another crate. 10 | -------------------------------------------------------------------------------- /crates/intl_database_exporter/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![feature(iter_collect_into)] 2 | 3 | pub use bundle::{ 4 | CompiledMessageFormat, IntlMessageBundler, IntlMessageBundlerError, IntlMessageBundlerOptions, 5 | }; 6 | pub use export::ExportTranslations; 7 | 8 | mod bundle; 9 | mod export; 10 | -------------------------------------------------------------------------------- /crates/intl_database_js_source/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_database_js_source" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | intl_database_core = { workspace = true } 8 | intl_message_utils = { workspace = true } 9 | swc_common = { workspace = true } 10 | swc_core = { workspace = true, features = [ 11 | "ecma_parser", 12 | "ecma_ast", 13 | "ecma_visit", 14 | ] } 15 | unescape_zero_copy = { workspace = true } 16 | -------------------------------------------------------------------------------- /crates/intl_database_js_source/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/intl_database_js_source/README.md: -------------------------------------------------------------------------------- 1 | # intl_database_js_source 2 | 3 | A Source definition for parsing message contents from JavaScript files. 4 | 5 | This is a library crate that is only built as part of another crate. 6 | -------------------------------------------------------------------------------- /crates/intl_database_js_source/src/lib.rs: -------------------------------------------------------------------------------- 1 | use intl_database_core::{ 2 | key_symbol, KeySymbol, MessageDefinitionSource, MessageSourceError, MessageSourceResult, 3 | RawMessageDefinition, SourceFileKind, SourceFileMeta, DEFAULT_LOCALE, 4 | }; 5 | use swc_common::sync::Lrc; 6 | use swc_common::SourceMap; 7 | use swc_core::ecma::ast::Module; 8 | 9 | use crate::extractor::{extract_message_definitions, parse_message_definitions_file}; 10 | 11 | mod extractor; 12 | 13 | pub struct JsMessageSource; 14 | 15 | impl MessageDefinitionSource for JsMessageSource { 16 | fn get_default_locale(&self, _file_name: &str) -> KeySymbol { 17 | key_symbol(DEFAULT_LOCALE) 18 | } 19 | 20 | fn extract_definitions( 21 | self, 22 | file_name: KeySymbol, 23 | content: &str, 24 | ) -> MessageSourceResult<(SourceFileMeta, impl Iterator)> { 25 | let (source, module) = parse_definitions_with_error_handling(file_name, content)?; 26 | let extractor = extract_message_definitions(&file_name, source, module); 27 | Ok(( 28 | extractor.root_meta, 29 | extractor.message_definitions.into_iter(), 30 | )) 31 | } 32 | } 33 | 34 | fn parse_definitions_with_error_handling( 35 | file_name: KeySymbol, 36 | content: &str, 37 | ) -> MessageSourceResult<(Lrc, Module)> { 38 | parse_message_definitions_file(&file_name, content).map_err(|error| { 39 | let kind = error.into_kind(); 40 | let message = kind.msg(); 41 | MessageSourceError::ParseError(SourceFileKind::Definition, message.to_string()) 42 | }) 43 | } 44 | -------------------------------------------------------------------------------- /crates/intl_database_json_source/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_database_json_source" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | intl_database_core = { workspace = true } 8 | intl_flat_json_parser = { workspace = true } -------------------------------------------------------------------------------- /crates/intl_database_json_source/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/intl_database_json_source/README.md: -------------------------------------------------------------------------------- 1 | # intl_database_json_source 2 | 3 | A Source definition for parsing message contents from JSON files. 4 | 5 | This is a library crate that is only built as part of another crate. 6 | -------------------------------------------------------------------------------- /crates/intl_database_json_source/src/lib.rs: -------------------------------------------------------------------------------- 1 | use intl_database_core::{ 2 | key_symbol, FilePosition, KeySymbol, MessageSourceResult, MessageTranslationSource, 3 | RawMessageTranslation, 4 | }; 5 | use intl_flat_json_parser::parse_flat_translation_json; 6 | use std::path::Path; 7 | 8 | pub struct JsonMessageSource; 9 | 10 | impl MessageTranslationSource for JsonMessageSource { 11 | fn get_locale_from_file_name(&self, file_name: &str) -> KeySymbol { 12 | Path::new(file_name) 13 | .file_name() 14 | .and_then(|p| p.to_str()) 15 | .map(|p| p.split_once(".").map_or(p, |(name, _ext)| name)) 16 | .unwrap_or("en-US") 17 | .into() 18 | } 19 | 20 | fn extract_translations( 21 | self, 22 | file_name: KeySymbol, 23 | content: &str, 24 | ) -> MessageSourceResult> { 25 | let iter = parse_flat_translation_json(&content); 26 | Ok(iter.map(move |item| { 27 | RawMessageTranslation::new( 28 | key_symbol(&item.key), 29 | FilePosition::new(file_name, item.position.line, item.position.col), 30 | item.value, 31 | ) 32 | })) 33 | } 34 | } 35 | 36 | #[test] 37 | fn test_locale_from_file_name() { 38 | assert_eq!( 39 | key_symbol("en-US"), 40 | JsonMessageSource.get_locale_from_file_name("foo/bar/baz/en-US.messages.jsona") 41 | ); 42 | assert_eq!( 43 | key_symbol("fr-FR"), 44 | JsonMessageSource.get_locale_from_file_name("foo/bar/baz/fr-FR.messages.jsona") 45 | ); 46 | assert_eq!( 47 | key_symbol("notareal__locale#1"), 48 | JsonMessageSource.get_locale_from_file_name("notareal__locale#1.messages.jsona") 49 | ); 50 | 51 | assert_eq!( 52 | key_symbol("da"), 53 | JsonMessageSource.get_locale_from_file_name("da.messages.jsona") 54 | ); 55 | assert_eq!( 56 | key_symbol("cz"), 57 | JsonMessageSource.get_locale_from_file_name("foo/bar/cz") 58 | ); 59 | } 60 | -------------------------------------------------------------------------------- /crates/intl_database_service/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_database_service" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | -------------------------------------------------------------------------------- /crates/intl_database_service/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/intl_database_service/README.md: -------------------------------------------------------------------------------- 1 | # intl_database_service 2 | 3 | Generic Trait definition for a Service that operates on a `MessageDatabase`. 4 | 5 | This is a library crate that is only built as part of another crate. 6 | -------------------------------------------------------------------------------- /crates/intl_database_service/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub trait IntlDatabaseService { 2 | type Result; 3 | 4 | fn run(&mut self) -> Self::Result; 5 | } 6 | -------------------------------------------------------------------------------- /crates/intl_database_types_generator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_database_types_generator" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | intl_database_core = { workspace = true } 8 | intl_database_service = { workspace = true } 9 | intl_message_utils = { workspace = true } 10 | rustc-hash = { workspace = true } 11 | thiserror = { workspace = true } 12 | anyhow = { workspace = true } 13 | ustr = { workspace = true } 14 | sourcemap = "9.0.0" 15 | -------------------------------------------------------------------------------- /crates/intl_database_types_generator/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/intl_database_types_generator/README.md: -------------------------------------------------------------------------------- 1 | # intl_database_types_generator 2 | 3 | Database services for generating TypeScript definition files mapping from a single `SourceFile`. Types are created by analyzing both definitions _and_ translations to create comprehensive types that represent all possibilities for a message and ensure content isn't missed or supplied incorrectly. 4 | 5 | This is a library crate that is only built as part of another crate. 6 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [target.x86_64-pc-windows-msvc] 2 | rustflags = ["-C", "target-feature=+crt-static"] -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_flat_json_parser" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | crate-type = ["cdylib", "lib"] 8 | 9 | [features] 10 | default = [] 11 | # Enable when building the NAPI extension version of this crate. 12 | node_addon = [] 13 | 14 | [dependencies] 15 | memchr = { workspace = true } 16 | intl_allocator = { workspace = true } 17 | unescape_zero_copy = { workspace = true } 18 | 19 | napi = { workspace = true } 20 | napi-derive = { workspace = true } 21 | 22 | [build-dependencies] 23 | napi-build = { workspace = true } 24 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/README.md: -------------------------------------------------------------------------------- 1 | # intl_flat_json_parser 2 | 3 | A Rust-based JSON parser with location information for flat objects, used by `@discord/intl-message-database` to provide more accurate diagnostics for values parsed out of JSON files. 4 | 5 | The parser is specifically built to only understand a flat, well-formed object structure and has minimal error recovery, but fully supports Unicode and other JSON string syntax like escapes. 6 | 7 | ## Install 8 | 9 | ``` 10 | pnpm add @discord/intl-flat-json-parser 11 | ``` 12 | 13 | ## Usage 14 | 15 | ```typescript 16 | import { parseJson, parseJsonFile } from '@discord/intl-flat-json-parser'; 17 | 18 | // Parse a string as flat JSON 19 | const jsonContent = `{ 20 | "MESSAGE_ONE": "Hello, this is the first message", 21 | "MESSAGE_TWO": "Another message!" 22 | }`; 23 | 24 | const messages = parseJson(jsonContent); 25 | console.log(messages[0]); 26 | //=> { 27 | // key: "MESSAGE_ONE", 28 | // value: "Hello, this is the first message", 29 | // position: { 30 | // line: 2, 31 | // col: 19 32 | // } 33 | // } 34 | 35 | // Or pass a file path to get parsed directly from the file system 36 | const directMessages = parseJsonFile('some/file/path.json'); 37 | ``` 38 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/bench/index.js: -------------------------------------------------------------------------------- 1 | const fs = require('node:fs'); 2 | const path = require('node:path'); 3 | 4 | const intlJson = require('..'); 5 | 6 | /** 7 | * @param {string} title 8 | * @param {() => unknown} callback 9 | * @param {boolean} log 10 | */ 11 | function bench(title, callback, log = true) { 12 | const start = performance.now(); 13 | callback(); 14 | const end = performance.now(); 15 | if (log) { 16 | console.log(title + ': ', end - start); 17 | } 18 | } 19 | 20 | const REPO_ROOT = path.resolve(__dirname, '../../..'); 21 | const DATA_DIR = path.resolve(REPO_ROOT, 'crates/intl_message_database/data/input'); 22 | 23 | /** @type {Record} */ 24 | const JSON_FILES = {}; 25 | bench('pre-reading content to start', () => { 26 | const files = fs.readdirSync(DATA_DIR, { withFileTypes: true }); 27 | for (const entry of files) { 28 | if (entry.isDirectory()) continue; 29 | if (!entry.name.endsWith('.jsona') && !entry.name.endsWith('.json')) continue; 30 | 31 | const filePath = path.join(entry.path, entry.name); 32 | JSON_FILES[filePath] = fs.readFileSync(filePath, 'utf8'); 33 | } 34 | console.log(`Found ${Object.keys(JSON_FILES).length} files`); 35 | }); 36 | 37 | bench('JSON.parse (default)', () => { 38 | for (const content of Object.values(JSON_FILES)) { 39 | JSON.parse(content); 40 | } 41 | }); 42 | bench('JSON.parse (mapped values)', () => { 43 | for (const content of Object.values(JSON_FILES)) { 44 | const result = []; 45 | const messages = JSON.parse(content); 46 | for (const [key, message] of Object.entries(messages)) { 47 | result.push({ key, value: message, position: undefined }); 48 | } 49 | } 50 | }); 51 | 52 | bench('intlJson.parseJson', () => { 53 | for (const content of Object.values(JSON_FILES)) { 54 | intlJson.parseJson(content); 55 | } 56 | }); 57 | 58 | bench('intlJson.parseJsonFile', () => { 59 | for (const filePath of Object.keys(JSON_FILES)) { 60 | intlJson.parseJsonFile(filePath); 61 | } 62 | }); 63 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/build.rs: -------------------------------------------------------------------------------- 1 | extern crate napi_build; 2 | 3 | fn main() { 4 | napi_build::setup(); 5 | } 6 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/index.d.ts: -------------------------------------------------------------------------------- 1 | /* auto-generated by NAPI-RS */ 2 | /* eslint-disable */ 3 | export declare class Message { 4 | key: string 5 | value: string 6 | line: number 7 | col: number 8 | } 9 | 10 | export declare function parseJson(text: string): Message[] 11 | 12 | export declare function parseJsonFile(filePath: string): Message[] 13 | 14 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/index.js: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | 4 | /* Originally auto-generated by NAPI-RS, adapted for working with an `npm` dir for each built package. */ 5 | 6 | const fs = require('node:fs'); 7 | const path = require('node:path'); 8 | 9 | function isMusl() { 10 | // For Node 10 11 | if (!process.report || typeof process.report.getReport !== 'function') { 12 | try { 13 | const lddPath = require('child_process').execSync('which ldd').toString().trim(); 14 | return fs.readFileSync(lddPath, 'utf8').includes('musl'); 15 | } catch (e) { 16 | return true; 17 | } 18 | } else { 19 | /** @type {any} */ 20 | let report = process.report.getReport(); 21 | if (typeof report === 'string') { 22 | report = JSON.parse(report); 23 | } 24 | const { glibcVersionRuntime } = report.header; 25 | return !glibcVersionRuntime; 26 | } 27 | } 28 | 29 | /** @type {Record>} */ 30 | const PACKAGE_NAMES = { 31 | android: { arm: 'android-arm-eabi', arm64: 'android-arm64' }, 32 | win32: { arm64: 'win32-arm64-msvc', ia32: 'win32-ia32-msvc', x64: 'win32-x64-msvc' }, 33 | darwin: { arm64: 'darwin-arm64', x64: 'darwin-x64' }, 34 | freebsd: { x64: 'freebsd-x64' }, 35 | 'linux-gnu': { 36 | arm: 'linux-arm-gnueabihf', 37 | arm64: 'linux-arm64-gnu', 38 | x64: 'linux-x64-gnu', 39 | riscv64: 'linux-riscv64-gnu', 40 | s390x: 'linux-s390x-gnu', 41 | }, 42 | 'linux-musl': { 43 | arm: 'linux-arm-musleabihf', 44 | arm64: 'linux-arm64-musl', 45 | x64: 'linux-x64-musl', 46 | riscv64: 'linux-riscv64-musl', 47 | }, 48 | }; 49 | 50 | const platform = 51 | process.platform !== 'linux' ? process.platform : 'linux-' + (isMusl() ? 'musl' : 'gnu'); 52 | const arch = process.arch; 53 | 54 | /** 55 | * @returns {string} 56 | */ 57 | function getPackageName() { 58 | if (!(platform in PACKAGE_NAMES)) { 59 | throw new Error(`Unsupported OS: ${platform}`); 60 | } 61 | if (!(arch in PACKAGE_NAMES[platform])) { 62 | throw new Error(`Unsupported architecture for ${platform}: ${arch}`); 63 | } 64 | return PACKAGE_NAMES[platform][arch]; 65 | } 66 | 67 | const packageName = getPackageName(); 68 | const localPath = path.join( 69 | __dirname, 70 | `npm/${packageName}/intl-flat-json-parser.${packageName}.node`, 71 | ); 72 | const packagePath = `@discord/intl-flat-json-parser-${packageName}`; 73 | const nativeBinding = fs.existsSync(localPath) ? require(localPath) : require(packagePath); 74 | 75 | const { parseJson, parseJsonFile } = nativeBinding; 76 | 77 | module.exports = { 78 | parseJson, 79 | parseJsonFile, 80 | }; 81 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/npm/darwin-arm64/README.md: -------------------------------------------------------------------------------- 1 | # `@discord/intl-flat-json-parser-darwin-arm64` 2 | 3 | This is the **aarch64-apple-darwin** binary for `@discord/intl-flat-json-parser` 4 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/npm/darwin-arm64/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-flat-json-parser-darwin-arm64", 3 | "version": "0.23.3", 4 | "os": [ 5 | "darwin" 6 | ], 7 | "cpu": [ 8 | "arm64" 9 | ], 10 | "main": "intl-flat-json-parser.darwin-arm64.node", 11 | "files": [ 12 | "intl-flat-json-parser.darwin-arm64.node" 13 | ], 14 | "license": "MIT", 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/discord/discord-intl" 21 | } 22 | } -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/npm/darwin-x64/README.md: -------------------------------------------------------------------------------- 1 | # `@discord/intl-flat-json-parser-darwin-x64` 2 | 3 | This is the **x86_64-apple-darwin** binary for `@discord/intl-flat-json-parser` 4 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/npm/darwin-x64/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-flat-json-parser-darwin-x64", 3 | "version": "0.23.3", 4 | "os": [ 5 | "darwin" 6 | ], 7 | "cpu": [ 8 | "x64" 9 | ], 10 | "main": "intl-flat-json-parser.darwin-x64.node", 11 | "files": [ 12 | "intl-flat-json-parser.darwin-x64.node" 13 | ], 14 | "license": "MIT", 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/discord/discord-intl" 21 | } 22 | } -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/npm/linux-arm64-gnu/README.md: -------------------------------------------------------------------------------- 1 | # `@discord/intl-flat-json-parser-linux-arm64-gnu` 2 | 3 | This is the **aarch64-unknown-linux-gnu** binary for `@discord/intl-flat-json-parser` 4 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/npm/linux-arm64-gnu/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-flat-json-parser-linux-arm64-gnu", 3 | "version": "0.23.3", 4 | "os": [ 5 | "linux" 6 | ], 7 | "cpu": [ 8 | "arm64" 9 | ], 10 | "main": "intl-flat-json-parser.linux-arm64-gnu.node", 11 | "files": [ 12 | "intl-flat-json-parser.linux-arm64-gnu.node" 13 | ], 14 | "license": "MIT", 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "libc": [ 19 | "glibc" 20 | ], 21 | "repository": { 22 | "type": "git", 23 | "url": "https://github.com/discord/discord-intl" 24 | } 25 | } -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/npm/linux-x64-gnu/README.md: -------------------------------------------------------------------------------- 1 | # `@discord/intl-flat-json-parser-linux-x64-gnu` 2 | 3 | This is the **x86_64-unknown-linux-gnu** binary for `@discord/intl-flat-json-parser` 4 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/npm/linux-x64-gnu/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-flat-json-parser-linux-x64-gnu", 3 | "version": "0.23.3", 4 | "os": [ 5 | "linux" 6 | ], 7 | "cpu": [ 8 | "x64" 9 | ], 10 | "main": "intl-flat-json-parser.linux-x64-gnu.node", 11 | "files": [ 12 | "intl-flat-json-parser.linux-x64-gnu.node" 13 | ], 14 | "license": "MIT", 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "libc": [ 19 | "glibc" 20 | ], 21 | "repository": { 22 | "type": "git", 23 | "url": "https://github.com/discord/discord-intl" 24 | } 25 | } -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/npm/win32-arm64-msvc/README.md: -------------------------------------------------------------------------------- 1 | # `@discord/intl-flat-json-parser-win32-arm64-msvc` 2 | 3 | This is the **aarch64-pc-windows-msvc** binary for `@discord/intl-flat-json-parser` 4 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/npm/win32-arm64-msvc/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-flat-json-parser-win32-arm64-msvc", 3 | "version": "0.23.3", 4 | "os": [ 5 | "win32" 6 | ], 7 | "cpu": [ 8 | "arm64" 9 | ], 10 | "main": "intl-flat-json-parser.win32-arm64-msvc.node", 11 | "files": [ 12 | "intl-flat-json-parser.win32-arm64-msvc.node" 13 | ], 14 | "license": "MIT", 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/discord/discord-intl" 21 | } 22 | } -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/npm/win32-x64-msvc/README.md: -------------------------------------------------------------------------------- 1 | # `@discord/intl-flat-json-parser-win32-x64-msvc` 2 | 3 | This is the **x86_64-pc-windows-msvc** binary for `@discord/intl-flat-json-parser` 4 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/npm/win32-x64-msvc/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-flat-json-parser-win32-x64-msvc", 3 | "version": "0.23.3", 4 | "os": [ 5 | "win32" 6 | ], 7 | "cpu": [ 8 | "x64" 9 | ], 10 | "main": "intl-flat-json-parser.win32-x64-msvc.node", 11 | "files": [ 12 | "intl-flat-json-parser.win32-x64-msvc.node" 13 | ], 14 | "license": "MIT", 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/discord/discord-intl" 21 | } 22 | } -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-flat-json-parser", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "description": "Rust-based JSON parser with location information for flat objects.", 6 | "author": "Jon Egeland", 7 | "main": "./index.js", 8 | "types": "./index.d.ts", 9 | "files": [ 10 | "index.d.ts", 11 | "index.js" 12 | ], 13 | "repository": { 14 | "type": "git", 15 | "url": "https://github.com/discord/discord-intl" 16 | }, 17 | "scripts": { 18 | "build:debug": "cargo build", 19 | "bench": "node ./bench/index.js" 20 | }, 21 | "optionalDependencies": { 22 | "@discord/intl-flat-json-parser-darwin-arm64": "workspace:*", 23 | "@discord/intl-flat-json-parser-darwin-x64": "workspace:*", 24 | "@discord/intl-flat-json-parser-linux-arm64-gnu": "workspace:*", 25 | "@discord/intl-flat-json-parser-linux-x64-gnu": "workspace:*", 26 | "@discord/intl-flat-json-parser-win32-arm64-msvc": "workspace:*", 27 | "@discord/intl-flat-json-parser-win32-x64-msvc": "workspace:*" 28 | }, 29 | "devDependencies": { 30 | "@napi-rs/cli": "^2.18.4" 31 | }, 32 | "engines": { 33 | "node": ">= 10" 34 | }, 35 | "napi": { 36 | "name": "intl-flat-json-parser", 37 | "triples": { 38 | "defaults": false, 39 | "additional": [ 40 | "aarch64-apple-darwin", 41 | "aarch64-unknown-linux-gnu", 42 | "aarch64-pc-windows-msvc", 43 | "x86_64-apple-darwin", 44 | "x86_64-pc-windows-msvc", 45 | "x86_64-unknown-linux-gnu" 46 | ] 47 | } 48 | } 49 | } -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod parser; 2 | mod util; 3 | 4 | pub use parser::{JsonMessage, JsonPosition, TranslationsJsonParser}; 5 | 6 | #[cfg(not(feature = "node_addon"))] 7 | pub mod napi; 8 | 9 | /// Parse the given `text` as a single, flat JSON object of message keys to 10 | /// message values. The JSON is assumed to be well-formed, and minimal error 11 | /// handling is implemented. 12 | /// 13 | /// Since sources are able to accept an iterator of translation values, this 14 | /// parser never stores the completed object in memory and instead yields each 15 | /// translation as it is parsed. 16 | /// 17 | /// This parser also handles tracking line and column positions within the 18 | /// source, which is the primary reason for this implementation over existing 19 | /// libraries like serde that only track that state internally. 20 | /// 21 | /// Note that some extra assumptions are made here for the sake of simplicity 22 | /// and efficient parsing that other implementations aren't able to make: 23 | /// - The given text is a well-formed, flat JSON object 24 | /// - Keys may not contain escaped quotes, `}`, or newlines. 25 | /// - The only important positional information is the first character of the value. 26 | /// - There will be no errors during parsing. The iterator will return None instead. 27 | pub fn parse_flat_translation_json(text: &str) -> impl Iterator + use<'_> { 28 | let mut parser = TranslationsJsonParser::new(text); 29 | parser.parse_start(); 30 | parser.into_iter() 31 | } 32 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/src/napi.rs: -------------------------------------------------------------------------------- 1 | use crate::JsonMessage; 2 | use napi::bindgen_prelude::Array; 3 | use napi::Env; 4 | use napi_derive::napi; 5 | 6 | // Use the mimalloc allocator explicitly when building the node addon. 7 | extern crate intl_allocator; 8 | 9 | #[napi] 10 | pub struct Message { 11 | pub key: String, 12 | pub value: String, 13 | // The Position properties are created directly inline on the message 14 | // because NAPI object construction is pretty slow (like 10x slower than 15 | // the engine creating an object directly), so moving these to direct 16 | // properties of the class makes the whole process ~30-35% faster. 17 | // 18 | // See https://github.com/nodejs/node/issues/45905 for future updates. 19 | pub line: u32, 20 | pub col: u32, 21 | } 22 | 23 | fn collect_messages(env: Env, iterator: impl Iterator) -> napi::Result { 24 | // This is an arbitrary size hint that should be suitable for a lot of use 25 | // cases. While it may inadvertently allocate extra memory for some, 26 | // avoiding repeated re-allocations that we're pretty confident will happen 27 | // ends up saving a lot more time in the end. 28 | let mut result = env.create_array(1024)?; 29 | let mut index = 0; 30 | for message in iterator { 31 | result.set( 32 | index, 33 | Message { 34 | key: message.key.to_string(), 35 | value: message.value.to_string(), 36 | line: message.position.line, 37 | col: message.position.col, 38 | }, 39 | )?; 40 | index += 1; 41 | } 42 | 43 | Ok(result) 44 | } 45 | 46 | #[napi(ts_return_type = "Message[]")] 47 | pub fn parse_json(env: Env, text: String) -> napi::Result { 48 | let messages = crate::parse_flat_translation_json(&text); 49 | Ok(collect_messages(env, messages)?) 50 | } 51 | 52 | #[napi(ts_return_type = "Message[]")] 53 | pub fn parse_json_file(env: Env, file_path: String) -> napi::Result { 54 | let content = std::fs::read_to_string(&file_path)?; 55 | parse_json(env, content) 56 | } 57 | -------------------------------------------------------------------------------- /crates/intl_flat_json_parser/src/util.rs: -------------------------------------------------------------------------------- 1 | use std::borrow::Cow; 2 | use unescape_zero_copy::Error; 3 | 4 | // COPIED FROM byte_lookup.rs in crates/intl_markdown. 5 | // Learned from: https://nullprogram.com/blog/2017/10/06/ 6 | #[rustfmt::skip] 7 | static UTF8_LENGTH_LOOKUP: [usize; 32] = [ 8 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9 | 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 3, 3, 4, 0, 10 | ]; 11 | 12 | /// Return the byte length of the complete UTF-8 code point that starts with `byte`. This can be 13 | /// done branchlessly and without computing the entire `char`. 14 | #[inline(always)] 15 | pub(crate) fn char_length_from_byte(byte: u8) -> usize { 16 | UTF8_LENGTH_LOOKUP[byte as usize >> 3] 17 | } 18 | 19 | pub(crate) fn unescape_json_str(str: &str) -> Result, Error> { 20 | unescape_zero_copy::unescape(json_escape_sequence, str) 21 | } 22 | 23 | pub fn json_escape_sequence(s: &str) -> Result<(char, &str), Error> { 24 | let mut chars = s.chars(); 25 | let next = chars.next().ok_or(Error::IncompleteSequence)?; 26 | match next { 27 | 'b' => Ok(('\x08', chars.as_str())), 28 | 'f' => Ok(('\x0C', chars.as_str())), 29 | 'n' => Ok(('\n', chars.as_str())), 30 | 'r' => Ok(('\r', chars.as_str())), 31 | 't' => Ok(('\t', chars.as_str())), 32 | '\r' | '\n' => Ok((next, chars.as_str())), 33 | 'u' => { 34 | let first = u32::from_str_radix(&s[1..5], 16)?; 35 | // This is the BMP surrogate range for the second character in a surrogate pair. If the 36 | // value is in this range and this step, then we know it's incorrect since it can't 37 | // stand on its own. 38 | if (0xDC00..=0xDFFF).contains(&first) { 39 | return Err(Error::InvalidUnicode(first)); 40 | } 41 | if !(0xD800..=0xDBFF).contains(&first) { 42 | // Characters outside the surrogate range should always be valid. 43 | let next = char::from_u32(first).unwrap(); 44 | return Ok((next, &s[5..])); 45 | } 46 | // Now the value must definitely be a surrogate pair, so the second one should follow 47 | // immediately, `\uXXXX\uXXXX`. We need to skip past the next `\u` as well to get the 48 | // following bytes. 49 | let second = u32::from_str_radix(&s[7..11], 16)?; 50 | // Taken from serde_json: https://github.com/serde-rs/json/blob/1d7378e8ee87e9225da28094329e06345b76cd99/src/read.rs#L969 51 | let next = 52 | char::from_u32((((first - 0xD800) << 10) | (second - 0xDC00)) + 0x1_0000).unwrap(); 53 | Ok((next, &s[11..])) 54 | } 55 | ch => Ok((ch, chars.as_str())), 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /crates/intl_markdown/.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [alias] 2 | test-markdown = "nextest run --no-fail-fast emphasis_and_strong_emphasis inlines links images textual_content code_spans entity_and_numeric_character_references backslash_escapes paragraphs fenced_code_blocks indented_code_blocks atx_headings setext_headings thematic_breaks tabs blank_lines autolinks soft_line_breaks hard_line_breaks" 3 | test-icu = "nextest run --no-fail-fast icu" 4 | test-subset = "nextest run --no-fail-fast" 5 | test-all = "nextest run" -------------------------------------------------------------------------------- /crates/intl_markdown/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/discord/discord-intl/18741b6efc8e9799abe47a7d94c9bc9baf9ad63d/crates/intl_markdown/.gitignore -------------------------------------------------------------------------------- /crates/intl_markdown/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_markdown" 3 | description = "A combination Markdown and ICU messageformat parser for i18n messages." 4 | version = "0.1.0" 5 | edition = "2021" 6 | publish = false 7 | 8 | [lib] 9 | crate-type = ["lib"] 10 | bench = false 11 | # Not using doctests currently 12 | doctest = false 13 | 14 | [dependencies] 15 | bitflags = "2" 16 | cjk = "0.2.5" 17 | intl_markdown_macros = { workspace = true } 18 | serde = { workspace = true } 19 | serde_json = { workspace = true } 20 | unicode-properties = "0" 21 | pulldown-cmark = "0.11.0" 22 | unescape_zero_copy = { workspace = true } 23 | memchr = { workspace = true } 24 | unicode-xid = { workspace = true } 25 | 26 | [dev-dependencies] 27 | test-case = "3" 28 | criterion = "0.5" 29 | keyless_json = { workspace = true } 30 | 31 | # Test generation script 32 | [[example]] 33 | name = "gen-spec-tests" 34 | path = "./examples/gen-spec-tests.rs" 35 | 36 | # HTML Entity generation script 37 | [[example]] 38 | name = "gen-html-entities" 39 | path = "./examples/gen-html-entities.rs" 40 | 41 | [[bench]] 42 | name = "long_documents" 43 | harness = false 44 | 45 | [[bench]] 46 | name = "icu_messageformat_parser_bench" 47 | harness = false -------------------------------------------------------------------------------- /crates/intl_markdown/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/intl_markdown/README.md: -------------------------------------------------------------------------------- 1 | # intl_markdown 2 | 3 | Core Markdown parsing implementation for `discord-intl`. Generally speaking, this implementation follows the CommonMark spec as closely as possible, with additions and extensions added to accommodate integrating ICU syntax for dynamic messages. This combination parser produces a fully-static syntax tree that can be analyzed ahead of time for accurate type generation, segmentation, validation, and more. 4 | 5 | The parser is implemented as a hand-built event-based parser. Source content is first analyzed as a whole for the overall block structure, which is then used to drive a loop of the inline parser. Parsing yields an Event buffer, which is then iterated to create a Concrete Syntax Tree of the content. Because Markdown output is non-linear with the input (e.g., spaces and newlines can change meaning or be omitted, escape characters are replaced, etc.), another conversion to an AST then happens that applies the necessary transformations to yield a final, static tree with the desired output. This AST is what all downstream libraries and services operate on. 6 | 7 | ## Development 8 | 9 | There are many tests for this crate to ensure there are no regressions against the CommonMark spec or any of our extensions to the syntax. They are broken out into various cargo commands that can be run from this directory: 10 | 11 | ```shell 12 | # Run all tests in this directory 13 | cargo test-all 14 | # Just run Markdown tests 15 | cargo test-markdown 16 | # Run an arbitrary subset of tests 17 | cargo test-subset tests::qualified::module::name 18 | ``` 19 | 20 | The aliases for these tests are defined in `./cargo/config.toml`. 21 | 22 | Additionally, a set of benchmarks is available to compare against `pulldown-cmark`, a standard CommonMark implementation in Rust known for being fast and efficient. 23 | 24 | ```shell 25 | cargo bench 26 | ``` 27 | -------------------------------------------------------------------------------- /crates/intl_markdown/benches/icu_messageformat_parser_bench.rs: -------------------------------------------------------------------------------- 1 | use criterion::{Criterion, criterion_group, criterion_main, Throughput}; 2 | 3 | use intl_markdown::parse_intl_message; 4 | 5 | fn parse_message(message: &str) { 6 | parse_intl_message(message, false); 7 | } 8 | 9 | fn parse_bench(c: &mut Criterion) { 10 | let mut group = c.benchmark_group("messages"); 11 | group.throughput(Throughput::Elements(1)); 12 | group.bench_function("complex messages", |b| { 13 | b.iter(|| { 14 | parse_message(r#" 15 | {gender_of_host, select, 16 | female { 17 | {num_guests, plural, 18 | =0 {{host} does not give a party.} 19 | =1 {{host} invites {guest} to her party.} 20 | =2 {{host} invites {guest} and one other person to her party.} 21 | other {{host} invites {guest} and # other people to her party.} 22 | } 23 | } 24 | male { 25 | {num_guests, plural, 26 | =0 {{host} does not give a party.} 27 | =1 {{host} invites {guest} to his party.} 28 | =2 {{host} invites {guest} and one other person to his party.} 29 | other {{host} invites {guest} and # other people to his party.} 30 | } 31 | } 32 | other { 33 | {num_guests, plural, 34 | =0 {{host} does not give a party.} 35 | =1 {{host} invites {guest} to their party.} 36 | =2 {{host} invites {guest} and one other person to their party.} 37 | other {{host} invites {guest} and # other people to their party.} 38 | } 39 | } 40 | }"# 41 | )}); 42 | }); 43 | 44 | group.bench_function("normal message", |b| { 45 | b.iter(|| { 46 | parse_message( 47 | r#" 48 | Yo, {firstName} {lastName} has 49 | {numBooks, number, integer} 50 | {numBooks, plural, 51 | one {book} 52 | other {books} 53 | } 54 | "#, 55 | ) 56 | }); 57 | }); 58 | group.bench_function("simple message", |b| { 59 | b.iter(|| parse_message(r#"Hello, {name}"#)); 60 | }); 61 | group.bench_function("string message", |b| { 62 | b.iter(|| parse_message(r#"Hello, world"#)); 63 | }); 64 | } 65 | criterion_group!(benches, parse_bench); 66 | criterion_main!(benches); 67 | -------------------------------------------------------------------------------- /crates/intl_markdown/examples/gen-html-entities.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, fmt::Write, path::PathBuf, str::FromStr}; 2 | 3 | use serde::Deserialize; 4 | 5 | #[derive(Deserialize, PartialEq, PartialOrd, Eq, Ord)] 6 | struct HTMLEntity { 7 | characters: String, 8 | #[allow(dead_code)] 9 | codepoints: Vec, 10 | } 11 | 12 | fn main() { 13 | let entities: HashMap = 14 | serde_json::from_str(include_str!("./html-entities.json")) 15 | .expect("Failed to parse html entities"); 16 | 17 | // Output the test contents into test case folders to read as individual 18 | // tests afterward. 19 | let html_file = PathBuf::from_str("./src/html_entities.rs").unwrap(); 20 | let mut output_buffer = String::new(); 21 | 22 | output_buffer.push_str(&format!( 23 | "//! This module is autogenerated with `cargo run --bin gen-html-entities`. The 24 | //! source for this script lives at `examples/gen-html-entities.rs`. 25 | 26 | const HTML_ENTITIES: [(&[u8], &str); {}] = [ 27 | ", 28 | entities.len() 29 | )); 30 | 31 | let mut sorted_entities = entities.into_iter().collect::>(); 32 | sorted_entities.sort(); 33 | 34 | for (name, entity) in sorted_entities { 35 | output_buffer 36 | .write_str(&format!( 37 | " (b\"{}\", \"{}\"),\n", 38 | name, 39 | entity.characters.escape_unicode(), 40 | )) 41 | .expect(&format!("failed to write entity {}", name)); 42 | } 43 | 44 | output_buffer 45 | .write_str( 46 | "]; 47 | 48 | pub fn is_html_entity(text: &[u8]) -> bool { 49 | HTML_ENTITIES 50 | .binary_search_by_key(&text, |&(key, _value)| key) 51 | .is_ok() 52 | } 53 | 54 | pub fn get_html_entity(text: &[u8]) -> Option<&'static str> { 55 | HTML_ENTITIES 56 | .binary_search_by_key(&text, |&(key, _value)| key) 57 | .ok() 58 | .map(|i| HTML_ENTITIES[i].1) 59 | } 60 | ", 61 | ) 62 | .expect("failed to write end of html entities"); 63 | 64 | std::fs::write(html_file, output_buffer).expect("Failed to write the specs file"); 65 | } 66 | -------------------------------------------------------------------------------- /crates/intl_markdown/src/byte_lookup.rs: -------------------------------------------------------------------------------- 1 | use intl_markdown_macros::generate_ascii_lookup_table; 2 | 3 | generate_ascii_lookup_table!( 4 | SIGNIFICANT_PUNCTUATION_BYTES, 5 | b"\n\x0C\r!\"$&'()*:<>[\\]_`{}~#" 6 | ); 7 | 8 | /// Returns true if the given byte represents a significant character that 9 | /// could become a new type of token. This effectively just includes 10 | /// punctuation and newline characters. 11 | /// 12 | /// Note that these are only the characters that are significant when they 13 | /// interrupt textual content. For example, a `-` could become a MINUS token, 14 | /// but within a word it can never be significant, e.g. the dash in `two-part` 15 | /// is not significant. 16 | /// 17 | /// Inline whitespace in this context _is not_ considered significant, but 18 | /// vertical whitespace _is_ significant. 19 | pub(crate) fn byte_is_significant_punctuation(byte: u8) -> bool { 20 | SIGNIFICANT_PUNCTUATION_BYTES[byte as usize] != 0 21 | } 22 | 23 | // Learned from: https://nullprogram.com/blog/2017/10/06/ 24 | #[rustfmt::skip] 25 | static UTF8_LENGTH_LOOKUP: [usize; 32] = [ 26 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27 | 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 3, 3, 4, 0, 28 | ]; 29 | 30 | /// Return the byte length of the complete UTF-8 code point that starts with `byte`. This can be 31 | /// done branchlessly and without computing the entire `char`. 32 | #[inline(always)] 33 | pub(crate) fn char_length_from_byte(byte: u8) -> usize { 34 | UTF8_LENGTH_LOOKUP[byte as usize >> 3] 35 | } 36 | 37 | /// Returns true if the char is valid as the starting character of a unicode identifier. 38 | #[inline(always)] 39 | pub(crate) fn is_unicode_identifier_start(c: char) -> bool { 40 | unicode_xid::UnicodeXID::is_xid_start(c) 41 | } 42 | 43 | /// Returns true if the char is valid as any character after the start of a unicode identifier. 44 | #[inline(always)] 45 | pub(crate) fn is_unicode_identifier_continue(c: char) -> bool { 46 | unicode_xid::UnicodeXID::is_xid_continue(c) 47 | } 48 | -------------------------------------------------------------------------------- /crates/intl_markdown/src/icu/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod compile; 2 | pub mod format; 3 | pub mod serialize; 4 | pub mod tags; 5 | -------------------------------------------------------------------------------- /crates/intl_markdown/src/icu/tags.rs: -------------------------------------------------------------------------------- 1 | pub struct TagNames<'a> { 2 | strong: &'a str, 3 | emphasis: &'a str, 4 | strike_through: &'a str, 5 | paragraph: &'a str, 6 | link: &'a str, 7 | code: &'a str, 8 | code_block: &'a str, 9 | br: &'a str, 10 | hr: &'a str, 11 | h1: &'a str, 12 | h2: &'a str, 13 | h3: &'a str, 14 | h4: &'a str, 15 | h5: &'a str, 16 | h6: &'a str, 17 | } 18 | 19 | impl<'a> TagNames<'a> { 20 | pub const fn strong(&self) -> &'a str { 21 | &self.strong 22 | } 23 | pub const fn emphasis(&self) -> &'a str { 24 | &self.emphasis 25 | } 26 | pub const fn strike_through(&self) -> &'a str { 27 | &self.strike_through 28 | } 29 | pub const fn paragraph(&self) -> &'a str { 30 | &self.paragraph 31 | } 32 | pub const fn link(&self) -> &'a str { 33 | &self.link 34 | } 35 | pub const fn code(&self) -> &'a str { 36 | &self.code 37 | } 38 | pub const fn code_block(&self) -> &'a str { 39 | &self.code_block 40 | } 41 | pub const fn br(&self) -> &'a str { 42 | &self.br 43 | } 44 | pub const fn hr(&self) -> &'a str { 45 | &self.hr 46 | } 47 | 48 | pub fn heading(&self, level: u8) -> &'a str { 49 | match level { 50 | 1 => self.h1, 51 | 2 => self.h2, 52 | 3 => self.h3, 53 | 4 => self.h4, 54 | 5 => self.h5, 55 | 6 => self.h6, 56 | _ => unreachable!(), 57 | } 58 | } 59 | } 60 | 61 | pub static DEFAULT_TAG_NAMES: TagNames<'static> = TagNames { 62 | strong: "$b", 63 | emphasis: "$i", 64 | strike_through: "$del", 65 | paragraph: "$p", 66 | link: "$link", 67 | code: "$code", 68 | code_block: "$codeBlock", 69 | br: "$br", 70 | hr: "$hr", 71 | h1: "$h1", 72 | h2: "$h2", 73 | h3: "$h3", 74 | h4: "$h4", 75 | h5: "$h5", 76 | h6: "$h6", 77 | }; 78 | -------------------------------------------------------------------------------- /crates/intl_markdown/src/lib.rs: -------------------------------------------------------------------------------- 1 | extern crate core; 2 | 3 | pub use ast::format::format_ast; 4 | pub use ast::process::process_cst_to_ast; 5 | pub use ast::*; 6 | pub use icu::compile::compile_to_format_js; 7 | pub use icu::format::format_icu_string; 8 | pub use icu::tags::DEFAULT_TAG_NAMES; 9 | pub use parser::ICUMarkdownParser; 10 | pub use syntax::SyntaxKind; 11 | pub use token::SyntaxToken; 12 | pub use tree_builder::cst::Document as CstDocument; 13 | 14 | pub mod ast; 15 | mod block_parser; 16 | mod byte_lookup; 17 | mod delimiter; 18 | mod event; 19 | mod html_entities; 20 | mod icu; 21 | mod lexer; 22 | mod parser; 23 | mod syntax; 24 | mod token; 25 | mod tree_builder; 26 | 27 | /// Parse an intl message into a final AST representing the semantics of the message. 28 | pub fn parse_intl_message(content: &str, include_blocks: bool) -> Document { 29 | let mut parser = ICUMarkdownParser::new(content, include_blocks); 30 | let source = parser.source().clone(); 31 | parser.parse(); 32 | let cst = parser.into_cst(); 33 | process_cst_to_ast(source, &cst) 34 | } 35 | 36 | /// Return a new Document with the given content as the only value, treated as a raw string with 37 | /// no parsing or semantics applied. 38 | pub fn raw_string_to_document(content: &str) -> Document { 39 | Document::from_literal(content) 40 | } 41 | 42 | pub fn format_to_icu_string(document: &Document) -> Result { 43 | format_icu_string(document) 44 | } 45 | -------------------------------------------------------------------------------- /crates/intl_markdown/src/parser/text.rs: -------------------------------------------------------------------------------- 1 | use super::ICUMarkdownParser; 2 | 3 | /// Consume as many sequential plain-text tokens as possible, merging them 4 | /// into a single token before pushing that new token onto the buffer. 5 | pub(super) fn parse_plain_text(p: &mut ICUMarkdownParser) -> Option<()> { 6 | p.bump(); 7 | Some(()) 8 | } 9 | -------------------------------------------------------------------------------- /crates/intl_markdown/tests/harness.rs: -------------------------------------------------------------------------------- 1 | use intl_markdown::{ 2 | compile_to_format_js, format_ast, format_icu_string, process_cst_to_ast, CstDocument, Document, 3 | ICUMarkdownParser, 4 | }; 5 | 6 | pub fn parse(content: &str, include_blocks: bool) -> CstDocument { 7 | let mut parser = ICUMarkdownParser::new(content, include_blocks); 8 | parser.parse(); 9 | parser.into_cst() 10 | } 11 | 12 | pub fn parse_to_ast(content: &str, include_blocks: bool) -> Document { 13 | let mut parser = ICUMarkdownParser::new(content, include_blocks); 14 | let source = parser.source().clone(); 15 | parser.parse(); 16 | process_cst_to_ast(source, &parser.into_cst()) 17 | } 18 | 19 | /// Test that the input is parsed and formatted as HTML as given. 20 | #[allow(unused)] 21 | pub fn run_spec_test(input: &str, expected: &str) { 22 | // AST-based formatting 23 | let ast = parse_to_ast(input, true); 24 | let output = format_ast(&ast).unwrap(); 25 | 26 | assert_eq!(expected, output); 27 | } 28 | 29 | /// Test that the input is parsed and formatted as an ICU string as given. 30 | #[allow(unused)] 31 | pub fn run_icu_string_test(input: &str, expected: &str, include_blocks: bool) { 32 | // AST-based formatting 33 | let ast = parse_to_ast(input, include_blocks); 34 | let output = format_icu_string(&ast).unwrap(); 35 | 36 | assert_eq!(expected, output); 37 | } 38 | 39 | /// Test that the input is parsed and formatted as an ICU AST as given. 40 | #[allow(unused)] 41 | pub fn run_icu_ast_test(input: &str, expected: &str, include_blocks: bool) { 42 | // AST-based formatting 43 | let ast = parse_to_ast(input, include_blocks); 44 | let output = keyless_json::to_string(&compile_to_format_js(&ast)).unwrap(); 45 | 46 | assert_eq!(expected, output); 47 | } 48 | 49 | macro_rules! ast_test { 50 | ($name:ident, $input:literal, $output:literal) => { 51 | #[test] 52 | fn $name() { 53 | crate::harness::run_icu_ast_test($input, $output, false); 54 | } 55 | }; 56 | } 57 | macro_rules! icu_string_test { 58 | ($name:ident, $input:literal, $output:literal) => { 59 | #[test] 60 | fn $name() { 61 | crate::harness::run_icu_string_test($input, $output, false); 62 | } 63 | }; 64 | } 65 | macro_rules! icu_block_string_test { 66 | ($name:ident, $input:literal, $output:literal) => { 67 | #[test] 68 | fn $name() { 69 | crate::harness::run_icu_string_test($input, $output, true); 70 | } 71 | }; 72 | } 73 | 74 | pub(crate) use ast_test; 75 | pub(crate) use icu_block_string_test; 76 | pub(crate) use icu_string_test; 77 | -------------------------------------------------------------------------------- /crates/intl_markdown/tests/test_ast.rs: -------------------------------------------------------------------------------- 1 | use intl_markdown::{compile_to_format_js, parse_intl_message}; 2 | 3 | #[test] 4 | #[ignore] 5 | fn test_ast() { 6 | let ast = parse_intl_message("Your server\n\nBoosts", true); 7 | println!("{:#?}", ast); 8 | 9 | let compiled = compile_to_format_js(&ast); 10 | println!("{:#?}", compiled); 11 | 12 | let serialized = serde_json::to_string(&compiled); 13 | println!("{}", serialized.unwrap()) 14 | } 15 | -------------------------------------------------------------------------------- /crates/intl_markdown_macros/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_markdown_macros" 3 | description = "Procedural macros for intl_markdown" 4 | version = "0.1.0" 5 | edition = "2021" 6 | publish = false 7 | 8 | [lib] 9 | proc-macro = true 10 | 11 | [dependencies] 12 | convert_case = "0" 13 | proc-macro2 = "1" 14 | quote = "1" 15 | syn = "2" -------------------------------------------------------------------------------- /crates/intl_markdown_macros/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/intl_markdown_macros/README.md: -------------------------------------------------------------------------------- 1 | # intl_markdown_macros 2 | 3 | Procedural macro definitions to help clean up the implementation of boilerplate in `intl_markdown`, like the event buffer to CST conversion and generating lookup tables. 4 | 5 | This is a library crate that is only built as part of another crate. 6 | -------------------------------------------------------------------------------- /crates/intl_markdown_visitor/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_markdown_visitor" 3 | description = "AST visitor utility traits and implementations for intl_markdown" 4 | version = "0.1.0" 5 | edition = "2021" 6 | 7 | [lib] 8 | crate-type = ["lib"] 9 | 10 | [dependencies] 11 | intl_markdown = { workspace = true } 12 | -------------------------------------------------------------------------------- /crates/intl_markdown_visitor/README.md: -------------------------------------------------------------------------------- 1 | # intl_markdown_visitor 2 | 3 | Abstract Visitor trait definition for the AST of `intl_markdown`. Any other crate can use this crate to define a new `Visitor` and drive a walk of a message AST to perform any kind of transformation or other operation. Visitors are _not_ mutable, meaning any transformation of the tree _must_ generate a new tree rather than mutating the existing one. 4 | 5 | This is a library crate that is only built as part of another crate. 6 | -------------------------------------------------------------------------------- /crates/intl_markdown_visitor/src/lib.rs: -------------------------------------------------------------------------------- 1 | use intl_markdown::Document; 2 | 3 | pub use crate::visit_with::VisitWith; 4 | pub use crate::visitor::Visit; 5 | 6 | mod visit_with; 7 | mod visitor; 8 | 9 | pub fn visit_with_mut(document: &Document, visitor: &mut V) { 10 | document.visit_with(visitor); 11 | } 12 | -------------------------------------------------------------------------------- /crates/intl_message_database/.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [target.x86_64-pc-windows-msvc] 2 | rustflags = ["-C", "target-feature=+crt-static"] 3 | 4 | [target.aarch64-unknown-linux-musl] 5 | linker = "aarch64-linux-musl-gcc" 6 | rustflags = ["-C", "target-feature=-crt-static"] 7 | 8 | [target.aarch64-apple-darwin.env] 9 | MACOSX_DEPLOYMENT_TARGET = "11" 10 | 11 | [target.x86_64-apple-darwin.env] 12 | MACOSX_DEPLOYMENT_TARGET = "10.13" -------------------------------------------------------------------------------- /crates/intl_message_database/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | ^target/ 3 | target 4 | test/cases/* 5 | 6 | data/* 7 | 8 | # This result is copied out from `target` to use from the node package 9 | *.node 10 | -------------------------------------------------------------------------------- /crates/intl_message_database/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_message_database" 3 | description = "An intl message management databae, supporting CRUD for messages with attribution to their source files, validation, type generation, exporting, and more" 4 | version = "0.1.0" 5 | edition = "2021" 6 | publish = false 7 | 8 | 9 | [lib] 10 | crate-type = ["cdylib", "lib"] 11 | 12 | [features] 13 | default = [] 14 | # Enable to compile the library as a static library 15 | static_link = [] 16 | 17 | [dependencies] 18 | anyhow = { workspace = true } 19 | ignore = { workspace = true } 20 | intl_allocator = { workspace = true } 21 | intl_database_core = { workspace = true } 22 | intl_database_exporter = { workspace = true } 23 | intl_database_js_source = { workspace = true } 24 | intl_database_json_source = { workspace = true } 25 | intl_database_service = { workspace = true } 26 | intl_database_types_generator = { workspace = true } 27 | intl_message_utils = { workspace = true } 28 | intl_validator = { workspace = true } 29 | napi = { workspace = true } 30 | napi-derive = { workspace = true } 31 | num_cpus = "1" 32 | rustc-hash = { workspace = true } 33 | serde = { workspace = true } 34 | threadpool = "1.8.1" 35 | 36 | [build-dependencies] 37 | napi-build = { workspace = true } 38 | -------------------------------------------------------------------------------- /crates/intl_message_database/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/intl_message_database/README.md: -------------------------------------------------------------------------------- 1 | # intl_message_database 2 | 3 | A Rust-based native Node extension module for quickly parsing, processing, and generating structured message definitions across an entire project. This is the core module powering all of the loaders, transformer plugins, and other tooling that relate to messages and translations. It includes a custom parser to handle a combination of ICU MessageFormat syntax for variable interpolations, along with (almost) full support for Markdown's inline styling syntax, and is able to output reformatted strings, ASTs, type definitions, perform validations across multiple translations, and more. 4 | 5 | This crate is primarily developed for the Node API, but is implemented in a way that allows for other interfaces in the future (or splitting into multiple crates). 6 | 7 | Business logic for this crate should be exposed as a Rust interface through the `public.rs` module. Interfaces for other languages should then be added as separate modules that proxy types from those languages to the `public` module. They should _not_ perform any other work or logic independently. 8 | 9 | ## Development 10 | 11 | This crate is managed by the `intl-cli`, including commands for building and testing it across platforms: 12 | 13 | ```shell 14 | # Build the library locally 15 | pnpm intl-cli db build --target local 16 | # Build specifically for a given target name 17 | pnpm intl-cli db build --target darwin-arm64 18 | # Run the benchmark tests on a local build to compare performance 19 | pnpm intl-cli db bench 20 | ``` 21 | -------------------------------------------------------------------------------- /crates/intl_message_database/bench/.gitignore: -------------------------------------------------------------------------------- 1 | test-en-US.js -------------------------------------------------------------------------------- /crates/intl_message_database/bench/util.js: -------------------------------------------------------------------------------- 1 | globalThis.window = globalThis; 2 | globalThis.navigator = { languages: ['en-US'] }; 3 | 4 | /** 5 | * @param {string} title 6 | * @param {() => unknown} callback 7 | * @param {boolean} log 8 | */ 9 | function bench(title, callback, log = true) { 10 | const start = performance.now(); 11 | callback(); 12 | const end = performance.now(); 13 | if (log) { 14 | console.log(title + ': ', end - start); 15 | } 16 | } 17 | 18 | const locales = [ 19 | 'bg', 20 | 'cs', 21 | 'da', 22 | 'de', 23 | 'el', 24 | 'en-GB', 25 | 'es-419', 26 | 'es-ES', 27 | 'fi', 28 | 'fr', 29 | 'hi', 30 | 'hr', 31 | 'hu', 32 | 'id', 33 | 'it', 34 | 'ja', 35 | 'ko', 36 | 'lt', 37 | 'nl', 38 | 'no', 39 | 'pl', 40 | 'pt-BR', 41 | 'ro', 42 | 'ru', 43 | 'sv-SE', 44 | 'th', 45 | 'tr', 46 | 'uk', 47 | 'vi', 48 | 'zh-CN', 49 | 'zh-TW', 50 | ]; 51 | 52 | module.exports = { 53 | bench, 54 | locales, 55 | }; 56 | -------------------------------------------------------------------------------- /crates/intl_message_database/build.rs: -------------------------------------------------------------------------------- 1 | extern crate napi_build; 2 | 3 | fn main() { 4 | napi_build::setup(); 5 | } 6 | -------------------------------------------------------------------------------- /crates/intl_message_database/examples/public-test.rs: -------------------------------------------------------------------------------- 1 | use intl_database_core::{DatabaseInsertStrategy, MessagesDatabase}; 2 | use intl_message_database::public::{process_definitions_file, validate_messages}; 3 | 4 | pub fn main() { 5 | let input_root = "./data/temp"; 6 | let output_root = "./data/output"; 7 | let mut database = MessagesDatabase::new(); 8 | // 9 | // let files = find_all_messages_files([&input_root].into_iter(), "en-US"); 10 | // process_all_messages_files(&mut database, files.into_iter()).expect("all files are processed"); 11 | // process_translation_file(&mut database, "./data/temp/es-ES.messages.jsona", "es-ES") 12 | // .expect("processed"); 13 | process_definitions_file( 14 | &mut database, 15 | "./data/temp/en-US.messages.js", 16 | None, 17 | DatabaseInsertStrategy::NewSourceFile, 18 | ) 19 | .expect("processed"); 20 | 21 | validate_messages(&database).expect("validated messages"); 22 | 23 | // let source = format!("{input_root}/en-US.messages.js"); 24 | // let output = format!("{output_root}/en-US.messages.d.ts"); 25 | // generate_types(&mut database, &source, &output).expect("types should be generated"); 26 | } 27 | -------------------------------------------------------------------------------- /crates/intl_message_database/index.js: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | 4 | /* Originally auto-generated by NAPI-RS, adapted for working with an `npm` dir for each built package. */ 5 | 6 | const fs = require('node:fs'); 7 | const path = require('node:path'); 8 | 9 | function isMusl() { 10 | // For Node 10 11 | if (!process.report || typeof process.report.getReport !== 'function') { 12 | try { 13 | const lddPath = require('child_process').execSync('which ldd').toString().trim(); 14 | return fs.readFileSync(lddPath, 'utf8').includes('musl'); 15 | } catch (e) { 16 | return true; 17 | } 18 | } else { 19 | /** @type {any} */ 20 | let report = process.report.getReport(); 21 | if (typeof report === 'string') { 22 | report = JSON.parse(report); 23 | } 24 | const { glibcVersionRuntime } = report.header; 25 | return !glibcVersionRuntime; 26 | } 27 | } 28 | 29 | /** @type {Record>} */ 30 | const PACKAGE_NAMES = { 31 | android: { arm: 'android-arm-eabi', arm64: 'android-arm64' }, 32 | win32: { arm64: 'win32-arm64-msvc', ia32: 'win32-ia32-msvc', x64: 'win32-x64-msvc' }, 33 | darwin: { arm64: 'darwin-arm64', x64: 'darwin-x64' }, 34 | freebsd: { x64: 'freebsd-x64' }, 35 | 'linux-gnu': { 36 | arm: 'linux-arm-gnueabihf', 37 | arm64: 'linux-arm64-gnu', 38 | x64: 'linux-x64-gnu', 39 | riscv64: 'linux-riscv64-gnu', 40 | s390x: 'linux-s390x-gnu', 41 | }, 42 | 'linux-musl': { 43 | arm: 'linux-arm-musleabihf', 44 | arm64: 'linux-arm64-musl', 45 | x64: 'linux-x64-musl', 46 | riscv64: 'linux-riscv64-musl', 47 | }, 48 | }; 49 | 50 | const platform = 51 | process.platform !== 'linux' ? process.platform : 'linux-' + (isMusl() ? 'musl' : 'gnu'); 52 | const arch = process.arch; 53 | 54 | /** 55 | * @returns {string} 56 | */ 57 | function getPackageName() { 58 | if (!(platform in PACKAGE_NAMES)) { 59 | throw new Error(`Unsupported OS: ${platform}`); 60 | } 61 | if (!(arch in PACKAGE_NAMES[platform])) { 62 | throw new Error(`Unsupported architecture for ${platform}: ${arch}`); 63 | } 64 | return PACKAGE_NAMES[platform][arch]; 65 | } 66 | 67 | const packageName = getPackageName(); 68 | const localPath = path.join( 69 | __dirname, 70 | `npm/${packageName}/intl-message-database.${packageName}.node`, 71 | ); 72 | const packagePath = `@discord/intl-message-database-${packageName}`; 73 | const nativeBinding = fs.existsSync(localPath) ? require(localPath) : require(packagePath); 74 | 75 | const { 76 | hashMessageKey, 77 | isMessageDefinitionsFile, 78 | isMessageTranslationsFile, 79 | IntlMessagesDatabase, 80 | IntlCompiledMessageFormat, 81 | IntlDatabaseInsertStrategy, 82 | } = nativeBinding; 83 | 84 | module.exports = { 85 | hashMessageKey, 86 | isMessageDefinitionsFile, 87 | isMessageTranslationsFile, 88 | IntlMessagesDatabase, 89 | IntlCompiledMessageFormat, 90 | IntlDatabaseInsertStrategy, 91 | }; 92 | -------------------------------------------------------------------------------- /crates/intl_message_database/npm/darwin-arm64/README.md: -------------------------------------------------------------------------------- 1 | # `intl-message-database-darwin-arm64` 2 | 3 | This is the **aarch64-apple-darwin** binary for `intl-message-database` 4 | -------------------------------------------------------------------------------- /crates/intl_message_database/npm/darwin-arm64/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-message-database-darwin-arm64", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "os": [ 6 | "darwin" 7 | ], 8 | "cpu": [ 9 | "arm64" 10 | ], 11 | "main": "intl-message-database.darwin-arm64.node", 12 | "files": [ 13 | "intl-message-database.darwin-arm64.node" 14 | ], 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/discord/discord-intl" 21 | } 22 | } -------------------------------------------------------------------------------- /crates/intl_message_database/npm/darwin-x64/README.md: -------------------------------------------------------------------------------- 1 | # `intl-message-database-darwin-x64` 2 | 3 | This is the **x86_64-apple-darwin** binary for `intl-message-database` 4 | -------------------------------------------------------------------------------- /crates/intl_message_database/npm/darwin-x64/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-message-database-darwin-x64", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "os": [ 6 | "darwin" 7 | ], 8 | "cpu": [ 9 | "x64" 10 | ], 11 | "main": "intl-message-database.darwin-x64.node", 12 | "files": [ 13 | "intl-message-database.darwin-x64.node" 14 | ], 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/discord/discord-intl" 21 | } 22 | } -------------------------------------------------------------------------------- /crates/intl_message_database/npm/linux-arm64-gnu/README.md: -------------------------------------------------------------------------------- 1 | # `intl-message-database-linux-arm64-gnu` 2 | 3 | This is the **aarch64-unknown-linux-gnu** binary for `intl-message-database` 4 | -------------------------------------------------------------------------------- /crates/intl_message_database/npm/linux-arm64-gnu/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-message-database-linux-arm64-gnu", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "os": [ 6 | "linux" 7 | ], 8 | "cpu": [ 9 | "arm64" 10 | ], 11 | "main": "intl-message-database.linux-arm64-gnu.node", 12 | "files": [ 13 | "intl-message-database.linux-arm64-gnu.node" 14 | ], 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "libc": [ 19 | "glibc" 20 | ], 21 | "repository": { 22 | "type": "git", 23 | "url": "https://github.com/discord/discord-intl" 24 | } 25 | } -------------------------------------------------------------------------------- /crates/intl_message_database/npm/linux-arm64-musl/README.md: -------------------------------------------------------------------------------- 1 | # `intl-message-database-linux-arm64-musl` 2 | 3 | This is the **aarch64-unknown-linux-musl** binary for `intl-message-database` 4 | -------------------------------------------------------------------------------- /crates/intl_message_database/npm/linux-arm64-musl/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-message-database-linux-arm64-musl", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "os": [ 6 | "linux" 7 | ], 8 | "cpu": [ 9 | "arm64" 10 | ], 11 | "main": "intl-message-database.linux-arm64-musl.node", 12 | "files": [ 13 | "intl-message-database.linux-arm64-musl.node" 14 | ], 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "libc": [ 19 | "musl" 20 | ], 21 | "repository": { 22 | "type": "git", 23 | "url": "https://github.com/discord/discord-intl" 24 | } 25 | } -------------------------------------------------------------------------------- /crates/intl_message_database/npm/linux-x64-gnu/README.md: -------------------------------------------------------------------------------- 1 | # `intl-message-database-linux-x64-gnu` 2 | 3 | This is the **x86_64-unknown-linux-gnu** binary for `intl-message-database` 4 | -------------------------------------------------------------------------------- /crates/intl_message_database/npm/linux-x64-gnu/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-message-database-linux-x64-gnu", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "os": [ 6 | "linux" 7 | ], 8 | "cpu": [ 9 | "x64" 10 | ], 11 | "main": "intl-message-database.linux-x64-gnu.node", 12 | "files": [ 13 | "intl-message-database.linux-x64-gnu.node" 14 | ], 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "libc": [ 19 | "glibc" 20 | ], 21 | "repository": { 22 | "type": "git", 23 | "url": "https://github.com/discord/discord-intl" 24 | } 25 | } -------------------------------------------------------------------------------- /crates/intl_message_database/npm/linux-x64-musl/README.md: -------------------------------------------------------------------------------- 1 | # `intl-message-database-linux-x64-musl` 2 | 3 | This is the **x86_64-unknown-linux-musl** binary for `intl-message-database` 4 | -------------------------------------------------------------------------------- /crates/intl_message_database/npm/linux-x64-musl/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-message-database-linux-x64-musl", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "os": [ 6 | "linux" 7 | ], 8 | "cpu": [ 9 | "x64" 10 | ], 11 | "main": "intl-message-database.linux-x64-musl.node", 12 | "files": [ 13 | "intl-message-database.linux-x64-musl.node" 14 | ], 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "libc": [ 19 | "musl" 20 | ], 21 | "repository": { 22 | "type": "git", 23 | "url": "https://github.com/discord/discord-intl" 24 | } 25 | } -------------------------------------------------------------------------------- /crates/intl_message_database/npm/win32-arm64-msvc/README.md: -------------------------------------------------------------------------------- 1 | # `intl-message-database-win32-arm64-msvc` 2 | 3 | This is the **aarch64-pc-windows-msvc** binary for `intl-message-database` 4 | -------------------------------------------------------------------------------- /crates/intl_message_database/npm/win32-arm64-msvc/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-message-database-win32-arm64-msvc", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "os": [ 6 | "win32" 7 | ], 8 | "cpu": [ 9 | "arm64" 10 | ], 11 | "main": "intl-message-database.win32-arm64-msvc.node", 12 | "files": [ 13 | "intl-message-database.win32-arm64-msvc.node" 14 | ], 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/discord/discord-intl" 21 | } 22 | } -------------------------------------------------------------------------------- /crates/intl_message_database/npm/win32-ia32-msvc/README.md: -------------------------------------------------------------------------------- 1 | # `intl-message-database-win32-ia32-msvc` 2 | 3 | This is the **i686-pc-windows-msvc** binary for `intl-message-database` -------------------------------------------------------------------------------- /crates/intl_message_database/npm/win32-ia32-msvc/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-message-database-win32-ia32-msvc", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "os": [ 6 | "win32" 7 | ], 8 | "cpu": [ 9 | "ia32" 10 | ], 11 | "main": "intl-message-database.win32-ia32-msvc.node", 12 | "files": [ 13 | "intl-message-database.win32-ia32-msvc.node" 14 | ], 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/discord/discord-intl" 21 | } 22 | } -------------------------------------------------------------------------------- /crates/intl_message_database/npm/win32-x64-msvc/README.md: -------------------------------------------------------------------------------- 1 | # `intl-message-database-win32-x64-msvc` 2 | 3 | This is the **x86_64-pc-windows-msvc** binary for `intl-message-database` 4 | -------------------------------------------------------------------------------- /crates/intl_message_database/npm/win32-x64-msvc/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-message-database-win32-x64-msvc", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "os": [ 6 | "win32" 7 | ], 8 | "cpu": [ 9 | "x64" 10 | ], 11 | "main": "intl-message-database.win32-x64-msvc.node", 12 | "files": [ 13 | "intl-message-database.win32-x64-msvc.node" 14 | ], 15 | "engines": { 16 | "node": ">= 10" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/discord/discord-intl" 21 | } 22 | } -------------------------------------------------------------------------------- /crates/intl_message_database/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-message-database", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "description": "Rust-based Node extension module for parsing, extracting, and managing messages and translations across a project.", 6 | "author": "Jon Egeland", 7 | "main": "./index.js", 8 | "types": "./index.d.ts", 9 | "files": [ 10 | "index.d.ts", 11 | "index.js" 12 | ], 13 | "repository": { 14 | "type": "git", 15 | "url": "https://github.com/discord/discord-intl" 16 | }, 17 | "scripts": { 18 | "build:debug": "cargo build", 19 | "test": "cargo test", 20 | "bench:native": "node ./bench/native.js" 21 | }, 22 | "optionalDependencies": { 23 | "@discord/intl-message-database-darwin-arm64": "workspace:*", 24 | "@discord/intl-message-database-darwin-x64": "workspace:*", 25 | "@discord/intl-message-database-linux-arm64-gnu": "workspace:*", 26 | "@discord/intl-message-database-linux-arm64-musl": "workspace:*", 27 | "@discord/intl-message-database-linux-x64-gnu": "workspace:*", 28 | "@discord/intl-message-database-linux-x64-musl": "workspace:*", 29 | "@discord/intl-message-database-win32-arm64-msvc": "workspace:*", 30 | "@discord/intl-message-database-win32-ia32-msvc": "workspace:*", 31 | "@discord/intl-message-database-win32-x64-msvc": "workspace:*" 32 | }, 33 | "devDependencies": { 34 | "@napi-rs/cli": "^2.18.3", 35 | "@discord/intl-ast": "workspace:*" 36 | }, 37 | "engines": { 38 | "node": ">= 10" 39 | }, 40 | "napi": { 41 | "name": "intl-message-database", 42 | "triples": { 43 | "additional": [ 44 | "aarch64-apple-darwin", 45 | "aarch64-unknown-linux-gnu", 46 | "aarch64-unknown-linux-musl", 47 | "x86_64-unknown-linux-musl", 48 | "aarch64-pc-windows-msvc", 49 | "i686-pc-windows-msvc" 50 | ] 51 | } 52 | } 53 | } -------------------------------------------------------------------------------- /crates/intl_message_database/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod sources; 2 | mod threading; 3 | 4 | pub mod public; 5 | 6 | #[cfg(not(feature = "static_link"))] 7 | pub mod napi; 8 | 9 | extern crate intl_allocator; 10 | -------------------------------------------------------------------------------- /crates/intl_message_utils/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_message_utils" 3 | description = "Utilities used across different crates in the intl package." 4 | version = "0.1.0" 5 | edition = "2021" 6 | publish = false 7 | 8 | [dependencies] 9 | xxhash-rust = { workspace = true } 10 | memchr = { workspace = true } 11 | once_cell = { workspace = true } -------------------------------------------------------------------------------- /crates/intl_message_utils/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/intl_message_utils/README.md: -------------------------------------------------------------------------------- 1 | # intl_message_utils 2 | 3 | Common utility functions used across multiple crates, including "constant" definitions like file type checks and hashing functions. 4 | 5 | This is a library crate that is only built as part of another crate. 6 | -------------------------------------------------------------------------------- /crates/intl_validator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "intl_validator" 3 | description = "Validation utilities for both individual message values and aggregate messages across an entire database" 4 | version = "0.1.0" 5 | edition = "2021" 6 | 7 | [dependencies] 8 | intl_database_core = { workspace = true } 9 | intl_markdown = { workspace = true } 10 | intl_markdown_visitor = { workspace = true } 11 | serde = { workspace = true } -------------------------------------------------------------------------------- /crates/intl_validator/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/intl_validator/README.md: -------------------------------------------------------------------------------- 1 | # intl_validator 2 | 3 | A message Visitor that runs various validations against the message AST to ensure consistency, accuracy, and validity of the content. Results are exposed as a list of `Diagnostic`s that can be serialized and interpreted by clients, like an adapter for ESLint and others. 4 | 5 | This is a library crate that is only built as part of another crate. 6 | -------------------------------------------------------------------------------- /crates/intl_validator/src/content.rs: -------------------------------------------------------------------------------- 1 | use intl_database_core::MessageValue; 2 | 3 | use crate::diagnostic::ValueDiagnostic; 4 | use crate::validators; 5 | use crate::validators::validator::Validator; 6 | 7 | pub fn validate_message_value(message: &MessageValue) -> Vec { 8 | let mut diagnostics: Vec = vec![]; 9 | let mut validators: Vec> = vec![ 10 | Box::new(validators::NoUnicodeVariableNames::new()), 11 | Box::new(validators::NoRepeatedPluralNames::new()), 12 | Box::new(validators::NoRepeatedPluralOptions::new()), 13 | Box::new(validators::NoTrimmableWhitespace::new()), 14 | ]; 15 | for validator in validators.iter_mut() { 16 | if let Some(result) = validator.validate_raw(message) { 17 | diagnostics.extend(result); 18 | } 19 | if let Some(result) = validator.validate_ast(message) { 20 | diagnostics.extend(result); 21 | } 22 | } 23 | 24 | diagnostics 25 | } 26 | -------------------------------------------------------------------------------- /crates/intl_validator/src/diagnostic.rs: -------------------------------------------------------------------------------- 1 | use intl_database_core::{FilePosition, KeySymbol}; 2 | 3 | use crate::DiagnosticSeverity; 4 | 5 | #[derive(Clone, Copy, Debug)] 6 | #[repr(u8)] 7 | pub enum DiagnosticName { 8 | NoExtraTranslationVariables, 9 | NoMissingSourceVariables, 10 | NoRepeatedPluralNames, 11 | NoRepeatedPluralOptions, 12 | NoTrimmableWhitespace, 13 | NoUnicodeVariableNames, 14 | } 15 | 16 | impl DiagnosticName { 17 | pub fn as_str(&self) -> &'static str { 18 | match self { 19 | DiagnosticName::NoExtraTranslationVariables => "NoExtraTranslationVariables", 20 | DiagnosticName::NoMissingSourceVariables => "NoMissingSourceVariables", 21 | DiagnosticName::NoRepeatedPluralNames => "NoRepeatedPluralNames", 22 | DiagnosticName::NoRepeatedPluralOptions => "NoRepeatedPluralOptions", 23 | DiagnosticName::NoTrimmableWhitespace => "NoTrimmableWhitespace", 24 | DiagnosticName::NoUnicodeVariableNames => "NoUnicodeVariableNames", 25 | } 26 | } 27 | } 28 | 29 | impl ToString for DiagnosticName { 30 | fn to_string(&self) -> String { 31 | self.as_str().into() 32 | } 33 | } 34 | 35 | pub struct MessageDiagnostic { 36 | pub key: KeySymbol, 37 | pub file_position: FilePosition, 38 | pub locale: KeySymbol, 39 | pub name: DiagnosticName, 40 | pub severity: DiagnosticSeverity, 41 | pub description: String, 42 | pub help: Option, 43 | } 44 | 45 | #[derive(Debug, Clone)] 46 | pub struct ValueDiagnostic { 47 | pub name: DiagnosticName, 48 | pub span: Option, 49 | pub severity: DiagnosticSeverity, 50 | pub description: String, 51 | pub help: Option, 52 | } 53 | 54 | pub struct MessageDiagnosticsBuilder { 55 | pub diagnostics: Vec, 56 | pub key: KeySymbol, 57 | } 58 | 59 | impl MessageDiagnosticsBuilder { 60 | pub fn new(key: KeySymbol) -> Self { 61 | Self { 62 | diagnostics: vec![], 63 | key, 64 | } 65 | } 66 | 67 | pub fn add(&mut self, diagnostic: MessageDiagnostic) { 68 | self.diagnostics.push(diagnostic); 69 | } 70 | 71 | pub fn extend_from_value_diagnostics( 72 | &mut self, 73 | value_diagnostics: Vec, 74 | file_position: FilePosition, 75 | locale: KeySymbol, 76 | ) { 77 | let converted_diagnostics = 78 | value_diagnostics 79 | .into_iter() 80 | .map(|diagnostic| MessageDiagnostic { 81 | key: self.key, 82 | file_position, 83 | locale, 84 | name: diagnostic.name, 85 | severity: diagnostic.severity, 86 | description: diagnostic.description, 87 | help: diagnostic.help, 88 | }); 89 | 90 | self.diagnostics.extend(converted_diagnostics); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /crates/intl_validator/src/severity.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Formatter; 2 | 3 | use serde::{Serialize, Serializer}; 4 | 5 | #[derive(Debug, Clone, Copy)] 6 | pub enum DiagnosticSeverity { 7 | Info, 8 | Warning, 9 | Error, 10 | } 11 | 12 | impl DiagnosticSeverity { 13 | pub fn as_str(&self) -> &str { 14 | match self { 15 | Self::Info => "info", 16 | Self::Warning => "warning", 17 | Self::Error => "error", 18 | } 19 | } 20 | } 21 | 22 | impl Serialize for DiagnosticSeverity { 23 | fn serialize(&self, serializer: S) -> Result 24 | where 25 | S: Serializer, 26 | { 27 | serializer.serialize_str(&self.as_str()) 28 | } 29 | } 30 | 31 | impl std::fmt::Display for DiagnosticSeverity { 32 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 33 | f.write_str(self.as_str()) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /crates/intl_validator/src/validators/mod.rs: -------------------------------------------------------------------------------- 1 | pub use no_repeated_plural_names::NoRepeatedPluralNames; 2 | pub use no_repeated_plural_options::NoRepeatedPluralOptions; 3 | pub use no_trimmable_whitespace::NoTrimmableWhitespace; 4 | pub use no_unicode_variable_names::NoUnicodeVariableNames; 5 | 6 | mod no_repeated_plural_names; 7 | mod no_repeated_plural_options; 8 | mod no_trimmable_whitespace; 9 | mod no_unicode_variable_names; 10 | 11 | pub mod validator; 12 | -------------------------------------------------------------------------------- /crates/intl_validator/src/validators/no_repeated_plural_options.rs: -------------------------------------------------------------------------------- 1 | use intl_database_core::MessageValue; 2 | use intl_markdown::IcuPlural; 3 | use intl_markdown_visitor::{visit_with_mut, Visit}; 4 | use std::collections::HashSet; 5 | 6 | use crate::diagnostic::{DiagnosticName, ValueDiagnostic}; 7 | use crate::validators::validator::Validator; 8 | use crate::DiagnosticSeverity; 9 | 10 | pub struct NoRepeatedPluralOptions { 11 | diagnostics: Vec, 12 | } 13 | 14 | impl NoRepeatedPluralOptions { 15 | pub fn new() -> Self { 16 | Self { 17 | diagnostics: vec![], 18 | } 19 | } 20 | } 21 | 22 | impl Validator for NoRepeatedPluralOptions { 23 | fn validate_ast(&mut self, message: &MessageValue) -> Option> { 24 | visit_with_mut(&message.parsed, self); 25 | Some(self.diagnostics.clone()) 26 | } 27 | } 28 | 29 | impl Visit for NoRepeatedPluralOptions { 30 | fn visit_icu_plural(&mut self, node: &IcuPlural) { 31 | let plural_name = node.name(); 32 | let arm_names = node.arms().iter().map(|arm| arm.selector().as_str()); 33 | let mut seen = HashSet::new(); 34 | // Allotting enough capacity to handle basically every possible case. More than 4 35 | // repetitions is egregious and there will almost never be more than 1, but this just 36 | // ensures it's always consistent allocation. 37 | let mut repeated_names: Vec<&str> = Vec::with_capacity(4); 38 | 39 | for name in arm_names { 40 | if seen.contains(name) { 41 | repeated_names.push(name); 42 | } else { 43 | seen.insert(name); 44 | } 45 | } 46 | 47 | for name in repeated_names { 48 | let diagnostic = ValueDiagnostic { 49 | name: DiagnosticName::NoRepeatedPluralOptions, 50 | span: None, 51 | severity: DiagnosticSeverity::Error, 52 | description: String::from( 53 | "Plural options must be unique within the plural selector", 54 | ), 55 | help: Some(format!("The option '{name}' is present more than once in the plural value '{plural_name}'. Remove or rename one of these options to fix it.")), 56 | }; 57 | 58 | self.diagnostics.push(diagnostic); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /crates/intl_validator/src/validators/no_trimmable_whitespace.rs: -------------------------------------------------------------------------------- 1 | use intl_database_core::MessageValue; 2 | 3 | use crate::diagnostic::{DiagnosticName, ValueDiagnostic}; 4 | use crate::validators::validator::Validator; 5 | use crate::DiagnosticSeverity; 6 | 7 | pub struct NoTrimmableWhitespace; 8 | impl NoTrimmableWhitespace { 9 | pub fn new() -> Self { 10 | Self 11 | } 12 | } 13 | 14 | impl Validator for NoTrimmableWhitespace { 15 | fn validate_raw(&mut self, message: &MessageValue) -> Option> { 16 | let mut diagnostics = vec![]; 17 | let content = &message.raw; 18 | if content.trim_start() != content { 19 | diagnostics.push(ValueDiagnostic { 20 | name: DiagnosticName::NoTrimmableWhitespace, 21 | span: None, 22 | severity: DiagnosticSeverity::Warning, 23 | description: "Avoid leading whitespace on messages".into(), 24 | help: Some("Leading whitespace is visually ambiguous when translating and leads to inconsistency".into()) 25 | }) 26 | } 27 | if content.trim_end() != content { 28 | diagnostics.push(ValueDiagnostic { 29 | name: DiagnosticName::NoTrimmableWhitespace, 30 | span: None, 31 | severity: DiagnosticSeverity::Warning, 32 | description: "Avoid trailing whitespace on messages".into(), 33 | help: Some("Trailing whitespace is visually ambiguous when translating and leads to inconsistency".into()) 34 | }) 35 | } 36 | Some(diagnostics) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /crates/intl_validator/src/validators/no_unicode_variable_names.rs: -------------------------------------------------------------------------------- 1 | use intl_database_core::MessageValue; 2 | use intl_markdown::IcuVariable; 3 | use intl_markdown_visitor::{visit_with_mut, Visit}; 4 | 5 | use crate::diagnostic::{DiagnosticName, ValueDiagnostic}; 6 | use crate::validators::validator::Validator; 7 | use crate::DiagnosticSeverity; 8 | 9 | pub struct NoUnicodeVariableNames { 10 | diagnostics: Vec, 11 | } 12 | 13 | impl NoUnicodeVariableNames { 14 | pub fn new() -> Self { 15 | Self { 16 | diagnostics: vec![], 17 | } 18 | } 19 | } 20 | 21 | impl Validator for NoUnicodeVariableNames { 22 | fn validate_ast(&mut self, message: &MessageValue) -> Option> { 23 | visit_with_mut(&message.parsed, self); 24 | Some(self.diagnostics.clone()) 25 | } 26 | } 27 | 28 | impl Visit for NoUnicodeVariableNames { 29 | fn visit_icu_variable(&mut self, node: &IcuVariable) { 30 | let name = node.name(); 31 | if !name.is_ascii() { 32 | let help_text = format!("\"{name}\" should be renamed to only use ASCII characters. If this is a translation, ensure the name matches the expected name in the source text"); 33 | self.diagnostics.push(ValueDiagnostic { 34 | name: DiagnosticName::NoUnicodeVariableNames, 35 | span: None, 36 | severity: DiagnosticSeverity::Error, 37 | description: "Variable names should not contain unicode characters to avoid ambiguity during translation".into(), 38 | help: Some(help_text), 39 | }); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /crates/intl_validator/src/validators/validator.rs: -------------------------------------------------------------------------------- 1 | use intl_database_core::MessageValue; 2 | 3 | use crate::diagnostic::ValueDiagnostic; 4 | 5 | pub trait Validator { 6 | fn validate_raw(&mut self, _message: &MessageValue) -> Option> { 7 | None 8 | } 9 | 10 | fn validate_ast(&mut self, _message: &MessageValue) -> Option> { 11 | None 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /crates/keyless_json/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "keyless_json" 3 | description = "A Serde serializer that allows well-structured JSON to be minified into keyless arrays of elements." 4 | version = "0.1.0" 5 | edition = "2021" 6 | publish = false 7 | 8 | [dependencies] 9 | serde = { workspace = true } 10 | itoa = "1" 11 | ryu = "1" 12 | thiserror = { workspace = true } -------------------------------------------------------------------------------- /crates/keyless_json/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /crates/keyless_json/README.md: -------------------------------------------------------------------------------- 1 | # Keyless JSON 2 | 3 | Keyless JSON is a minified, array-based serialization format that allows well-known structures to be efficiently 4 | represented without any duplication of key names. Keyless JSON is similar to MessagePack with the Record extension, 5 | using structures known ahead of time to greatly reduce the size of the serialized data. 6 | 7 | Deserializing Keyless JSON requires knowing the expected structure and _will fail_ if the data does not match the 8 | expected structure. 9 | 10 | For example, an AST for a localized message might look like this when serialized to plain JSON: 11 | 12 | ```text 13 | Hello, {username}. You have {messageCount, plural, one {# new message.} other {# new messages.}} 14 | ``` 15 | 16 | ```json 17 | [ 18 | { 19 | "type": 0, 20 | "value": "Hello, " 21 | }, 22 | { 23 | "type": 1, 24 | "value": "username" 25 | }, 26 | { 27 | "type": 0, 28 | "value": ". You have " 29 | }, 30 | { 31 | "type": 6, 32 | "value": "messageCount", 33 | "options": { 34 | "one": { 35 | "value": [ 36 | { 37 | "type": 7 38 | }, 39 | { 40 | "type": 0, 41 | "value": " new message." 42 | } 43 | ] 44 | }, 45 | "other": { 46 | "value": [ 47 | { 48 | "type": 7 49 | }, 50 | { 51 | "type": 0, 52 | "value": " new messages." 53 | } 54 | ] 55 | } 56 | } 57 | } 58 | ] 59 | ``` 60 | 61 | When serialized with keyless JSON, the repeated `type` and `value` keys can be left out thanks to the well-known 62 | structure of each node in this tree (each node has a `type` and `value` in the same order): 63 | 64 | ```json 65 | [ 66 | [0, "Hello, "], 67 | [1, "username"], 68 | [0, ". You have"], 69 | [ 70 | 6, 71 | "messageCount", 72 | { 73 | "one": { 74 | "value": [[7], [0, "new message."]] 75 | }, 76 | "other": { 77 | "value": [[7], [0, "new messages."]] 78 | } 79 | } 80 | ] 81 | ] 82 | ``` 83 | 84 | These examples are not minified fully, but showcase just how much repetition can be omitted by pre-defining the keys. 85 | 86 | This is a library crate that is only built as part of another crate. 87 | -------------------------------------------------------------------------------- /crates/keyless_json/src/error.rs: -------------------------------------------------------------------------------- 1 | use serde::ser; 2 | use thiserror::Error as ThisError; 3 | 4 | #[derive(Debug, ThisError)] 5 | pub enum Error { 6 | #[error("IOError: {0}")] 7 | IoError(std::io::Error), 8 | #[error("{0}")] 9 | CustomError(String), 10 | } 11 | 12 | pub type Result = core::result::Result; 13 | 14 | impl From for Error { 15 | fn from(value: std::io::Error) -> Self { 16 | Error::IoError(value) 17 | } 18 | } 19 | 20 | impl ser::Error for Error { 21 | #[cold] 22 | fn custom(msg: T) -> Error { 23 | Error::CustomError(msg.to_string()) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /crates/keyless_json/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub use serializer::{to_string, to_writer, Serializer}; 2 | pub use string::write_escaped_str_contents; 3 | 4 | mod error; 5 | mod serializer; 6 | mod string; 7 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "discord-intl-root", 3 | "description": "The monorepo workspace package for managing tools across the project.", 4 | "version": "none", 5 | "private": true, 6 | "devDependencies": { 7 | "@discord/intl-tools": "workspace:*", 8 | "@types/node": "22.0.0", 9 | "@types/react": "*", 10 | "eslint": "*", 11 | "prettier": "*", 12 | "react": "*", 13 | "typescript": "*" 14 | }, 15 | "pnpm": { 16 | "overrides": { 17 | "@types/react": "18", 18 | "react": "18", 19 | "typescript": "5.5.4", 20 | "prettier": "3.3.3" 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /packages/babel-plugin-transform-discord-intl/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /packages/babel-plugin-transform-discord-intl/README.md: -------------------------------------------------------------------------------- 1 | # babel-plugin-transform-discord-intl 2 | 3 | A Babel plugin for transforming intl message _usages_ to obfuscated, minified strings, saving on bundle size and allowing for complete anonymity of string names. 4 | 5 | # Development 6 | 7 | This package is plain CommonJS with JSDoc types that requires no compilation. It can be tested locally by using `file:` or `link:` references from another project. No build step is required. 8 | 9 | # Usage 10 | 11 | This plugin is usable with Babel 7+. There is minimal configuration other than `extraImports` for adding additional names to check and transform usages for. 12 | 13 | ```js 14 | [ 15 | [ 16 | require('@discord/babel-plugin-transform-discord-intl'), 17 | { 18 | extraImports: { 19 | [path.resolve('some/source/file')]: ['t', 'otherMessagesName'], 20 | }, 21 | }, 22 | ], 23 | ]; 24 | ``` 25 | -------------------------------------------------------------------------------- /packages/babel-plugin-transform-discord-intl/index.js: -------------------------------------------------------------------------------- 1 | const { hashMessageKey } = require('@discord/intl-loader-core'); 2 | const { traverseMessageAccesses } = require('./traverse'); 3 | 4 | /** 5 | * Babel plugin for obfuscating and minifying intl message _usages_, like 6 | * turning `intl.format(t.SOME_LONG_MESSAGE_KEY_NAME)` into `intl.format(t['f21c/3'])`. 7 | * This transform gets applied to every file in the project, but only member 8 | * expressions of imports from `.messages.js` files are considered and affected. 9 | * 10 | * Configuration for `extraImports` differs from swc-intl-message-transformer in that 11 | * paths here are resolved, _absolute_ paths to the desired file, since babel 12 | * will often have re-written the AST to resolve import aliases by the time 13 | * this plugin runs. 14 | * 15 | * @param {any} babel - The Babel core object. 16 | * @returns {{visitor: import("babel__traverse").Visitor}} A visitor object for the Babel transform. 17 | */ 18 | module.exports = function babelPluginTransformDiscordIntl(babel) { 19 | /** @type {{types: import("@babel/types")}} */ 20 | const { types: t } = babel; 21 | 22 | return { 23 | visitor: traverseMessageAccesses((access, messageName) => { 24 | if (messageName == null) { 25 | throw new Error( 26 | '[INTL] Encountered a member expression with neither an identifier nor string literal member node', 27 | ); 28 | } 29 | 30 | // Then hash it up and re-write the member with the hashed version. 31 | access.computed = true; 32 | access.property = t.stringLiteral(hashMessageKey(messageName)); 33 | }), 34 | }; 35 | }; 36 | -------------------------------------------------------------------------------- /packages/babel-plugin-transform-discord-intl/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/babel-plugin-transform-discord-intl", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "type": "commonjs", 6 | "description": "Babel plugin for transforming message accesses and definitions", 7 | "main": "index.js", 8 | "exports": { 9 | ".": "./index.js", 10 | "./traverse": "./traverse.js" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "https://github.com/discord/discord-intl" 15 | }, 16 | "dependencies": { 17 | "@discord/intl-loader-core": "workspace:*", 18 | "@typescript-eslint/scope-manager": "^8.8.1" 19 | }, 20 | "devDependencies": { 21 | "@babel/types": "^7.24.9", 22 | "@types/babel__traverse": "7.20.6" 23 | } 24 | } -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/README.md: -------------------------------------------------------------------------------- 1 | # eslint-plugin-discord-intl 2 | 3 | An ESLint (v8) plugin for linting intl messages in the `@discord/intl` system. Currently this linter only checks best 4 | practices for messages like whitespace and access notation in other source files, but will eventually be able to lint 5 | the content of messages themselves using the database's `validateMessages` API. 6 | 7 | ## Install 8 | 9 | ```shell 10 | pnpm add -D @discord/eslint-plugin-discord-intl 11 | ``` 12 | 13 | ## Configure 14 | 15 | ```javascript 16 | module.exports = { 17 | // Extend the recommended plugin configuration 18 | extends: ['plugin:@discord/eslint-plugin-discord-intl/recommended'], 19 | // Optionally add configuration for all rules 20 | settings: { 21 | '@discord/discord-intl': { 22 | extraImports: { 23 | '@app/intl': ['t'], 24 | }, 25 | }, 26 | }, 27 | }; 28 | ``` 29 | -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | rules: { 3 | 'no-repeated-plural-names': require('./rules/native/no-repeated-plural-names'), 4 | 'no-repeated-plural-options': require('./rules/native/no-repeated-plural-options'), 5 | 'no-trimmable-whitespace': require('./rules/native/no-trimmable-whitespace'), 6 | 'no-unicode-variable-names': require('./rules/native/no-unicode-variable-names'), 7 | 'no-duplicate-message-keys': require('./rules/native/no-duplicate-message-keys'), 8 | 9 | 'use-static-access': require('./rules/use-static-access'), 10 | 'no-opaque-messages-objects': require('./rules/no-opaque-messages-objects'), 11 | }, 12 | configs: { 13 | recommended: { 14 | plugins: ['@discord/discord-intl'], 15 | rules: { 16 | // Native rules 17 | '@discord/discord-intl/no-trimmable-whitespace': 'error', 18 | '@discord/discord-intl/no-repeated-plural-names': 'error', 19 | '@discord/discord-intl/no-repeated-plural-options': 'error', 20 | '@discord/discord-intl/no-unicode-variable-names': 'error', 21 | 22 | // JS rules 23 | '@discord/discord-intl/use-static-access': 'error', 24 | '@discord/discord-intl/no-duplicate-message-keys': 'error', 25 | '@discord/discord-intl/no-opaque-messages-objects': 'error', 26 | }, 27 | }, 28 | }, 29 | }; 30 | -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/lib/is-typescript.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Returns true if the current execution is using a TypeScript parser, using a best-effort guess 3 | * based on the given context. 4 | * @param {import('eslint').Rule.RuleContext} context 5 | * @returns {boolean} 6 | */ 7 | function isTypeScript(context) { 8 | return context.parserPath?.includes('@typescript-eslint') ?? false; 9 | } 10 | 11 | module.exports = { isTypeScript }; 12 | -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/eslint-plugin-discord-intl", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "type": "commonjs", 6 | "description": "ESLint plugin for validating and linting messages and translations using @discord/intl", 7 | "main": "index.js", 8 | "repository": { 9 | "type": "git", 10 | "url": "https://github.com/discord/discord-intl" 11 | }, 12 | "dependencies": { 13 | "@discord/intl-loader-core": "workspace:*", 14 | "@typescript-eslint/scope-manager": "^8.8.1" 15 | }, 16 | "devDependencies": { 17 | "@types/eslint": "^8.56.12", 18 | "@types/estree": "^1.0.6", 19 | "@typescript-eslint/parser": "^8.8.1", 20 | "eslint": "8.57.1" 21 | } 22 | } -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/rules/native/no-duplicate-message-keys.js: -------------------------------------------------------------------------------- 1 | const { traverseAndReportMatchingNativeValidations } = require('../../lib/native-validation'); 2 | 3 | module.exports = /** @type {import('eslint').Rule.RuleModule} */ ({ 4 | meta: { 5 | fixable: 'code', 6 | type: 'problem', 7 | docs: { 8 | description: 'Prevent message keys from being repeated across the entire database.', 9 | }, 10 | }, 11 | create(context) { 12 | return traverseAndReportMatchingNativeValidations( 13 | context, 14 | (diagnostic) => diagnostic.name === 'Processing::AlreadyDefined', 15 | ); 16 | }, 17 | }); 18 | -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/rules/native/no-repeated-plural-names.js: -------------------------------------------------------------------------------- 1 | const { traverseAndReportMatchingNativeValidations } = require('../../lib/native-validation'); 2 | 3 | module.exports = /** @type {import('eslint').Rule.RuleModule} */ ({ 4 | meta: { 5 | fixable: 'code', 6 | docs: { 7 | description: 'Disallow whitespace at the beginning and end of intl messages', 8 | category: 'Best Practices', 9 | }, 10 | }, 11 | create(context) { 12 | return traverseAndReportMatchingNativeValidations( 13 | context, 14 | (diagnostic) => diagnostic.name === 'NoRepeatedPluralNames', 15 | ); 16 | }, 17 | }); 18 | -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/rules/native/no-repeated-plural-options.js: -------------------------------------------------------------------------------- 1 | const { traverseAndReportMatchingNativeValidations } = require('../../lib/native-validation'); 2 | 3 | module.exports = /** @type {import('eslint').Rule.RuleModule} */ ({ 4 | meta: { 5 | fixable: 'code', 6 | docs: { 7 | description: 'Disallow whitespace at the beginning and end of intl messages', 8 | category: 'Best Practices', 9 | }, 10 | }, 11 | create(context) { 12 | return traverseAndReportMatchingNativeValidations( 13 | context, 14 | (diagnostic) => diagnostic.name === 'NoRepeatedPluralOptions', 15 | ); 16 | }, 17 | }); 18 | -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/rules/native/no-trimmable-whitespace.js: -------------------------------------------------------------------------------- 1 | const { traverseAndReportMatchingNativeValidations } = require('../../lib/native-validation'); 2 | 3 | module.exports = /** @type {import('eslint').Rule.RuleModule} */ ({ 4 | meta: { 5 | fixable: 'code', 6 | docs: { 7 | description: 'Disallow whitespace at the beginning and end of intl messages', 8 | category: 'Best Practices', 9 | }, 10 | }, 11 | create(context) { 12 | return traverseAndReportMatchingNativeValidations( 13 | context, 14 | (diagnostic) => diagnostic.name === 'NoTrimmableWhitespace', 15 | ); 16 | }, 17 | }); 18 | -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/rules/native/no-trimmable-whitespace.test.js: -------------------------------------------------------------------------------- 1 | // enforce-foo-bar.test.js 2 | const { RuleTester } = require('eslint'); 3 | const noTrimmableWhitespace = require('./no-trimmable-whitespace'); 4 | 5 | const ruleTester = new RuleTester({ 6 | // Must use at least ecmaVersion 2015 because 7 | // that's when `const` variables were introduced. 8 | parserOptions: { ecmaVersion: 2015, sourceType: 'module' }, 9 | }); 10 | 11 | /** 12 | * @param {string} messages 13 | */ 14 | function defineMessages(messages) { 15 | return ` 16 | import {defineMessages} from '@discord/intl'; 17 | 18 | export default defineMessages(${messages}); 19 | `; 20 | } 21 | 22 | ruleTester.run('no-trimmable-whitespace', noTrimmableWhitespace, { 23 | valid: [ 24 | { 25 | name: 'normal strings', 26 | code: defineMessages("{ A: 'no trimmed whitespace' }"), 27 | }, 28 | { 29 | name: 'templates', 30 | code: defineMessages( 31 | '{ A: `no trimmed whitespace`, QUASI: `${ space }`, MULTILINE: `hi\n yes` }', 32 | ), 33 | }, 34 | { 35 | name: 'multi-line', 36 | code: defineMessages( 37 | `{ A: \`no trimmed 38 | whitespace\`}`, 39 | ), 40 | }, 41 | { 42 | name: 'object', 43 | code: defineMessages( 44 | '{ A: { message: "no whitespace", description: " does not matter here " }}', 45 | ), 46 | }, 47 | ], 48 | invalid: [ 49 | { 50 | code: defineMessages(`{ 51 | A: ' leading whitespace', 52 | B: 'trailing whitespace ', 53 | C: ' surrounding whitespace ', 54 | }`), 55 | errors: 4, 56 | }, 57 | { 58 | code: defineMessages(`{ 59 | TABS: '\tleading whitespace', 60 | B: 'trailing whitespace\t', 61 | C: '\tsurrounding whitespace\t', 62 | }`), 63 | errors: 4, 64 | }, 65 | { 66 | code: defineMessages(`{ 67 | NEWLINES: '\\nleading whitespace', 68 | B: 'trailing whitespace\\n', 69 | C: '\\nsurrounding whitespace\\n', 70 | }`), 71 | errors: 4, 72 | }, 73 | { 74 | code: defineMessages(`{ 75 | MIXED: '\\n \\t leading\\n \\twhitespace \\t\\n', 76 | }`), 77 | errors: 2, 78 | }, 79 | { 80 | code: defineMessages(`{ 81 | ONLY_BROKEN: '\\n \\t leading\\n \\twhitespace \\t\\n', 82 | VALID: 'valid string', 83 | }`), 84 | errors: 2, 85 | }, 86 | { 87 | name: 'object', 88 | code: defineMessages( 89 | '{ A: { message: " no whitespace", description: " does not matter here " }}', 90 | ), 91 | errors: 1, 92 | }, 93 | ], 94 | }); 95 | 96 | console.log('All tests passed!'); 97 | -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/rules/native/no-unicode-variable-names.js: -------------------------------------------------------------------------------- 1 | const { traverseAndReportMatchingNativeValidations } = require('../../lib/native-validation'); 2 | 3 | module.exports = /** @type {import('eslint').Rule.RuleModule} */ ({ 4 | meta: { 5 | fixable: 'code', 6 | docs: { 7 | description: 'Disallow whitespace at the beginning and end of intl messages', 8 | category: 'Best Practices', 9 | }, 10 | }, 11 | create(context) { 12 | return traverseAndReportMatchingNativeValidations( 13 | context, 14 | (diagnostic) => diagnostic.name === 'NoUnicodeVariableNames', 15 | ); 16 | }, 17 | }); 18 | -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/rules/no-duplicate-message-keys.test.js: -------------------------------------------------------------------------------- 1 | const { RuleTester } = require('eslint'); 2 | const noDuplicateMessageKeys = require('./no-duplicate-message-keys'); 3 | 4 | const typescriptParser = require.resolve('@typescript-eslint/parser'); 5 | const ruleTester = new RuleTester({ 6 | // Must use at least ecmaVersion 2015 because 7 | // that's when `const` variables were introduced. 8 | parserOptions: { ecmaVersion: 2015, sourceType: 'module' }, 9 | }); 10 | 11 | ruleTester.run('no-duplicate-message-keys', noDuplicateMessageKeys, { 12 | valid: [ 13 | { 14 | name: 'different messages', 15 | code: ` 16 | import {defineMessages} from '@discord/intl'; 17 | export default defineMessages({ 18 | MESSAGE_ONE: 'hello', 19 | MESSAGE_TWO: 'world', 20 | }); 21 | `, 22 | }, 23 | { 24 | name: 'different message keys with same value', 25 | code: ` 26 | import {defineMessages} from '@discord/intl'; 27 | export default defineMessages({ 28 | MESSAGE_ONE: 'hello', 29 | MESSAGE_TWO: 'hello', 30 | }); 31 | `, 32 | }, 33 | ], 34 | invalid: [ 35 | { 36 | name: 'repeated message key', 37 | code: ` 38 | import {defineMessages} from '@discord/intl'; 39 | export default defineMessages({ 40 | MESSAGE_ONE: 'hello', 41 | MESSAGE_ONE: 'world', 42 | }); 43 | `, 44 | errors: 1, 45 | }, 46 | { 47 | name: 'repeated with others present', 48 | code: ` 49 | import {defineMessages} from '@discord/intl'; 50 | export default defineMessages({ 51 | MESSAGE_ONE: 'hello', 52 | MESSAGE_TWO: 'world', 53 | MESSAGE_ONE: 'world', 54 | }); 55 | `, 56 | errors: 1, 57 | }, 58 | { 59 | name: 'multiple repeated messages', 60 | code: ` 61 | import {defineMessages} from '@discord/intl'; 62 | export default defineMessages({ 63 | MESSAGE_ONE: 'hello', 64 | MESSAGE_TWO: 'world', 65 | MESSAGE_ONE: 'linting', 66 | MESSAGE_TWO: 'messages', 67 | }); 68 | `, 69 | errors: 2, 70 | }, 71 | { 72 | name: 'repeated with the same value', 73 | code: ` 74 | import {defineMessages} from '@discord/intl'; 75 | export default defineMessages({ 76 | MESSAGE_ONE: 'hello', 77 | MESSAGE_ONE: 'hello', 78 | }); 79 | `, 80 | errors: 1, 81 | }, 82 | ], 83 | }); 84 | 85 | console.log('All tests passed!'); 86 | -------------------------------------------------------------------------------- /packages/eslint-plugin-discord-intl/rules/no-opaque-messages-objects.js: -------------------------------------------------------------------------------- 1 | const { traverseMessageObjectReferences } = require('../lib/traverse'); 2 | const { isTypeScript } = require('../lib/is-typescript'); 3 | 4 | module.exports = /** @type {import('eslint').Rule.RuleModule} */ ({ 5 | meta: { 6 | docs: { 7 | description: 8 | 'Disallow using whole messages objects as singular values, through passing as arguments to functions, taking the type of the object, and more.', 9 | category: 'Best Practices', 10 | }, 11 | messages: { 12 | noObjectArgument: 13 | 'Avoid passing message objects around as parameters. Use messages individually', 14 | noTypeof: 15 | 'Avoid requesting the type of an entire messages object. Use messages individually.', 16 | noSpread: 'Avoid spreading a messages object into another object', 17 | noReferenceHolding: 18 | 'Avoid holding references to entire message objects. Use messages individually', 19 | }, 20 | }, 21 | create(context) { 22 | return traverseMessageObjectReferences(context, (reference) => { 23 | const parent = reference.parent; 24 | switch (parent.type) { 25 | case 'CallExpression': 26 | context.report({ 27 | node: reference, 28 | messageId: 'noObjectArgument', 29 | }); 30 | return; 31 | case 'SpreadElement': 32 | context.report({ 33 | node: reference, 34 | messageId: 'noSpread', 35 | }); 36 | return; 37 | case 'Property': 38 | context.report({ 39 | node: reference, 40 | messageId: 'noReferenceHolding', 41 | }); 42 | return; 43 | case 'UnaryExpression': 44 | if (parent.operator === 'typeof') { 45 | context.report({ 46 | node: parent, 47 | messageId: 'noTypeof', 48 | }); 49 | return; 50 | } 51 | } 52 | 53 | if (isTypeScript(context)) { 54 | // @ts-expect-error TSNodes 55 | if (parent.type === 'TSTypeQuery') { 56 | context.report({ 57 | node: parent, 58 | messageId: 'noTypeof', 59 | }); 60 | return; 61 | } 62 | } 63 | 64 | // Any other expression, if it's not a message access, is "opaque", so we want to report on 65 | // it with _some_ kind of generic info. 66 | if ( 67 | parent.type !== 'MemberExpression' && 68 | // @ts-expect-error TSNodes 69 | parent.type !== 'TSQualifiedName' 70 | ) { 71 | context.report({ 72 | node: reference, 73 | messageId: 'noReferenceHolding', 74 | }); 75 | } 76 | }); 77 | }, 78 | }); 79 | -------------------------------------------------------------------------------- /packages/intl-ast/.gitignore: -------------------------------------------------------------------------------- 1 | dist/ -------------------------------------------------------------------------------- /packages/intl-ast/README.md: -------------------------------------------------------------------------------- 1 | # intl-ast 2 | 3 | AST type definitions and conversion utilities for compiled messages to be used at runtime. 4 | -------------------------------------------------------------------------------- /packages/intl-ast/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-ast", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "description": "Types and utilities for working with the ICU+Markdown AST format from @discord/intl", 6 | "main": "./index.ts", 7 | "exports": { 8 | ".": { 9 | "import": "./index.ts", 10 | "require": "./dist/index.js" 11 | } 12 | }, 13 | "publishConfig": { 14 | "main": "./dist/index.js", 15 | "exports": { 16 | ".": { 17 | "types": "./dist/index.d.ts", 18 | "default": "./dist/index.js" 19 | } 20 | } 21 | }, 22 | "files": [ 23 | "dist", 24 | "src" 25 | ], 26 | "repository": { 27 | "type": "git", 28 | "url": "https://github.com/discord/discord-intl" 29 | }, 30 | "scripts": { 31 | "build": "tsc", 32 | "build:release": "tsc", 33 | "prepublishOnly": "pnpm build:release" 34 | }, 35 | "devDependencies": { 36 | "typescript": "*" 37 | } 38 | } -------------------------------------------------------------------------------- /packages/intl-ast/tsconfig.json: -------------------------------------------------------------------------------- 1 | // Configuration for projects that run in Browser/client environments. 2 | { 3 | "compilerOptions": { 4 | "lib": ["es2015"], 5 | "noEmit": false, 6 | "outDir": "./dist", 7 | "declaration": true, 8 | "declarationDir": "./dist", 9 | "module": "commonjs", 10 | "target": "ES2015", 11 | }, 12 | } 13 | -------------------------------------------------------------------------------- /packages/intl-loader-core/.gitignore: -------------------------------------------------------------------------------- 1 | types/ -------------------------------------------------------------------------------- /packages/intl-loader-core/README.md: -------------------------------------------------------------------------------- 1 | # @discord/intl-loader-core 2 | 3 | This package acts as a core set of utilities and processes that loaders and transformers can rely on to implement translation discovery, compilation, and management in a consistent way. It implements a core transformer to compile a source file into a loader runtime for `@discord/intl`, as well as functions for scanning the file system for translations, watching for changes, emitting typescript type definitions, and more. 4 | 5 | For the most part, consuming packages should never have to interact with the message database directly when using this package, which allows changes in the native extension to be masked and swapped out as needed. However, a `database` instance is exposed for cases where additional functionality is needed. 6 | -------------------------------------------------------------------------------- /packages/intl-loader-core/index.js: -------------------------------------------------------------------------------- 1 | const { 2 | hashMessageKey, 3 | isMessageDefinitionsFile, 4 | isMessageTranslationsFile, 5 | IntlCompiledMessageFormat, 6 | IntlDatabaseInsertStrategy, 7 | } = require('@discord/intl-message-database'); 8 | 9 | const { database } = require('./src/database'); 10 | const { 11 | findAllDefinitionsFilesForTranslations, 12 | findAllMessagesFiles, 13 | filterAllMessagesFiles, 14 | processAllMessagesFiles, 15 | generateTypeDefinitions, 16 | processDefinitionsFile, 17 | processTranslationsFile, 18 | precompileFileForLocale, 19 | } = require('./src/processing'); 20 | const { MessageDefinitionsTransformer } = require('./src/transformer'); 21 | const { findAllTranslationFiles, getLocaleFromTranslationsFileName } = require('./src/util'); 22 | const watcher = require('./src/watcher'); 23 | 24 | module.exports = { 25 | // @ts-expect-error This is a const enum, which TypeScript doesn't like letting you export, even 26 | // though it's a tangible object that can be accessed just fine from normal JS. 27 | IntlCompiledMessageFormat, 28 | // @ts-expect-error This is a const enum, which TypeScript doesn't like letting you export, even 29 | // though it's a tangible object that can be accessed just fine from normal JS. 30 | IntlDatabaseInsertStrategy, 31 | MessageDefinitionsTransformer, 32 | database, 33 | findAllTranslationFiles, 34 | findAllDefinitionsFilesForTranslations, 35 | getLocaleFromTranslationsFileName, 36 | generateTypeDefinitions, 37 | hashMessageKey, 38 | isMessageDefinitionsFile, 39 | isMessageTranslationsFile, 40 | processDefinitionsFile, 41 | processTranslationsFile, 42 | precompileFileForLocale, 43 | findAllMessagesFiles, 44 | filterAllMessagesFiles, 45 | processAllMessagesFiles, 46 | watcher, 47 | }; 48 | -------------------------------------------------------------------------------- /packages/intl-loader-core/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-loader-core", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "description": "Core utilities for writing loaders and transformers using @discord/intl", 6 | "author": "Jon Egeland", 7 | "main": "index.js", 8 | "types": "types/index.d.ts", 9 | "files": [ 10 | "index.js", 11 | "types.d.ts", 12 | "src", 13 | "types" 14 | ], 15 | "exports": { 16 | ".": { 17 | "types": "./types/index.d.ts", 18 | "default": "./index.js" 19 | }, 20 | "./types": { 21 | "types": "./types/types.d.ts" 22 | } 23 | }, 24 | "repository": { 25 | "type": "git", 26 | "url": "https://github.com/discord/discord-intl" 27 | }, 28 | "scripts": { 29 | "build": "tsc && mkdir -p types && cp types.d.ts types/", 30 | "build:release": "tsc && mkdir -p types && cp types.d.ts types/", 31 | "prepublishOnly": "pnpm build:release" 32 | }, 33 | "dependencies": { 34 | "@discord/intl-message-database": "workspace:*", 35 | "chokidar": "^3.6.0", 36 | "debug": "^4.3.6" 37 | }, 38 | "devDependencies": { 39 | "@types/debug": "^4.1.12", 40 | "typescript": "*" 41 | } 42 | } -------------------------------------------------------------------------------- /packages/intl-loader-core/src/database.js: -------------------------------------------------------------------------------- 1 | const { IntlMessagesDatabase } = require('@discord/intl-message-database'); 2 | 3 | /** 4 | * A shared message database instance that's used and shared across all parts 5 | * of the loader and plugin together. 6 | */ 7 | const database = new IntlMessagesDatabase(); 8 | 9 | module.exports = { database }; 10 | -------------------------------------------------------------------------------- /packages/intl-loader-core/src/util.js: -------------------------------------------------------------------------------- 1 | const path = require('node:path'); 2 | const fs = require('node:fs'); 3 | const { isMessageTranslationsFile } = require('@discord/intl-message-database'); 4 | 5 | const IGNORED_MESSAGE_FILE_PATTERNS = [/.*\.compiled.messages\..*/]; 6 | 7 | /** 8 | * Return the presumed locale for a translations file from it's name. The convention follows the 9 | * format: `some/path/to/.messages.jsona`, so the locale is determined by taking the content 10 | * of the basename up until the first `.`. 11 | * 12 | * @param {string} fileName 13 | * @returns {string} 14 | */ 15 | function getLocaleFromTranslationsFileName(fileName) { 16 | return path.basename(fileName).split('.')[0]; 17 | } 18 | 19 | /** 20 | * Scan the given `translationsPath` to discover all translation files that exist, returning them 21 | * as a map from locale name to the path for importing. 22 | * 23 | * @param {string} translationsPath 24 | * @returns {Record | Error} 25 | */ 26 | function findAllTranslationFiles(translationsPath) { 27 | /** @type {Record} */ 28 | const localeMap = {}; 29 | 30 | try { 31 | const translationFiles = fs.readdirSync(translationsPath, { encoding: 'utf-8' }); 32 | for (const foundFile of translationFiles) { 33 | const filePath = path.join(translationsPath, foundFile); 34 | // Only include translation files, not definitions files. 35 | if (!isMessageTranslationsFile(filePath)) continue; 36 | // Some files are excluded, like pre-compiled artifacts. 37 | if (IGNORED_MESSAGE_FILE_PATTERNS.some((pattern) => pattern.test(filePath))) continue; 38 | 39 | const locale = getLocaleFromTranslationsFileName(filePath); 40 | localeMap[locale] = filePath; 41 | } 42 | } catch (e) { 43 | return new Error( 44 | `The translations directory ${translationsPath} was not found. No translations will be loaded for these messages`, 45 | ); 46 | } 47 | 48 | return localeMap; 49 | } 50 | 51 | module.exports = { 52 | IGNORED_MESSAGE_FILE_PATTERNS, 53 | findAllTranslationFiles, 54 | getLocaleFromTranslationsFileName, 55 | }; 56 | -------------------------------------------------------------------------------- /packages/intl-loader-core/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["../../tsconfig.node.json"], 3 | "compilerOptions": { 4 | "lib": ["es2015"], 5 | "noEmit": false, 6 | "declaration": true, 7 | "emitDeclarationOnly": true, 8 | "declarationDir": "./types", 9 | "target": "ES2015", 10 | }, 11 | "include": ["src", "index.js", "types.d.ts"] 12 | } 13 | -------------------------------------------------------------------------------- /packages/intl/.gitignore: -------------------------------------------------------------------------------- 1 | dist/ -------------------------------------------------------------------------------- /packages/intl/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /packages/intl/README.md: -------------------------------------------------------------------------------- 1 | # `@discord/intl` 2 | 3 | Redefined internationalization support, created by Discord to manage millions of messages across multiple projects, multiple programming languages, and multiple client platforms. discord-intl supports an expanded message format that combines ICU MessageFormat syntax with Markdown to make authoring rich, stylized messages as clear and concise as possible. 4 | 5 | This project was born out of a need for more flexibility with our translations, a desire for granular ownership, and a need to refactor to keep up with modern versions of `intl-messageformat` and others that power the system. 6 | 7 | ## Usage 8 | 9 | You most likely only need to know about how to use `defineMessages` to write new strings and string modules. A message 10 | definition file _must_ have the extension `.messages.js`, and _must_ have a default export that calls `defineMessages` 11 | inline, like this: 12 | 13 | ```typescript 14 | // Import the defineMessages magic function from the package. 15 | import { defineMessages } from '@discord/intl'; 16 | 17 | /** 18 | * Meta information about the strings contained in this file. Each string will 19 | * be tagged with this information by default, and it will be sent along to 20 | * translators to help provide context, categorization, and some special 21 | * features like hiding "secret" strings that shouldn't be bundled until a 22 | * specific release date. 23 | * 24 | * Future additions would be able to specify visual context for each string 25 | * (e.g. a screenshot image where the string is used) and more. 26 | */ 27 | export const meta = { 28 | project: 'custom-status', 29 | secret: true, 30 | translate: true, 31 | }; 32 | 33 | /** 34 | * Messages are "defined" by creating a default export with `defineMessages`. 35 | * This function provides the typeguards for the shape of each message, and 36 | * ensures a consistent format for the external tooling to rely on when finding 37 | * all messages in the codebase. 38 | * 39 | * This _must_ be defined inline as a default export, and will be transformed 40 | * by bundlers both in development and production builds. 41 | */ 42 | export default defineMessages({ 43 | HELLO_WORLD: { 44 | defaultMessage: 'Hello, world!', 45 | description: 'The standard greeting for new computers.', 46 | }, 47 | }); 48 | ``` 49 | 50 | ## Development 51 | 52 | This project is managed with the `intl-cli`: 53 | 54 | ```shell 55 | # Compile the TypeScript source to distributable JavaScript once 56 | pnpm intl-cli runtime build 57 | # For development, run in watch mode to compile changes on the fly 58 | pnpm intl-cli rt watch 59 | ``` 60 | -------------------------------------------------------------------------------- /packages/intl/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "description": "Client runtime for managing messages and translations in a React project.", 6 | "main": "./src/index.ts", 7 | "exports": { 8 | ".": "./src/index.ts" 9 | }, 10 | "publishConfig": { 11 | "main": "./dist/index.js", 12 | "exports": { 13 | ".": { 14 | "types": "./dist/index.d.ts", 15 | "default": "./dist/index.js" 16 | } 17 | } 18 | }, 19 | "files": [ 20 | "dist", 21 | "src" 22 | ], 23 | "repository": { 24 | "type": "git", 25 | "url": "https://github.com/discord/discord-intl" 26 | }, 27 | "scripts": { 28 | "build": "tsc", 29 | "build:release": "tsc && cp ./src/*.d.ts dist/", 30 | "prepublishOnly": "pnpm build:release" 31 | }, 32 | "dependencies": { 33 | "@discord/intl-ast": "workspace:*", 34 | "@formatjs/icu-skeleton-parser": "1.8.2", 35 | "@formatjs/intl": "^2.10.1", 36 | "@intrnl/xxhash64": "^0.1.2", 37 | "intl-messageformat": "^10.5.11" 38 | }, 39 | "devDependencies": { 40 | "@formatjs/intl-durationformat": "^0.7.3", 41 | "@swc/cli": "^0.3.12", 42 | "@swc/core": "^1.4.11", 43 | "typescript": "*" 44 | }, 45 | "peerDependencies": { 46 | "react": "> 16" 47 | } 48 | } -------------------------------------------------------------------------------- /packages/intl/src/data-formatters/cache.ts: -------------------------------------------------------------------------------- 1 | import type { DurationFormat as FormatJsDurationFormat } from '@formatjs/intl-durationformat'; 2 | 3 | type Cache = Map; 4 | 5 | type TemporaryIntlDurationFormat = typeof FormatJsDurationFormat; 6 | 7 | class FormatterCache { 8 | dateTime: Cache = new Map(); 9 | duration: Cache = new Map(); 10 | list: Cache = new Map(); 11 | number: Cache = new Map(); 12 | pluralRules: Cache = new Map(); 13 | relativeTime: Cache = new Map(); 14 | 15 | getDateTimeFormatter(...args: ConstructorParameters) { 16 | return this._getCached(this.dateTime, args, (args) => new Intl.DateTimeFormat(...args)); 17 | } 18 | 19 | getDurationFormatter(...args: ConstructorParameters) { 20 | return this._getCached( 21 | this.duration, 22 | args, 23 | // @ts-expect-error DurationFormat is _not_ included in typescript 24 | // https://github.com/microsoft/TypeScript/issues/60608 25 | (args) => new Intl.DurationFormat(...args), 26 | ); 27 | } 28 | 29 | getListFormatter(...args: ConstructorParameters) { 30 | return this._getCached(this.list, args, (args) => new Intl.ListFormat(...args)); 31 | } 32 | 33 | getNumberFormatter(...args: ConstructorParameters) { 34 | return this._getCached(this.number, args, (args) => new Intl.NumberFormat(...args)); 35 | } 36 | 37 | getPluralRules(...args: ConstructorParameters) { 38 | return this._getCached(this.pluralRules, args, (args) => new Intl.PluralRules(...args)); 39 | } 40 | 41 | getRelativeTimeFormatter(...args: ConstructorParameters) { 42 | return this._getCached(this.relativeTime, args, (args) => new Intl.RelativeTimeFormat(...args)); 43 | } 44 | 45 | _getCached(cache: Cache, args: Args, constructor: (args: Args) => T): T { 46 | const key = this._getKey(args); 47 | const cached = cache.get(key); 48 | if (cached) return cached; 49 | 50 | const created = constructor(args); 51 | cache.set(key, created); 52 | return created; 53 | } 54 | 55 | _getKey(...args: any): string { 56 | return JSON.stringify(args); 57 | } 58 | } 59 | 60 | export const dataFormatterCache = new FormatterCache(); 61 | -------------------------------------------------------------------------------- /packages/intl/src/data-formatters/config.ts: -------------------------------------------------------------------------------- 1 | import type { DurationFormatOptions, DurationInput } from '@formatjs/intl-durationformat/src/types'; 2 | 3 | export type TemporaryDurationInput = DurationInput; 4 | export type TemporaryDurationFormatOptions = DurationFormatOptions; 5 | 6 | export interface FormatConfigType { 7 | date: Record; 8 | duration: Record; 9 | list: Record; 10 | number: Record; 11 | relativeTime: Record; 12 | time: Record; 13 | } 14 | 15 | export function resolveFormatConfigOptions( 16 | config: Record, 17 | style?: T & { format?: K }, 18 | ): T { 19 | if (typeof style?.format === 'string') { 20 | return { 21 | ...config[style.format], 22 | ...style, 23 | }; 24 | } 25 | return style; 26 | } 27 | 28 | export const DEFAULT_FORMAT_CONFIG = { 29 | // These have no known defaults, so they can't be applied easily. 30 | duration: {}, 31 | list: {}, 32 | relativeTime: {}, 33 | 34 | /** 35 | * Default formatting configuration options for common date, time, and number formats. This is 36 | * taken almost directly from FormatJS's defaults here, for the sake of compatibility. 37 | * https://github.com/formatjs/formatjs/blob/c30975bfbe2db7eb62f4dbe6c8ad6ca5e786dcb3/packages/intl-messageformat/src/core.ts#L229-L296 38 | */ 39 | number: { 40 | integer: { maximumFractionDigits: 0 }, 41 | currency: { style: 'currency' }, 42 | percent: { style: 'percent' }, 43 | }, 44 | 45 | date: { 46 | short: { 47 | month: 'numeric', 48 | day: 'numeric', 49 | year: '2-digit', 50 | }, 51 | 52 | medium: { 53 | month: 'short', 54 | day: 'numeric', 55 | year: 'numeric', 56 | }, 57 | 58 | long: { 59 | month: 'long', 60 | day: 'numeric', 61 | year: 'numeric', 62 | }, 63 | 64 | full: { 65 | weekday: 'long', 66 | month: 'long', 67 | day: 'numeric', 68 | year: 'numeric', 69 | }, 70 | }, 71 | 72 | time: { 73 | short: { 74 | hour: 'numeric', 75 | minute: 'numeric', 76 | }, 77 | 78 | medium: { 79 | hour: 'numeric', 80 | minute: 'numeric', 81 | second: 'numeric', 82 | }, 83 | 84 | long: { 85 | hour: 'numeric', 86 | minute: 'numeric', 87 | second: 'numeric', 88 | timeZoneName: 'short', 89 | }, 90 | 91 | full: { 92 | hour: 'numeric', 93 | minute: 'numeric', 94 | second: 'numeric', 95 | timeZoneName: 'short', 96 | }, 97 | }, 98 | } satisfies FormatConfigType; 99 | -------------------------------------------------------------------------------- /packages/intl/src/formatters/index.ts: -------------------------------------------------------------------------------- 1 | export { astFormatter, type AstFunctionTypes, type RichTextNode, RichTextNodeType } from './ast'; 2 | export { markdownFormatter, type MarkdownFunctionTypes } from './markdown'; 3 | export { 4 | reactFormatter, 5 | makeReactFormatter, 6 | DEFAULT_REACT_RICH_TEXT_ELEMENTS, 7 | type ReactFunctionTypes, 8 | ReactIntlMessage, 9 | ReactIntlPlainString, 10 | ReactIntlRichText, 11 | } from './react'; 12 | export { stringFormatter, type StringFunctionTypes } from './string'; 13 | -------------------------------------------------------------------------------- /packages/intl/src/formatters/markdown.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Similar to `formatToPlainString`, format the given message with the provided 3 | * values, but convert all rich text formatting back to Markdown syntax rather 4 | * than rendering the actual rich content. The result is a plain string that 5 | * can be sent through a separate Markdown renderer to get an equivalent 6 | * result to formatting this message directly. 7 | */ 8 | import { 9 | AnyIntlMessage, 10 | FormatterImplementation, 11 | FunctionTypes, 12 | RichTextFormattingMap, 13 | RichTextTagNames, 14 | } from '../types'; 15 | import { IntlManager } from '../intl-manager'; 16 | import { FormatBuilderConstructor } from '../format'; 17 | import { StringBuilder } from './string'; 18 | 19 | export type MarkdownFunctionTypes = FunctionTypes; 20 | 21 | const MARKDOWN_RICH_TEXT_ELEMENTS: RichTextFormattingMap = { 22 | $b: (content) => '**' + content.join('') + '**', 23 | $i: (content) => '*' + content.join('') + '*', 24 | $del: (content) => '~~' + content.join('') + '~~', 25 | $code: (content) => '`' + content.join('') + '`', 26 | $link: (content, _, [target]) => '[' + content.join('') + '](' + target + ')', 27 | $p: (content) => content.join('') + '\n\n', 28 | }; 29 | 30 | class MarkdownBuilder extends StringBuilder { 31 | result: string = ''; 32 | 33 | pushRichTextTag(tag: RichTextTagNames, children: string[], control: string[]) { 34 | this.result += MARKDOWN_RICH_TEXT_ELEMENTS[tag](children, '', control); 35 | } 36 | } 37 | 38 | export function formatToMarkdownString( 39 | this: IntlManager, 40 | message: AnyIntlMessage, 41 | values: object, 42 | Builder: FormatBuilderConstructor = MarkdownBuilder, 43 | ): string { 44 | if (typeof message === 'string') return message; 45 | 46 | const result = this.bindFormatValues(Builder, message, values); 47 | // MarkdownBuilder always creates a single-element array with the string value. 48 | return result[0]; 49 | } 50 | 51 | export const markdownFormatter: FormatterImplementation = { 52 | format: formatToMarkdownString, 53 | builder: MarkdownBuilder, 54 | }; 55 | -------------------------------------------------------------------------------- /packages/intl/src/formatters/string.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Format the given message with the provided values, removing any styling 3 | * and non-textual content from the message, returning a plain string. 4 | */ 5 | import { AnyIntlMessage, FormatterImplementation, FunctionTypes, RichTextTagNames } from '../types'; 6 | import { FormatBuilder } from '../format'; 7 | import { IntlManager } from '../intl-manager'; 8 | 9 | /** 10 | * Types for formatting functions when calling `bindFormatValues`, ensuring the 11 | * functions always yield plain strings. 12 | */ 13 | export type StringFunctionTypes = FunctionTypes; 14 | 15 | export class StringBuilder extends FormatBuilder { 16 | result: string = ''; 17 | 18 | pushRichTextTag(_tag: RichTextTagNames, children: string[], _control: string[]) { 19 | // Plain string formatting ignores rich text tags and just takes the 20 | // visible content from the children. This means the control element is not 21 | // important for string rendering, so the result is always just the 22 | // children joined together. 23 | for (const child of children) { 24 | this.result += child; 25 | } 26 | } 27 | 28 | pushLiteralText(text: string) { 29 | this.result += text; 30 | } 31 | 32 | pushObject(value: object) { 33 | // Objects are only included in the result if they specify a toString value directly. 34 | // Otherwise, they would be rendered as `[object Object]`, which is never helpful. 35 | if (value != null && 'toString' in value) { 36 | this.result += value.toString(); 37 | } 38 | } 39 | 40 | finish(): string[] { 41 | return [this.result]; 42 | } 43 | } 44 | 45 | export function formatToPlainString( 46 | this: IntlManager, 47 | message: AnyIntlMessage, 48 | values: object, 49 | ): string { 50 | if (typeof message === 'string') return message; 51 | 52 | const result = this.bindFormatValues(StringBuilder, message, values); 53 | // StringBuilder always creates a single element array with the string value. 54 | return result[0]; 55 | } 56 | 57 | export const stringFormatter: FormatterImplementation = { 58 | format: formatToPlainString, 59 | builder: StringBuilder, 60 | }; 61 | -------------------------------------------------------------------------------- /packages/intl/src/hash.ts: -------------------------------------------------------------------------------- 1 | import { hash as h64 } from '@intrnl/xxhash64'; 2 | 3 | const BASE64_TABLE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'.split(''); 4 | const IS_BIG_ENDIAN = (() => { 5 | const array = new Uint8Array(4); 6 | const view = new Uint32Array(array.buffer); 7 | return !((view[0] = 1) & array[0]); 8 | })(); 9 | 10 | function numberToBytes(number) { 11 | number = BigInt(number); 12 | const array = []; 13 | const byteCount = Math.ceil(Math.floor(Math.log2(Number(number)) + 1) / 8); 14 | for (let i = 0; i < byteCount; i++) { 15 | array.unshift(Number((number >> BigInt(8 * i)) & BigInt(255))); 16 | } 17 | 18 | const bytes = new Uint8Array(array); 19 | // The native `hashToMessageKey` always works in Big/Network Endian bytes, so this array 20 | // needs to be converted to the same endianness to get the same base64 result. 21 | return IS_BIG_ENDIAN ? bytes : bytes.reverse(); 22 | } 23 | 24 | /** 25 | * Returns a consistent, short hash of the given key by first processing it through a hash digest, 26 | * then encoding the first few bytes to base64. 27 | * 28 | * This function is specifically written to mirror the native backend hashing function used by 29 | * `@discord/intl-loader-core`, to be able to hash names at runtime. 30 | */ 31 | export function runtimeHashMessageKey(key: string): string { 32 | const hash = h64(key, 0); 33 | const bytes = numberToBytes(hash); 34 | return [ 35 | BASE64_TABLE[bytes[0] >> 2], 36 | BASE64_TABLE[((bytes[0] & 0x03) << 4) | (bytes[1] >> 4)], 37 | BASE64_TABLE[((bytes[1] & 0x0f) << 2) | (bytes[2] >> 6)], 38 | BASE64_TABLE[bytes[2] & 0x3f], 39 | BASE64_TABLE[bytes[3] >> 2], 40 | BASE64_TABLE[((bytes[3] & 0x03) << 4) | (bytes[3] >> 4)], 41 | ].join(''); 42 | } 43 | -------------------------------------------------------------------------------- /packages/intl/src/index.ts: -------------------------------------------------------------------------------- 1 | export { type DataFormatters, makeDataFormatters } from './data-formatters'; 2 | export { dataFormatterCache } from './data-formatters/cache'; 3 | export { FormatBuilder, FormatBuilderConstructor, bindFormatValues } from './format'; 4 | export * from './formatters'; 5 | export { runtimeHashMessageKey } from './hash'; 6 | export { IntlManager, DEFAULT_LOCALE, type FormatFunction } from './intl-manager'; 7 | export { 8 | createLoader, 9 | loadAllMessagesInLocale, 10 | waitForAllDefaultIntlMessagesLoaded, 11 | MessageLoader, 12 | } from './message-loader'; 13 | export type * from './types.d.ts'; 14 | 15 | export { chainMessagesObjects, makeMessagesProxy } from './runtime-utils'; 16 | 17 | /** 18 | * The return value of `formatToParts` from `@discord/intl`, this type 19 | * represents any AST structure for a message rendered using this system. 20 | * The AST generally follows a Markdown-like structure, with text nodes 21 | * interspersed within and around rich text formatting nodes. 22 | * 23 | * ASTs are created _after_ all placeholders have been filled by values 24 | * from a call to `formatToParts`, meaning they are intended to be fully static 25 | * structures passed around for custom rendering functions to use. 26 | */ 27 | import { type RichTextNode } from './formatters'; 28 | export type IntlMessageAst = RichTextNode; 29 | -------------------------------------------------------------------------------- /packages/intl/tsconfig.json: -------------------------------------------------------------------------------- 1 | // Configuration for projects that run in Browser/client environments. 2 | { 3 | "compilerOptions": { 4 | // Including the latest Intl libraries means we can use things like 5 | // `Intl.DurationFormat` before they've been fully standardized, and just 6 | // inform users to polyfill them as needed. 7 | // This should _not_ be `ESNext`, since we want to know which structures 8 | // are currently "out there" and have good polyfills (like from FormatJS). 9 | "lib": ["es2015", "ES2020.Intl", "ES2021.Intl", "ES2022.Intl"], 10 | "noEmit": false, 11 | "outDir": "./dist", 12 | "declaration": true, 13 | "declarationDir": "./dist", 14 | "module": "commonjs", 15 | "jsx": "react-jsx", 16 | "target": "ES2015" 17 | }, 18 | } 19 | -------------------------------------------------------------------------------- /packages/jest-processor-discord-intl/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /packages/jest-processor-discord-intl/README.md: -------------------------------------------------------------------------------- 1 | # @discord/jest-processor-discord-intl 2 | 3 | A processor plugin for Jest to handle intl message definitions and translations from `@discord/intl`. 4 | 5 | In your Jest config, add this as a transformer for message file patterns: 6 | 7 | ```javascript 8 | module.exports = { 9 | transform: { 10 | // Order matters here! Even though this is an Object, Jest still processes these first to 11 | // last (object insertion order). These patterns can overlap, so we want to put all of the 12 | // special handling first. 13 | [INTL_MESSAGES_FILE_PATTERN]: require.resolve('@discord/jest-processor-discord-intl'), 14 | // ...other transforms, like `*.tsx?` and more. 15 | }, 16 | }; 17 | ``` 18 | 19 | You'll also need to use the babel plugin to transform consuming code like normal bundling: 20 | 21 | ```javascript 22 | // In a custom processor or wherever you configure Babel for Jest: 23 | babel.transform({ 24 | // ... 25 | plugins: [ 26 | [ 27 | require.resolve('@discord/babel-plugin-transform-discord-intl'), 28 | { 29 | // Jest does _not_ resolve these to absolute paths, so tell the plugin to use the 30 | // original import paths instead of resolving them like it does for Metro. 31 | preserveImportSource: true, 32 | extraImports: { 33 | '@app/intl': ['t', 'untranslated', 'international'], 34 | }, 35 | }, 36 | ], 37 | ], 38 | }); 39 | ``` 40 | -------------------------------------------------------------------------------- /packages/jest-processor-discord-intl/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/jest-processor-discord-intl", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "description": "Processor plugin for Jest to handle intl message definitions and translations from @discord/intl", 6 | "exports": { 7 | ".": "./index.js" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/discord/discord-intl" 12 | }, 13 | "dependencies": { 14 | "@discord/intl-loader-core": "workspace:*" 15 | }, 16 | "devDependencies": { 17 | "@jest/types": "^29.6.3" 18 | } 19 | } -------------------------------------------------------------------------------- /packages/metro-intl-transformer/.gitignore: -------------------------------------------------------------------------------- 1 | .cache/ -------------------------------------------------------------------------------- /packages/metro-intl-transformer/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /packages/metro-intl-transformer/README.md: -------------------------------------------------------------------------------- 1 | # metro-intl-transformer 2 | 3 | A Metro loader for intl message definition files using `@discord/intl`. This loader handles both definitions and translations as a single group, emitting the appropriate file types and contents based on the kind of file provided. 4 | 5 | Note that you'll also want/need the `@discord/babel-plugin-transform-discord-intl` plugin applied to your JS compilation to ensure that message usages compile to the same keys that match the compiled definitions. 6 | -------------------------------------------------------------------------------- /packages/metro-intl-transformer/index.js: -------------------------------------------------------------------------------- 1 | const { 2 | isMessageDefinitionsFile, 3 | isMessageTranslationsFile, 4 | processDefinitionsFile, 5 | MessageDefinitionsTransformer, 6 | processTranslationsFile, 7 | precompileFileForLocale, 8 | IntlCompiledMessageFormat, 9 | } = require('@discord/intl-loader-core'); 10 | const debug = require('debug')('intl:metro-intl-transformer'); 11 | 12 | /** 13 | * @param {{ 14 | * filename: string, 15 | * src: string, 16 | * getPrelude: () => string 17 | * getTranslationAssetExtension: () => string, 18 | * getTranslationImport: (importPath: string) => string, 19 | * format?: IntlCompiledMessageFormat, 20 | * bundleSecrets?: boolean, 21 | * bindMode?: 'proxy' | 'literal', 22 | * sourceLocale?: string, 23 | * }} options 24 | * @returns {string | Buffer} 25 | */ 26 | function transformToString({ 27 | filename, 28 | src, 29 | getPrelude, 30 | getTranslationAssetExtension, 31 | getTranslationImport, 32 | format = IntlCompiledMessageFormat.KeylessJson, 33 | bundleSecrets = false, 34 | bindMode = 'proxy', 35 | sourceLocale = 'en-US', 36 | }) { 37 | if (isMessageDefinitionsFile(filename)) { 38 | debug(`[${filename}] Processing as a definitions file`); 39 | const result = processDefinitionsFile(filename, src, { locale: sourceLocale }); 40 | if (!result.succeeded) { 41 | throw new Error('Intl processing error in ' + filename + ': ' + result.errors[0].message); 42 | } 43 | const compiledSourcePath = filename.replace( 44 | /\.messages\.js$/, 45 | `.compiled.messages.${getTranslationAssetExtension()}`, 46 | ); 47 | 48 | debug( 49 | `[${filename}] Resolving source file to compiled translations file ${compiledSourcePath}`, 50 | ); 51 | result.translationsLocaleMap[result.locale] = compiledSourcePath; 52 | debug('Locale map created: %O', result.translationsLocaleMap); 53 | 54 | return new MessageDefinitionsTransformer({ 55 | messageKeys: result.messageKeys, 56 | localeMap: result.translationsLocaleMap, 57 | defaultLocale: result.locale, 58 | getTranslationImport, 59 | getPrelude, 60 | debug: process.env.NODE_ENV === 'development', 61 | bindMode, 62 | }).getOutput(); 63 | } else if (isMessageTranslationsFile(filename)) { 64 | debug(`[${filename}] Processing as a translations file`); 65 | const result = processTranslationsFile(filename, src); 66 | if (!result.succeeded) { 67 | throw new Error('Intl processing error in ' + filename + ': ' + result.errors[0]); 68 | } 69 | // @ts-expect-error Without the `outputFile` option, this always returns a Buffer, but the 70 | // option allows the function to return void instead. 71 | return precompileFileForLocale(filename, result.locale, undefined, { 72 | format, 73 | bundleSecrets, 74 | }); 75 | } 76 | 77 | return src; 78 | } 79 | 80 | module.exports = { transformToString }; 81 | -------------------------------------------------------------------------------- /packages/metro-intl-transformer/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/metro-intl-transformer", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "description": "Metro/Babel transformer plugin for compiling, bundling, and minifying intl message definitions and usages using intl-message-database.", 6 | "exports": { 7 | ".": "./index.js", 8 | "./asset-plugin": "./asset-plugin.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "https://github.com/discord/discord-intl" 13 | }, 14 | "dependencies": { 15 | "@discord/babel-plugin-transform-discord-intl": "workspace:*", 16 | "@discord/intl-loader-core": "workspace:*", 17 | "debug": "^4.3.6", 18 | "metro-config": "^0.81.0" 19 | }, 20 | "devDependencies": { 21 | "@types/debug": "^4.1.12" 22 | } 23 | } -------------------------------------------------------------------------------- /packages/rspack-intl-loader/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /packages/rspack-intl-loader/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/rspack-intl-loader", 3 | "version": "0.23.3", 4 | "license": "MIT", 5 | "type": "commonjs", 6 | "description": "Webpack/Rspack loader for i18n message definition files using @discord/intl", 7 | "exports": { 8 | ".": "./rspack-loader.js", 9 | "./types-plugin": "./types-plugin.js" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "https://github.com/discord/discord-intl" 14 | }, 15 | "peerDependencies": { 16 | "@rspack/core": "0.x || 1.x", 17 | "webpack": "^5.0.0" 18 | }, 19 | "peerDependenciesMeta": { 20 | "@rspack/core": { 21 | "optional": true 22 | }, 23 | "webpack": { 24 | "optional": true 25 | } 26 | }, 27 | "dependencies": { 28 | "@discord/intl-loader-core": "workspace:*", 29 | "debug": "^4.3.6" 30 | }, 31 | "devDependencies": { 32 | "@types/debug": "^4.1.12" 33 | } 34 | } -------------------------------------------------------------------------------- /packages/rspack-intl-loader/types-plugin.js: -------------------------------------------------------------------------------- 1 | const { 2 | database, 3 | isMessageDefinitionsFile, 4 | generateTypeDefinitions, 5 | } = require('@discord/intl-loader-core'); 6 | 7 | /** 8 | * A plugin that watches for changes to I18n strings and updates messages.d.ts (and its sourcemap) automatically. 9 | */ 10 | class IntlTypeGeneratorPlugin { 11 | /** 12 | * @param {string} filePath 13 | * @returns {number} How long it took to generate the type definitions file. 14 | */ 15 | generateTypeDefinitions(filePath) { 16 | const start = performance.now(); 17 | generateTypeDefinitions(filePath, undefined); 18 | const end = performance.now(); 19 | 20 | return end - start; 21 | } 22 | 23 | generateAllTypes() { 24 | const paths = database.getAllSourceFilePaths(); 25 | let totalDuration = 0; 26 | 27 | for (const path of paths) { 28 | if (isMessageDefinitionsFile(path)) { 29 | totalDuration += this.generateTypeDefinitions(path); 30 | } 31 | } 32 | 33 | console.error( 34 | `🌍 Updated all intl type definitions (${paths.length} files, ${totalDuration.toFixed(3)}ms)`, 35 | ); 36 | } 37 | 38 | /** @param {import('webpack').Compiler} compiler */ 39 | apply(compiler) { 40 | let isFirstCompilation = true; 41 | compiler.hooks.afterCompile.tap('IntlTypeGeneratorPlugin', () => { 42 | if (isFirstCompilation) { 43 | this.generateAllTypes(); 44 | isFirstCompilation = false; 45 | } 46 | }); 47 | compiler.hooks.invalid.tap('IntlTypeGeneratorPlugin', (filePath) => { 48 | if (filePath != null && isMessageDefinitionsFile(filePath)) { 49 | const duration = this.generateTypeDefinitions(filePath); 50 | console.error( 51 | `🌍 Updated intl type definitions for ${filePath} (${duration.toFixed(3)}ms)`, 52 | ); 53 | } 54 | }); 55 | } 56 | } 57 | 58 | module.exports = { IntlTypeGeneratorPlugin }; 59 | -------------------------------------------------------------------------------- /packages/swc-intl-message-transformer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "swc-intl-message-transformer" 3 | description = "" 4 | version = "0.1.0" 5 | edition = "2021" 6 | 7 | [lib] 8 | crate-type = ["cdylib"] 9 | 10 | [dependencies] 11 | getrandom = { version = "0.2", features = ["js"] } 12 | pathdiff = "0.2.1" 13 | intl_message_utils = { workspace = true } 14 | serde = "1" 15 | serde_json = "1" 16 | # This version is roughly tied to our rspack version: https://swc.rs/docs/plugin/selecting-swc-core 17 | swc_core = { workspace = true, features = ["ecma_plugin_transform"] } 18 | tracing = "0.1.40" 19 | ustr = "1.0.0" 20 | -------------------------------------------------------------------------------- /packages/swc-intl-message-transformer/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Discord, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /packages/swc-intl-message-transformer/README.md: -------------------------------------------------------------------------------- 1 | # swc-intl-message-transformer 2 | 3 | An SWC plugin for transforming intl message _usages_ to obfuscated, minified strings, saving on bundle size and allowing for complete anonymity of string names. 4 | 5 | The runtime of this package is a single `.wasm` file, which is loaded as the main file when requiring the package. 6 | 7 | # Development 8 | 9 | ``` 10 | pnpm intl-cli swc build 11 | ``` 12 | 13 | This will automatically ensure you have the appropriate Rust toolchains and targets installed, compile the plugin to a `.wasm` file, and copy it to the appropriate location for use as an npm package. 14 | 15 | # Usage 16 | 17 | This plugin is usable in any project using SWC >= 1.0. Add it to the transpiler configuration through the `options.jsc.experimental.plugins` setting: 18 | 19 | ```js 20 | { 21 | jsc: { 22 | experimental: { 23 | plugins: [ 24 | [ 25 | require.resolve('@discord/swc-intl-message-transformer'), 26 | // Optional extra configuration for customized usage. 27 | { extraImports: { './custom-module': ['additional', 'imported', 'names'] } }, 28 | ], 29 | ]; 30 | } 31 | } 32 | } 33 | ``` 34 | -------------------------------------------------------------------------------- /packages/swc-intl-message-transformer/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/swc-intl-message-transformer", 3 | "license": "MIT", 4 | "version": "0.23.3", 5 | "description": "SWC plugin for minifying intl message usages.", 6 | "author": "Jon Egeland", 7 | "keywords": [ 8 | "swc-plugin" 9 | ], 10 | "main": "./swc_intl_message_transformer.wasm", 11 | "repository": { 12 | "type": "git", 13 | "url": "https://github.com/discord/discord-intl" 14 | }, 15 | "scripts": { 16 | "build": "cargo build --target wasm32-wasip1 --release && cp ../../target/wasm32-wasip1/release/swc_intl_message_transformer.wasm .", 17 | "build:dev": "cargo build --target wasm32-wasip1 && cp ../../target/wasm32-wasip1/debug/swc_intl_message_transformer.wasm .", 18 | "prepublishOnly": "ls swc_intl_message_transformer.wasm" 19 | }, 20 | "files": [ 21 | "./swc_intl_message_transformer.wasm" 22 | ] 23 | } -------------------------------------------------------------------------------- /packages/swc-intl-message-transformer/src/config.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use serde::Deserialize; 4 | 5 | #[derive(Default, Deserialize)] 6 | #[serde(rename_all = "camelCase")] 7 | pub(crate) struct IntlMessageTransformerConfig { 8 | pub extra_imports: Option>>, 9 | } 10 | 11 | impl IntlMessageTransformerConfig { 12 | pub fn get_configured_names_for_import_specifier( 13 | &self, 14 | specifier: &str, 15 | ) -> Option<&Vec> { 16 | match &self.extra_imports { 17 | Some(extras) => extras.get(specifier), 18 | None => None, 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /packages/swc-intl-message-transformer/src/lib.rs: -------------------------------------------------------------------------------- 1 | use swc_core::ecma::ast::Program; 2 | use swc_core::ecma::visit::VisitMutWith; 3 | use swc_core::plugin::{plugin_transform, proxies::TransformPluginProgramMetadata}; 4 | 5 | use config::IntlMessageTransformerConfig; 6 | 7 | mod config; 8 | mod transformer; 9 | 10 | #[plugin_transform] 11 | pub fn process_transform( 12 | mut program: Program, 13 | metadata: TransformPluginProgramMetadata, 14 | ) -> Program { 15 | let config = serde_json::from_str::( 16 | &metadata 17 | .get_transform_plugin_config() 18 | .expect("failed to get swc-intl-message-transformer plugin config"), 19 | ) 20 | .expect("failed to parse swc-intl-message-transformer config"); 21 | 22 | program.visit_mut_with(&mut transformer::IntlMessageConsumerTransformer::new( 23 | config, 24 | )); 25 | 26 | program 27 | } 28 | -------------------------------------------------------------------------------- /pnpm-workspace.yaml: -------------------------------------------------------------------------------- 1 | packages: 2 | - ./crates/intl_message_database 3 | - ./crates/intl_message_database/npm/* 4 | - ./crates/intl_flat_json_parser 5 | - ./crates/intl_flat_json_parser/npm/* 6 | - ./packages/* 7 | - ./tools 8 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "nightly-2024-10-13" 3 | components = ["rustfmt", "clippy"] 4 | profile = "minimal" -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | let 2 | # Using an unreleased version here to get pnpm_9, since it's not released in a stable yet. 3 | nixpkgs = builtins.fetchTarball "https://github.com/NixOS/nixpkgs/tarball/0e0ab06610ca2a9a266bf7272f818e628059a2d9"; 4 | rust-overlay = builtins.fetchTarball "https://github.com/oxalica/rust-overlay/archive/master.tar.gz"; 5 | 6 | pkgs = import nixpkgs { 7 | config = {}; 8 | overlays = [(import rust-overlay)]; 9 | }; 10 | 11 | rust-toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; 12 | in 13 | pkgs.mkShell { 14 | # nativeBuildInputs is usually what you want -- tools you need to run 15 | nativeBuildInputs = with pkgs.buildPackages; [ 16 | # Node for all of the client packages 17 | pkgs.nodejs_22 18 | # pnpm for managing the workspace 19 | pkgs.pnpm_9 20 | # Rust for crates 21 | rust-toolchain 22 | # Zig for cross compilation 23 | pkgs.zig 24 | ]; 25 | } 26 | -------------------------------------------------------------------------------- /tools/index.js: -------------------------------------------------------------------------------- 1 | import { program } from 'commander'; 2 | import { cd } from 'zx'; 3 | 4 | import { REPO_ROOT } from './src/constants.js'; 5 | import utilCommands from './src/util-commands.js'; 6 | import ciCommands from './src/ci/commands.js'; 7 | import dbCommands from './src/db/commands.js'; 8 | import jsonCommands from './src/json/commands.js'; 9 | import ecosystemCommands from './src/ecosystem/commands.js'; 10 | import { createJsPackageCommands } from './src/js-package.js'; 11 | import { rustup } from './src/util/rustup.js'; 12 | 13 | process.chdir(REPO_ROOT); 14 | cd(REPO_ROOT); 15 | 16 | (async () => { 17 | program 18 | .description('Internal tooling for managing the discord-intl repo and packages.') 19 | .addCommand(await ciCommands()) 20 | .addCommand(await dbCommands()) 21 | .addCommand(await jsonCommands()) 22 | .addCommand(await ecosystemCommands()) 23 | .addCommand(await utilCommands()) 24 | .addCommand( 25 | await createJsPackageCommands('eslint-plugin-discord-intl', { aliases: ['eslint'] }), 26 | ) 27 | .addCommand( 28 | await createJsPackageCommands('babel-plugin-transform-discord-intl', { aliases: ['babel'] }), 29 | ) 30 | .addCommand(await createJsPackageCommands('jest-processor-discord-intl', { aliases: ['jest'] })) 31 | .addCommand(await createJsPackageCommands('metro-intl-transformer', { aliases: ['metro'] })) 32 | .addCommand(await createJsPackageCommands('rspack-intl-loader', { aliases: ['rspack'] })) 33 | .addCommand(await createJsPackageCommands('intl-ast', { build: true, watch: true })) 34 | .addCommand( 35 | await createJsPackageCommands('intl', { 36 | aliases: ['rt', 'runtime'], 37 | build: true, 38 | watch: true, 39 | }), 40 | ) 41 | .addCommand( 42 | await createJsPackageCommands('intl-loader-core', { 43 | aliases: ['loader'], 44 | build: true, 45 | watch: ['index.js', 'src/**'], 46 | }), 47 | ) 48 | .addCommand( 49 | await createJsPackageCommands('swc-intl-message-transformer', { 50 | aliases: ['swc'], 51 | prebuild: rustup.ensureWasmSetup, 52 | build: true, 53 | }), 54 | ) 55 | .parse(); 56 | })(); 57 | -------------------------------------------------------------------------------- /tools/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@discord/intl-tools", 3 | "private": true, 4 | "version": "0.0.0", 5 | "description": "Internal tooling for managing the discord-intl repository and packages", 6 | "type": "module", 7 | "main": "index.js", 8 | "bin": { 9 | "intl-cli": "index.js" 10 | }, 11 | "scripts": { 12 | "intl-cli": "node index.js" 13 | }, 14 | "dependencies": { 15 | "@inquirer/prompts": "^5.3.8", 16 | "@napi-rs/cli": "3.0.0-alpha.62", 17 | "chokidar": "^3.6.0", 18 | "commander": "^12.1.0", 19 | "semver": "^7.6.3", 20 | "zx": "^8.1.4" 21 | }, 22 | "devDependencies": { 23 | "@types/semver": "^7.5.8" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tools/src/constants.js: -------------------------------------------------------------------------------- 1 | import { fileURLToPath } from 'node:url'; 2 | import path from 'node:path'; 3 | 4 | export const __filename = fileURLToPath(import.meta.url); 5 | export const __dirname = path.dirname(__filename); 6 | 7 | export const REPO_ROOT = path.resolve(__dirname, '..', '..'); 8 | 9 | export const NPM_PACKAGES = { 10 | ESLINT_PLUGIN: '@discord/eslint-plugin-discord-intl', 11 | SWC_TRANSFORMER: '@discord/swc-intl-message-transformer', 12 | METRO_TRANSFORMER: '@discord/metro-intl-transformer', 13 | RSPACK_LOADER: '@discord/rspack-intl-loader', 14 | LOADER_CORE: '@discord/intl-loader-core', 15 | RUNTIME: '@discord/intl', 16 | DATABASE: '@discord/intl-message-database', 17 | JSON_PARSER: '@discord/intl-flat-json-parser', 18 | }; 19 | -------------------------------------------------------------------------------- /tools/src/util/git.js: -------------------------------------------------------------------------------- 1 | import { $ } from 'zx'; 2 | 3 | /** 4 | * Return the full SHA of the current commit. If `short` is specified then the shortened version is 5 | * returned instead. 6 | * 7 | * @param {{ 8 | * short?: boolean, 9 | * }} options 10 | * @returns {string} 11 | */ 12 | function currentHead(options = {}) { 13 | const { short = false } = options; 14 | return $.sync`git rev-parse ${short ? '--short' : ''} HEAD`.stdout.trim(); 15 | } 16 | /** 17 | * Return the name of the current branch (ref). 18 | */ 19 | function currentBranch(options = {}) { 20 | return $.sync`git rev-parse --abbrev-ref HEAD`.stdout.trim(); 21 | } 22 | 23 | /** 24 | * Returns true if the current git state contains any changes. 25 | * 26 | * @returns {Promise} 27 | */ 28 | async function hasChanges() { 29 | const status = await $`git status --porcelain`; 30 | return status.stdout.trim().length > 0; 31 | } 32 | 33 | /** 34 | * If the current git state has any changes, log an error message and reject execution. The caller 35 | * can either allow the Promise rejection to propagate, or handle it separately. If `hardExit` is 36 | * true, the process will be aborted directly after logging the message. 37 | * 38 | * @param {boolean=} hardExit 39 | * @returns {Promise} 40 | */ 41 | async function rejectIfHasChanges(hardExit = false) { 42 | if (!(await hasChanges())) return; 43 | 44 | const errorMessage = 45 | 'There are uncommited changes. Commit and push them before running this command'; 46 | 47 | if (hardExit) { 48 | console.log(errorMessage); 49 | process.exit(1); 50 | } 51 | 52 | return Promise.reject(errorMessage); 53 | } 54 | 55 | export const git = { 56 | currentHead, 57 | currentBranch, 58 | hasChanges, 59 | rejectIfHasChanges, 60 | }; 61 | -------------------------------------------------------------------------------- /tools/src/util/platform.js: -------------------------------------------------------------------------------- 1 | import fs from 'node:fs'; 2 | import path from 'node:path'; 3 | 4 | import { __dirname } from '../constants.js'; 5 | import { NAPI_TARGET_MAP } from '../napi.js'; 6 | 7 | function isMusl() { 8 | // For Node 10 9 | if (!process.report || typeof process.report.getReport !== 'function') { 10 | try { 11 | const lddPath = require('child_process').execSync('which ldd').toString().trim(); 12 | return fs.readFileSync(lddPath, 'utf8').includes('musl'); 13 | } catch (e) { 14 | return true; 15 | } 16 | } else { 17 | /** @type {any} */ 18 | let report = process.report.getReport(); 19 | if (typeof report === 'string') { 20 | report = JSON.parse(report); 21 | } 22 | const { glibcVersionRuntime } = report.header; 23 | return !glibcVersionRuntime; 24 | } 25 | } 26 | 27 | /** @type {Record>} */ 28 | const PACKAGE_NAMES = { 29 | android: { arm: 'android-arm-eabi', arm64: 'android-arm64' }, 30 | win32: { arm64: 'win32-arm64-msvc', ia32: 'win32-ia32-msvc', x64: 'win32-x64-msvc' }, 31 | darwin: { arm64: 'darwin-arm64', x64: 'darwin-x64' }, 32 | freebsd: { x64: 'freebsd-x64' }, 33 | 'linux-gnu': { 34 | arm: 'linux-arm-gnueabihf', 35 | arm64: 'linux-arm64-gnu', 36 | x64: 'linux-x64-gnu', 37 | riscv64: 'linux-riscv64-gnu', 38 | s390x: 'linux-s390x-gnu', 39 | }, 40 | 'linux-musl': { 41 | arm: 'linux-arm-musleabihf', 42 | arm64: 'linux-arm64-musl', 43 | x64: 'linux-x64-musl', 44 | riscv64: 'linux-riscv64-musl', 45 | }, 46 | }; 47 | 48 | const platform = 49 | process.platform !== 'linux' ? process.platform : 'linux-' + (isMusl() ? 'musl' : 'gnu'); 50 | const arch = process.arch; 51 | 52 | /** 53 | * @returns {keyof NAPI_TARGET_MAP} 54 | */ 55 | function getPackageName() { 56 | if (!(platform in PACKAGE_NAMES)) { 57 | throw new Error(`Unsupported OS: ${platform}`); 58 | } 59 | if (!(arch in PACKAGE_NAMES[platform])) { 60 | throw new Error(`Unsupported architecture for ${platform}: ${arch}`); 61 | } 62 | return PACKAGE_NAMES[platform][arch]; 63 | } 64 | 65 | const packageName = getPackageName(); 66 | const localPath = path.join( 67 | __dirname, 68 | `npm/${packageName}/intl-message-database.${packageName}.node`, 69 | ); 70 | const packagePath = `@discord/intl-message-database-${packageName}`; 71 | 72 | /** 73 | * Information about the host system that's running this command, usable for creating default 74 | * targets and argument values that rely on a target platform or package. 75 | * @type {{packagePath: string, triple: *, localPath: string, target: keyof NAPI_TARGET_MAP}} 76 | */ 77 | const hostPlatform = { 78 | target: packageName, 79 | triple: NAPI_TARGET_MAP[packageName], 80 | packagePath, 81 | localPath, 82 | }; 83 | 84 | export { hostPlatform }; 85 | -------------------------------------------------------------------------------- /tools/src/util/rustup.js: -------------------------------------------------------------------------------- 1 | import { $ } from 'zx'; 2 | 3 | /** 4 | * Check that the given rustup toolchain `target` is installed on the host system. 5 | * 6 | * @param {string} targetTriple 7 | * @returns {Promise} 8 | */ 9 | async function hasTargetInstalled(targetTriple) { 10 | return (await $`rustup target list --installed`).stdout.split('\n').includes(targetTriple); 11 | } 12 | 13 | /** 14 | * Install the given `targetTriple` for the current rustup toolchain. If the target fails to install 15 | * the returned Promise will be rejected. 16 | * 17 | * @param {string} targetTriple 18 | * @returns {Promise} 19 | */ 20 | async function installTarget(targetTriple) { 21 | const result = await $`rustup target add ${targetTriple}`; 22 | if (result.exitCode !== 0) return Promise.reject(); 23 | } 24 | 25 | /** 26 | * Ensure that everything needed to build WASM targets is setup on the host system. 27 | * @returns {Promise} 28 | */ 29 | async function ensureWasmSetup() { 30 | console.log('Ensuring wasm environment setup for building'); 31 | const hasTargets = 32 | // (await rustup.hasTargetInstalled('wasm32-wasip1')) && 33 | await rustup.hasTargetInstalled('wasm32-unknown-unknown'); 34 | if (hasTargets) return; 35 | console.log('Installing wasm32 rust targets'); 36 | 37 | // await installTarget('wasm32-unknown-unknown'); 38 | await installTarget('wasm32-wasip1'); 39 | console.log('wasm build environment successfully installed'); 40 | } 41 | 42 | export const rustup = { 43 | hasTargetInstalled, 44 | ensureWasmSetup, 45 | }; 46 | -------------------------------------------------------------------------------- /tools/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "types": ["node"], 4 | "target": "ESNext", 5 | "module": "NodeNext", 6 | "moduleResolution": "NodeNext", 7 | "composite": true, 8 | "noEmit": true, 9 | "allowJs": true, 10 | "checkJs": true, 11 | "lib": [], 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tsconfig.base.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "incremental": true, 4 | // Targeting 5 | "target": "esnext", 6 | "moduleResolution": "bundler", 7 | // JSDoc type checking 8 | "allowJs": true, 9 | "checkJs": true, 10 | "noEmit": true, 11 | "strict": true, 12 | // Misc. 13 | "jsx": "preserve", 14 | "esModuleInterop": true, 15 | "resolveJsonModule": true 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tsconfig.client.json: -------------------------------------------------------------------------------- 1 | // Configuration for projects that run in Browser/client environments. 2 | { 3 | "extends": "./tsconfig.base.json", 4 | "compilerOptions": { 5 | "module": "commonjs", 6 | "moduleResolution": "bundler", 7 | "jsx": "preserve", 8 | }, 9 | } 10 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.base.json", 3 | } -------------------------------------------------------------------------------- /tsconfig.node.json: -------------------------------------------------------------------------------- 1 | // Configuration for projects that run in Node.js/server environments. 2 | { 3 | "extends": "./tsconfig.base.json", 4 | "compilerOptions": { 5 | "types": ["node"], 6 | "target": "ESNext", 7 | "module": "NodeNext", 8 | "moduleResolution": "NodeNext", 9 | "composite": true, 10 | "noEmit": true 11 | }, 12 | "include": [ 13 | "crates/intl_message_database/", 14 | "packages/babel-plugin-transform-discord-intl/", 15 | "packages/eslint-plugin-discord-intl/", 16 | "packages/intl-loader-core/", 17 | "packages/jest-processor-discord-intl/", 18 | "packages/metro-intl-transformer/", 19 | "packages/rspack-intl-loader/", 20 | "packages/swc-intl-message-transformer/", 21 | ], 22 | } 23 | --------------------------------------------------------------------------------