├── .github └── workflows │ └── main.yml ├── .gitignore ├── .prettierignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── bin └── docs-sync ├── fixtures ├── source │ ├── 1 │ │ └── en │ │ │ └── doc.md │ └── .gitignore └── target │ ├── 1 │ └── fr │ │ └── doc.md │ ├── .gitignore │ ├── attribution.json │ └── localize.json ├── package.json ├── src ├── commands │ ├── getEnglishFiles.ts │ ├── pull.ts │ ├── updateIssues.ts │ └── validate.ts ├── index.ts └── util │ ├── cloneRepo.ts │ ├── getGHTar.ts │ ├── recursiveReadDirSync.ts │ ├── refForPath.ts │ └── setupFolders.ts ├── test └── blah.test.ts ├── tsconfig.json └── yarn.lock /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: [push] 3 | jobs: 4 | build: 5 | name: Build, lint, and test on Node ${{ matrix.node }} and ${{ matrix.os }} 6 | 7 | runs-on: ${{ matrix.os }} 8 | strategy: 9 | matrix: 10 | node: ['10.x', '12.x', '14.x'] 11 | os: [ubuntu-latest, windows-latest, macOS-latest] 12 | 13 | steps: 14 | - name: Checkout repo 15 | uses: actions/checkout@v2 16 | 17 | - name: Use Node ${{ matrix.node }} 18 | uses: actions/setup-node@v1 19 | with: 20 | node-version: ${{ matrix.node }} 21 | 22 | - name: Install deps and build (with cache) 23 | uses: bahmutov/npm-install@v1 24 | 25 | - name: Test 26 | run: yarn test --ci --coverage --maxWorkers=2 27 | 28 | - name: Build 29 | run: yarn build 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | .DS_Store 3 | node_modules 4 | dist 5 | tmp -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | src/index.ts -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | This repo uses TSDX. 3 | 4 | ## Commands 5 | 6 | To run TSDX, use: 7 | 8 | ```bash 9 | yarn start 10 | ``` 11 | 12 | This builds to `/dist` and runs the project in watch mode so any edits you save inside `src` causes a rebuild to `/dist`. 13 | 14 | To do a one-off build, use `npm run build` or `yarn build`. 15 | 16 | To run tests, use `npm test` or `yarn test`. 17 | 18 | ## Configuration 19 | 20 | Code quality is set up for you with `prettier`, `husky`, and `lint-staged`. Adjust the respective fields in `package.json` accordingly. 21 | 22 | ### Jest 23 | 24 | Jest tests are set up to run with `npm test` or `yarn test`. 25 | 26 | ### Bundle Analysis 27 | 28 | [`size-limit`](https://github.com/ai/size-limit) is set up to calculate the real cost of your library with `npm run size` and visualize the bundle with `npm run analyze`. 29 | 30 | #### Setup Files 31 | 32 | This is the folder structure we set up for you: 33 | 34 | ```txt 35 | /src 36 | index.tsx # EDIT THIS 37 | /test 38 | blah.test.tsx # EDIT THIS 39 | .gitignore 40 | package.json 41 | README.md # EDIT THIS 42 | tsconfig.json 43 | ``` 44 | 45 | ### Rollup 46 | 47 | TSDX uses [Rollup](https://rollupjs.org) as a bundler and generates multiple rollup configs for various module formats and build settings. See [Optimizations](#optimizations) for details. 48 | 49 | ### TypeScript 50 | 51 | `tsconfig.json` is set up to interpret `dom` and `esnext` types, as well as `react` for `jsx`. Adjust according to your needs. 52 | 53 | ## Continuous Integration 54 | 55 | ### GitHub Actions 56 | 57 | Two actions are added by default: 58 | 59 | - `main` which installs deps w/ cache, lints, tests, and builds on all pushes against a Node and OS matrix 60 | - `size` which comments cost comparison of your library on every pull request using [`size-limit`](https://github.com/ai/size-limit) 61 | 62 | ## Optimizations 63 | 64 | Please see the main `tsdx` [optimizations docs](https://github.com/palmerhq/tsdx#optimizations). In particular, know that you can take advantage of development-only optimizations: 65 | 66 | ```js 67 | // ./types/index.d.ts 68 | declare var __DEV__: boolean; 69 | 70 | // inside your code... 71 | if (__DEV__) { 72 | console.log('foo'); 73 | } 74 | ``` 75 | 76 | You can also choose to install and use [invariant](https://github.com/palmerhq/tsdx#invariant) and [warning](https://github.com/palmerhq/tsdx#warning) functions. 77 | 78 | ## Module Formats 79 | 80 | CJS, ESModules, and UMD module formats are supported. 81 | 82 | The appropriate paths are configured in `package.json` and `dist/index.js` accordingly. Please report if any issues are found. 83 | 84 | ## Named Exports 85 | 86 | Per Palmer Group guidelines, [always use named exports.](https://github.com/palmerhq/typescript#exports) Code split inside your React app instead of your React library. 87 | 88 | ## Including Styles 89 | 90 | There are many ways to ship styles, including with CSS-in-JS. TSDX has no opinion on this, configure how you like. 91 | 92 | For vanilla CSS, you can include it at the root directory and add it to the `files` section in your `package.json`, so that it can be imported separately by your users and run through their bundler's loader. 93 | 94 | ## Publishing to NPM 95 | 96 | We recommend using [np](https://github.com/sindresorhus/np). 97 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 github@orta.io 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # @oss-docs/sync 2 | 3 | This is the CLI which powers a docs translation repo. It's job is to handle a few things: 4 | 5 | - Pulling translations into the app repo (`docs-sync pull`) 6 | - Grabbing the english files from the localization repo (`docs-sync get-en`) 7 | - Validating the changes in the localization repo (`docs-sync validate-against-en`) 8 | - Updating a set of GitHub issues (`update-github-issues`) 9 | 10 | ### `localize.json` 11 | 12 | This CLI expects you to have a `localize.json` in the root of your localization repo. 13 | 14 | ```json 15 | { 16 | "app": "microsft/TypeScript-Website#v2", 17 | "issues": { 18 | "pt": 23, 19 | "zh": 44 20 | }, 21 | "docsRoots": [ 22 | { "name": "Playground", "from": "packages/playground-examples/copy/", "to": "docs/playground" }, 23 | { "name": "TSConfig Reference", "from": "packages/tsconfig-reference/copy/", "to": "docs/tsconfig" }, 24 | { "name": "Website Copy", "from": "packages/typescriptlang-org/src/copy/", "to": "docs/typescriptlang" }, 25 | { "name": "Handbook", "from": "packages/documentation/copy/", "to": "docs/documentation" } 26 | ], 27 | "validate": { 28 | "ignoreFiles": ["docs/playground/*.js"] 29 | } 30 | } 31 | ``` 32 | -------------------------------------------------------------------------------- /bin/docs-sync: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | 3 | var path = require('path'); 4 | var fs = require('fs'); 5 | 6 | var potentialPaths = [ 7 | path.join(process.cwd(), 'node_modules/@oss-docs/sync/dist/index.js'), 8 | path.join(__dirname, '../dist/index.js') 9 | ]; 10 | 11 | for (var i = 0, len = potentialPaths.length; i < len; i++) { 12 | if (fs.existsSync(potentialPaths[i])) { 13 | require(potentialPaths[i]) 14 | break; 15 | } 16 | } -------------------------------------------------------------------------------- /fixtures/source/.gitignore: -------------------------------------------------------------------------------- 1 | **/1/fr -------------------------------------------------------------------------------- /fixtures/source/1/en/doc.md: -------------------------------------------------------------------------------- 1 | hello in eng -------------------------------------------------------------------------------- /fixtures/target/.gitignore: -------------------------------------------------------------------------------- 1 | **/en -------------------------------------------------------------------------------- /fixtures/target/1/fr/doc.md: -------------------------------------------------------------------------------- 1 | french -------------------------------------------------------------------------------- /fixtures/target/attribution.json: -------------------------------------------------------------------------------- 1 | { "/1/fr/doc.md": { "top": [{}], "total": 1 } } 2 | -------------------------------------------------------------------------------- /fixtures/target/localize.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "microsoft/TypeScript-Website#v2", 3 | "issues": { 4 | "pt": 1, 5 | "ja": 3, 6 | "es": 4, 7 | "zh": 5, 8 | "ko": 6, 9 | "id": 7, 10 | "uk": 8, 11 | "pl": 9, 12 | "it": 10 13 | }, 14 | "docsRoots": [ 15 | { "name": "docs", "from": "./1", "to": "./1" } 16 | ], 17 | "validate": { 18 | "ignoreFiles": ["a/b"] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.1.1", 3 | "license": "MIT", 4 | "repository": "OSS-Docs-Tools/sync", 5 | "main": "dist/index.js", 6 | "typings": "dist/index.d.ts", 7 | "files": [ 8 | "dist", 9 | "src" 10 | ], 11 | "bin": { 12 | "docs-sync": "bin/docs-sync" 13 | }, 14 | "engines": { 15 | "node": ">=10" 16 | }, 17 | "scripts": { 18 | "start": "tsdx watch", 19 | "build": "tsdx build", 20 | "test": "tsdx test", 21 | "lint": "tsdx lint", 22 | "prepare": "tsdx build", 23 | "size": "size-limit", 24 | "analyze": "size-limit --why" 25 | }, 26 | "peerDependencies": {}, 27 | "husky": { 28 | "hooks": { 29 | "pre-commit": "tsdx test" 30 | } 31 | }, 32 | "prettier": { 33 | "printWidth": 140, 34 | "semi": false, 35 | "singleQuote": false, 36 | "trailingComma": "es5" 37 | }, 38 | "name": "@oss-docs/sync", 39 | "author": "github@orta.io", 40 | "module": "dist/oss-docs-sync.esm.js", 41 | "size-limit": [ 42 | { 43 | "path": "dist/oss-docs-sync.cjs.production.min.js", 44 | "limit": "10 KB" 45 | }, 46 | { 47 | "path": "dist/oss-docs-sync.esm.js", 48 | "limit": "10 KB" 49 | } 50 | ], 51 | "devDependencies": { 52 | "@size-limit/preset-small-lib": "^4.9.1", 53 | "@types/gunzip-maybe": "^1.4.0", 54 | "@types/micromatch": "^4.0.1", 55 | "@types/tar-fs": "^2.0.0", 56 | "@types/yargs": "^15.0.12", 57 | "husky": "^4.3.6", 58 | "size-limit": "^4.9.1", 59 | "tsdx": "^0.14.1", 60 | "tslib": "^2.1.0", 61 | "typescript": "^4.1.3" 62 | }, 63 | "dependencies": { 64 | "cachedir": "^2.3.0", 65 | "chalk": "^4.1.0", 66 | "cross-spawn": "^7.0.1", 67 | "gunzip-maybe": "^1.4.1", 68 | "micromatch": "^4.0.2", 69 | "mvdir": "^1.0.21", 70 | "tar-fs": "^2.1.0", 71 | "yargs": "^16.2.0" 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/commands/getEnglishFiles.ts: -------------------------------------------------------------------------------- 1 | import { readFileSync, readdirSync } from "fs" 2 | import { moveEnFoldersIn, moveAllFoldersIn } from "../util/setupFolders" 3 | import { existsSync, mkdirSync } from "fs" 4 | import { join } from "path" 5 | import { ghRepresentationForPath } from "../util/refForPath" 6 | import { getGHTar } from "../util/getGHTar" 7 | import { Settings } from ".." 8 | 9 | // node dist/index.js get-en --from-cwd ./fixtures/source --to-cwd fixtures/target 10 | 11 | export const getEnglish = async (opts: { source: string; toCwd: string; fromCwd?: string; all: boolean }) => { 12 | const toDir = opts.toCwd 13 | const localizeJSONPath = join(toDir, "localize.json") 14 | if (!existsSync(localizeJSONPath)) { 15 | throw new Error(`There isn't a localize.json file in the root of the current working dir (expected at ${localizeJSONPath})`) 16 | } 17 | 18 | const settings = JSON.parse(readFileSync(localizeJSONPath, "utf8")) as Settings 19 | const ghRep = ghRepresentationForPath(opts.source) 20 | 21 | const cachedir: string = require("cachedir")("oss-doc-sync") 22 | const [user, repo] = ghRep.repoSlug!.split("/") 23 | let localCopy = opts.fromCwd 24 | 25 | // Grab a copy of the other repo, and pull in the files 26 | if (!localCopy) { 27 | if (!existsSync(cachedir)) mkdirSync(cachedir, { recursive: true }) 28 | if (!existsSync(join(cachedir, user))) mkdirSync(join(cachedir, user)) 29 | 30 | await getGHTar({ 31 | user, 32 | repo, 33 | branch: ghRep.branch, 34 | to: join(cachedir, user, repo), 35 | }) 36 | 37 | const unzipped = join(cachedir, user, repo) 38 | localCopy = join(unzipped, readdirSync(unzipped).find(p => !p.startsWith("."))!) 39 | } 40 | 41 | if (opts.all) { 42 | moveAllFoldersIn(localCopy, toDir, settings) 43 | } else { 44 | moveEnFoldersIn(localCopy, toDir, settings) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/commands/pull.ts: -------------------------------------------------------------------------------- 1 | // node dist/index.js pull asda/asdasd 111 --from-cwd ./fixtures/target --to-cwd fixtures/source 2 | // node dist/index.js delete-translations asda/asdasd 111 --from-cwd ./fixtures/target --to-cwd fixtures/source 3 | 4 | import { readFileSync, readdirSync } from "fs" 5 | import { deleteLocaleFiles, moveLocaleFoldersIn } from "../util/setupFolders" 6 | import { existsSync, mkdirSync } from "fs" 7 | import { join } from "path" 8 | import { ghRepresentationForPath } from "../util/refForPath" 9 | import { getGHTar } from "../util/getGHTar" 10 | 11 | export const setup = async (opts: { target: string; toCwd: string; fromCwd?: string }) => { 12 | const ghRep = ghRepresentationForPath(opts.target) 13 | 14 | const cachedir: string = require("cachedir")("oss-doc-sync") 15 | const [user, repo] = ghRep.repoSlug!.split("/") 16 | let localCopy = opts.fromCwd 17 | const toDir = opts.toCwd 18 | 19 | // Grab a copy of the other repo, and pull in the files 20 | if (!localCopy) { 21 | if (!existsSync(cachedir)) mkdirSync(cachedir, { recursive: true }) 22 | if (!existsSync(join(cachedir, user))) mkdirSync(join(cachedir, user)) 23 | 24 | await getGHTar({ 25 | user, 26 | repo, 27 | branch: ghRep.branch, 28 | to: join(cachedir, user, repo), 29 | }) 30 | 31 | const unzipped = join(cachedir, user, repo) 32 | localCopy = join(unzipped, readdirSync(unzipped).find(p => !p.startsWith("."))!) 33 | } 34 | 35 | const localizeJSONPath = join(localCopy, "localize.json") 36 | if (!existsSync(localizeJSONPath)) { 37 | throw new Error( 38 | `There isn't a localize.json file in the root of this: ${user}/${repo}#${ghRep.branch} (locally found at ${localizeJSONPath})` 39 | ) 40 | } 41 | 42 | const settings = JSON.parse(readFileSync(localizeJSONPath, "utf8")) 43 | return { toDir, localCopy, settings } 44 | } 45 | 46 | 47 | export const rmCommand = async (opts: { target: string; toCwd: string; fromCwd?: string }) => { 48 | const {toDir, localCopy, settings } = await setup(opts) 49 | deleteLocaleFiles(toDir, localCopy, settings) 50 | } 51 | 52 | 53 | export const pullCommand = async (opts: { target: string; toCwd: string; fromCwd?: string }) => { 54 | const {toDir, localCopy, settings } = await setup(opts) 55 | moveLocaleFoldersIn(toDir, localCopy, settings) 56 | } 57 | -------------------------------------------------------------------------------- /src/commands/updateIssues.ts: -------------------------------------------------------------------------------- 1 | import chalk from "chalk" 2 | import { join, basename } from "path" 3 | import {existsSync, readFileSync} from "fs" 4 | import { Settings } from ".." 5 | import { recursiveReadDirSync } from "../util/recursiveReadDirSync" 6 | import { ghRepresentationForPath } from "../util/refForPath" 7 | import { cloneRepo } from "../util/cloneRepo" 8 | import * as https from "https" 9 | 10 | const cwd = "." 11 | 12 | export const updateIssues = async (args: {thisRepo: string}) => { 13 | const localizeJSONPath = join(cwd, "localize.json") 14 | if (!existsSync(localizeJSONPath)) { 15 | throw new Error(`There isn't a localize.json file in the root of the current working dir (expected at ${localizeJSONPath})`) 16 | } 17 | 18 | const settings = JSON.parse(readFileSync(localizeJSONPath, "utf8")) as Settings 19 | const ghRep = ghRepresentationForPath(settings.app) 20 | 21 | // Grab a copy of the other repo, and pull in the files 22 | const appRoot = await cloneRepo(ghRep) 23 | 24 | const langs = Object.keys(settings.issues) 25 | for (const lang of langs) { 26 | 27 | const todo = getAllTODOFiles(appRoot, settings, lang) 28 | const md = toMarkdown(todo, args.thisRepo) 29 | if (!process.env.GITHUB_ACCESS_TOKEN) { 30 | console.log(chalk.bold("Printing to console because GITHUB_ACCESS_TOKEN is not set")) 31 | console.log(md) 32 | } else { 33 | await updateGitHubIssue({ number: settings.issues[lang], repo: args.thisRepo, markdown: md, lang, token: process.env.GITHUB_ACCESS_TOKEN }) 34 | } 35 | } 36 | } 37 | 38 | const getAllTODOFiles = (appCWD: string, settings:Settings, lang: string) => { 39 | const diffFolders = (en: string, thisLang: string) => { 40 | 41 | const englishFiles = recursiveReadDirSync(en) 42 | const thisLangFiles = recursiveReadDirSync(thisLang) 43 | 44 | const todo: string[] = [] 45 | const done: string[] = [] 46 | englishFiles.forEach(enFile => { 47 | const localFile = enFile.replace(en, "").replace("/en/", `/${lang}/`) 48 | const reRooted = join(thisLang, localFile) 49 | if (thisLangFiles.includes(reRooted)) { 50 | done.push(localFile) 51 | } else { 52 | todo.push(enFile) 53 | } 54 | }) 55 | 56 | return { todo, done } 57 | } 58 | 59 | const todos: Record = {} 60 | for (const root of settings.docsRoots) { 61 | process.stderr.write(`\n ${chalk.bold(root.to)}:`) 62 | 63 | const appDir = join(appCWD, root.from) 64 | const toDir = join(cwd, root.to) 65 | 66 | const en = join(appDir, "en") 67 | if (!existsSync(en)) { 68 | throw new Error(`No en folder found at ${en}.`) 69 | } 70 | 71 | const langRoot = join(toDir, lang) 72 | todos[root.name] = diffFolders(en, langRoot) 73 | 74 | } 75 | 76 | return todos 77 | } 78 | 79 | let totalDone = 0 80 | let totalTodo = 0 81 | 82 | const toMarkdown = (files: Record, repo: string) => { 83 | const md = [""] 84 | 85 | const markdownLink = (f: string, done: boolean) => { 86 | const name = basename(f) 87 | const url = `https://github.com/${repo}/blob/` 88 | const check = done ? "x" : " " 89 | return `- [${check}] [\`${name}\`](${url}${f.replace(/ /g, "%20")})` 90 | } 91 | 92 | Object.keys(files).forEach(section => { 93 | const todo = files[section].todo 94 | const done = files[section].done 95 | 96 | md.push("\n\n## " + section + "\n") 97 | md.push(`Done: ${done.length}, TODO: ${todo.length}.\n\n`) 98 | done.forEach(f => { 99 | md.push(markdownLink(f, true)) 100 | }) 101 | 102 | todo.forEach(f => { 103 | md.push(markdownLink(f, false)) 104 | }) 105 | 106 | totalDone += done.length 107 | totalTodo += todo.length 108 | }) 109 | 110 | md[0] = `For this language there are ${totalDone} translated files, with ${totalTodo} TODO.\n\n` 111 | return md.join("\n") 112 | } 113 | function updateGitHubIssue(args: { number: number; repo: string; markdown: string, lang: string, token: string }) { 114 | // https://docs.github.com/en/rest/reference/issues#update-an-issue 115 | 116 | const data = JSON.stringify({ 117 | body: args.markdown, 118 | labels: ["Translation Summary", args.lang] 119 | }) 120 | 121 | const options = { 122 | hostname: 'api.github.com', 123 | port: 443, 124 | path: `/repos/${args.repo}/issues/${args.number}`, 125 | method: 'PATCH', 126 | headers: { 127 | 'Content-Type': 'application/json', 128 | 'Content-Length': data.length, 129 | "Accept": "application/vnd.github.v3+json", 130 | 'User-Agent': "OSS Docs Sync Issue Auto-Updater", 131 | Authorization: `token ${args.token}` 132 | } 133 | } 134 | 135 | const req = https.request(options, res => { console.log(`statusCode: ${res.statusCode}`) }) 136 | req.on('error', error => { 137 | console.error(error) 138 | }) 139 | req.write(data) 140 | req.end() 141 | } 142 | 143 | -------------------------------------------------------------------------------- /src/commands/validate.ts: -------------------------------------------------------------------------------- 1 | import chalk from "chalk" 2 | import { existsSync, statSync, readFileSync, readdirSync } from "fs" 3 | import { join } from "path" 4 | import { ghRepresentationForPath } from "../util/refForPath" 5 | import { Settings } from ".." 6 | import { recursiveReadDirSync } from "../util/recursiveReadDirSync" 7 | import { cloneRepo } from "../util/cloneRepo" 8 | import micromatch from 'micromatch' 9 | 10 | // node dist/index.js validate-against-en --to-cwd fixtures/target --from-cwd ./fixtures/source 11 | 12 | const tick = chalk.bold.greenBright("✓") 13 | const cross = chalk.bold.redBright("⤫") 14 | 15 | export const validate = async (opts: { toCwd: string; fromCwd?: string }) => { 16 | console.error(`Comparing that all locale files match the en version:`) 17 | const toDir = opts.toCwd 18 | 19 | const localizeJSONPath = join(toDir, "localize.json") 20 | if (!existsSync(localizeJSONPath)) { 21 | throw new Error(`There isn't a localize.json file in the root of the current working dir (expected at ${localizeJSONPath})`) 22 | } 23 | 24 | const settings = JSON.parse(readFileSync(localizeJSONPath, "utf8")) as Settings 25 | const ghRep = ghRepresentationForPath(settings.app) 26 | 27 | // Grab a copy of the other repo, and pull in the files 28 | let localCopy = opts.fromCwd 29 | if (!localCopy) { 30 | localCopy = await cloneRepo(ghRep) 31 | } 32 | 33 | const wrong: { path: string; lang: string; from: string, expected: string}[] = [] 34 | 35 | for (const root of settings.docsRoots) { 36 | process.stderr.write(`\n ${chalk.bold(root.to)}:`) 37 | 38 | const appDir = join(localCopy, root.from) 39 | const toDir = join(opts.toCwd, root.to) 40 | 41 | const en = join(appDir, "en") 42 | if (!existsSync(en)) { 43 | throw new Error(`No en folder found at ${en}.`) 44 | } 45 | 46 | const englishTree = recursiveReadDirSync(en) 47 | let allFolders = readdirSync(toDir) 48 | 49 | const languages = allFolders.filter(f => statSync(join(toDir, f)).isDirectory()).filter(f => f !== "en") 50 | languages.forEach(lang => { 51 | process.stderr.write(` ${lang}`) 52 | 53 | const fullpath = join(toDir, lang) 54 | let langTree = recursiveReadDirSync(fullpath) 55 | 56 | if (settings.validate?.ignoreFiles) { 57 | const toRemove = micromatch(langTree, settings.validate?.ignoreFiles) 58 | langTree = langTree.filter(i => !toRemove.includes(i)) 59 | } 60 | 61 | let error = false 62 | langTree.forEach(path => { 63 | const enRelative = path.replace(fullpath, "") 64 | const reRooted = join(appDir, "en", enRelative) 65 | if (!englishTree.includes(reRooted)) { 66 | error = true 67 | process.exitCode = 1 68 | wrong.push({ path, lang, from: root.from, expected: reRooted }) 69 | } 70 | }) 71 | 72 | process.stderr.write(" " + (error ? cross : tick)) 73 | }) 74 | } 75 | 76 | console.error("") 77 | 78 | if (wrong.length) { 79 | console.error(chalk.bold.red("\nFiles with paths that aren't the same as English files:\n")) 80 | 81 | wrong.forEach(w => { 82 | console.error(" " + w.path) 83 | console.error(" > Expected " + w.expected) 84 | }) 85 | 86 | console.error("\n# All English files\n") 87 | 88 | for (const root of settings.docsRoots) { 89 | console.error(`### ${root.name}\n`) 90 | 91 | const appDir = join(localCopy, root.from) 92 | const en = join(appDir, "en") 93 | const enLang = recursiveReadDirSync(en) 94 | process.stderr.write(" - ") 95 | console.error(enLang.join(" - ")) 96 | } 97 | 98 | console.error("\n") 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import yargs from 'yargs/yargs' 2 | import { getEnglish } from './commands/getEnglishFiles'; 3 | import { pullCommand, rmCommand } from './commands/pull'; 4 | import { updateIssues } from './commands/updateIssues'; 5 | import { validate } from './commands/validate'; 6 | 7 | export interface CLIOpts { 8 | _: [string, string] 9 | target: string 10 | source: string 11 | toCwd: string 12 | fromCwd: string 13 | } 14 | 15 | export interface Settings { 16 | app: string, 17 | issues: Record 18 | docsRoots: Array<{ name: string, from: string, to: string }> 19 | validate?: { 20 | ignoreFiles?: string[] 21 | } 22 | } 23 | 24 | yargs(process.argv.slice(2)).scriptName("docs-sync") 25 | 26 | .command('pull ', 'Used in your repo to grab localizations', (argv) => { 27 | argv.options("to-cwd", { type: "string", default: ".", description: "What folder do you want to treat as the base for the localize.json to use" }) 28 | argv.options("from-cwd", { type: "string", description: "Instead of downloading from GitHub, you can use a local dir as the place to pull from"}) 29 | }, (argv: CLIOpts) => { 30 | pullCommand(argv) 31 | }) 32 | 33 | .command('delete-translations ', 'Remove translation files from an app repo', (argv) => { 34 | argv.options("to-cwd", { type: "string", default: ".", description: "What folder do you want to treat as the base for the localize.json to use" }) 35 | argv.options("from-cwd", { type: "string", description: "Instead of downloading from GitHub, you can use a local dir as the place to pull from"}) 36 | }, (argv: CLIOpts) => { 37 | rmCommand(argv) 38 | }) 39 | 40 | .command('get-en ', 'Used in localizations to get the english versions', (argv) => { 41 | argv.options("to-cwd", { type: "string", default: ".", description: "What folder do you want to treat as the base for the localize.json to use" }) 42 | argv.options("from-cwd", { type: "string", description: "Instead of downloading from GitHub, you can use a local dir as the place to pull from"}) 43 | argv.options("all", { type: "boolean", default: false, description: "Also include the other folders"}) 44 | }, (argv: CLIOpts & { all: boolean }) => { 45 | getEnglish(argv) 46 | }) 47 | 48 | .command('validate-against-en', 'Used in localizations to validate against english versions', (argv) => { 49 | argv.options("to-cwd", { type: "string", default: ".", description: "What folder do you want to treat as the base for the localize.json to use" }) 50 | argv.options("from-cwd", { type: "string", description: "Instead of downloading from GitHub, you can use a local dir as the place to pull from"}) 51 | }, (argv: CLIOpts) => { 52 | validate(argv) 53 | }) 54 | 55 | .command('update-github-issues ', 'Uses info in localization.json to create issues covering the state of translations', () => {}, (argv: { thisRepo: string}) => { 56 | updateIssues(argv) 57 | }) 58 | .argv; 59 | 60 | -------------------------------------------------------------------------------- /src/util/cloneRepo.ts: -------------------------------------------------------------------------------- 1 | import { existsSync, mkdirSync, readdirSync } from "fs" 2 | import { join } from "path" 3 | import { getGHTar } from "../util/getGHTar" 4 | import { GHRep } from "./refForPath" 5 | 6 | export const cloneRepo = async (ghRep: GHRep) => { 7 | const cachedir: string = require("cachedir")("oss-doc-sync") 8 | const [user, repo] = ghRep.repoSlug!.split("/") 9 | 10 | if (!existsSync(cachedir)) mkdirSync(cachedir); 11 | if (!existsSync(join(cachedir, user))) mkdirSync(join(cachedir, user)); 12 | 13 | await getGHTar({ 14 | user, 15 | repo, 16 | branch: ghRep.branch, 17 | to: join(cachedir, user, repo), 18 | }) 19 | 20 | 21 | const unzipped = join(cachedir, user, repo) 22 | return join(unzipped, readdirSync(unzipped).find(p => !p.startsWith("."))!) 23 | } 24 | -------------------------------------------------------------------------------- /src/util/getGHTar.ts: -------------------------------------------------------------------------------- 1 | // Based on fork of https://github.com/c8r/initit/blob/master/index.js 2 | // which is MIT licensed 3 | 4 | import path from "path" 5 | import tar from "tar-fs" 6 | import gunzip from "gunzip-maybe" 7 | import https from "https" 8 | 9 | interface TarOpts { 10 | user: string 11 | repo: string 12 | to: string 13 | branch: string 14 | templatepath?: string 15 | } 16 | 17 | export const getGHTar = ({ user, branch, repo, templatepath = "", to }: TarOpts) => { 18 | return new Promise((resolve, reject) => { 19 | const ignorePrefix = "__INITIT_IGNORE__/" 20 | const ignorepath = path.join(to, ignorePrefix) 21 | const extractTar = tar.extract(to, { 22 | ignore: (filepath: string) => { 23 | const isInIgnoreFolder = !path.relative(ignorepath, filepath).startsWith("..") 24 | return isInIgnoreFolder 25 | }, 26 | }) 27 | const url = `https://codeload.github.com/${user}/${repo}/tar.gz/${branch}` 28 | https.get(url, (response: any) => response.pipe(gunzip()).pipe(extractTar)) 29 | extractTar.on("error", err => { 30 | console.error(`Could not download from ${url} - maybe wrong branch?`) 31 | reject(err) 32 | }) 33 | extractTar.on("finish", resolve) 34 | }) 35 | } 36 | -------------------------------------------------------------------------------- /src/util/recursiveReadDirSync.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs" 2 | import * as path from "path" 3 | 4 | /** Recursively retrieve file paths from a given folder and its subfolders. */ 5 | // https://gist.github.com/kethinov/6658166#gistcomment-2936675 6 | export const recursiveReadDirSync = (folderPath: string): string[] => { 7 | if (!fs.existsSync(folderPath)) return [] 8 | 9 | const entryPaths = fs.readdirSync(folderPath).map(entry => path.join(folderPath, entry)) 10 | const filePaths = entryPaths.filter(entryPath => fs.statSync(entryPath).isFile()) 11 | const dirPaths = entryPaths.filter(entryPath => !filePaths.includes(entryPath)) 12 | const dirFiles = dirPaths.reduce( 13 | // @ts-ignore 14 | (prev, curr) => prev.concat(recursiveReadDirSync(curr)), 15 | [] 16 | ) 17 | 18 | return [...filePaths, ...dirFiles] 19 | .filter(f => !f.endsWith(".DS_Store") && !f.endsWith("README.md")) 20 | .map(f => { 21 | const root = path.join(__dirname, "..", "..", "..") 22 | return f.replace(root, "") 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /src/util/refForPath.ts: -------------------------------------------------------------------------------- 1 | 2 | export const ghRepresentationForPath = (value: string) => { 3 | const afterAt = value.includes("@") ? value.split("@")[1] : value 4 | return { 5 | branch: value.includes("#") ? value.split("#")[1] : "master", 6 | filePath: afterAt.split("#")[0], 7 | repoSlug: value.includes("@") ? value.split("@")[0] : value.split("#")[0], 8 | referenceString: value, 9 | } 10 | } 11 | 12 | export type GHRep = ReturnType 13 | -------------------------------------------------------------------------------- /src/util/setupFolders.ts: -------------------------------------------------------------------------------- 1 | import chalk from "chalk" 2 | import { readdirSync, existsSync, statSync, rmdirSync } from "fs" 3 | import { join } from "path" 4 | import type { Settings } from ".." 5 | 6 | const mvdir = require("mvdir") 7 | 8 | export const moveAllFoldersIn = async (fromWD: string, toWD: string, settings: Settings) => { 9 | for (const root of settings.docsRoots) { 10 | const fromDir = join(fromWD, root.from) 11 | const toDir = join(toWD, root.to) 12 | 13 | const allFolders = readdirSync(fromDir) 14 | const folders = allFolders.filter(f => statSync(join(fromDir, f)).isDirectory()).filter(f => f !== "en") 15 | 16 | for (const lang of folders) { 17 | await mvdir(join(fromDir, lang), join(toDir, lang), { copy: true }) 18 | } 19 | } 20 | } 21 | 22 | export const moveEnFoldersIn = async (fromWD: string, toWD: string, settings: Settings) => { 23 | const name = fromWD.includes("tmp") || fromWD.includes("Caches") ? settings.app : fromWD 24 | console.error(`Moved en files from ${chalk.bold(name)}:\n`) 25 | 26 | for (const root of settings.docsRoots) { 27 | const fromDir = join(fromWD, root.from) 28 | const toDir = join(toWD, root.to) 29 | const toEN = join(toDir, "en") 30 | 31 | const en = join(fromDir, "en") 32 | if (!existsSync(en)) { 33 | throw new Error(`No en folder found at ${en}.`) 34 | } 35 | 36 | await mvdir(en, toEN, { copy: true }) 37 | console.error(` ${chalk.bold(root.from)} -> ${chalk.bold(toEN)}`) 38 | } 39 | console.error("") 40 | } 41 | 42 | export const moveLocaleFoldersIn = async (appWD: string, lclWD: string, settings: Settings) => { 43 | const isFromTmp = lclWD.includes("tmp") || lclWD.includes("Caches") 44 | const name = isFromTmp ? settings.app : lclWD 45 | console.error(`Moved locale files from ${chalk.bold(name)}:\n`) 46 | 47 | for (const root of settings.docsRoots) { 48 | const fromDir = join(lclWD, root.to) 49 | const toDir = join(appWD, root.from) 50 | 51 | const allFolders = readdirSync(fromDir) 52 | const folders = allFolders.filter(f => statSync(join(fromDir, f)).isDirectory()).filter(f => f !== "en") 53 | for (const lang of folders) { 54 | await mvdir(join(fromDir, lang), join(toDir, lang), { copy: true }) 55 | } 56 | 57 | const displayFrom = isFromTmp ? "[cache]" + fromDir.replace(root.from, "") : fromDir 58 | console.error(` ${chalk.bold(displayFrom)} [${folders.join(", ")}] -> ${chalk.bold(toDir)}`) 59 | } 60 | 61 | console.error("") 62 | } 63 | 64 | export const deleteLocaleFiles = async (appWD: string, lclWD: string, settings: Settings) => { 65 | const isFromTmp = lclWD.includes("tmp") || lclWD.includes("Caches") 66 | const name = isFromTmp ? settings.app : lclWD 67 | console.error(`Moved preparing to delete locales from ${chalk.bold(name)}:\n`) 68 | 69 | for (const root of settings.docsRoots) { 70 | const fromDir = join(lclWD, root.to) 71 | const toDir = join(appWD, root.from) 72 | 73 | 74 | const allFolders = readdirSync(fromDir) 75 | const folders = allFolders.filter(f => statSync(join(fromDir, f)).isDirectory()).filter(f => f !== "en") 76 | for (const lang of folders) { 77 | rmdirSync(join(toDir, lang), { recursive: true }) 78 | } 79 | 80 | console.error(` ${chalk.bold("rm'd")} [${folders.join(", ")}]`) 81 | } 82 | 83 | console.error("") 84 | } 85 | 86 | -------------------------------------------------------------------------------- /test/blah.test.ts: -------------------------------------------------------------------------------- 1 | describe("blah", () => { 2 | it("works", () => { 3 | expect(2).toEqual(2) 4 | }) 5 | }) 6 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | // see https://www.typescriptlang.org/tsconfig to better understand tsconfigs 3 | "include": ["src", "types"], 4 | "compilerOptions": { 5 | "module": "esnext", 6 | "lib": ["dom", "esnext"], 7 | "importHelpers": true, 8 | // output .d.ts declaration files for consumers 9 | "declaration": true, 10 | // output .js.map sourcemap files for consumers 11 | "sourceMap": true, 12 | // match output dir to input dir. e.g. dist/index instead of dist/src/index 13 | "rootDir": "./src", 14 | // stricter type-checking for stronger correctness. Recommended by TS 15 | "strict": true, 16 | // linter checks for common issues 17 | "noImplicitReturns": true, 18 | "noFallthroughCasesInSwitch": true, 19 | // noUnused* overlap with @typescript-eslint/no-unused-vars, can disable if duplicative 20 | // "noUnusedLocals": true, 21 | // "noUnusedParameters": true, 22 | // use Node's module resolution algorithm, instead of the legacy TS one 23 | "moduleResolution": "node", 24 | // transpile JSX to React.createElement 25 | "jsx": "react", 26 | // interop between ESM and CJS modules. Recommended by TS 27 | "esModuleInterop": true, 28 | // significant perf increase by skipping checking .d.ts files, particularly those in node_modules. Recommended by TS 29 | "skipLibCheck": true, 30 | // error out if import and file system have a casing mismatch. Recommended by TS 31 | "forceConsistentCasingInFileNames": true, 32 | // `tsdx build` ignores this option, but it is commonly used when type-checking separately with `tsc` 33 | "noEmit": true, 34 | } 35 | } 36 | --------------------------------------------------------------------------------