├── .apibuilder ├── .tracked_files └── config ├── .eslintignore ├── .eslintrc.json ├── .gitattributes ├── .gitignore ├── .nvmrc ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── readme-api.json ├── result.png ├── sync ├── generated │ └── readme.ts ├── index.ts ├── util.test.ts ├── util.ts └── validation.ts └── tsconfig.json /.apibuilder/.tracked_files: -------------------------------------------------------------------------------- 1 | --- 2 | flow: 3 | readme: 4 | ts_sdk: 5 | - sync/generated/readme.ts 6 | -------------------------------------------------------------------------------- /.apibuilder/config: -------------------------------------------------------------------------------- 1 | code: 2 | flow: 3 | readme: 4 | version: latest 5 | generators: 6 | ts_sdk: sync/generated 7 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist/ 3 | sync/generated 4 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "es6": true, 4 | "node": true 5 | }, 6 | "extends": [ 7 | "eslint:recommended", 8 | "plugin:@typescript-eslint/eslint-recommended", 9 | "plugin:@typescript-eslint/recommended" 10 | ], 11 | "globals": { 12 | "Atomics": "readonly", 13 | "SharedArrayBuffer": "readonly" 14 | }, 15 | "parser": "@typescript-eslint/parser", 16 | "parserOptions": { 17 | "ecmaVersion": 2018, 18 | "sourceType": "module" 19 | }, 20 | "plugins": [ 21 | "@typescript-eslint" 22 | ], 23 | "rules": { 24 | "indent": [ 25 | "error", 26 | 4 27 | ], 28 | "linebreak-style": [ 29 | "error", 30 | "unix" 31 | ], 32 | "quotes": [ 33 | "error", 34 | "single" 35 | ], 36 | "semi": [ 37 | "error", 38 | "never" 39 | ] 40 | } 41 | } -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.ts linguist-language=TypeScript 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/node,macos 2 | # Edit at https://www.gitignore.io/?templates=node,macos 3 | 4 | ### macOS ### 5 | # General 6 | .DS_Store 7 | .AppleDouble 8 | .LSOverride 9 | 10 | # Icon must end with two \r 11 | Icon 12 | 13 | # Thumbnails 14 | ._* 15 | 16 | # Files that might appear in the root of a volume 17 | .DocumentRevisions-V100 18 | .fseventsd 19 | .Spotlight-V100 20 | .TemporaryItems 21 | .Trashes 22 | .VolumeIcon.icns 23 | .com.apple.timemachine.donotpresent 24 | 25 | # Directories potentially created on remote AFP share 26 | .AppleDB 27 | .AppleDesktop 28 | Network Trash Folder 29 | Temporary Items 30 | .apdisk 31 | 32 | ### Node ### 33 | # Logs 34 | logs 35 | *.log 36 | npm-debug.log* 37 | yarn-debug.log* 38 | yarn-error.log* 39 | lerna-debug.log* 40 | 41 | # Diagnostic reports (https://nodejs.org/api/report.html) 42 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 43 | 44 | # Runtime data 45 | pids 46 | *.pid 47 | *.seed 48 | *.pid.lock 49 | 50 | # Directory for instrumented libs generated by jscoverage/JSCover 51 | lib-cov 52 | 53 | # Coverage directory used by tools like istanbul 54 | coverage 55 | *.lcov 56 | 57 | # nyc test coverage 58 | .nyc_output 59 | 60 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 61 | .grunt 62 | 63 | # Bower dependency directory (https://bower.io/) 64 | bower_components 65 | 66 | # node-waf configuration 67 | .lock-wscript 68 | 69 | # Compiled binary addons (https://nodejs.org/api/addons.html) 70 | build/Release 71 | 72 | # Dependency directories 73 | node_modules/ 74 | jspm_packages/ 75 | 76 | # TypeScript v1 declaration files 77 | typings/ 78 | 79 | # TypeScript cache 80 | *.tsbuildinfo 81 | 82 | # Optional npm cache directory 83 | .npm 84 | 85 | # Optional eslint cache 86 | .eslintcache 87 | 88 | # Optional REPL history 89 | .node_repl_history 90 | 91 | # Output of 'npm pack' 92 | *.tgz 93 | 94 | # Yarn Integrity file 95 | .yarn-integrity 96 | 97 | # dotenv environment variables file 98 | .env 99 | .env.test 100 | 101 | # parcel-bundler cache (https://parceljs.org/) 102 | .cache 103 | 104 | # next.js build output 105 | .next 106 | 107 | # nuxt.js build output 108 | .nuxt 109 | 110 | # react / gatsby 111 | public/ 112 | 113 | # vuepress build output 114 | .vuepress/dist 115 | 116 | # Serverless directories 117 | .serverless/ 118 | 119 | # FuseBox cache 120 | .fusebox/ 121 | 122 | # DynamoDB Local files 123 | .dynamodb/ 124 | 125 | # End of https://www.gitignore.io/api/node,macos 126 | 127 | dist/ 128 | docs/ 129 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | lts/* 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Flow Commerce 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # readme.com Sync Tool 2 | 3 | This is a CLI tool that synchronizes markdown files from a local directory (typically in a git repo) to https://readme.com. 4 | 5 | ## Usage 6 | 7 | `npx readme-sync --apiKey --version --docs ` 8 | 9 | or, to just validate the files: 10 | 11 | `npx readme-sync --apiKey --version --docs --validateOnly` 12 | 13 | ## Expected Directory Structure 14 | 15 | Top level folders are mapped to categories. Second and third level `.md` files are synced as docs. Readme only supports two levels of nesting (Category > Parent Doc > Child Doc). If you want a doc with children, create a folder with the doc name, and create an `index.md` file inside it. 16 | 17 | The folder and file names are turned into the slugs. 18 | 19 | Example: 20 | 21 | ``` 22 | docs 23 | ├── Welcome 24 | │ ├── 00-introduction.md 25 | │ └── 10-license.md 26 | └── Integration 27 | ├── 00-installation.md 28 | ├── 10-setup.md 29 | └── 20-configuration 30 | ├── index.md 31 | ├── 00-database.md 32 | └── 10-proxy.md 33 | ``` 34 | 35 | Becomes 36 | 37 | ![](result.png) 38 | 39 | ## File Contents 40 | 41 | Markdown, with front matter: 42 | 43 | ```markdown 44 | --- 45 | title: "Installation" 46 | excerpt: "How to Install Arch Linux" # optional 47 | hidden: true # optional 48 | --- 49 | 50 | # Installation 51 | 52 | ... 53 | ``` 54 | 55 | ## Limitations 56 | 57 | - Categories cannot yet be created automatically. They must be manually created in Readme. You can fetch the existing category slugs with 58 | ```bash 59 | curl 'https://dash.readme.io/api/v1/categories?perPage=100' -u '': -H 'x-readme-version: ' 60 | ``` 61 | Note that category slugs may differ from the category titles you see on dash.readme.io, so this API call is a good way to troubleshoot the error message "can't create categories." 62 | 63 | ## Syncing Behavior 64 | 65 | - If you have a category on readme.com that you don't have locally, the category and its contents will remain untouched on readme.com. 66 | - If you have a doc on readme.com that you don't have locally (but you have the category), it will be deleted from readme.com. 67 | - If you have a doc locally that is not on readme.com, it will be uploaded to readme.com 68 | - If you try to create two docs with the same name, you'll get an error about document slugs not being unique, even if the files are in separate categories. 69 | - The publishing order is alphanumeric. You can force ordering by prefixing your files with `01-`, `02-`, etc. Then, these ordered pages go first in the table of contents (stripped of their `01-`, `02-` ordering prefixes). 70 | 71 | ## Development 72 | 73 | 1. `git clone https://github.com/flowcommerce/readme-sync` 74 | 1. `nvm install` 75 | 1. `npm install` 76 | 1. `npx ts-node sync/index.ts --apiKey --version --docs ` 77 | 78 | ## Releasing a new version 79 | 80 | ```bash 81 | $ npm version patch 82 | $ npm publish 83 | ``` 84 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "readme-sync", 3 | "author": "Ben Iofel ", 4 | "version": "0.0.24", 5 | "repository": { 6 | "url": "https://github.com/flowcommerce/readme-sync", 7 | "type": "git" 8 | }, 9 | "license": "MIT", 10 | "scripts": { 11 | "prepare": "tsc", 12 | "lint": "eslint --ext .ts sync", 13 | "test": "tsc && jest dist/" 14 | }, 15 | "bin": { 16 | "readme-sync": "dist/index.js" 17 | }, 18 | "files": [ 19 | "dist/" 20 | ], 21 | "dependencies": { 22 | "chalk": "^3.0.0", 23 | "debug": "^4.1.1", 24 | "gray-matter": "^4.0.2", 25 | "isomorphic-fetch": "^3.0.0", 26 | "yargs": "^15.0.2" 27 | }, 28 | "devDependencies": { 29 | "@jest/globals": "^29.3.1", 30 | "@types/debug": "^4.1.5", 31 | "@types/isomorphic-fetch": "0.0.35", 32 | "@types/node": "^13.1.0", 33 | "@typescript-eslint/eslint-plugin": "^2.13.0", 34 | "@typescript-eslint/parser": "^2.13.0", 35 | "eslint": "^6.8.0", 36 | "jest": "^29.3.1", 37 | "ts-node": "^10.9.1", 38 | "typescript": "^4.9.4" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /readme-api.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "readme", 3 | "base_url": "https://dash.readme.io/api/v1", 4 | "models": { 5 | "category": { 6 | "fields": [ 7 | { "name": "_id", "type": "string" }, 8 | { "name": "version", "type": "string" }, 9 | { "name": "project", "type": "string" }, 10 | { "name": "slug", "type": "string" }, 11 | { "name": "title", "type": "string" }, 12 | { "name": "reference", "type": "boolean" }, 13 | { "name": "order", "type": "long" }, 14 | { "name": "createdAt", "type": "date-time-iso8601" } 15 | ] 16 | }, 17 | 18 | "doc_summary_parent": { 19 | "fields": [ 20 | { "name": "_id", "type": "string" }, 21 | { "name": "hidden", "type": "boolean" }, 22 | { "name": "order", "type": "long" }, 23 | { "name": "slug", "type": "string" }, 24 | { "name": "title", "type": "string" }, 25 | { "name": "children", "type": "[doc_summary_child]" } 26 | ] 27 | }, 28 | 29 | "doc_summary_child": { 30 | "fields": [ 31 | { "name": "_id", "type": "string" }, 32 | { "name": "hidden", "type": "boolean" }, 33 | { "name": "order", "type": "long" }, 34 | { "name": "slug", "type": "string" }, 35 | { "name": "title", "type": "string" } 36 | ] 37 | }, 38 | 39 | "doc": { 40 | "fields": [ 41 | { "name": "_id", "type": "string" }, 42 | { "name": "body", "type": "string" }, 43 | { "name": "category", "type": "string" }, 44 | { "name": "hidden", "type": "boolean" }, 45 | { "name": "order", "type": "integer" }, 46 | { "name": "parentDoc", "type": "string" }, 47 | { "name": "project", "type": "string" }, 48 | { "name": "slug", "type": "string" }, 49 | { "name": "title", "type": "string" }, 50 | { "name": "type", "type": "string" }, 51 | { "name": "version", "type": "string" } 52 | ] 53 | }, 54 | 55 | "doc_form": { 56 | "fields": [ 57 | { "name": "slug", "type": "string", "required": false }, 58 | { "name": "title", "type": "string", "required": false }, 59 | { "name": "body", "type": "string", "required": false }, 60 | { "name": "excerpt", "type": "string", "required": false }, 61 | { "name": "category", "type": "string", "required": false }, 62 | { "name": "parentDoc", "type": "string", "required": false }, 63 | { "name": "hidden", "type": "boolean", "required": false }, 64 | { "name": "order", "type": "integer" } 65 | ] 66 | }, 67 | 68 | "error": { 69 | "fields": [ 70 | { "name": "description", "type": "string" }, 71 | { "name": "error", "type": "string" }, 72 | { "name": "errors", "type": "json", "required": false } 73 | ] 74 | } 75 | }, 76 | 77 | "resources": { 78 | "category": { 79 | "path": "/categories", 80 | "operations": [ 81 | { 82 | "method": "GET", 83 | "path": "/:slug", 84 | "responses": { 85 | "200": { "type": "category" }, 86 | "404": { "type": "error" } 87 | } 88 | }, 89 | { 90 | "method": "GET", 91 | "path": "/:slug/docs", 92 | "responses": { 93 | "200": { "type": "[doc_summary_parent]" }, 94 | "404": { "type": "error" } 95 | } 96 | } 97 | ] 98 | }, 99 | 100 | "doc": { 101 | "path": "/docs", 102 | "operations": [ 103 | { 104 | "method": "POST", 105 | "body": { "type": "doc_form" }, 106 | "responses": { 107 | "200": { "type": "doc" }, 108 | "400": { "type": "error" } 109 | } 110 | }, 111 | { 112 | "method": "PUT", 113 | "path": "/:slug", 114 | "body": { "type": "doc_form" }, 115 | "responses": { 116 | "200": { "type": "doc" }, 117 | "400": { "type": "error" } 118 | } 119 | }, 120 | { 121 | "method": "DELETE", 122 | "path": "/:slug", 123 | "responses": { 124 | "200": { "type": "unit" }, 125 | "404": { "type": "error" } 126 | } 127 | } 128 | ] 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flowcommerce/readme-sync/5d9fcd93164875b897c038743a2fca46bd8f14c4/result.png -------------------------------------------------------------------------------- /sync/generated/readme.ts: -------------------------------------------------------------------------------- 1 | import * as url from 'url'; 2 | 3 | declare namespace io.flow.readme.v0.models { 4 | interface Category { 5 | readonly '_id': string; 6 | readonly 'version': string; 7 | readonly 'project': string; 8 | readonly 'slug': string; 9 | readonly 'title': string; 10 | readonly 'reference': boolean; 11 | readonly 'order': number; 12 | readonly 'createdAt': string; 13 | } 14 | 15 | interface Doc { 16 | readonly '_id': string; 17 | readonly 'body': string; 18 | readonly 'category': string; 19 | readonly 'hidden': boolean; 20 | readonly 'order': number; 21 | readonly 'parentDoc': string; 22 | readonly 'project': string; 23 | readonly 'slug': string; 24 | readonly 'title': string; 25 | readonly 'type': string; 26 | readonly 'version': string; 27 | } 28 | 29 | interface DocForm { 30 | readonly 'slug'?: string; 31 | readonly 'title'?: string; 32 | readonly 'body'?: string; 33 | readonly 'excerpt'?: string; 34 | readonly 'category'?: string; 35 | readonly 'parentDoc'?: string; 36 | readonly 'hidden'?: boolean; 37 | readonly 'order': number; 38 | } 39 | 40 | interface DocSummaryChild { 41 | readonly '_id': string; 42 | readonly 'hidden': boolean; 43 | readonly 'order': number; 44 | readonly 'slug': string; 45 | readonly 'title': string; 46 | } 47 | 48 | interface DocSummaryParent { 49 | readonly '_id': string; 50 | readonly 'hidden': boolean; 51 | readonly 'order': number; 52 | readonly 'slug': string; 53 | readonly 'title': string; 54 | readonly 'children': io.flow.readme.v0.models.DocSummaryChild[]; 55 | } 56 | 57 | interface Error { 58 | readonly 'description': string; 59 | readonly 'error': string; 60 | readonly 'errors'?: any/*json*/; 61 | } 62 | } 63 | 64 | export type Category = io.flow.readme.v0.models.Category; 65 | export type Doc = io.flow.readme.v0.models.Doc; 66 | export type DocForm = io.flow.readme.v0.models.DocForm; 67 | export type DocSummaryChild = io.flow.readme.v0.models.DocSummaryChild; 68 | export type DocSummaryParent = io.flow.readme.v0.models.DocSummaryParent; 69 | export type Error = io.flow.readme.v0.models.Error; 70 | 71 | export interface $FetchOptions { 72 | body?: string; 73 | headers?: $HttpHeaders; 74 | method?: $HttpMethod; 75 | } 76 | 77 | export type $FetchFunction = (url: string, options?: $FetchOptions) => Promise; 78 | 79 | export interface $HttpHeaders { 80 | [key: string]: string; 81 | } 82 | 83 | export type $HttpMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'HEAD' | 'CONNECT' | 'OPTIONS' | 'TRACE'; 84 | 85 | export interface $HttpQuery { 86 | [key: string]: string | number | boolean | string[] | number[] | boolean[] | undefined | null; 87 | } 88 | 89 | export interface $HttpRequest { 90 | body?: any; 91 | url: string; 92 | headers: $HttpHeaders; 93 | method: $HttpMethod; 94 | } 95 | 96 | export interface $HttpRequestOptions { 97 | body?: any; 98 | endpoint: string; 99 | headers?: $HttpHeaders; 100 | method: $HttpMethod; 101 | query?: $HttpQuery; 102 | } 103 | 104 | export interface $HttpResponse { 105 | body: B; 106 | headers: $HttpHeaders; 107 | ok: O; 108 | request: $HttpRequest; 109 | status: S; 110 | statusText: string; 111 | } 112 | 113 | export type $HttpContinue = $HttpResponse; 114 | export type $HttpSwitchingProtocol = $HttpResponse; 115 | export type $HttpProcessing = $HttpResponse; 116 | export type $HttpOk = $HttpResponse; 117 | export type $HttpCreated = $HttpResponse; 118 | export type $HttpAccepted = $HttpResponse; 119 | export type $HttpNonAuthoritativeInformation = $HttpResponse; 120 | export type $HttpNoContent = $HttpResponse; 121 | export type $HttpResetContent = $HttpResponse; 122 | export type $HttpPartialContent = $HttpResponse; 123 | export type $HttpMultiStatus = $HttpResponse; 124 | export type $HttpAlreadyReported = $HttpResponse; 125 | export type $HttpImUsed = $HttpResponse; 126 | export type $HttpMultipleChoices = $HttpResponse; 127 | export type $HttpMovedPermanently = $HttpResponse; 128 | export type $HttpFound = $HttpResponse; 129 | export type $HttpSeeOther = $HttpResponse; 130 | export type $HttpNotModified = $HttpResponse; 131 | export type $HttpUseProxy = $HttpResponse; 132 | export type $HttpTemporaryRedirect = $HttpResponse; 133 | export type $HttpPermanentRedirect = $HttpResponse; 134 | export type $HttpBadRequest = $HttpResponse; 135 | export type $HttpUnauthorized = $HttpResponse; 136 | export type $HttpPaymentRequired = $HttpResponse; 137 | export type $HttpForbidden = $HttpResponse; 138 | export type $HttpNotFound = $HttpResponse; 139 | export type $HttpMethodNotAllowed = $HttpResponse; 140 | export type $HttpNotAcceptable = $HttpResponse; 141 | export type $HttpProxyAuthenticationRequired = $HttpResponse; 142 | export type $HttpRequestTimeout = $HttpResponse; 143 | export type $HttpConflict = $HttpResponse; 144 | export type $HttpGone = $HttpResponse; 145 | export type $HttpLengthRequired = $HttpResponse; 146 | export type $HttpPreconditionFailed = $HttpResponse; 147 | export type $HttpRequestEntityTooLarge = $HttpResponse; 148 | export type $HttpRequestUriTooLong = $HttpResponse; 149 | export type $HttpUnsupportedMediaType = $HttpResponse; 150 | export type $HttpRequestedRangeNotSatisfiable = $HttpResponse; 151 | export type $HttpExpectationFailed = $HttpResponse; 152 | export type $HttpMisdirectedRequest = $HttpResponse; 153 | export type $HttpUnprocessableEntity = $HttpResponse; 154 | export type $HttpLocked = $HttpResponse; 155 | export type $HttpFailedDependency = $HttpResponse; 156 | export type $HttpUpgradeRequired = $HttpResponse; 157 | export type $HttpPreconditionRequired = $HttpResponse; 158 | export type $HttpTooManyRequests = $HttpResponse; 159 | export type $HttpRequestHeaderFieldsTooLarge = $HttpResponse; 160 | export type $HttpNoResponse = $HttpResponse; 161 | export type $HttpRetryWith = $HttpResponse; 162 | export type $HttpBlockedByWindowsParentalControls = $HttpResponse; 163 | export type $HttpUnavailableForLegalReasons = $HttpResponse; 164 | export type $HttpClientClosedRequest = $HttpResponse; 165 | export type $HttpInternalServerError = $HttpResponse; 166 | export type $HttpNotImplemented = $HttpResponse; 167 | export type $HttpBadGateway = $HttpResponse; 168 | export type $HttpServiceUnavailable = $HttpResponse; 169 | export type $HttpGatewayTimeout = $HttpResponse; 170 | export type $HttpHttpVersionNotSupported = $HttpResponse; 171 | export type $HttpInsufficientStorage = $HttpResponse; 172 | export type $HttpLoopDetected = $HttpResponse; 173 | export type $HttpBandwidthLimitExceeded = $HttpResponse; 174 | export type $HttpNotExtended = $HttpResponse; 175 | export type $HttpNetworkAuthenticationRequired = $HttpResponse; 176 | export type $HttpNetworkReadTimeoutError = $HttpResponse; 177 | export type $HttpNetworkConnectTimeoutError = $HttpResponse; 178 | 179 | export interface $HttpClientOptions { 180 | fetch: $FetchFunction; 181 | } 182 | 183 | export function isResponseEmpty(response: Response): boolean { 184 | const contentLength = response.headers.get('Content-Length'); 185 | return contentLength != null && Number.parseInt(contentLength, 10) === 0; 186 | } 187 | 188 | export function isResponseJson(response: Response): boolean { 189 | const contentType = response.headers.get('Content-Type'); 190 | return contentType != null && contentType.indexOf('json') >= 0; 191 | } 192 | 193 | export function parseJson(response: Response): Promise { 194 | return !isResponseEmpty(response) && isResponseJson(response) ? response.json() : Promise.resolve(); 195 | } 196 | 197 | export function parseHeaders(response: Response): Record { 198 | const headers: Record = {}; 199 | 200 | response.headers.forEach((value, key) => { 201 | headers[key.toLowerCase()] = value; 202 | }); 203 | 204 | return headers; 205 | } 206 | 207 | export function stripQuery(query: $HttpQuery = {}): $HttpQuery { 208 | const initialValue: $HttpQuery = {}; 209 | 210 | return Object.keys(query).reduce((previousValue, key) => { 211 | const value = query[key]; 212 | 213 | if (value != null) 214 | previousValue[key] = value; 215 | 216 | return previousValue; 217 | }, initialValue); 218 | } 219 | 220 | export class $HttpClient { 221 | private options: $HttpClientOptions; 222 | 223 | constructor(options: $HttpClientOptions) { 224 | this.options = options; 225 | } 226 | 227 | public request(options: $HttpRequestOptions): Promise<$HttpResponse> { 228 | const finalUrl: string = url.format({ 229 | hostname: 'dash.readme.io', 230 | pathname: '/api/v1' + options.endpoint, 231 | protocol: 'https:', 232 | query: stripQuery(options.query), 233 | }); 234 | 235 | const finalHeaders: $HttpHeaders = { 236 | accept: 'application/json', 237 | 'content-type': 'application/json', 238 | ...options.headers, 239 | }; 240 | 241 | const request: $HttpRequest = { 242 | body: options.body, 243 | headers: finalHeaders, 244 | method: options.method, 245 | url: finalUrl, 246 | }; 247 | 248 | return this.options.fetch(request.url, { 249 | body: JSON.stringify(request.body), 250 | headers: request.headers, 251 | method: request.method, 252 | }).then((response) => { 253 | return parseJson(response).then((json) => { 254 | return { 255 | body: json, 256 | headers: parseHeaders(response), 257 | ok: response.ok, 258 | request, 259 | status: response.status, 260 | statusText: response.statusText, 261 | }; 262 | }); 263 | }); 264 | } 265 | } 266 | 267 | export class $Resource { 268 | protected client: $HttpClient; 269 | 270 | constructor(options: $HttpClientOptions) { 271 | this.client = new $HttpClient(options); 272 | } 273 | } 274 | 275 | export interface CategoriesGetBySlugParameters { 276 | headers?: $HttpHeaders; 277 | slug: string; 278 | } 279 | 280 | export interface CategoriesGetDocsBySlugParameters { 281 | headers?: $HttpHeaders; 282 | slug: string; 283 | } 284 | 285 | export interface DocsPostParameters { 286 | body: io.flow.readme.v0.models.DocForm; 287 | headers?: $HttpHeaders; 288 | } 289 | 290 | export interface DocsPutBySlugParameters { 291 | body: io.flow.readme.v0.models.DocForm; 292 | headers?: $HttpHeaders; 293 | slug: string; 294 | } 295 | 296 | export interface DocsDeleteBySlugParameters { 297 | headers?: $HttpHeaders; 298 | slug: string; 299 | } 300 | 301 | export type CategoriesGetBySlugResponse = $HttpOk | $HttpNotFound; 302 | export type CategoriesGetDocsBySlugResponse = $HttpOk | $HttpNotFound; 303 | export type DocsPostResponse = $HttpOk | $HttpBadRequest; 304 | export type DocsPutBySlugResponse = $HttpOk | $HttpBadRequest; 305 | export type DocsDeleteBySlugResponse = $HttpOk | $HttpNotFound; 306 | 307 | export class CategoriesResource extends $Resource { 308 | public getBySlug(params: CategoriesGetBySlugParameters): Promise { 309 | return this.client.request({ 310 | endpoint: `/categories/${encodeURIComponent(params.slug)}`, 311 | headers: params.headers, 312 | method: 'GET', 313 | }); 314 | } 315 | 316 | public getDocsBySlug(params: CategoriesGetDocsBySlugParameters): Promise { 317 | return this.client.request({ 318 | endpoint: `/categories/${encodeURIComponent(params.slug)}/docs`, 319 | headers: params.headers, 320 | method: 'GET', 321 | }); 322 | } 323 | } 324 | 325 | export class DocsResource extends $Resource { 326 | public post(params: DocsPostParameters): Promise { 327 | return this.client.request({ 328 | body: params.body, 329 | endpoint: '/docs', 330 | headers: params.headers, 331 | method: 'POST', 332 | }); 333 | } 334 | 335 | public putBySlug(params: DocsPutBySlugParameters): Promise { 336 | return this.client.request({ 337 | body: params.body, 338 | endpoint: `/docs/${encodeURIComponent(params.slug)}`, 339 | headers: params.headers, 340 | method: 'PUT', 341 | }); 342 | } 343 | 344 | public deleteBySlug(params: DocsDeleteBySlugParameters): Promise { 345 | return this.client.request({ 346 | endpoint: `/docs/${encodeURIComponent(params.slug)}`, 347 | headers: params.headers, 348 | method: 'DELETE', 349 | }); 350 | } 351 | } 352 | 353 | export interface ClientInstance { 354 | categories: CategoriesResource; 355 | docs: DocsResource; 356 | } 357 | 358 | export function createClient(options: $HttpClientOptions): ClientInstance { 359 | return { 360 | categories: new CategoriesResource(options), 361 | docs: new DocsResource(options), 362 | }; 363 | } -------------------------------------------------------------------------------- /sync/index.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import fs from 'fs' 3 | import path from 'path' 4 | import yargs from 'yargs' 5 | import matter from 'gray-matter' 6 | import assert from 'assert' 7 | import { DocForm, Doc, createClient as createReadmeClient } from './generated/readme' 8 | import { slugify, orderFromName, nameWithoutOrder, findSlugInTree, RemoteTree, RemoteTreeDoc, RemoteTreeEntry, removeSlugFromTree, addDocUnderSlug } from './util' 9 | import { blueBright, green, yellow, redBright } from 'chalk' 10 | import _debug from 'debug' 11 | import fetch from 'isomorphic-fetch' 12 | import { ensureFrontMatter, ensureUniqueSlugs, ensureLinksAreValid, ensureIndexMdExists, ensureNoWeirdFiles, ensureMaxTwoLevels } from './validation' 13 | 14 | const info = _debug('readme-sync:info') 15 | const verbose = _debug('readme-sync:verbose') 16 | 17 | const argv = yargs 18 | .version(false) 19 | .options({ 20 | 'apiKey': { type: 'string', demandOption: true }, 21 | 'docs': { type: 'string', demandOption: true }, 22 | 'version': { type: 'string', demandOption: true }, 23 | 'validateOnly': { type: 'boolean' }, 24 | 'category': { type: 'string', array: true }, 25 | 'dryRun': { type: 'boolean', default: false }, 26 | }).argv 27 | 28 | const client = createReadmeClient({ 29 | fetch: async (url, options) => { 30 | info(`${options.method} ${url}`) 31 | verbose('body', options.body) 32 | verbose('headers', options.headers) 33 | const response = await fetch(url, { 34 | ...options, 35 | headers: { 36 | ...options.headers, 37 | 'x-readme-version': argv.version, 38 | 'authorization': `Basic ${Buffer.from(argv.apiKey + ':').toString('base64')}`, 39 | } 40 | }) 41 | verbose('response', response) 42 | return response 43 | } 44 | }) 45 | 46 | async function upsertDoc(remoteTree: RemoteTree, categoryName: string, filepath: string, parent: Doc | null, options: { slug?: string; order?: number } = {}): Promise { 47 | assert(fs.statSync(filepath).isFile()) 48 | 49 | const docFileName = path.parse(filepath).name 50 | 51 | const slug = options.slug ?? slugify(nameWithoutOrder(docFileName)) 52 | 53 | const existing = findSlugInTree(remoteTree, slug) 54 | const targetCategory = remoteTree.get(slugify(categoryName)) 55 | 56 | const metadata = matter(fs.readFileSync(filepath)) 57 | 58 | const form: DocForm = { 59 | slug, 60 | title: metadata.data.title, 61 | body: metadata.content, 62 | excerpt: metadata.data.excerpt, 63 | order: options.order ?? orderFromName(docFileName), 64 | category: targetCategory.category._id, 65 | parentDoc: parent === null ? null : parent._id, 66 | hidden: metadata.data.hidden ?? false, 67 | } 68 | 69 | const destination = `${slugify(targetCategory.category.title)}${parent ? ` / ${parent.slug}` : ''} / ${slug}` 70 | 71 | if (existing !== null) { 72 | console.log(`\tUpdating ${blueBright(filepath)} -> ${green(destination)}`) 73 | 74 | if (argv.dryRun) { 75 | console.log(`\t${redBright('DRY RUN')} PUT ${slug}`) 76 | return { 77 | _id: 'id', 78 | slug, 79 | body: form.body, 80 | category: targetCategory.category._id, 81 | hidden: form.hidden, 82 | order: form.order, 83 | parentDoc: form.parentDoc, 84 | project: 'proj', 85 | title: form.title, 86 | type: 'type', 87 | version: '1', 88 | } 89 | } 90 | 91 | const doc = await client.docs.putBySlug({ slug, body: form }) 92 | info(`updated - ${doc.status}`) 93 | verbose(doc.body) 94 | if (doc.status == 400) { 95 | console.error(`Error: ${doc.body.error} - ${doc.body.description}`) 96 | if (doc.body.errors != null) 97 | console.error(doc.body.errors) 98 | throw new Error(doc.body.description) 99 | } 100 | 101 | const removed = removeSlugFromTree(existing.category, slug) 102 | assert(removed != null) 103 | assert(addDocUnderSlug(targetCategory, removed, parent?.slug)) 104 | info(targetCategory) 105 | 106 | return doc.body 107 | } else { 108 | console.log(`\tCreating ${blueBright(filepath)} -> ${green(destination)}`) 109 | 110 | if (argv.dryRun) { 111 | console.log(`\t${redBright('DRY RUN')} POST ${slug}`) 112 | return { 113 | _id: 'id', 114 | slug, 115 | body: form.body, 116 | category: targetCategory.category._id, 117 | hidden: form.hidden, 118 | order: form.order, 119 | parentDoc: form.parentDoc, 120 | project: 'proj', 121 | title: form.title, 122 | type: 'type', 123 | version: '1', 124 | } 125 | } 126 | 127 | const doc = await client.docs.post({ body: form }) 128 | info(`created - ${doc.status}`) 129 | verbose(doc.body) 130 | if (doc.status == 400) { 131 | console.error(`Error: ${doc.body.error} - ${doc.body.description}`) 132 | if (doc.body.errors != null) 133 | console.error(doc.body.errors) 134 | throw new Error(doc.body.description) 135 | } 136 | if (doc.body.slug !== slug) { 137 | console.error(doc.body) 138 | throw new Error('Bug. Existing document not updated.') 139 | } 140 | assert(addDocUnderSlug(targetCategory, {slug, children: []}, parent?.slug)) 141 | info(targetCategory) 142 | return doc.body 143 | } 144 | } 145 | 146 | /** 147 | * Insert and update a doc and its children 148 | * 149 | * integration/ 150 | * +- index.md 151 | * +- setup.md 152 | * +- config.md 153 | */ 154 | async function upsertDir(remoteTree: RemoteTree, categoryName: string, dirpath: string): Promise { 155 | assert(fs.statSync(dirpath).isDirectory()) 156 | 157 | const children = fs.readdirSync(dirpath) 158 | if (!children.includes('index.md')) { 159 | console.error(`ERROR: ${dirpath} requires an index.md page`) 160 | return 161 | } 162 | 163 | const parentName = path.basename(dirpath) 164 | 165 | const parent = await upsertDoc(remoteTree, categoryName, path.join(dirpath, 'index.md'), null, { 166 | slug: slugify(nameWithoutOrder(parentName)), 167 | order: orderFromName(parentName), 168 | }) 169 | 170 | for (const child of children) { 171 | if (child === 'index.md') 172 | continue 173 | 174 | await upsertDoc(remoteTree, categoryName, path.join(dirpath, child), parent) 175 | } 176 | } 177 | 178 | /** 179 | * Delete remote docs that are not present locally. 180 | */ 181 | async function deleteNotPresent({ docs }: RemoteTreeEntry): Promise { 182 | const getSlug = (f: string): string => slugify(nameWithoutOrder(path.parse(f).name)) 183 | 184 | function findLocalBySlug(slug: string): boolean { 185 | for (const category of fs.readdirSync(argv.docs)) { 186 | for (const page of fs.readdirSync(`${argv.docs}/${category}`)) { 187 | 188 | const stat = fs.lstatSync(`${argv.docs}/${category}/${page}`) 189 | 190 | if (getSlug(page) === slug) // category/slug.md or category/slug/index.md 191 | return true 192 | else if (stat.isDirectory()) { 193 | for (const subpage of fs.readdirSync(`${argv.docs}/${category}/${page}`)) { 194 | if (getSlug(subpage) === slug) // category/x/slug.md 195 | return true 196 | } 197 | } 198 | } 199 | } 200 | return false 201 | } 202 | 203 | async function deleteIfNotFoundLocally(doc: RemoteTreeDoc): Promise { 204 | if (!findLocalBySlug(doc.slug)) { 205 | console.log(`\tDeleting ${doc.slug} - not found locally`) 206 | if (argv.dryRun) 207 | console.log(`\t${redBright('DRY RUN')} DELETE ${doc.slug}`) 208 | else 209 | await client.docs.deleteBySlug({ slug: doc.slug }) 210 | } 211 | } 212 | 213 | for (const page of docs) { 214 | for (const subpage of page.children) 215 | await deleteIfNotFoundLocally(subpage) 216 | await deleteIfNotFoundLocally(page) 217 | } 218 | } 219 | 220 | /** 221 | * Insert, update, and delete remote docs. 222 | * 223 | * Only two layers of nesting supported 224 | * 225 | * category/ 226 | * +- doc1.md 227 | * +- doc2.md 228 | * +- group/ 229 | * +- child.md 230 | * +- index.md 231 | */ 232 | async function sync(remoteTree: RemoteTree): Promise { 233 | for (const category of fs.readdirSync(argv.docs)) { 234 | if (category.startsWith('.') || !fs.statSync(path.join(argv.docs, category)).isDirectory()) 235 | continue 236 | 237 | if (argv.category != null && !argv.category.includes(slugify(category))) { 238 | console.log(`Skipping ${redBright(category)}`) 239 | continue 240 | } 241 | 242 | console.log(category) 243 | const categoryPath = path.join(argv.docs, category) 244 | for (const doc of fs.readdirSync(categoryPath)) { 245 | const docPath = path.join(categoryPath, doc) 246 | if (doc.startsWith('.')) { 247 | continue 248 | } else if (doc.endsWith('.md')) { 249 | await upsertDoc(remoteTree, category, docPath, null) 250 | } else { 251 | await upsertDir(remoteTree, category, path.join(argv.docs, category, doc)) 252 | } 253 | } 254 | 255 | await deleteNotPresent(remoteTree.get(slugify(category))) 256 | } 257 | } 258 | 259 | async function main(): Promise { 260 | const remoteTree: RemoteTree = new Map() 261 | let errored = false 262 | 263 | const checks = [ 264 | ensureNoWeirdFiles, 265 | ensureMaxTwoLevels, 266 | ensureIndexMdExists, 267 | ensureUniqueSlugs, 268 | ensureFrontMatter, 269 | ensureLinksAreValid, 270 | ] 271 | 272 | for (const check of checks) 273 | if (!check(argv.docs)) 274 | process.exit(1) 275 | 276 | console.log('Docs look good') 277 | if (argv.validateOnly) { 278 | return 279 | } 280 | 281 | // we need to fetch the categories from local dir names because there is no API to get this from readme.com 282 | // TODO: use /api/v1/categories 283 | console.log('Fetching categories') 284 | for (const localCategoryName of fs.readdirSync(argv.docs)) { 285 | if (localCategoryName.startsWith('.') || !fs.statSync(path.join(argv.docs, localCategoryName)).isDirectory()) 286 | continue 287 | 288 | const slug = slugify(localCategoryName) 289 | 290 | const [remoteCategory, remoteDocs] = await Promise.all([ 291 | client.categories.getBySlug({ slug }), 292 | client.categories.getDocsBySlug({ slug }), 293 | ]) 294 | if (remoteCategory.status == 200 && remoteDocs.status == 200) { 295 | assert(remoteCategory.body.slug === slug) 296 | console.log(`Got ${yellow(localCategoryName)}`) 297 | remoteTree.set(slug, { 298 | category: remoteCategory.body, 299 | docs: remoteDocs.body.map(parent => ({ 300 | slug: parent.slug, 301 | children: parent.children.map(child => ({ 302 | slug: child.slug, 303 | children: [], 304 | })), 305 | })), 306 | }) 307 | } else { 308 | if (remoteCategory.status == 404) { 309 | console.error(`I cannot create categories yet. Please manually create the category ${localCategoryName} (slug ${slug}) in Readme.`) 310 | errored = true 311 | } else { 312 | console.error(remoteCategory) 313 | console.error(remoteDocs) 314 | throw new Error('something happened') 315 | } 316 | } 317 | } 318 | 319 | if (errored) 320 | process.exit(1) 321 | 322 | info(remoteTree) 323 | await sync(remoteTree) 324 | } 325 | 326 | main().catch((err) => { 327 | console.error(err) 328 | process.exit(1) 329 | }) 330 | -------------------------------------------------------------------------------- /sync/util.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, test } from '@jest/globals' 2 | import { findSlugInCategory, removeSlugFromTree, addDocUnderSlug } from './util' 3 | 4 | test('findSlugInCategory', () => { 5 | const doc = { 6 | slug: 'slug', 7 | children: [] 8 | } 9 | expect(findSlugInCategory({ 10 | category: null, 11 | docs: [doc] 12 | }, 'slug')).toBe(doc) 13 | 14 | expect(findSlugInCategory({ 15 | category: null, 16 | docs: [{ 17 | slug: 'a', 18 | children: [{ 19 | slug: 'b', 20 | children: [ 21 | { 22 | slug: 'c', 23 | children: [] 24 | }, 25 | doc 26 | ] 27 | }] 28 | }] 29 | }, 'slug')).toBe(doc) 30 | 31 | expect(findSlugInCategory({ 32 | category: null, 33 | docs: [{ 34 | slug: 'a', 35 | children: [{ 36 | slug: 'b', 37 | children: [ 38 | { 39 | slug: 'c', 40 | children: [] 41 | }, 42 | { 43 | slug: 'd', 44 | children: [], 45 | } 46 | ] 47 | }] 48 | }] 49 | }, 'slug')).toBe(null) 50 | }) 51 | 52 | test('removeSlugFromTree', () => { 53 | const b = { 54 | slug: 'b', 55 | children: [], 56 | } 57 | const a = { 58 | slug: 'a', 59 | children: [b], 60 | } 61 | const tree = { 62 | category: null, 63 | docs: [a], 64 | } 65 | 66 | expect(removeSlugFromTree(tree, 'd')).toBeNull() 67 | 68 | expect(removeSlugFromTree(tree, 'b')).toBe(b) 69 | expect(a.children).toHaveLength(0) 70 | 71 | expect(removeSlugFromTree(tree, 'a')).toBe(a) 72 | expect(tree.docs).toHaveLength(0) 73 | }) 74 | 75 | test('addDocUnderSlug', () => { 76 | const tree = { 77 | category: null, 78 | docs: [{ 79 | slug: 'a', 80 | children: [ 81 | { 82 | slug: 'b', 83 | children: [] 84 | }, 85 | { 86 | slug: 'c', 87 | children: [] 88 | } 89 | ] 90 | }] 91 | } 92 | 93 | const newDoc = { 94 | slug: 'd', 95 | children: [] 96 | } 97 | 98 | expect(addDocUnderSlug(tree, newDoc, 'c')).toBeTruthy() 99 | expect(tree.docs[0].children[1].children[0]).toBe(newDoc) 100 | 101 | expect(addDocUnderSlug(tree, { 102 | slug: 'x', 103 | children: [] 104 | }, 'g')).toBeFalsy() 105 | 106 | expect(addDocUnderSlug(tree, { 107 | slug: 'x', 108 | children: [] 109 | }, null)).toBe(true) 110 | expect(tree.docs[1].slug).toBe('x') 111 | }) 112 | -------------------------------------------------------------------------------- /sync/util.ts: -------------------------------------------------------------------------------- 1 | import { Category } from './generated/readme' 2 | 3 | export type RemoteTreeDoc = { 4 | slug: string; 5 | children: RemoteTreeDoc[]; 6 | } 7 | export type RemoteTreeEntry = { category: Category; docs: RemoteTreeDoc[] } 8 | export type RemoteTree = Map 9 | 10 | /** 11 | * Call fn on each element of arr until it returns non-null, then pass 12 | * that return value back. 13 | */ 14 | function arrayTryEach(arr: E[], fn: (t: E) => R | null): R | null { 15 | for (const elem of arr) { 16 | const res = fn(elem) 17 | if (res != null) 18 | return res 19 | } 20 | return null 21 | } 22 | 23 | /** 24 | * Return whether a slug was found in a RemoteTreeEntry 25 | */ 26 | export function findSlugInCategory(tree: RemoteTreeEntry, slug: string): RemoteTreeDoc | null { 27 | function findInDocs(doc: RemoteTreeDoc): RemoteTreeDoc | null { 28 | if (doc.slug === slug) 29 | return doc 30 | else 31 | return arrayTryEach(doc.children, findInDocs) 32 | } 33 | 34 | return arrayTryEach(tree.docs, findInDocs) 35 | } 36 | 37 | 38 | export function findSlugInTree(tree: RemoteTree, slug: string): { doc: RemoteTreeDoc; category: RemoteTreeEntry } | null { 39 | for (const [, entry] of tree) { 40 | const found = findSlugInCategory(entry, slug) 41 | if (found != null) 42 | return { 43 | doc: found, 44 | category: entry, 45 | } 46 | } 47 | return null 48 | } 49 | 50 | /** 51 | * Modify the tree to remove a doc and all of it's children, returning the doc 52 | */ 53 | export function removeSlugFromTree(tree: RemoteTreeEntry, slug: string): RemoteTreeDoc | null { 54 | function remove(doc: RemoteTreeDoc): RemoteTreeDoc | null { 55 | const index = doc.children.findIndex(doc => doc.slug === slug) 56 | if (index === -1) { 57 | return arrayTryEach(doc.children, remove) 58 | } else { 59 | return doc.children.splice(index, 1)[0] 60 | } 61 | } 62 | 63 | const rootIndex = tree.docs.findIndex(doc => doc.slug === slug) 64 | if (rootIndex !== -1) { 65 | return tree.docs.splice(rootIndex, 1)[0] 66 | } else { 67 | return arrayTryEach(tree.docs, remove) 68 | } 69 | } 70 | 71 | /** 72 | * Insert a doc into the tree under a given slug 73 | * Returns true if successful 74 | */ 75 | export function addDocUnderSlug(tree: RemoteTreeEntry, newDoc: RemoteTreeDoc, parent?: string): boolean { 76 | function add(doc: RemoteTreeDoc): boolean { 77 | if (doc.slug === parent) { 78 | doc.children.push(newDoc) 79 | return true 80 | } else { 81 | return doc.children.some(add) 82 | } 83 | } 84 | 85 | if (parent == null) { 86 | tree.docs.push(newDoc) 87 | return true 88 | } else { 89 | return tree.docs.some(add) 90 | } 91 | } 92 | 93 | export function slugify(name: string): string { 94 | return name.toLowerCase().replace(/[^A-Za-z0-9-]/g, '-').replace(/--+/g, '-') 95 | } 96 | 97 | function parseNameWithOrder(name: string): {name: string; order?: number} { 98 | const match = name.match(/^(\d+)\s*-\s*(.+)/) 99 | if (match != null) 100 | return { 101 | order: parseInt(match[1]), 102 | name: match[2], 103 | } 104 | else 105 | return { 106 | order: undefined, 107 | name, 108 | } 109 | } 110 | 111 | export function nameWithoutOrder(name: string): string { 112 | return parseNameWithOrder(name).name 113 | } 114 | 115 | export function orderFromName(name: string): number | undefined { 116 | return parseNameWithOrder(name).order 117 | } 118 | -------------------------------------------------------------------------------- /sync/validation.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import path from 'path' 3 | import { redBright, green, underline, blueBright } from 'chalk' 4 | import matter from 'gray-matter' 5 | import { slugify, nameWithoutOrder } from './util' 6 | 7 | const CATEGORY_LEVEL = 0 8 | const PAGE_LEVEL = 1 9 | const SUBPAGE_LEVEL = 2 10 | 11 | function walkDocTree( 12 | root: string, 13 | cb: (filepath: string, level: number, stat: fs.Stats) => boolean, 14 | level = 0, 15 | ): boolean { 16 | let pass = true 17 | 18 | for (const node of fs.readdirSync(root)) { 19 | if (node.startsWith('.')) 20 | continue 21 | 22 | const fullpath = path.join(root, node) 23 | const stat = fs.lstatSync(fullpath) 24 | const valid = cb(fullpath, level, stat) 25 | if (!valid) 26 | pass = false 27 | 28 | if (stat.isDirectory()) { 29 | const validSubtree = walkDocTree(fullpath, cb, level + 1) 30 | if (!validSubtree) 31 | pass = false 32 | } 33 | 34 | } 35 | 36 | return pass 37 | } 38 | 39 | export function ensureNoWeirdFiles(root: string): boolean { 40 | return walkDocTree(root, (filepath, _, stat) => { 41 | if (stat.isFile()) { 42 | if (filepath.endsWith('.md')) 43 | return true 44 | 45 | console.error(`Stray file at ${filepath}. All files are expected to end in .md`) 46 | return false 47 | } 48 | 49 | if (stat.isDirectory()) 50 | return true 51 | 52 | const type = 53 | stat.isFile() ? 'file' 54 | : stat.isDirectory() ? 'directory' 55 | : stat.isBlockDevice() ? 'block device' 56 | : stat.isCharacterDevice() ? 'character device' 57 | : stat.isFIFO() ? 'fifo' 58 | : stat.isSocket() ? 'socket' 59 | : stat.isSymbolicLink() ? 'symbolic link' 60 | : 'unknown' 61 | 62 | console.error(`Node of type ${type} at ${filepath} not supported`) 63 | return false 64 | }) 65 | } 66 | 67 | export function ensureMaxTwoLevels(root: string): boolean { 68 | return walkDocTree(root, (filepath, level, stat) => { 69 | if (level === CATEGORY_LEVEL && stat.isDirectory()) 70 | return true 71 | if (level === PAGE_LEVEL && (stat.isDirectory() || stat.isFile())) 72 | return true 73 | if (level === SUBPAGE_LEVEL && stat.isFile()) 74 | return true 75 | console.error(`${redBright(filepath)} not allowed. Readme only supports 2 levels of pages.`) 76 | return false 77 | }) 78 | } 79 | 80 | function validateFrontMatter(docPath: string, content: Buffer): boolean { 81 | const frontmatter = matter(content) 82 | const { title, hidden } = frontmatter.data 83 | let passed = true 84 | 85 | for (const key of Object.keys(frontmatter.data)) { 86 | if (!['title', 'hidden', 'excerpt'].includes(key)) { 87 | console.log(`Error: ${redBright(docPath)}: invalid frontmatter key ${key}`) 88 | passed = false 89 | } 90 | } 91 | 92 | if (title == null || typeof title !== 'string') { 93 | console.error(`Error: ${redBright(docPath)}: title missing or invalid`) 94 | passed = false 95 | } 96 | 97 | if (hidden != null && typeof hidden !== 'boolean') { 98 | console.error(`Error: ${redBright(docPath)}: hidden must be true or false`) 99 | passed = false 100 | } 101 | 102 | return passed 103 | } 104 | 105 | /** Ensure that all files have valid frontmatter */ 106 | export function ensureFrontMatter(root: string): boolean { 107 | return walkDocTree(root, (filepath, _, stat) => { 108 | if (stat.isFile()) 109 | return validateFrontMatter(filepath, fs.readFileSync(filepath)) 110 | return true 111 | }) 112 | } 113 | 114 | export function ensureUniqueSlugs(docs: string): boolean { 115 | const slugs = {} 116 | 117 | return walkDocTree(docs, (filepath, level, stat) => { 118 | if (stat.isDirectory()) 119 | return true 120 | 121 | let parsedPath = path.parse(filepath) 122 | 123 | if (level === SUBPAGE_LEVEL && parsedPath.base === 'index.md') { 124 | parsedPath = path.parse(parsedPath.dir) // use parent slug 125 | } 126 | 127 | const slug = slugify(nameWithoutOrder(parsedPath.name)) 128 | if (Object.keys(slugs).includes(slug)) { 129 | console.error(`Error: ${redBright(filepath)} has the same slug as ${redBright(slugs[slug])}`) 130 | return false 131 | } else { 132 | slugs[slug] = filepath 133 | return true 134 | } 135 | }) 136 | } 137 | 138 | export function ensureLinksAreValid(docs: string): boolean { 139 | const slugs = [] 140 | const link = /\[(?[^)\n]+)\]\(doc:(?[A-Za-z0-9-]+)(#[A-Za-z0-9-]+)?\)/g 141 | 142 | // Step 1: Gather all doc slugs 143 | walkDocTree(docs, (filepath, level, stat) => { 144 | if (stat.isFile()) { 145 | if (level == SUBPAGE_LEVEL && path.basename(filepath) == 'index.md') 146 | slugs.push(slugify(nameWithoutOrder(path.parse(path.dirname(filepath)).name))) 147 | else 148 | slugs.push(slugify(nameWithoutOrder(path.parse(filepath).name))) 149 | } 150 | 151 | return true 152 | }) 153 | 154 | // Step 2: Check that each link points to a valid slug 155 | return walkDocTree(docs, (filepath, _, stat) => { 156 | if (stat.isDirectory()) 157 | return true 158 | 159 | const contents = fs.readFileSync(filepath).toString() 160 | let hasBadLink = false 161 | for (const match of contents.matchAll(link)) { 162 | if (!slugs.includes(match.groups.target)) { 163 | hasBadLink = true 164 | console.error(`Broken link ${underline(blueBright(`[${match.groups.text}](doc:${match.groups.target})`))} in ${green(filepath)}`) 165 | } 166 | } 167 | return !hasBadLink 168 | }) 169 | } 170 | 171 | export function ensureIndexMdExists(root: string): boolean { 172 | return walkDocTree(root, (filepath, level, stat) => { 173 | if (stat.isDirectory() && level == PAGE_LEVEL) { 174 | if (!fs.readdirSync(filepath).includes('index.md')) { 175 | console.error(`Error: "${filepath}" has no index.md`) 176 | return false 177 | } 178 | } 179 | return true 180 | }) 181 | } -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "downlevelIteration": true, 5 | "outDir": "dist/", 6 | "module": "commonjs", 7 | "esModuleInterop": true, 8 | }, 9 | "include": [ 10 | "sync" 11 | ], 12 | "exclude": [ 13 | "node_modules" 14 | ] 15 | } 16 | --------------------------------------------------------------------------------