├── templates ├── week.md.mu ├── file-by-day.md.mu ├── day.md.mu ├── index.html.mu ├── root-readme.md.mu └── search.html.mu ├── static ├── CNAME ├── robots.txt ├── icon.png ├── favicon.ico ├── favicon-16x16.png ├── favicon-32x32.png ├── mstile-150x150.png ├── apple-touch-icon.png ├── android-chrome-192x192.png ├── android-chrome-512x512.png ├── browserconfig.xml ├── site.webmanifest ├── 404.html ├── safari-pinned-tab.svg ├── icon.svg ├── badge-flat.svg └── badge.svg ├── .github ├── FUNDING.yml └── workflows │ └── cron.yml ├── example ├── public │ └── EbookFoundation │ │ └── free-programming-books │ │ ├── readme.md │ │ ├── index.html │ │ └── books │ │ ├── free-programming-books-zh.md │ │ └── free-programming-books-zh │ │ ├── index.html │ │ └── feed.json ├── data │ ├── 2-formated │ │ └── EbookFoundation │ │ │ └── free-programming-books │ │ │ ├── readme.json │ │ │ └── books │ │ │ └── free-programming-books-zh.json │ ├── github_awesome_nodejs.json │ ├── data.json │ └── 1-raw │ │ └── EbookFoundation │ │ └── free-programming-books │ │ └── books │ │ ├── markdownlist_free-programming-books-langs.md │ │ └── markdownlist_free-programming-books-zh.md ├── simple.md ├── mac.md ├── books.md ├── repo-meta.json └── public-apis-simple.md ├── local.kak ├── .vscode └── settings.json ├── db-meta-init.json ├── .editorconfig ├── error.ts ├── render-markdown_test.ts ├── serve-public.ts ├── test-deps.ts ├── adapters ├── api.ts └── github.ts ├── format-category.ts ├── parser ├── markdown │ ├── list_test.ts │ ├── table_test.ts │ ├── util.ts │ ├── table.ts │ ├── list.ts │ └── heading.ts └── mod.ts ├── README.md ├── util_test.ts ├── scripts ├── install-morsels.sh ├── check-404.ts └── install-mdbook.sh ├── deno.json ├── init-db.ts ├── book.toml ├── constant.ts ├── morsels_config.json ├── lib └── gemoji.js ├── migrations └── to-new-config.ts ├── render-markdown.ts ├── serve-markdown.ts ├── tal.ts ├── log.ts ├── main.ts ├── .gitignore ├── deps.ts ├── get-git-blame.ts ├── Makefile ├── init-items.ts ├── interface.ts ├── format-markdown-item.ts ├── db.ts └── fetch-sources.ts /templates/week.md.mu: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /static/CNAME: -------------------------------------------------------------------------------- 1 | www.trackawesomelist.com -------------------------------------------------------------------------------- /static/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: theowenyoung 2 | -------------------------------------------------------------------------------- /example/public/EbookFoundation/free-programming-books/readme.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/public/EbookFoundation/free-programming-books/index.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/data/2-formated/EbookFoundation/free-programming-books/readme.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /local.kak: -------------------------------------------------------------------------------- 1 | set-option global lsp_toml_path "~/.config/kak-lsp/kak-lsp-deno.toml" 2 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "deno.enable": true, 3 | "deno.unstable": true 4 | } 5 | -------------------------------------------------------------------------------- /example/public/EbookFoundation/free-programming-books/books/free-programming-books-zh.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /db-meta-init.json: -------------------------------------------------------------------------------- 1 | { 2 | "sources": {}, 3 | "checked_at": "2019-01-01T00:00:00Z" 4 | } 5 | -------------------------------------------------------------------------------- /static/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahseema/trackawesomelist-source/HEAD/static/icon.png -------------------------------------------------------------------------------- /example/public/EbookFoundation/free-programming-books/books/free-programming-books-zh/index.html: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahseema/trackawesomelist-source/HEAD/static/favicon.ico -------------------------------------------------------------------------------- /static/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahseema/trackawesomelist-source/HEAD/static/favicon-16x16.png -------------------------------------------------------------------------------- /static/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahseema/trackawesomelist-source/HEAD/static/favicon-32x32.png -------------------------------------------------------------------------------- /static/mstile-150x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahseema/trackawesomelist-source/HEAD/static/mstile-150x150.png -------------------------------------------------------------------------------- /static/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahseema/trackawesomelist-source/HEAD/static/apple-touch-icon.png -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [config.yml] 4 | indent_style = space 5 | indent_size = 2 6 | trim_trailing_whitespace = true 7 | -------------------------------------------------------------------------------- /static/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahseema/trackawesomelist-source/HEAD/static/android-chrome-192x192.png -------------------------------------------------------------------------------- /static/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahseema/trackawesomelist-source/HEAD/static/android-chrome-512x512.png -------------------------------------------------------------------------------- /example/data/2-formated/EbookFoundation/free-programming-books/books/free-programming-books-zh.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "", 3 | "items": {} 4 | } 5 | -------------------------------------------------------------------------------- /error.ts: -------------------------------------------------------------------------------- 1 | export class NotFound extends Error { 2 | constructor(message: string) { 3 | super(message); 4 | this.name = "NotFound"; 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /example/simple.md: -------------------------------------------------------------------------------- 1 | # Title 2 | 3 | Introduction 4 | 5 | ## Subtitle1 6 | 7 | Subintroduction1 8 | 9 | - Item1 10 | - Item2 11 | 12 | ## Subtitle2 13 | 14 | Subintroduction2 15 | 16 | - Item1 17 | - Item2 18 | -------------------------------------------------------------------------------- /templates/file-by-day.md.mu: -------------------------------------------------------------------------------- 1 | # {{{title}}} 2 | 3 | {{{description}}} 4 | 5 | {{{_nav_text}}} 6 | 7 | {{#items}} 8 | ## [{{{title}}}{{{_title_suffix}}}]({{{url}}}) 9 | 10 | {{{content_text}}} 11 | {{/items}} 12 | 13 | -------------------------------------------------------------------------------- /example/public/EbookFoundation/free-programming-books/books/free-programming-books-zh/feed.json: -------------------------------------------------------------------------------- 1 | { 2 | "items": [ 3 | { 4 | "id": "1", 5 | "title": "Item 1", 6 | "content_html": "This is a description for Item 1" 7 | } 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /example/data/github_awesome_nodejs.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "awesome", 3 | "url": "github.com/sindresorhus/awesome", 4 | "description": "", 5 | "description_html": "", 6 | "items": { 7 | "item": { 8 | "date_published": "2019-01-01" 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /render-markdown_test.ts: -------------------------------------------------------------------------------- 1 | import render from "./render-markdown.ts"; 2 | import { assertEquals } from "./test-deps.ts"; 3 | Deno.test("renderMarkdown", () => { 4 | const result = render("[Hello](/test/README.md)"); 5 | assertEquals(result, `

Hello

\n`); 6 | }); 7 | -------------------------------------------------------------------------------- /serve-public.ts: -------------------------------------------------------------------------------- 1 | import { serve, serveDir } from "./deps.ts"; 2 | import { getPublicPath } from "./util.ts"; 3 | export default function servePublic() { 4 | serve((req) => { 5 | return serveDir(req, { 6 | fsRoot: getPublicPath(), 7 | showIndex: true, 8 | }); 9 | }); 10 | } 11 | -------------------------------------------------------------------------------- /static/browserconfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | #da532c 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /templates/day.md.mu: -------------------------------------------------------------------------------- 1 | {{{nav}}} 2 | 3 | {{#groups}} 4 | ## [{{{group_name}}}{{{group_suffix}}}]({{{group_url}}}) 5 | 6 | {{#items}} 7 | {{#category}} 8 | ### {{{category}}} 9 | 10 | {{/category}} 11 | {{#items}}{{{markdown}}} 12 | {{/items}} 13 | 14 | {{/items}} 15 | {{/groups}} 16 | 17 | {{{footer}}} 18 | -------------------------------------------------------------------------------- /test-deps.ts: -------------------------------------------------------------------------------- 1 | export { 2 | assert, 3 | assertAlmostEquals, 4 | assertArrayIncludes, 5 | assertEquals, 6 | assertExists, 7 | assertMatch, 8 | assertNotEquals, 9 | assertNotMatch, 10 | assertObjectMatch, 11 | assertRejects, 12 | assertStringIncludes, 13 | assertThrows, 14 | } from "https://deno.land/std@0.151.0/testing/asserts.ts"; 15 | -------------------------------------------------------------------------------- /example/mac.md: -------------------------------------------------------------------------------- 1 | # Awesome Mac 2 | 3 | ## Text Editors 4 | 5 | - [Bootstrap Studio](https://bootstrapstudio.io/) - A powerful desktop app for creating responsive websites using the Bootstrap framework. 6 | - [Brackets](http://brackets.io) - A modern, open source text editor that understands web design. [![Open-Source Software][oss icon]](https://github.com/adobe/brackets/) ![Freeware][freeware icon] 7 | -------------------------------------------------------------------------------- /adapters/api.ts: -------------------------------------------------------------------------------- 1 | import { RepoMeta, Source } from "../interface.ts"; 2 | export default class API { 3 | source: Source; 4 | constructor(source: Source) { 5 | this.source = source; 6 | } 7 | getConent(_filePath: string): Promise { 8 | return Promise.reject("not implemented"); 9 | } 10 | getRepoMeta(): Promise { 11 | return Promise.reject("not implemented"); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /example/data/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "sources": { 3 | "xxxx/xxx": { 4 | "updated": "2019-01-01T00:00:00.000Z" 5 | }, 6 | "yxxxx/xxx": { 7 | "updated": "2019-01-01T00:00:00.000Z" 8 | } 9 | }, 10 | "items": { 11 | "xxxx/xxx": { 12 | "README.md": { 13 | "sha1": { 14 | "updated_at": 12341, 15 | "updates_day": 1234, 16 | "update_week": 234234 17 | } 18 | } 19 | } 20 | }, 21 | 22 | -------------------------------------------------------------------------------- /static/site.webmanifest: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Track Awesome List", 3 | "short_name": "Track Awesome List", 4 | "icons": [ 5 | { 6 | "src": "/android-chrome-192x192.png", 7 | "sizes": "192x192", 8 | "type": "image/png" 9 | }, 10 | { 11 | "src": "/android-chrome-512x512.png", 12 | "sizes": "512x512", 13 | "type": "image/png" 14 | } 15 | ], 16 | "theme_color": "#ffffff", 17 | "background_color": "#ffffff", 18 | "display": "standalone" 19 | } 20 | -------------------------------------------------------------------------------- /static/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 404 Not Found 8 | 9 | 10 |

404 Not Found

11 |

12 | Sorry for this, you may want to go 13 | Home Page. 14 |

15 | 16 | 17 | -------------------------------------------------------------------------------- /format-category.ts: -------------------------------------------------------------------------------- 1 | import { Content, remove, Root, toMarkdown } from "./deps.ts"; 2 | 3 | export default function formatItemMarkdown( 4 | item: Content | Root, 5 | ): string { 6 | // visit and remove sup item 7 | remove(item, (node, _n) => { 8 | // remove hash link 9 | // remote html 10 | if (node.type === "html") { 11 | return true; 12 | } 13 | if (node.type === "link" && node.url.startsWith("#")) { 14 | return true; 15 | } else { 16 | return false; 17 | } 18 | }); 19 | return toMarkdown(item).trim(); 20 | } 21 | -------------------------------------------------------------------------------- /parser/markdown/list_test.ts: -------------------------------------------------------------------------------- 1 | import markdownlist from "./list.ts"; 2 | import { getFakeFileInfo } from "./util.ts"; 3 | import { 4 | getDbCachedStars, 5 | readTextFile, 6 | writeDbCachedStars, 7 | } from "../../util.ts"; 8 | import { assertEquals } from "../../test-deps.ts"; 9 | Deno.test("markdown list test #3", async () => { 10 | const content = await readTextFile("./example/mac.md"); 11 | const dbCachedStars = await getDbCachedStars(); 12 | const items = await markdownlist(content, getFakeFileInfo(), dbCachedStars); 13 | await writeDbCachedStars(dbCachedStars); 14 | }); 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Track Awesome List Source 2 | 3 | This repo is for generating [trackawesomelist](https://github.com/trackawesomelist/trackawesomelist), runing with Deno, `json` files as database. 4 | 5 | > Documentation is being improved. 6 | 7 | ## Dev 8 | 9 | 1. Install [Deno](https://deno.land/manual@v1.26.2/getting_started/installation) 10 | 2. `git clone git@github.com:trackawesomelist/trackawesomelist-source.git` 11 | 3. Add github awesome repo to `config.yml` -> `sources` 12 | 4. `make startsource source=owner/repo` 13 | 14 | Open 15 | 16 | ### Rebuild Single Repo 17 | 18 | `make prod-build args="--rebuild xxx/repo"` 19 | -------------------------------------------------------------------------------- /util_test.ts: -------------------------------------------------------------------------------- 1 | import { getDayNumber, readTextFile, sha1, titleCase } from "./util.ts"; 2 | import { assertEquals } from "./test-deps.ts"; 3 | 4 | Deno.test("sha1 #1", async () => { 5 | const content = "hello world"; 6 | const sum = await sha1(content); 7 | assertEquals(sum, "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed"); 8 | }); 9 | 10 | Deno.test("getDayNumber", () => { 11 | const date = new Date("2020-01-01"); 12 | const dayNumber = getDayNumber(date); 13 | assertEquals(dayNumber, 20200101); 14 | }); 15 | 16 | Deno.test("title case", () => { 17 | const title = titleCase("free-for-dev"); 18 | assertEquals(title, "Free for Dev"); 19 | }); 20 | -------------------------------------------------------------------------------- /scripts/install-morsels.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) 4 | BIN_DIR="$SCRIPT_DIR/../bin" 5 | binname="morsels" 6 | mkdir -p $BIN_DIR 7 | if [[ "$OSTYPE" == "linux-gnu"* ]]; then 8 | cd -- /tmp 9 | curl -OL https://github.com/ang-zeyu/morsels/releases/download/v0.7.3/indexer.x86_64-unknown-linux-gnu.zip 10 | unzip indexer.x86_64-unknown-linux-gnu.zip -d $BIN_DIR 11 | chmod +x $BIN_DIR/morsels 12 | elif [[ "$OSTYPE" == "darwin"* ]]; then 13 | # Mac OSX 14 | cd -- /tmp/ 15 | curl -OL https://github.com/ang-zeyu/morsels/releases/download/v0.7.3/indexer.x86_64-apple-darwin.zip 16 | unzip indexer.x86_64-apple-darwin.zip -d $BIN_DIR 17 | fi 18 | 19 | chmod +x $BIN_DIR/* 20 | 21 | echo Install Success. 22 | -------------------------------------------------------------------------------- /deno.json: -------------------------------------------------------------------------------- 1 | { 2 | "lint": { 3 | "files": { 4 | "exclude": ["workers-site/"] 5 | } 6 | }, 7 | "fmt": { 8 | "files": { 9 | "exclude": [ 10 | "node_modules/", 11 | "public/", 12 | "example/", 13 | "example/**/*", 14 | "workers-site/", 15 | "current/", 16 | "dev-current/", 17 | "dev-archive/", 18 | "archive/", 19 | "s3/", 20 | "zip/" 21 | ] 22 | } 23 | }, 24 | "test": { 25 | "files": { 26 | "exclude": [ 27 | "s3", 28 | "node_modules/", 29 | "public/", 30 | "example/", 31 | "workers-site/", 32 | "current/", 33 | "dev-current/", 34 | "dev-archive/", 35 | "archive/" 36 | ] 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /init-db.ts: -------------------------------------------------------------------------------- 1 | import { 2 | getDbIndexFilePath, 3 | getDbMetaFilePath, 4 | writeJSONFile, 5 | } from "./util.ts"; 6 | import log from "./log.ts"; 7 | import dbInitMeta from "./db-meta-init.json" assert { type: "json" }; 8 | export default async function initDb() { 9 | const dbMetaFilePath = getDbMetaFilePath(); 10 | const dbIndexFilePath = getDbIndexFilePath(); 11 | if (!await Deno.stat(dbMetaFilePath).catch(() => false)) { 12 | log.info("db meta not found, auto init"); 13 | // copy db-meta-init.json 14 | await writeJSONFile(dbMetaFilePath, dbInitMeta); 15 | } 16 | if (!await Deno.stat(dbIndexFilePath).catch(() => false)) { 17 | log.info("db index not found, auto init"); 18 | // copy db-meta-init.json 19 | await writeJSONFile(dbIndexFilePath, {}); 20 | } 21 | } 22 | 23 | if (import.meta.main) { 24 | initDb(); 25 | } 26 | -------------------------------------------------------------------------------- /parser/mod.ts: -------------------------------------------------------------------------------- 1 | import markdownList from "./markdown/list.ts"; 2 | import markdownTable from "./markdown/table.ts"; 3 | import markdownHeading from "./markdown/heading.ts"; 4 | import { DocItem, ExpiredValue, FileInfo } from "../interface.ts"; 5 | 6 | export default function ( 7 | content: string, 8 | options: FileInfo, 9 | dbCachedStars: Record, 10 | ): Promise { 11 | const fileConfig = options.sourceConfig.files[options.filepath]; 12 | const type = fileConfig.options.type; 13 | if (type === "list") { 14 | return markdownList(content, options, dbCachedStars); 15 | } 16 | if (type === "table") { 17 | return markdownTable(content, options, dbCachedStars); 18 | } 19 | if (type === "heading") { 20 | return markdownHeading(content, options, dbCachedStars); 21 | } 22 | throw new Error(`unknown type ${type}`); 23 | } 24 | -------------------------------------------------------------------------------- /parser/markdown/table_test.ts: -------------------------------------------------------------------------------- 1 | import markdowntable from "./markdowntable.ts"; 2 | import { readTextFile } from "../util.ts"; 3 | import { assertEquals } from "../test-deps.ts"; 4 | Deno.test("markdown table test #1", async () => { 5 | const content = await readTextFile("./example/public-apis-simple.md"); 6 | 7 | const items = await markdowntable(content); 8 | // assertEquals(items, [ 9 | // { markdown: "* Item1\n", categories: ["Subtitle1\n"] }, 10 | // { markdown: "* Item2\n", categories: ["Subtitle1\n"] }, 11 | // { markdown: "* Item1\n", categories: ["Subtitle2\n"] }, 12 | // { markdown: "* Item2\n", categories: ["Subtitle2\n"] }, 13 | // ]); 14 | 15 | // console.log("items", items); 16 | }); 17 | 18 | Deno.test("markdown table test #2", async () => { 19 | const content = await readTextFile("./example/books.md"); 20 | 21 | const items = markdowntable(content); 22 | }); 23 | -------------------------------------------------------------------------------- /book.toml: -------------------------------------------------------------------------------- 1 | [book] 2 | title = "Track Awesome List" 3 | description = "Track Awesome List" 4 | src = "dist/content" 5 | language = "zh" 6 | authors = ["Owen Young"] 7 | 8 | [build] 9 | create-missing = true 10 | 11 | [preprocessor.morsels] 12 | command = "mdbook-morsels" 13 | 14 | [preprocessor.toc] 15 | command = "mdbook-toc" 16 | renderer = ["html"] 17 | 18 | [output.html] 19 | git-repository-url = "https://github.com/theowenyoung/blog" 20 | edit-url-template = "https://github.com/theowenyoung/blog/edit/main/{path}" 21 | 22 | [output.html.fold] 23 | enable = true # whether or not to enable section folding 24 | level = 1 25 | [output.html.search] 26 | enable = false 27 | [output.html.print] 28 | enable = false 29 | 30 | # Plugin configuration options (optional) 31 | [output.morsels] 32 | # Relative path to a Morsels indexer configuration file from the project directory. 33 | # The config file will also automatically be created here if it dosen't exist. 34 | config = "morsels_config.json" 35 | -------------------------------------------------------------------------------- /constant.ts: -------------------------------------------------------------------------------- 1 | export const INDEX_MARKDOWN_PATH = "README.md"; 2 | export const RECENTLY_UPDATED_COUNT = 10; 3 | export const TOP_REPOS_COUNT = 50; 4 | export const PROD_DOMAIN = "https://www.trackawesomelist.com"; 5 | export const DEV_DOMAIN = "http://localhost:8000"; 6 | export const INDEX_HTML_PATH = "index.html"; 7 | export const DEFAULT_CATEGORY = "Miscellaneous"; 8 | export const CONTENT_DIR = "content"; 9 | export const SUBSCRIPTION_URL = 10 | "https://trackawesomelist.us17.list-manage.com/subscribe?u=d2f0117aa829c83a63ec63c2f&id=36a103854c"; 11 | 12 | export const HOME_NAV = "🏠 Home"; 13 | export const SEARCH_NAV = "🔍 Search"; 14 | export const FEED_NAV = "🔥 Feed"; 15 | export const SUBSCRIBE_NAV = "📮 Subscribe"; 16 | export const SPONSOR_NAV = "❤️ Sponsor"; 17 | export const SPONSOR_URL = "https://github.com/sponsors/theowenyoung"; 18 | 19 | export const GITHUB_NAV = "😺 Github"; 20 | export const WEBSITE_NAV = "🌐 Website"; 21 | export const GITHUB_REPO = 22 | "https://github.com/trackawesomelist/trackawesomelist/"; 23 | -------------------------------------------------------------------------------- /scripts/check-404.ts: -------------------------------------------------------------------------------- 1 | import { getConfig, gotWithCache } from "../util.ts"; 2 | import { PROD_DOMAIN } from "../constant.ts"; 3 | 4 | async function check() { 5 | const config = await getConfig(); 6 | const sources = config.sources; 7 | const sourcesKeys = Object.keys(sources); 8 | for (const siteIdentifier of sourcesKeys) { 9 | const site = sources[siteIdentifier]; 10 | const files = site.files; 11 | const filesKeys = Object.keys(files); 12 | for (const fileIdentifier of filesKeys) { 13 | const file = files[fileIdentifier]; 14 | const url = new URL(file.pathname, PROD_DOMAIN); 15 | try { 16 | const response = await gotWithCache(url.href, {}); 17 | console.log("ok", url.href); 18 | } catch (e) { 19 | const ignored = ["PatrickJS/awesome-angular"]; 20 | if (ignored.includes(file.pathname.slice(1, -1))) { 21 | console.warn(`ignored ${url.href}`); 22 | continue; 23 | } 24 | 25 | console.log(`Error: ${url}`); 26 | throw e; 27 | } 28 | } 29 | } 30 | } 31 | 32 | if (import.meta.main) { 33 | await check(); 34 | } 35 | -------------------------------------------------------------------------------- /morsels_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "indexing_config": { 3 | "exclude": [ 4 | "index.html", 5 | "print.html", 6 | "404.html", 7 | "morsels_config.json" 8 | ], 9 | "include": [ 10 | "*/*/readme/index.html" 11 | ], 12 | "loaders": { 13 | "HtmlLoader": { 14 | "exclude_selectors": [ 15 | "script,style,#sidebar,#menu-bar" 16 | ], 17 | "selectors": [ 18 | { 19 | "attr_map": {}, 20 | "field_name": "title", 21 | "selector": "title" 22 | }, 23 | { 24 | "attr_map": {}, 25 | "field_name": "h1", 26 | "selector": "h1" 27 | }, 28 | { 29 | "attr_map": {}, 30 | "field_name": "body", 31 | "selector": "body" 32 | }, 33 | { 34 | "attr_map": { 35 | "id": "headingLink" 36 | }, 37 | "field_name": "heading", 38 | "selector": "h1,h2,h3,h4,h5,h6" 39 | } 40 | ], 41 | "type": "HtmlLoader" 42 | } 43 | } 44 | }, 45 | "preset": "small" 46 | } 47 | -------------------------------------------------------------------------------- /templates/index.html.mu: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | {{{_seo_title}}} 14 | 15 | 16 | 17 | 18 | 19 | 27 | 28 | 29 |
30 | {{{body}}} 31 |
32 | 33 | 34 | -------------------------------------------------------------------------------- /lib/gemoji.js: -------------------------------------------------------------------------------- 1 | import { visit } from "https://esm.sh/unist-util-visit@4.1.1"; 2 | import { gemoji, nameToEmoji } from "https://cdn.skypack.dev/gemoji@7?dts"; 3 | const find = /:(\+1|[-\w]+):/g; 4 | 5 | const own = {}.hasOwnProperty; 6 | 7 | export default function remarkGemoji() { 8 | return (tree) => { 9 | visit(tree, "text", (node) => { 10 | const value = node.value; 11 | /** @type {string[]} */ 12 | const slices = []; 13 | find.lastIndex = 0; 14 | let match = find.exec(value); 15 | let start = 0; 16 | 17 | while (match) { 18 | const emoji = /** @type {keyof nameToEmoji} */ (match[1]); 19 | const position = match.index; 20 | 21 | if (own.call(nameToEmoji, emoji) || emoji === "octocat") { 22 | if (start !== position) { 23 | slices.push(value.slice(start, position)); 24 | } 25 | let finalEmoji = nameToEmoji[emoji]; 26 | if (!finalEmoji && emoji === "octocat") { 27 | finalEmoji = "🐙"; 28 | } 29 | 30 | slices.push(finalEmoji); 31 | start = position + match[0].length; 32 | } else { 33 | find.lastIndex = position + 1; 34 | } 35 | 36 | match = find.exec(value); 37 | } 38 | 39 | if (slices.length > 0) { 40 | slices.push(value.slice(start)); 41 | node.value = slices.join(""); 42 | } 43 | }); 44 | }; 45 | } 46 | -------------------------------------------------------------------------------- /migrations/to-new-config.ts: -------------------------------------------------------------------------------- 1 | import oldMeta from "./old-meta.json" assert { type: "json" }; 2 | import { YAML } from "../deps.ts"; 3 | import { Config, RawSource } from "../interface.ts"; 4 | import { DEFAULT_CATEGORY } from "../constant.ts"; 5 | export function migrate() { 6 | const awesomelist = oldMeta.awesomeList; 7 | const sources: Record = {}; 8 | const newConfig = YAML.parse(Deno.readTextFileSync("./config.yml")) as Config; 9 | const newSources = newConfig.sources; 10 | for (const repo of awesomelist) { 11 | const source: RawSource = { 12 | category: repo.category, 13 | default_branch: repo.defaultBranch, 14 | files: { 15 | [repo.readmePath]: { 16 | index: true, 17 | }, 18 | }, 19 | }; 20 | sources[repo.repo] = source; 21 | } 22 | const mergedSources = { 23 | ...sources, 24 | ...newSources, 25 | }; 26 | 27 | //resort the sources keys, by category 28 | const sortedSources = Object.fromEntries( 29 | Object.entries(mergedSources).sort((a, b) => { 30 | const aCategory = a[1]?.category || DEFAULT_CATEGORY; 31 | const bCategory = b[1]?.category || DEFAULT_CATEGORY; 32 | if (aCategory > bCategory) { 33 | return 1; 34 | } else if (aCategory < bCategory) { 35 | return -1; 36 | } else { 37 | return 0; 38 | } 39 | }), 40 | ); 41 | 42 | const yamlSource = YAML.stringify({ 43 | sources: sortedSources, 44 | }); 45 | Deno.writeTextFileSync("./temp-config.yml", yamlSource); 46 | } 47 | 48 | if (import.meta.main) { 49 | migrate(); 50 | } 51 | -------------------------------------------------------------------------------- /scripts/install-mdbook.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ); 4 | BIN_DIR="$SCRIPT_DIR/../bin" 5 | binname="mdbook-epub" 6 | mkdir -p $BIN_DIR 7 | if [[ "$OSTYPE" == "linux-gnu"* ]]; then 8 | cd -- /tmp 9 | # curl -OL https://github.com/theowenyoung/$binname/releases/latest/download/$binname-x86_64-unknown-linux-gnu.tar.gz 10 | # tar -xf /tmp/$binname-x86_64-unknown-linux-gnu.tar.gz -C $BIN_DIR 11 | curl -OL https://github.com/rust-lang/mdBook/releases/download/v0.4.21/mdbook-v0.4.21-x86_64-unknown-linux-gnu.tar.gz 12 | tar -xf /tmp/mdbook-v0.4.21-x86_64-unknown-linux-gnu.tar.gz -C $BIN_DIR 13 | # curl -OL https://github.com/HollowMan6/mdbook-pdf/releases/download/v0.1.3/mdbook-pdf-v0.1.3-x86_64-unknown-linux-gnu.zip 14 | # unzip /tmp/mdbook-pdf-v0.1.3-x86_64-unknown-linux-gnu.zip -d $BIN_DIR 15 | elif [[ "$OSTYPE" == "darwin"* ]]; then 16 | # Mac OSX 17 | cd -- /tmp/ 18 | # curl -OL https://github.com/theowenyoung/$binname/releases/latest/download/$binname-x86_64-apple-darwin.zip 19 | # unzip -o /tmp/$binname-x86_64-apple-darwin.zip -d $BIN_DIR 20 | curl -OL https://github.com/rust-lang/mdBook/releases/download/v0.4.21/mdbook-v0.4.21-x86_64-apple-darwin.tar.gz 21 | tar -xf /tmp/mdbook-v0.4.21-x86_64-apple-darwin.tar.gz -C $BIN_DIR 22 | # curl -OL https://github.com/HollowMan6/mdbook-pdf/releases/download/v0.1.3/mdbook-pdf-v0.1.3-x86_64-apple-darwin.zip 23 | # unzip -o /tmp/mdbook-pdf-v0.1.3-x86_64-apple-darwin.zip -d $BIN_DIR 24 | fi; 25 | 26 | chmod +x $BIN_DIR/* 27 | 28 | echo Install Success. 29 | 30 | echo Run \`make buildbook\` to build to book-dist folder 31 | -------------------------------------------------------------------------------- /render-markdown.ts: -------------------------------------------------------------------------------- 1 | import { DocItem, FileInfo, ParseOptions } from "./interface.ts"; 2 | import { 3 | Content, 4 | fromMarkdown, 5 | render, 6 | TableRow, 7 | toMarkdown, 8 | visit, 9 | } from "./deps.ts"; 10 | import { childrenToRoot, getDomain } from "./util.ts"; 11 | import _log from "./log.ts"; 12 | import { 13 | gfm, 14 | gfmFromMarkdown, 15 | gfmToMarkdown, 16 | remarkEmoji, 17 | remarkGemoji, 18 | } from "./deps.ts"; 19 | import { CONTENT_DIR, INDEX_MARKDOWN_PATH } from "./constant.ts"; 20 | export default function renderMarkdown(content: string): string { 21 | const domain = getDomain(); 22 | const tree = fromMarkdown(content, "utf8", { 23 | // @ts-ignore: remarkInlineLinks is not typed 24 | extensions: [gfm()], 25 | mdastExtensions: [gfmFromMarkdown()], 26 | }); 27 | // @ts-ignore: node function 28 | const remarkEmojiPlugin = remarkEmoji(); 29 | // @ts-ignore: node function 30 | remarkEmojiPlugin(tree); 31 | const remarkGemojiPlugin = remarkGemoji(); 32 | // @ts-ignore: node function 33 | remarkGemojiPlugin(tree); 34 | 35 | visit(tree, "link", (node) => { 36 | const { url } = node; 37 | if ( 38 | url && 39 | (url.startsWith("/" + CONTENT_DIR + "/")) && 40 | url.endsWith(INDEX_MARKDOWN_PATH) 41 | ) { 42 | node.url = url.slice(CONTENT_DIR.length + 1, -INDEX_MARKDOWN_PATH.length); 43 | } else if ( 44 | url && (url.startsWith("/")) && url.endsWith(INDEX_MARKDOWN_PATH) 45 | ) { 46 | node.url = url.slice(0, -INDEX_MARKDOWN_PATH.length); 47 | } 48 | }); 49 | 50 | const markdownDist = toMarkdown(tree, { 51 | extensions: [gfmToMarkdown()], 52 | }); 53 | return render(markdownDist, { 54 | allowIframes: true, 55 | }); 56 | } 57 | -------------------------------------------------------------------------------- /serve-markdown.ts: -------------------------------------------------------------------------------- 1 | import { CSS, mustache, path, serve, serveFile } from "./deps.ts"; 2 | 3 | import { 4 | getDistRepoPath, 5 | getStaticPath, 6 | readTextFile, 7 | urlToFilePath, 8 | } from "./util.ts"; 9 | import log from "./log.ts"; 10 | import { RunOptions } from "./interface.ts"; 11 | import render from "./render-markdown.ts"; 12 | export default async function serveSite(runOptions: RunOptions) { 13 | const port = runOptions.port; 14 | const BASE_PATH = getDistRepoPath(); 15 | const staticpath = getStaticPath(); 16 | const htmlTemplate = await readTextFile("./templates/index.html.mu"); 17 | const handler = async (request: Request): Promise => { 18 | const filepath = urlToFilePath(request.url); 19 | 20 | log.debug(`Request for ${filepath}`); 21 | let localPath = BASE_PATH + filepath; 22 | if (!filepath.endsWith(".md")) { 23 | // serve static fold 24 | localPath = path.join(staticpath, filepath); 25 | return await serveFile(request, localPath); 26 | } 27 | // check if file exists 28 | let finalPath: string | undefined; 29 | try { 30 | const fileInfo = Deno.statSync(localPath); 31 | if (fileInfo.isFile) { 32 | finalPath = localPath; 33 | } 34 | } catch (e) { 35 | log.warn(e); 36 | } 37 | if (finalPath) { 38 | const fileContent = await readTextFile(finalPath); 39 | log.debug(`serving file: ${finalPath}`); 40 | const body = render(fileContent); 41 | const htmlContent = mustache.render(htmlTemplate, { CSS, body }); 42 | return new Response(htmlContent, { 43 | status: 200, 44 | headers: { 45 | "content-type": "text/html", 46 | }, 47 | }); 48 | } else { 49 | return Promise.resolve(new Response("Not Found", { status: 404 })); 50 | } 51 | }; 52 | log.info( 53 | `HTTP webserver running. Access it at: http://localhost:${port}/`, 54 | ); 55 | serve(handler, { port }); 56 | } 57 | -------------------------------------------------------------------------------- /example/data/1-raw/EbookFoundation/free-programming-books/books/markdownlist_free-programming-books-langs.md: -------------------------------------------------------------------------------- 1 | ## BY PROGRAMMING LANGUAGE 2 | 3 | Originally, this list included a section called "Language Agnostic" for books about programming subjects not restricted to a specific programming language. 4 | That section got so big, we decided to split it into its own file, the [BY SUBJECT file](free-programming-books-subjects.md). 5 | 6 | ### ABAP 7 | 8 | - [SAP Code Style Guides - Clean ABAP](https://github.com/SAP/styleguides/blob/master/clean-abap/CleanABAP.md) 9 | 10 | ### Ada 11 | 12 | - [A Guide to Ada for C and C++ Programmers](http://www.cs.uni.edu/~mccormic/4740/guide-c2ada.pdf) (PDF) 13 | - [Ada Distilled](http://www.adapower.com/pdfs/AdaDistilled07-27-2003.pdf) (PDF) 14 | - [Ada for the C++ or Java Developer](https://www.adacore.com/uploads/books/pdf/Ada_for_the_C_or_Java_Developer-cc.pdf) - Quentin Ochem (PDF) 15 | - [Ada Programming](https://en.wikibooks.org/wiki/Ada_Programming) - Wikibooks 16 | - [Ada Reference Manual - ISO/IEC 8652:2012(E) Language and Standard Libraries](http://www.ada-auth.org/standards/12rm/RM-Final.pdf) (PDF) 17 | - [Introduction To Ada](https://learn.adacore.com/courses/intro-to-ada/index.html) 18 | - [Introduction To SPARK](https://learn.adacore.com/courses/SPARK_for_the_MISRA_C_Developer/index.html) 19 | - [The Big Online Book of Linux Ada Programming](http://www.pegasoft.ca/resources/boblap/book.html) 20 | 21 | ### Workflow 22 | 23 | - [Declare Peace on Virtual Machines. A guide to simplifying vm-based development on a Mac](https://leanpub.com/declarepeaceonvms/read) 24 | 25 | ### xBase (dBase / Clipper / Harbour) 26 | 27 | - [Application Development with Harbour](https://en.wikibooks.org/wiki/Application_Development_with_Harbour) - Wikibooks 28 | - [CA-Clipper 5.2 Norton Guide](https://web.archive.org/web/20190516192814/http://www.ousob.com/ng/clguide/) 29 | - [Clipper Tutorial: a Guide to Open Source Clipper(s)]() - Wikibooks 30 | -------------------------------------------------------------------------------- /tal.ts: -------------------------------------------------------------------------------- 1 | import { Command } from "./deps.ts"; 2 | import main from "./main.ts"; 3 | export default async function tal() { 4 | await new Command() 5 | .name("tal") 6 | .version("0.1.0") 7 | .description("Track Markdown Files Changes") 8 | .env("DEBUG=", "Enable debug output.") 9 | .env("FORCE=", "Enable force update.") 10 | .env("FORCE_FETCH=", "Enable force update fetch.") 11 | .env("PUSH=", "Enable push to remote repo.") 12 | .env("REBUILD=", "Enable rebuild.") 13 | .env("LIMIT=", "Limit sources to build, for debug.") 14 | .env("DAY_MARKDOWN=", "Disable day markdown output.") 15 | .env( 16 | "FETCH_REPO_UPDATES=", 17 | "fetch repo updates when init there is a cache. for dev fast test", 18 | ) 19 | .option("-d, --debug", "Enable debug output.") 20 | .option("-f, --force", "Force update markdown.") 21 | .option("--force-fetch", "Force update sources.") 22 | .option("--rebuild", "rebuild updates from git repo") 23 | .option("-p, --push", "Push markdown to remote.") 24 | .option("--no-fetch", "Don't fetch remote sources.") 25 | .option("--no-markdown", "do not build markdown file.") 26 | .option("--clean-html", "clean html files.") 27 | .option("--clean-markdown", "clean markdown files.") 28 | .option("--no-day-markdown", "do not build day markdown file.") 29 | .option("--no-fetch-repo-updates", "do not fetch repo updates.") 30 | .option("--html", "Build html files.") 31 | .option("--no-serve", "Serve site.") 32 | .option("--limit ", "Limit number of sources to process.") 33 | .option( 34 | "--auto-init", 35 | "auto init db meta, for avoid load remote db failed", 36 | ).option( 37 | "--port ", 38 | "Serve site port.", 39 | { 40 | default: 8000, 41 | }, 42 | ) 43 | .arguments("[files...:string]") 44 | .action((options, ...args) => { 45 | main(options, ...args); 46 | }) 47 | .parse(Deno.args); 48 | } 49 | 50 | if (import.meta.main) { 51 | await tal(); 52 | } 53 | -------------------------------------------------------------------------------- /static/safari-pinned-tab.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 7 | 8 | Created by potrace 1.14, written by Peter Selinger 2001-2017 9 | 10 | 12 | 19 | 30 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /templates/root-readme.md.mu: -------------------------------------------------------------------------------- 1 | # {{{feed.title}}} 2 | 3 | {{{feed.description}}} This repo is generated by [trackawesomelist-source](https://github.com/trackawesomelist/trackawesomelist-source), visit it [Online](https://www.trackawesomelist.com) or with [Github](https://github.com/trackawesomelist/trackawesomelist/). 4 | 5 | {{{navText}}} 6 | 7 | ## Table of Contents 8 | 9 | - [Recently Updated](#recently-updated) 10 | - [Top 50 Awesome List](#top-50-awesome-list) 11 | - [All Tracked List](#all-tracked-list) 12 | - [Social Media](#social-media) 13 | - [Contribution](#contribution) 14 | 15 | ## Recently Updated{{#items}} 16 | 17 | ### [{{{_short_title}}}]({{{_filepath}}}) 18 | 19 | {{{content_text}}}{{/items}}{{paginationText}} 20 | 21 | ## Top 50 Awesome List 22 | 23 | {{#sortedRepos}} 24 | {{order}}. [{{{name}}}]({{{url}}}) - ([Source ⭐ {{star}} 📝 {{updated}} ]({{{source_url}}})) - {{{meta.description}}} 25 | {{/sortedRepos}} 26 | 27 | ## All Tracked List 28 | {{#list}} 29 | 30 | ### {{category}} 31 | 32 | {{#items}} 33 | - [{{{name}}}]({{{url}}}) - ([Source ⭐ {{star}}, 📝 {{updated}} ]({{{source_url}}})) - {{{meta.description}}} 34 | {{/items}} 35 | {{/list}} 36 | 37 | 38 | ## Social Media 39 | 40 | - [Twitter](https://twitter.com/trackawesome) 41 | - [Telegram](https://t.me/trackawesomelist) 42 | 43 | 44 | ## Contribution 45 | 46 | This repo is generated by [trackawesomelist-source](https://github.com/trackawesomelist/trackawesomelist-source), if you want to add your awesome list here, please edit [config.yml](https://github.com/trackawesomelist/trackawesomelist-source/blob/main/config.yml), and send a pull request, or just open an [issue](https://github.com/trackawesomelist/trackawesomelist-source/issues), I'll add it manually. 47 | 48 | If you want to add badge ([![Track Awesome List](https://www.trackawesomelist.com/badge.svg)](https://www.trackawesomelist.com/ripienaar/free-for-dev/) 49 | ) to your awesome list, please add the following code to your README.md: 50 | 51 | ```markdown 52 | [![Track Awesome List](https://www.trackawesomelist.com/badge.svg)](https://www.trackawesomelist.com/your_repo_pathname/) 53 | ``` 54 | 55 | 56 | The doc is still under construction, if you have any question, please open an [issue](https://github.com/trackawesomelist/trackawesomelist-source/issues) 57 | -------------------------------------------------------------------------------- /log.ts: -------------------------------------------------------------------------------- 1 | import { colors } from "./deps.ts"; 2 | import { Level, LevelName } from "./interface.ts"; 3 | 4 | export class Timing { 5 | #t = performance.now(); 6 | 7 | reset() { 8 | this.#t = performance.now(); 9 | } 10 | 11 | stop(message: string) { 12 | const now = performance.now(); 13 | const d = Math.round(now - this.#t); 14 | let cf = colors.green; 15 | if (d > 10000) { 16 | cf = colors.red; 17 | } else if (d > 1000) { 18 | cf = colors.yellow; 19 | } 20 | console.debug(colors.dim("TIMING"), message, "in", cf(d + "ms")); 21 | this.#t = now; 22 | } 23 | } 24 | 25 | export class Logger { 26 | #level: Level = Level.Info; 27 | 28 | get level(): Level { 29 | return this.#level; 30 | } 31 | 32 | setLevel(level: LevelName): void { 33 | switch (level) { 34 | case "debug": 35 | this.#level = Level.Debug; 36 | break; 37 | case "info": 38 | this.#level = Level.Info; 39 | break; 40 | case "warn": 41 | this.#level = Level.Warn; 42 | break; 43 | case "error": 44 | this.#level = Level.Error; 45 | break; 46 | case "fatal": 47 | this.#level = Level.Fatal; 48 | break; 49 | } 50 | } 51 | 52 | debug(...args: unknown[]): void { 53 | if (this.#level <= Level.Debug) { 54 | console.debug(colors.dim("DEBUG"), ...args); 55 | } 56 | } 57 | 58 | info(...args: unknown[]): void { 59 | if (this.#level <= Level.Info) { 60 | console.log(colors.green("INFO"), ...args); 61 | } 62 | } 63 | 64 | warn(...args: unknown[]): void { 65 | if (this.#level <= Level.Warn) { 66 | console.warn(colors.yellow("WARN"), ...args); 67 | } 68 | } 69 | 70 | error(...args: unknown[]): void { 71 | if (this.#level <= Level.Error) { 72 | console.error(colors.red("ERROR"), ...args); 73 | } 74 | } 75 | 76 | fatal(...args: unknown[]): void { 77 | if (this.#level <= Level.Fatal) { 78 | console.error(colors.red("FATAL"), ...args); 79 | Deno.exit(1); 80 | } 81 | } 82 | 83 | timing(): { reset(): void; stop(message: string): void } { 84 | if (this.level === Level.Debug) { 85 | return new Timing(); 86 | } 87 | return { reset: () => {}, stop: () => {} }; 88 | } 89 | } 90 | 91 | export default new Logger(); 92 | -------------------------------------------------------------------------------- /main.ts: -------------------------------------------------------------------------------- 1 | import log from "./log.ts"; 2 | import fetchSources from "./fetch-sources.ts"; 3 | import build from "./build.ts"; 4 | import serverMarkdown from "./serve-markdown.ts"; 5 | import servePublic from "./serve-public.ts"; 6 | import { getConfig, getFormatedSource, getSqlitePath, isDev } from "./util.ts"; 7 | import { CliOptions, RunOptions } from "./interface.ts"; 8 | import initDb from "./init-db.ts"; 9 | import buildHtml from "./build-html.ts"; 10 | // import db init meta json 11 | export default async function main(cliOptions: CliOptions, ...args: string[]) { 12 | if (cliOptions.debug) { 13 | log.setLevel("debug"); 14 | } 15 | const config = await getConfig(); 16 | let sourceIdentifiers: string[] = args.length > 0 17 | ? args 18 | : Object.keys(config.sources); 19 | if ( 20 | cliOptions.limit && cliOptions.limit > 0 21 | ) { 22 | sourceIdentifiers = sourceIdentifiers.slice(0, cliOptions.limit); 23 | } 24 | // check if source exists 25 | for (const sourceIdentifier of sourceIdentifiers) { 26 | if (config.sources[sourceIdentifier] === undefined) { 27 | config.sources[sourceIdentifier] = getFormatedSource( 28 | sourceIdentifier, 29 | null, 30 | ); 31 | } 32 | } 33 | const isBuildHtml = cliOptions.html || false; 34 | const autoInit = cliOptions.autoInit; 35 | if (autoInit || (isDev())) { 36 | await initDb(); 37 | } 38 | // init sqlite db 39 | // te 40 | // Open a database 41 | const runOptions: RunOptions = { 42 | config: config, 43 | sourceIdentifiers: args, 44 | ...cliOptions, 45 | }; 46 | log.info( 47 | `run options: ${ 48 | JSON.stringify({ sourceIdentifiers: args, ...cliOptions }, null, 2) 49 | }`, 50 | ); 51 | if (cliOptions.fetch) { 52 | await fetchSources(runOptions); 53 | } else { 54 | log.info("skip fetch sources"); 55 | } 56 | // 2. build markdowns, and htmls 57 | await build(runOptions); 58 | 59 | // 3. build html 60 | // 61 | // if (isBuildHtml) { 62 | // await buildHtml(runOptions); 63 | // } 64 | 65 | // 3. serve site 66 | if (runOptions.serve) { 67 | log.info("serve site"); 68 | // check is there is html 69 | if (isBuildHtml) { 70 | servePublic(); 71 | } else { 72 | // serve to markdown preview files 73 | await serverMarkdown(runOptions); 74 | } 75 | } else { 76 | log.info("skip serve site"); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /.github/workflows/cron.yml: -------------------------------------------------------------------------------- 1 | name: cron tasks 2 | on: 3 | repository_dispatch: 4 | types: [schedule] 5 | workflow_dispatch: 6 | inputs: 7 | args: 8 | default: "" 9 | description: "args to build" 10 | type: string 11 | required: false 12 | push: 13 | branches: 14 | - main 15 | paths-ignore: 16 | - "**.md" 17 | schedule: 18 | - cron: "9 */12 * * *" 19 | jobs: 20 | build: 21 | runs-on: ubuntu-latest 22 | concurrency: build 23 | steps: 24 | - name: Check out repository code 25 | uses: actions/checkout@v3 26 | - uses: actions/setup-node@v3 27 | with: 28 | node-version: 16 29 | cache: "npm" 30 | - name: install wrangler 31 | run: npm install -g wrangler 32 | - uses: denoland/setup-deno@v1 33 | with: 34 | deno-version: v1.29.4 35 | - uses: actions/cache@v3 36 | with: 37 | path: | 38 | ~/.deno 39 | ~/.cache/deno 40 | key: ${{ runner.os }}-deno-${{ hashFiles('**/*deps.ts') }} 41 | - run: make prod-load 42 | env: 43 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 44 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 45 | AWS_DEFAULT_REGION: ${{secrets.AWS_DEFAULT_REGION}} 46 | AWS_ENDPOINT: ${{secrets.AWS_ENDPOINT}} 47 | - run: make install 48 | - run: "make prod-build args='${{ github.event.inputs.args }}'" 49 | id: source 50 | continue-on-error: true 51 | env: 52 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 53 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 54 | AWS_DEFAULT_REGION: ${{secrets.AWS_DEFAULT_REGION}} 55 | AWS_ENDPOINT: ${{secrets.AWS_ENDPOINT}} 56 | PUSH: 1 57 | DIST_REPO: ${{ secrets.DIST_REPO }} 58 | PERSONAL_GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }} 59 | - name: upload files 60 | run: make prod-upload 61 | env: 62 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 63 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 64 | AWS_DEFAULT_REGION: ${{secrets.AWS_DEFAULT_REGION}} 65 | AWS_ENDPOINT: ${{secrets.AWS_ENDPOINT}} 66 | - name: upload temp folder to github action for debug 67 | uses: actions/upload-artifact@v3 68 | with: 69 | name: temp 70 | path: temp 71 | if-no-files-found: ignore 72 | - name: throw if build failed 73 | if: steps.source.outcome == 'failure' 74 | run: | 75 | echo "::error::prod-build failed" 76 | exit 1 77 | - name: Publish pages 78 | if: true 79 | run: make prod-publish 80 | env: 81 | CLOUDFLARE_ACCOUNT_ID: ${{secrets.CLOUDFLARE_ACCOUNT_ID}} 82 | CLOUDFLARE_API_TOKEN: ${{secrets.CLOUDFLARE_API_TOKEN}} 83 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /prod-current/ 2 | deno.lock 3 | *.zip 4 | /bin/ 5 | /book/ 6 | /current/ 7 | /temp/ 8 | /cache/ 9 | /prod-temp/ 10 | /prod-*/ 11 | /public/ 12 | /prod-public/ 13 | /db/ 14 | temp-* 15 | # Logs 16 | logs 17 | *.log 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* 21 | lerna-debug.log* 22 | .pnpm-debug.log* 23 | 24 | # Diagnostic reports (https://nodejs.org/api/report.html) 25 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 26 | 27 | # Runtime data 28 | pids 29 | *.pid 30 | *.seed 31 | *.pid.lock 32 | 33 | # Directory for instrumented libs generated by jscoverage/JSCover 34 | lib-cov 35 | 36 | # Coverage directory used by tools like istanbul 37 | coverage 38 | *.lcov 39 | 40 | # nyc test coverage 41 | .nyc_output 42 | 43 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 44 | .grunt 45 | 46 | # Bower dependency directory (https://bower.io/) 47 | bower_components 48 | 49 | # node-waf configuration 50 | .lock-wscript 51 | 52 | # Compiled binary addons (https://nodejs.org/api/addons.html) 53 | build/Release 54 | 55 | # Dependency directories 56 | node_modules/ 57 | jspm_packages/ 58 | 59 | # Snowpack dependency directory (https://snowpack.dev/) 60 | web_modules/ 61 | 62 | # TypeScript cache 63 | *.tsbuildinfo 64 | 65 | # Optional npm cache directory 66 | .npm 67 | 68 | # Optional eslint cache 69 | .eslintcache 70 | 71 | # Optional stylelint cache 72 | .stylelintcache 73 | 74 | # Microbundle cache 75 | .rpt2_cache/ 76 | .rts2_cache_cjs/ 77 | .rts2_cache_es/ 78 | .rts2_cache_umd/ 79 | 80 | # Optional REPL history 81 | .node_repl_history 82 | 83 | # Output of 'npm pack' 84 | *.tgz 85 | 86 | # Yarn Integrity file 87 | .yarn-integrity 88 | 89 | # dotenv environment variable files 90 | .env 91 | .env.development.local 92 | .env.test.local 93 | .env.production.local 94 | .env.local 95 | 96 | # parcel-bundler cache (https://parceljs.org/) 97 | .cache 98 | .parcel-cache 99 | 100 | # Next.js build output 101 | .next 102 | out 103 | 104 | # Nuxt.js build / generate output 105 | .nuxt 106 | dist 107 | 108 | # Gatsby files 109 | .cache/ 110 | # Comment in the public line in if your project uses Gatsby and not Next.js 111 | # https://nextjs.org/blog/next-9-1#public-directory-support 112 | # public 113 | 114 | # vuepress build output 115 | .vuepress/dist 116 | 117 | # vuepress v2.x temp and cache directory 118 | .temp 119 | .cache 120 | 121 | # Docusaurus cache and generated files 122 | .docusaurus 123 | 124 | # Serverless directories 125 | .serverless/ 126 | 127 | # FuseBox cache 128 | .fusebox/ 129 | 130 | # DynamoDB Local files 131 | .dynamodb/ 132 | 133 | # TernJS port file 134 | .tern-port 135 | 136 | # Stores VSCode versions used for testing VSCode extensions 137 | .vscode-test 138 | 139 | # yarn v2 140 | .yarn/cache 141 | .yarn/unplugged 142 | .yarn/build-state.yml 143 | .yarn/install-state.gz 144 | .pnp.* 145 | -------------------------------------------------------------------------------- /adapters/github.ts: -------------------------------------------------------------------------------- 1 | import API from "./api.ts"; 2 | import { RepoMeta, RepoMetaOverride, Source } from "../interface.ts"; 3 | import { base64 } from "../deps.ts"; 4 | import { 5 | got, 6 | gotWithCache, 7 | gotWithDbCache, 8 | isUseCache, 9 | readTextFile, 10 | } from "../util.ts"; 11 | 12 | export default class github extends API { 13 | repo: string; 14 | headers: Headers; 15 | apiPrefix = `https://api.github.com`; 16 | constructor(source: Source) { 17 | super(source); 18 | const githubToken = Deno.env.get("PERSONAL_GITHUB_TOKEN"); 19 | if (!githubToken) { 20 | throw new Error("PERSONAL_GITHUB_TOKEN is not set"); 21 | } 22 | const headerAuthorization = `token ${githubToken}`; 23 | this.headers = new Headers({ 24 | Authorization: headerAuthorization, 25 | }); 26 | const urlObj = new URL(source.url); 27 | this.repo = urlObj.pathname.slice(1); 28 | if (this.repo.endsWith(".git")) { 29 | this.repo = this.repo.slice(0, -4); 30 | } 31 | if (this.repo.endsWith("/")) { 32 | this.repo = this.repo.slice(0, -1); 33 | } 34 | } 35 | getCloneUrl(): string { 36 | return `https://github.com/${this.repo}.git`; 37 | } 38 | async getConent(filePath: string, branch?: string): Promise { 39 | const baseurl = `${this.apiPrefix}/repos/${this.repo}/contents/${filePath}`; 40 | const baseUrlObj = new URL(baseurl); 41 | if (branch) { 42 | baseUrlObj.searchParams.set("ref", branch); 43 | } 44 | const url = baseUrlObj.toString(); 45 | 46 | let result; 47 | if (isUseCache()) { 48 | result = await gotWithCache( 49 | url, 50 | { 51 | headers: this.headers, 52 | }, 53 | { 54 | expires: 4 * 60 * 60 * 1000, 55 | }, 56 | ); 57 | } else { 58 | result = await got( 59 | url, 60 | { 61 | headers: this.headers, 62 | }, 63 | ); 64 | } 65 | 66 | const data = JSON.parse(result); 67 | const content = base64.decode(data.content); 68 | const finalContent = new TextDecoder().decode(content); 69 | return finalContent; 70 | } 71 | async getRepoMeta(overrieds?: RepoMetaOverride): Promise { 72 | const url = `${this.apiPrefix}/repos/${this.repo}`; 73 | const json = await gotWithDbCache( 74 | url, 75 | { 76 | headers: this.headers, 77 | }, 78 | ); 79 | const data = JSON.parse(json); 80 | 81 | let repoMeta: RepoMeta = { 82 | default_branch: data.default_branch, 83 | name: data.name, 84 | description: data.description, 85 | url: data.html_url, 86 | language: data.language, 87 | stargazers_count: data.stargazers_count, 88 | subscribers_count: data.subscribers_count, 89 | forks_count: data.forks_count, 90 | tags: data.topics, 91 | updated_at: data.pushed_at, 92 | created_at: data.created_at, 93 | checked_at: new Date().toISOString(), 94 | }; 95 | // add overrides 96 | if (overrieds) { 97 | repoMeta = Object.assign(repoMeta, overrieds); 98 | } 99 | return repoMeta; 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /deps.ts: -------------------------------------------------------------------------------- 1 | // std 2 | export * as YAML from "https://deno.land/std@0.158.0/encoding/yaml.ts"; 3 | export * as TOML from "https://deno.land/std@0.158.0/encoding/toml.ts"; 4 | export * as path from "https://deno.land/std@0.158.0/path/mod.ts"; 5 | export * as fs from "https://deno.land/std@0.158.0/fs/mod.ts"; 6 | export * as dotenv from "https://deno.land/std@0.158.0/dotenv/mod.ts"; 7 | export * as datetime from "https://deno.land/std@0.158.0/datetime/mod.ts"; 8 | export * as async from "https://deno.land/std@0.158.0/async/mod.ts"; 9 | export * as flags from "https://deno.land/std@0.158.0/flags/mod.ts"; 10 | export * as colors from "https://deno.land/std@0.158.0/fmt/colors.ts"; 11 | export { delay } from "https://deno.land/std@0.158.0/async/delay.ts"; 12 | export { DateTimeFormatter } from "https://deno.land/std@0.158.0/datetime/formatter.ts"; 13 | export { Command } from "https://deno.land/x/cliffy@v0.25.2/command/mod.ts"; 14 | export { serve } from "https://deno.land/std@0.158.0/http/server.ts"; 15 | export { contentType } from "https://deno.land/std@0.158.0/media_types/mod.ts"; 16 | export { 17 | serveDir, 18 | serveFile, 19 | } from "https://deno.land/x/std@0.159.0/http/file_server.ts"; 20 | export * as posixPath from "https://deno.land/std@0.158.0/path/posix.ts"; 21 | export { config as dotenvConfig } from "https://deno.land/std@0.158.0/dotenv/mod.ts"; 22 | export { readLines } from "https://deno.land/std@0.153.0/io/buffer.ts"; 23 | export * as base64 from "https://deno.land/std@0.153.0/encoding/base64.ts"; 24 | // third party 25 | export { titleCase } from "https://esm.sh/title-case@3.0.3"; 26 | export { default as camelCase } from "https://deno.land/x/lodash@4.17.15-es/camelCase.js"; 27 | export { default as groupBy } from "https://deno.land/x/lodash@4.17.15-es/groupBy.js"; 28 | export { CSS, render } from "https://deno.land/x/gfm@0.1.22/mod.ts"; 29 | // npm modules 30 | export { default as mustache } from "https://esm.sh/mustache@4.2.0"; 31 | export { default as pLimit } from "https://esm.sh/p-limit@4.0.0"; 32 | export { gfm } from "https://esm.sh/micromark-extension-gfm@2.0.1"; 33 | export { 34 | gfmFromMarkdown, 35 | gfmToMarkdown, 36 | } from "https://esm.sh/mdast-util-gfm@2.0.2"; 37 | // export { default as kebabCase } from "https://jspm.dev/lodash@4.17.21/kebabCase"; 38 | export { toMarkdown } from "https://esm.sh/mdast-util-to-markdown@1.5.0"; 39 | export { fromMarkdown } from "https://esm.sh/mdast-util-from-markdown@1.3.0"; 40 | export { EXIT, visit } from "https://esm.sh/unist-util-visit@4.1.2"; 41 | export { selectAll } from "https://esm.sh/unist-util-select@4.0.3"; 42 | export { remove } from "https://esm.sh/unist-util-remove@3.1.1"; 43 | export { u } from "https://esm.sh/unist-builder@3.0.1"; 44 | export { default as remarkInlineLinks } from "https://esm.sh/remark-inline-links@6.0.1"; 45 | export { default as remarkEmoji } from "https://esm.sh/remark-emoji@3.1.0"; 46 | export { default as remarkGemoji } from "./lib/gemoji.js"; 47 | export type { 48 | Content, 49 | Link, 50 | Root, 51 | TableCell, 52 | TableRow, 53 | } from "https://esm.sh/v92/@types/mdast@3.0.10/index.d.ts"; 54 | export { default as jsonfeedToAtom } from "https://jspm.dev/jsonfeed-to-atom@1.2.2"; 55 | import transliteration from "https://jspm.dev/transliteration@2.3.5"; 56 | // @ts-ignore: npm module 57 | const slug = transliteration.slugify; 58 | export { slug }; 59 | export { default as kebabCase } from "https://jspm.dev/lodash@4.17.21/kebabCase"; 60 | -------------------------------------------------------------------------------- /templates/search.html.mu: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | {{{_seo_title}}} 14 | 15 | 16 | 17 | 18 | 19 | 44 | 45 | 46 | 47 |
48 |

Search Awesome Stuff

49 |
50 | 51 |
52 |
53 |
54 | 55 |
56 | 57 | 58 | 59 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /static/icon.svg: -------------------------------------------------------------------------------- 1 | 9 | 10 | 16 | 22 | 30 | 35 | 43 | 48 | 53 | 54 | -------------------------------------------------------------------------------- /parser/markdown/util.ts: -------------------------------------------------------------------------------- 1 | import { DocItem, FileInfo, ParseOptions } from "../../interface.ts"; 2 | import { 3 | Content, 4 | EXIT, 5 | fromMarkdown, 6 | Root, 7 | TableRow, 8 | toMarkdown, 9 | visit, 10 | } from "../../deps.ts"; 11 | import { childrenToRoot, promiseLimit, writeTextFile } from "../../util.ts"; 12 | import _log from "../../log.ts"; 13 | import formatMarkdownItem from "../../format-markdown-item.ts"; 14 | import { gfm, gfmFromMarkdown, gfmToMarkdown } from "../../deps.ts"; 15 | export function getValidSections(tree: Root, options: ParseOptions): Content[] { 16 | let currentLevel = 0; 17 | let currentSubCategory = ""; 18 | let currentCategory = ""; 19 | let lowestHeadingLevel = 3; 20 | // first check valided sections 21 | const validSections: Content[] = []; 22 | for (const rootNode of tree.children) { 23 | if (rootNode.type === "heading") { 24 | currentLevel = rootNode.depth; 25 | if (currentLevel > lowestHeadingLevel) { 26 | lowestHeadingLevel = currentLevel; 27 | } 28 | validSections.push(rootNode); 29 | } else if (rootNode.type === "list") { 30 | // check if all links is author link 31 | // if so, it's a table of content 32 | // ignore it 33 | let isToc = true; 34 | visit(childrenToRoot(rootNode.children), "link", (node) => { 35 | if (!node.url.startsWith("#")) { 36 | isToc = false; 37 | } 38 | }); 39 | if (!isToc) { 40 | validSections.push(rootNode); 41 | } 42 | } 43 | } 44 | return validSections; 45 | } 46 | export function uglyFormatItemIdentifier( 47 | _fileInfo: FileInfo, 48 | item: Content | Root, 49 | ): string { 50 | // use link name as identifier 51 | let linkItem; 52 | visit(item, "link", (node) => { 53 | linkItem = node; 54 | return EXIT; 55 | }); 56 | 57 | if (linkItem) { 58 | const finalMarkdown = toMarkdown(linkItem, { 59 | extensions: [gfmToMarkdown()], 60 | }).trim(); 61 | return finalMarkdown; 62 | } else { 63 | return toMarkdown(item, { 64 | extensions: [gfmToMarkdown()], 65 | }).trim(); 66 | } 67 | } 68 | 69 | export function getFakeFileInfo(): FileInfo { 70 | return { 71 | "sourceConfig": { 72 | "identifier": "jaywcjlove/awesome-mac", 73 | "url": "https://github.com/jaywcjlove/awesome-mac", 74 | "files": { 75 | "README.md": { 76 | "filepath": "README.md", 77 | "pathname": "/jaywcjlove/awesome-mac/", 78 | "name": "Awesome Mac", 79 | "index": true, 80 | "options": { "type": "list" }, 81 | }, 82 | }, 83 | "category": "Platforms", 84 | }, 85 | "sourceMeta": { 86 | "created_at": "2022-10-24T18:24:24.090Z", 87 | "updated_at": "2022-10-24T18:35:54.686Z", 88 | "meta": { 89 | "default_branch": "master", 90 | "name": "awesome-mac", 91 | "description": 92 | " Now we have become very big, Different from the original idea. Collect premium software in various categories.", 93 | "url": "https://github.com/jaywcjlove/awesome-mac", 94 | "language": "JavaScript", 95 | "stargazers_count": 54167, 96 | "subscribers_count": 1410, 97 | "forks_count": 5538, 98 | "tags": [ 99 | "apple", 100 | "awesome", 101 | "awesome-list", 102 | "awesome-lists", 103 | "list", 104 | "mac", 105 | "mac-osx", 106 | "macos", 107 | "macosx", 108 | "software", 109 | ], 110 | "updated_at": "2022-10-22T02:50:51Z", 111 | "created_at": "2016-07-17T15:33:47Z", 112 | "checked_at": "2022-10-24T18:24:22.929Z", 113 | }, 114 | "files": { 115 | "README.md": { 116 | "sha1": "0049556161b8f5ddc0a3f89dbb9fb952826fd605", 117 | "updated_at": "2022-10-22T02:50:11.000Z", 118 | "meta_created_at": "2022-10-24T18:27:22.017Z", 119 | "created_at": "2016-07-17T15:34:53.000Z", 120 | "checked_at": "2022-10-24T18:27:22.017Z", 121 | }, 122 | }, 123 | }, 124 | "filepath": "README.md", 125 | }; 126 | } 127 | -------------------------------------------------------------------------------- /get-git-blame.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-non-null-assertion */ 2 | /** 3 | * Spawns git blame and parses results into JSON, via stream (so, no problem on huge files) 4 | */ 5 | 6 | import { camelCase, readLines } from "./deps.ts"; 7 | 8 | interface BlameOptions { 9 | /** 10 | * Annotate only the given line range. May be specified multiple times. Overlapping ranges are allowed. 11 | * @see {@link https://git-scm.com/docs/git-blame#_specifying_ranges} 12 | */ 13 | range: string; 14 | ignoreWhitespace: boolean; 15 | workTree: string; 16 | gitDir: string; 17 | rev: string; 18 | } 19 | 20 | interface LineInfo { 21 | sourceLine: number; 22 | resultLine: number; 23 | hash: string; 24 | numberOfLines: number; 25 | author: string; 26 | authorMail: string; 27 | authorTime: number; 28 | authorTz: string; 29 | commiter: string; 30 | commiterMail: string; 31 | commiterTime: number; 32 | commiterTz: string; 33 | summary: string; 34 | previous: string; 35 | filename: string; 36 | [k: string]: string | number; 37 | } 38 | 39 | export default async function getGitBlame( 40 | filename: string, 41 | options: Partial = {}, 42 | gitPath = "git", 43 | ): Promise> { 44 | /** 45 | * @see {@link https://git-scm.com/docs/git-blame#_options} 46 | */ 47 | const args = ["--no-pager", "blame", "--line-porcelain"]; 48 | if (typeof options.workTree === "string") { 49 | args.unshift(`--work-tree=${options.workTree}`); 50 | } 51 | if (typeof options.gitDir === "string") { 52 | args.unshift(`--git-dir=${options.gitDir}`); 53 | } 54 | if (typeof options.ignoreWhitespace === "boolean") { 55 | args.push("-w"); 56 | } 57 | if (typeof options.range === "string") { 58 | args.push(`-L${options.range}`); 59 | } 60 | if (typeof options.rev === "string") { 61 | args.push(options.rev); 62 | } 63 | const cmd = [gitPath, ...args, "--", filename]; 64 | const process = Deno.run({ 65 | cmd, 66 | cwd: options.workTree, 67 | stdin: "piped", 68 | stdout: "piped", 69 | }); 70 | let currentLine: Partial; 71 | const linesMap: Map> = new Map(); 72 | // return linesMap; 73 | 74 | for await (const line of readLines(process.stdout)) { 75 | // https://git-scm.com/docs/git-blame#_the_porcelain_format 76 | // Each blame entry always starts with a line of: 77 | // <40-byte hex sha1> 78 | // like: 49790775624c422f67057f7bb936f35df920e391 94 120 3 79 | 80 | const parsedLine = 81 | /^(?[a-f0-9]{40,40})\s(?\d+)\s(?\d+)\s(?\d+)$/ 82 | .exec( 83 | line, 84 | ); 85 | if (parsedLine?.groups) { 86 | // this is a new line info 87 | const sourceLine = parseInt(parsedLine.groups.sourceline, 10); 88 | const resultLine = parseInt(parsedLine?.groups.resultLine, 10); 89 | const numberOfLines = parseInt(parsedLine?.groups.numLines, 10); 90 | currentLine = { 91 | hash: parsedLine.groups.hash, 92 | sourceLine, 93 | resultLine, 94 | numberOfLines, 95 | }; 96 | // set for all lines 97 | for (let i = resultLine; i < resultLine + numberOfLines; i++) { 98 | linesMap.set(i, currentLine); 99 | } 100 | } else { 101 | if (currentLine!) { 102 | const commitInfo = 103 | /^(?[a-z]+(-(?[a-z]+))?)\s(?.+)$/.exec( 104 | line, 105 | ); 106 | if (commitInfo?.groups) { 107 | const property = camelCase(commitInfo.groups.token); 108 | let value: string | number = commitInfo.groups.data; 109 | switch (commitInfo.groups.subtoken) { 110 | case "mail": 111 | // remove <> from email 112 | value = value.slice(1, -1); 113 | break; 114 | 115 | case "time": 116 | // parse datestamp into number 117 | value = parseInt(value, 10); 118 | break; 119 | } 120 | currentLine![property] = value; 121 | } 122 | } 123 | } 124 | } 125 | return linesMap as Map; 126 | } 127 | -------------------------------------------------------------------------------- /example/books.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | Introduction text. 4 | 5 | # Table of Contents 6 | 7 | - [Major Cloud Providers' Always-Free Limits](#major-cloud-providers) 8 | - [Cloud management solutions](#cloud-management-solutions) 9 | - [Analytics, Events and Statistics](#analytics-events-and-statistics) 10 | - [APIs, Data and ML](#apis-data-and-ml) 11 | - [Artifact Repos](#artifact-repos) 12 | - [BaaS](#baas) 13 | 14 | ## BY PROGRAMMING LANGUAGE 15 | 16 | Originally, this list included a section called "Language Agnostic" for books about programming subjects not restricted to a specific programming language. 17 | That section got so big, we decided to split it into its own file, the [BY SUBJECT file](free-programming-books-subjects.md). 18 | 19 | ### ABAP 20 | 21 | - [SAP Code Style Guides - Clean ABAP](https://github.com/SAP/styleguides/blob/master/clean-abap/CleanABAP.md) 22 | 23 | ### Ada 24 | 25 | - [A Guide to Ada for C and C++ Programmers](http://www.cs.uni.edu/~mccormic/4740/guide-c2ada.pdf) (PDF) 26 | - [Ada Distilled](http://www.adapower.com/pdfs/AdaDistilled07-27-2003.pdf) (PDF) 27 | - [Ada for the C++ or Java Developer](https://www.adacore.com/uploads/books/pdf/Ada_for_the_C_or_Java_Developer-cc.pdf) - Quentin Ochem (PDF) 28 | - [Ada Programming](https://en.wikibooks.org/wiki/Ada_Programming) - Wikibooks 29 | - [Ada Reference Manual - ISO/IEC 8652:2012(E) Language and Standard Libraries](http://www.ada-auth.org/standards/12rm/RM-Final.pdf) (PDF) 30 | - [Introduction To Ada](https://learn.adacore.com/courses/intro-to-ada/index.html) 31 | - [Introduction To SPARK](https://learn.adacore.com/courses/SPARK_for_the_MISRA_C_Developer/index.html) 32 | - [The Big Online Book of Linux Ada Programming](http://www.pegasoft.ca/resources/boblap/book.html) 33 | 34 | ### Workflow 35 | 36 | - [Declare Peace on Virtual Machines. A guide to simplifying vm-based development on a Mac](https://leanpub.com/declarepeaceonvms/read) 37 | 38 | ### xBase (dBase / Clipper / Harbour) 39 | 40 | - [Application Development with Harbour](https://en.wikibooks.org/wiki/Application_Development_with_Harbour) - Wikibooks 41 | - [CA-Clipper 5.2 Norton Guide](https://web.archive.org/web/20190516192814/http://www.ousob.com/ng/clguide/) 42 | - [Clipper Tutorial: a Guide to Open Source Clipper(s)]() - Wikibooks 43 | 44 | ## BY PROGRAMMING LANGUAGE 45 | 46 | Originally, this list included a section called "Language Agnostic" for books about programming subjects not restricted to a specific programming language. 47 | That section got so big, we decided to split it into its own file, the [BY SUBJECT file](free-programming-books-subjects.md). 48 | 49 | ### ABAP 50 | 51 | - [SAP Code Style Guides - Clean ABAP](https://github.com/SAP/styleguides/blob/master/clean-abap/CleanABAP.md) 52 | 53 | ### Ada 54 | 55 | - [A Guide to Ada for C and C++ Programmers](http://www.cs.uni.edu/~mccormic/4740/guide-c2ada.pdf) (PDF) 56 | - [Ada Distilled](http://www.adapower.com/pdfs/AdaDistilled07-27-2003.pdf) (PDF) 57 | - [Ada for the C++ or Java Developer](https://www.adacore.com/uploads/books/pdf/Ada_for_the_C_or_Java_Developer-cc.pdf) - Quentin Ochem (PDF) 58 | - [Ada Programming](https://en.wikibooks.org/wiki/Ada_Programming) - Wikibooks 59 | - [Ada Reference Manual - ISO/IEC 8652:2012(E) Language and Standard Libraries](http://www.ada-auth.org/standards/12rm/RM-Final.pdf) (PDF) 60 | - [Introduction To Ada](https://learn.adacore.com/courses/intro-to-ada/index.html) 61 | - [Introduction To SPARK](https://learn.adacore.com/courses/SPARK_for_the_MISRA_C_Developer/index.html) 62 | - [The Big Online Book of Linux Ada Programming](http://www.pegasoft.ca/resources/boblap/book.html) 63 | 64 | ### Workflow 65 | 66 | - [Declare Peace on Virtual Machines. A guide to simplifying vm-based development on a Mac](https://leanpub.com/declarepeaceonvms/read) 67 | 68 | ### xBase (dBase / Clipper / Harbour) 69 | 70 | - [Application Development with Harbour](https://en.wikibooks.org/wiki/Application_Development_with_Harbour) - Wikibooks 71 | - [CA-Clipper 5.2 Norton Guide](https://web.archive.org/web/20190516192814/http://www.ousob.com/ng/clguide/) 72 | - [Clipper Tutorial: a Guide to Open Source Clipper(s)]() - Wikibooks 73 | -------------------------------------------------------------------------------- /static/badge-flat.svg: -------------------------------------------------------------------------------- 1 | 3 | Track Awesome List 4 | 5 | 6 | 7 | 8 | 9 | 10 | Track Awesome List 11 | 12 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ifneq (,$(wildcard ./.env)) 2 | include .env 3 | export 4 | endif 5 | 6 | .Phony: start 7 | start: 8 | deno run -A tal.ts "ripienaar/free-for-dev" 9 | 10 | .Phony: startall 11 | startall: 12 | deno run -A tal.ts 13 | 14 | .Phony: build 15 | build: 16 | deno run -A tal.ts --html --no-serve 17 | 18 | .Phony: prod-start 19 | prod-start: 20 | FORCE=1 PROD=1 deno run -A tal.ts ${args} 21 | .Phony: prod-builddemo 22 | prod-builddemo: 23 | FORCE=1 PROD=1 deno run -A tal.ts --no-fetch --html ripienaar/free-for-dev ${args} 24 | .Phony: prod-buildindex 25 | prod-buildindex: 26 | PROD=1 deno run -A tal.ts --no-fetch --html ripienaar/free-for-dev 27 | 28 | .Phony: prod-buildsource 29 | prod-buildsource: 30 | FORCE=1 PROD=1 deno run -A tal.ts --no-fetch --html ${source} 31 | 32 | .Phony: prod-build 33 | prod-build: 34 | PROD=1 deno run -A tal.ts --html --no-serve ${args} && make prod-buildsearch 35 | 36 | .Phony: prod-run 37 | prod-run: 38 | FORCE=1 PROD=1 deno run -A tal.ts --html ${args} 39 | 40 | 41 | .Phony: startsource 42 | startsource: 43 | deno run -A tal.ts --html ${source} 44 | 45 | .Phony: prod-startsource 46 | prod-startsource: 47 | PROD=1 deno run -A tal.ts ${source} 48 | 49 | .Phony: all 50 | all: 51 | FORCE=1 deno run -A tal.ts --html "ripienaar/free-for-dev" 52 | 53 | .Phony: allall 54 | allall: 55 | FORCE=1 deno run -A tal.ts --html 56 | .Phony: startallforce 57 | startallforce: 58 | deno run -A tal.ts --force 59 | .Phony: fetch 60 | fetch: 61 | deno run -A tal.ts --no-markdown --no-serve "ripienaar/free-for-dev" --force 62 | .Phony: fetchall 63 | fetchall: 64 | deno run -A tal.ts --no-markdown --no-serve 65 | .Phony: fetchsource 66 | fetchsource: 67 | deno run -A tal.ts --no-markdown --no-serve ${source} 68 | 69 | .Phony: buildmarkdown 70 | buildmarkdown: 71 | FORCE=1 deno run -A tal.ts --no-fetch --no-serve "ripienaar/free-for-dev" 72 | .Phony: buildsource 73 | buildsource: 74 | FORCE=1 deno run -A tal.ts --no-serve --no-fetch ${source} 75 | .Phony: buildmarkdownall 76 | buildmarkdownall: 77 | FORCE=1 deno run -A tal.ts --no-fetch --no-serve 78 | 79 | .Phony: serve 80 | serve: 81 | deno run -A --watch=tal.ts,templates/ tal.ts --no-fetch --no-markdown 82 | 83 | .Phony: run 84 | run: 85 | LIMIT=3 FORCE=1 deno run -A tal.ts --no-fetch --html 86 | 87 | .Phony: siteall 88 | siteall: 89 | FORCE=1 deno run -A tal.ts --no-fetch --html 90 | .Phony: initdb 91 | initdb: 92 | [[ ! -d /db/meta.json ]] && mkdir -p ./db && cat db-meta-init.json > ./db/meta.json && deno run -A init-db.ts 93 | 94 | .Phony: prod-initdb 95 | prod-initdb: 96 | [[ ! -d /prod-db/meta.json ]] && mkdir -p ./prod-db && cat db-meta-init.json > ./prod-db/meta.json && PROD=1 deno run -A init-db.ts 97 | 98 | .Phony: clean 99 | clean: 100 | rm -rf ./db rm -rf ./public && rm -rf ./dist && make initdb 101 | 102 | 103 | .Phony: cleanall 104 | cleanall: 105 | rm -rf ./db rm -rf ./public && rm -rf ./dist && rm -rf ./prod-db && rm -rf ./prod-dist && rm -rf ./prod-public && make initdb && make prod-initdb 106 | .Phony: push 107 | push: 108 | cd -- ./dist/repo && git add . && git commit -m "update" && git push 109 | 110 | .Phony: testbooks 111 | testbooks: 112 | deno test -A parsers/markdownlist_test.ts --filter="#2" 113 | .Phony: buildsite 114 | buildsite: 115 | FORCE=1 deno run -A tal.ts --no-fetch --html "ripienaar/free-for-dev" 116 | .Phony: buildsitesource 117 | buildsitesource: 118 | FORCE=1 deno run -A tal.ts --no-fetch --html ${source} 119 | .Phony: buildsiteall 120 | buildsiteall: 121 | FORCE=1 deno run -A tal.ts --no-fetch --html 122 | 123 | .Phony: prod-buildsiteall 124 | prod-buildsiteall: 125 | PROD=1 deno run -A tal.ts --no-fetch --html --no-serve 126 | .Phony: buildhtmlall 127 | buildhtmlall: 128 | deno run -A tal.ts --no-fetch --no-markdown --html --no-serve 129 | 130 | .Phony: servepublic 131 | servepublic: 132 | deno run -A https://deno.land/std@0.159.0/http/file_server.ts ./public -p 8000 133 | 134 | 135 | .Phony: install 136 | install: 137 | ./scripts/install-morsels.sh 138 | 139 | .Phony: servebook 140 | servebook: 141 | ./bin/mdbook serve --port 8000 142 | 143 | .Phony: buildbook 144 | buildbook: 145 | ./bin/mdbook build 146 | .Phony: publish 147 | publish: 148 | wrangler pages publish db/public --project-name trackawesomelist 149 | 150 | .Phony: prod-publish 151 | prod-publish: 152 | wrangler pages publish prod-db/public --project-name trackawesomelist 153 | 154 | .Phony: prod-upload 155 | prod-upload: 156 | make prod-zipdb && aws s3 cp ./prod-db.zip s3://trackawesomelist/prod-db.zip --endpoint-url $(AWS_ENDPOINT) 157 | 158 | .Phony: prod-load 159 | prod-load: 160 | aws s3 cp s3://trackawesomelist/prod-db.zip ./prod-db.zip --endpoint-url $(AWS_ENDPOINT) && make prod-unzipdb 161 | 162 | .Phony: prod-zipdb 163 | prod-zipdb: 164 | zip -r -q -FS prod-db.zip ./prod-db -x "*/.*" 165 | 166 | .Phony: prod-unzipdb 167 | prod-unzipdb: 168 | unzip -q -o prod-db.zip 169 | 170 | .Phony: prod-dbclean 171 | prod-dbclean: 172 | rm -rf ./prod-db/public && rm -rf ./prod-db/repos && rm ./prod-db/index.json && rm ./prod-db/meta.json && make prod-initdb 173 | 174 | .Phony: buildsearch 175 | buildsearch: 176 | ./bin/morsels ./db/public ./temp-morsels -c morsels_config.json && deno run -A ./build-search.ts 177 | .Phony: prod-buildsearch 178 | prod-buildsearch: 179 | ./bin/morsels ./prod-db/public ./temp-morsels -c morsels_config.json && PROD=1 deno run -A ./build-search.ts 180 | -------------------------------------------------------------------------------- /static/badge.svg: -------------------------------------------------------------------------------- 1 | 3 | Track Awesome Litt 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | Track Awesome List 20 | 21 | -------------------------------------------------------------------------------- /init-items.ts: -------------------------------------------------------------------------------- 1 | import { 2 | DBIndex, 3 | DBMeta, 4 | ExpiredValue, 5 | FileInfo, 6 | Item, 7 | RepoMetaOverride, 8 | RunOptions, 9 | Source, 10 | } from "./interface.ts"; 11 | import renderMarkdown from "./render-markdown.ts"; 12 | import Github from "./adapters/github.ts"; 13 | import { 14 | exists, 15 | getCachePath, 16 | getDayNumber, 17 | getWeekNumber, 18 | readTextFile, 19 | sha1, 20 | } from "./util.ts"; 21 | import log from "./log.ts"; 22 | import { fs, path } from "./deps.ts"; 23 | import parser from "./parser/mod.ts"; 24 | import getGitBlame from "./get-git-blame.ts"; 25 | import { updateFile, updateItems } from "./db.ts"; 26 | export default async function initItems( 27 | source: Source, 28 | options: RunOptions, 29 | dbMeta: DBMeta, 30 | dbIndex: DBIndex, 31 | dbCachedStars: Record, 32 | ) { 33 | // first get repo meta info from api 34 | const api = new Github(source); 35 | const metaOverrides: RepoMetaOverride = {}; 36 | if (source.default_branch) { 37 | metaOverrides.default_branch = source.default_branch; 38 | } 39 | const meta = await api.getRepoMeta(metaOverrides); 40 | const sources = dbMeta.sources; 41 | //check repo folder is empty 42 | const repoPath = path.join(getCachePath(false), "repos", source.identifier); 43 | 44 | const isExist = await exists(repoPath); 45 | log.debug(`repo ${repoPath} exist cache, try to pull updates`); 46 | 47 | // then git clone the entire repo, and parse the files 48 | if (isExist) { 49 | // try to update 50 | if (options.fetchRepoUpdates) { 51 | const args: string[] = [ 52 | "--work-tree", 53 | repoPath, 54 | "--git-dir", 55 | path.join(repoPath, ".git"), 56 | ]; 57 | 58 | const p = Deno.run({ 59 | cmd: ["git"].concat(args).concat(["pull"]), 60 | }); 61 | await p.status(); 62 | } 63 | } else { 64 | // ensure parent folder exists 65 | await fs.ensureDir(path.dirname(repoPath)); 66 | log.info(`cloning ${api.getCloneUrl()} to ${repoPath}`); 67 | // try to clone 68 | const p = Deno.run({ 69 | cmd: [ 70 | "git", 71 | "clone", 72 | "-b", 73 | meta.default_branch, 74 | api.getCloneUrl(), 75 | repoPath, 76 | ], 77 | }); 78 | await p.status(); 79 | } 80 | const now = new Date(); 81 | sources[source.identifier] = sources[source.identifier] || { 82 | created_at: now.toISOString(), 83 | updated_at: now.toISOString(), 84 | meta, 85 | files: {}, 86 | }; 87 | 88 | for (const file of Object.keys(source.files)) { 89 | const fileConfig = source.files[file]; 90 | const blameInfoMap = await getGitBlame(file, { 91 | workTree: repoPath, 92 | gitDir: path.join(repoPath, ".git"), 93 | }); 94 | const items: Record = {}; 95 | const cachedFilePath = path.join(repoPath, file); 96 | const content = await readTextFile(cachedFilePath); 97 | const fileInfo: FileInfo = { 98 | sourceConfig: source, 99 | sourceMeta: sources[source.identifier], 100 | filepath: file, 101 | }; 102 | const docItems = await parser(content, fileInfo, dbCachedStars); 103 | // console.log("docItems", docItems); 104 | let latestUpdatedAt = new Date(0); 105 | for (const docItem of docItems) { 106 | const now = new Date(); 107 | const commitInfo = blameInfoMap.get(docItem.line); 108 | if (commitInfo) { 109 | const itemSha1 = await sha1(docItem.rawMarkdown); 110 | const commitTime = commitInfo.committerTime; 111 | const commitDate = new Date(Number(commitTime) * 1000); 112 | const updatedAt = commitDate.toISOString(); 113 | items[itemSha1] = { 114 | category: docItem.category, 115 | category_html: renderMarkdown(docItem.category), 116 | updated_at: updatedAt, 117 | source_identifier: source.identifier, 118 | file, 119 | markdown: docItem.formatedMarkdown, 120 | html: renderMarkdown(docItem.formatedMarkdown), 121 | sha1: itemSha1, 122 | checked_at: now.toISOString(), 123 | updated_day: getDayNumber(new Date(updatedAt)), 124 | updated_week: getWeekNumber(new Date(updatedAt)), 125 | }; 126 | if (commitDate.getTime() > latestUpdatedAt.getTime()) { 127 | latestUpdatedAt = commitDate; 128 | } 129 | } else { 130 | throw new Error( 131 | `no commit info for ${source.identifier} ${file} ${docItem.line}`, 132 | ); 133 | } 134 | } 135 | const contentSha1 = await sha1(content); 136 | // try to get items updated time 137 | // get created time and updated time from blameinfo 138 | let createdAt = now; 139 | for (const blame of blameInfoMap.values()) { 140 | const commitTime = blame.committerTime; 141 | const commitDate = new Date(Number(commitTime) * 1000); 142 | if (commitDate < createdAt) { 143 | createdAt = commitDate; 144 | } 145 | } 146 | 147 | sources[source.identifier].files[file] = { 148 | sha1: contentSha1, 149 | updated_at: latestUpdatedAt.toISOString(), 150 | meta_created_at: now.toISOString(), 151 | created_at: createdAt.toISOString(), 152 | checked_at: now.toISOString(), 153 | }; 154 | //write to file 155 | // await writeJSONFile(formatedPath, itemsJson); 156 | // write to db 157 | 158 | await updateFile(fileInfo, content); 159 | await updateItems(fileInfo, items, dbIndex); 160 | 161 | log.info( 162 | `init ${source.identifier}/${file} success, total ${ 163 | Object.keys(items).length 164 | } items`, 165 | ); 166 | } 167 | dbMeta.sources = sources; 168 | } 169 | -------------------------------------------------------------------------------- /parser/markdown/table.ts: -------------------------------------------------------------------------------- 1 | import { 2 | DocItem, 3 | ExpiredValue, 4 | FileConfigInfo, 5 | FileInfo, 6 | ParseOptions, 7 | } from "../../interface.ts"; 8 | import { 9 | Content, 10 | fromMarkdown, 11 | remarkInlineLinks, 12 | TableRow, 13 | toMarkdown, 14 | visit, 15 | } from "../../deps.ts"; 16 | import { childrenToRoot, promiseLimit, writeTextFile } from "../../util.ts"; 17 | import _log from "../../log.ts"; 18 | import formatMarkdownItem from "../../format-markdown-item.ts"; 19 | import { gfm, gfmFromMarkdown, gfmToMarkdown } from "../../deps.ts"; 20 | 21 | import { uglyFormatItemIdentifier } from "./util.ts"; 22 | export default async function ( 23 | content: string, 24 | fileInfo: FileInfo, 25 | dbCachedStars: Record, 26 | ): Promise { 27 | const sourceConfig = fileInfo.sourceConfig; 28 | const fileConfig = sourceConfig.files[fileInfo.filepath]; 29 | const options = fileConfig.options; 30 | const parseOptions = fileConfig.options; 31 | const isParseCategory = parseOptions.is_parse_category === undefined 32 | ? true 33 | : parseOptions.is_parse_category; 34 | 35 | const items: DocItem[] = []; 36 | const tree = fromMarkdown(content, "utf8", { 37 | extensions: [gfm()], 38 | mdastExtensions: [gfmFromMarkdown()], 39 | }); 40 | // @ts-ignore: remarkInlineLinks is not typed 41 | remarkInlineLinks()(tree); 42 | 43 | let index = 0; 44 | let currentLevel = 0; 45 | let currentSubCategory = ""; 46 | let currentCategory = ""; 47 | let lowestHeadingLevel = 3; 48 | // first check valided sections 49 | const validSections: Content[] = []; 50 | for (const rootNode of tree.children) { 51 | if (rootNode.type === "heading") { 52 | currentLevel = rootNode.depth; 53 | if (currentLevel > lowestHeadingLevel) { 54 | lowestHeadingLevel = currentLevel; 55 | } 56 | validSections.push(rootNode); 57 | } else if (rootNode.type === "table") { 58 | validSections.push(rootNode); 59 | } 60 | } 61 | const min_heading_level = options.min_heading_level || lowestHeadingLevel; 62 | const max_heading_level = options.max_heading_level || 2; 63 | const funcs: (() => Promise)[] = []; 64 | // console.log("validSections", validSections); 65 | for (const rootNode of validSections) { 66 | // console.log("rootNode", rootNode); 67 | if (rootNode.type === "heading") { 68 | currentLevel = rootNode.depth; 69 | if ( 70 | currentLevel < min_heading_level && currentLevel >= max_heading_level 71 | ) { 72 | currentCategory = toMarkdown(childrenToRoot(rootNode.children)); 73 | } else if (currentLevel === min_heading_level) { 74 | currentSubCategory = toMarkdown(childrenToRoot(rootNode.children)); 75 | } 76 | } else if (rootNode.type === "table") { 77 | // console.log("rootNode", rootNode); 78 | // await writeTextFile("temp.json", JSON.stringify(rootNode)); 79 | let rowIndex = 0; 80 | for (const item of rootNode.children) { 81 | // console.log("item", item); 82 | if (item.type === "tableRow") { 83 | if (rowIndex === 0) { 84 | // first row is header 85 | rowIndex++; 86 | continue; 87 | } 88 | let category = ""; 89 | if (currentCategory) { 90 | category = currentCategory.trim().replace(/\n/g, " "); 91 | } 92 | if (currentSubCategory) { 93 | if (category) { 94 | category += " / "; 95 | } 96 | category += currentSubCategory.trim().replace(/\n/g, " "); 97 | } 98 | const itemIdentifier = uglyFormatItemIdentifier(fileInfo, item); 99 | funcs.push(() => { 100 | return formatMarkdownItem(item as TableRow, fileInfo, dbCachedStars) 101 | .then( 102 | (formatedItem) => { 103 | let markdown = "- "; 104 | // transform table row to item 105 | (formatedItem as TableRow).children.forEach( 106 | (child, cellIndex) => { 107 | const tableHeaderCell = 108 | rootNode.children[0].children[cellIndex]; 109 | let tableHeaderCellMarkdown = ""; 110 | try { 111 | tableHeaderCellMarkdown = toMarkdown( 112 | tableHeaderCell, 113 | { 114 | extensions: [gfmToMarkdown()], 115 | }, 116 | ).trim(); 117 | } catch (e) { 118 | console.log("e", e); 119 | console.log("tableHeaderCell", tableHeaderCell); 120 | } 121 | const rowCellMarkdown = toMarkdown( 122 | child, 123 | { 124 | extensions: [gfmToMarkdown()], 125 | }, 126 | ).trim(); 127 | if (cellIndex > 0) { 128 | markdown += 129 | ` ${tableHeaderCellMarkdown}: ${rowCellMarkdown}\n\n`; 130 | } else { 131 | markdown += 132 | `${tableHeaderCellMarkdown}: ${rowCellMarkdown}\n\n`; 133 | } 134 | }, 135 | ); 136 | 137 | return { 138 | formatedMarkdown: markdown, 139 | rawMarkdown: itemIdentifier, 140 | category: isParseCategory ? category : "", 141 | line: item.position!.end.line, 142 | }; 143 | }, 144 | ); 145 | }); 146 | rowIndex++; 147 | } 148 | } 149 | } 150 | } 151 | 152 | return promiseLimit(funcs); 153 | } 154 | -------------------------------------------------------------------------------- /parser/markdown/list.ts: -------------------------------------------------------------------------------- 1 | import { 2 | DocItem, 3 | ExpiredValue, 4 | FileInfo, 5 | ParseOptions, 6 | } from "../../interface.ts"; 7 | import { 8 | Content, 9 | fromMarkdown, 10 | gfm, 11 | gfmFromMarkdown, 12 | gfmToMarkdown, 13 | Link, 14 | remarkInlineLinks, 15 | toMarkdown, 16 | visit, 17 | } from "../../deps.ts"; 18 | import { childrenToRoot, getRepoHTMLURL, promiseLimit } from "../../util.ts"; 19 | import log from "../../log.ts"; 20 | import formatMarkdownItem from "../../format-markdown-item.ts"; 21 | import formatCategory from "../../format-category.ts"; 22 | import { uglyFormatItemIdentifier } from "./util.ts"; 23 | export default function ( 24 | content: string, 25 | fileInfo: FileInfo, 26 | dbCachedStars: Record, 27 | ): Promise { 28 | const sourceConfig = fileInfo.sourceConfig; 29 | const fileConfig = sourceConfig.files[fileInfo.filepath]; 30 | const parseOptions = fileConfig.options; 31 | const isParseCategory = parseOptions.is_parse_category === undefined 32 | ? true 33 | : parseOptions.is_parse_category; 34 | const items: DocItem[] = []; 35 | const tree = fromMarkdown(content, "utf8", { 36 | extensions: [gfm()], 37 | mdastExtensions: [gfmFromMarkdown()], 38 | }); 39 | // transform inline links to link 40 | // @ts-ignore: remarkInlineLinks is not typed 41 | remarkInlineLinks()(tree); 42 | let index = 0; 43 | let currentLevel = 0; 44 | let currentSubCategory = ""; 45 | let currentCategory = ""; 46 | let lowestHeadingLevel = 3; 47 | // first check valided sections 48 | const validSections: Content[] = []; 49 | let isReachedValidSection = false; 50 | const max_heading_level = parseOptions.max_heading_level || 2; 51 | for (const rootNode of tree.children) { 52 | // start with the first valid ma x_heading_level 53 | 54 | if (!isReachedValidSection) { 55 | // check is valid now 56 | if ( 57 | rootNode.type === "heading" && 58 | rootNode.depth === max_heading_level 59 | ) { 60 | isReachedValidSection = true; 61 | } else { 62 | continue; 63 | } 64 | } 65 | 66 | if (rootNode.type === "heading") { 67 | currentLevel = rootNode.depth; 68 | 69 | if ( 70 | currentLevel > lowestHeadingLevel 71 | ) { 72 | lowestHeadingLevel = currentLevel; 73 | } 74 | validSections.push(rootNode); 75 | } else if (rootNode.type === "list") { 76 | // check if all links is author link 77 | // if so, it's a table of content 78 | // ignore it 79 | let internalLinkCount = 0; 80 | let externalLinkCount = 0; 81 | visit(childrenToRoot(rootNode.children), "link", (node) => { 82 | if (!node.url.startsWith("#")) { 83 | internalLinkCount++; 84 | } else { 85 | externalLinkCount++; 86 | } 87 | }); 88 | // for fix some repo's toc include a little external links 89 | // we still treat it as toc if internal link count is more than 80% 90 | // for example: https://github.com/EbookFoundation/free-programming-books/blob/main/books/free-programming-books-langs.md#bootstrap 91 | if ( 92 | externalLinkCount === 0 || 93 | (internalLinkCount > 10 && externalLinkCount < 2) 94 | ) { 95 | validSections.push(rootNode); 96 | } 97 | } 98 | } 99 | const min_heading_level = parseOptions.min_heading_level || 100 | lowestHeadingLevel; 101 | const funcs: (() => Promise)[] = []; 102 | for (const rootNode of validSections) { 103 | if (rootNode.type === "heading") { 104 | currentLevel = rootNode.depth; 105 | 106 | if ( 107 | currentLevel < min_heading_level && currentLevel >= max_heading_level 108 | ) { 109 | currentCategory = formatCategory( 110 | childrenToRoot(rootNode.children), 111 | ); 112 | } else if (currentLevel === min_heading_level) { 113 | currentSubCategory = formatCategory( 114 | childrenToRoot(rootNode.children), 115 | ); 116 | } 117 | } else if (rootNode.type === "list") { 118 | for (const item of rootNode.children) { 119 | if (item.type === "listItem") { 120 | let category = ""; 121 | if (currentCategory) { 122 | category = currentCategory.trim().replace(/\n/g, " "); 123 | } 124 | if (currentSubCategory) { 125 | if (category) { 126 | category += " / "; 127 | } 128 | category += currentSubCategory.trim().replace(/\n/g, " "); 129 | } 130 | const itemIdentifier = uglyFormatItemIdentifier(fileInfo, item); 131 | // console.log("itemIdentifier", itemIdentifier); 132 | if (uglyIsValidCategory(fileInfo, category)) { 133 | funcs.push(() => { 134 | return formatMarkdownItem(item, fileInfo, dbCachedStars).then( 135 | (formatedItem) => { 136 | return { 137 | formatedMarkdown: toMarkdown(formatedItem, { 138 | extensions: [gfmToMarkdown()], 139 | }).trim(), 140 | rawMarkdown: itemIdentifier, 141 | category: isParseCategory ? category : "", 142 | line: item.position!.end.line, 143 | }; 144 | }, 145 | ); 146 | }); 147 | } 148 | } 149 | } 150 | } 151 | } 152 | 153 | return promiseLimit(funcs); 154 | } 155 | 156 | function uglyIsValidCategory( 157 | fileInfo: FileInfo, 158 | category: string, 159 | ): boolean { 160 | const sourceConfig = fileInfo.sourceConfig; 161 | const fileConfig = sourceConfig.files[fileInfo.filepath]; 162 | const sourceIdentifier = sourceConfig.identifier; 163 | if (sourceIdentifier === "KotlinBy/awesome-kotlin") { 164 | if (category.startsWith("Github Trending / ")) { 165 | return false; 166 | } 167 | } 168 | return true; 169 | } 170 | -------------------------------------------------------------------------------- /interface.ts: -------------------------------------------------------------------------------- 1 | import { Content } from "./deps.ts"; 2 | export type ExpiredValue = [number, string]; 3 | export interface WeekOfYear { 4 | year: number; 5 | week: number; 6 | number: number; 7 | path: string; 8 | date: Date; 9 | id: string; 10 | name: string; 11 | } 12 | export interface CustomRequestOptions { 13 | expires?: number; 14 | } 15 | export interface BuiltMarkdownInfo { 16 | commitMessage: string; 17 | } 18 | export interface RepoMetaOverride { 19 | default_branch?: string; 20 | } 21 | export interface ParseOptions { 22 | min_heading_level?: number; 23 | max_heading_level?: number; 24 | heading_level?: number; // only need for heading type 25 | type: "table" | "list" | "heading"; 26 | is_parse_category?: boolean; 27 | } 28 | export interface DayInfo { 29 | year: number; 30 | month: number; 31 | day: number; 32 | number: number; 33 | path: string; 34 | name: string; 35 | id: string; 36 | date: Date; 37 | } 38 | export type LevelName = "debug" | "info" | "warn" | "error" | "fatal"; 39 | export interface File { 40 | source_identifier: string; 41 | file: string; 42 | } 43 | export enum Level { 44 | Debug = 0, 45 | Info = 1, 46 | Warn = 2, 47 | Error = 3, 48 | Fatal = 4, 49 | } 50 | export interface ApiInfo { 51 | url: string; 52 | headers: Headers; 53 | } 54 | export interface RawSource { 55 | category?: string; 56 | default_branch?: string; 57 | url?: string; 58 | files?: Record | string; 59 | } 60 | export interface ParsedItemsFilePath { 61 | originalFilepath: string; 62 | sourceIdentifier: string; 63 | } 64 | export interface Site { 65 | title: string; 66 | description: string; 67 | url: string; 68 | } 69 | export interface RawConfig { 70 | sources: Record; 71 | file_min_updated_hours: number; 72 | site: Site; 73 | } 74 | export interface RawSourceFile { 75 | index?: boolean; 76 | name?: string; 77 | options?: ParseOptions; 78 | } 79 | 80 | export interface FileConfigInfo { 81 | sourceConfig: Source; 82 | filepath: string; 83 | } 84 | export interface FileInfo extends FileConfigInfo { 85 | sourceMeta: DbMetaSource; 86 | filepath: string; 87 | } 88 | export interface FormatMarkdownItemOptions { 89 | repoUrl: string; 90 | defaultBranch: string; 91 | filepath: string; 92 | } 93 | export interface RawSourceFileWithType extends RawSourceFile { 94 | options: ParseOptions; 95 | } 96 | export interface Nav { 97 | name: string; 98 | active?: boolean; 99 | markdown_url?: string; 100 | url?: string; 101 | } 102 | export interface FeedConfig { 103 | nav1: Nav[]; 104 | nav2?: Nav[]; 105 | } 106 | export interface FileConfig extends RawSourceFile { 107 | filepath: string; 108 | pathname: string; 109 | name: string; 110 | options: ParseOptions; 111 | } 112 | export interface Source { 113 | identifier: string; 114 | url: string; 115 | default_branch?: string; 116 | category: string; 117 | files: Record; 118 | } 119 | export interface ListItem { 120 | name: string; 121 | updated: string; 122 | url: string; 123 | meta: RepoMeta; 124 | star: string; 125 | source_url: string; 126 | } 127 | export interface List { 128 | category: string; 129 | items: ListItem[]; 130 | } 131 | export interface Config extends RawConfig { 132 | sources: Record; 133 | } 134 | export interface RunOptions extends CliOptions { 135 | config: Config; 136 | sourceIdentifiers: string[]; 137 | } 138 | export interface CliOptions { 139 | debug?: boolean; 140 | force?: boolean; 141 | forceFetch?: boolean; 142 | push?: boolean; 143 | autoInit?: boolean; 144 | fetchRepoUpdates: boolean; 145 | markdown: boolean; 146 | fetch: boolean; 147 | cleanMarkdown?: boolean; 148 | cleanHtml?: boolean; 149 | dayMarkdown: boolean; 150 | rebuild?: boolean; 151 | html?: boolean; 152 | serve: boolean; 153 | port: number; 154 | limit?: number; 155 | } 156 | export interface Item { 157 | updated_at: string; 158 | updated_day: number; 159 | updated_week: number; 160 | category: string; 161 | category_html: string; 162 | markdown: string; 163 | html: string; 164 | sha1: string; 165 | source_identifier: string; 166 | file: string; 167 | checked_at: string; 168 | } 169 | export interface ItemDetail extends Item { 170 | updated_day: number; 171 | updated_week: number; 172 | updated_day_info: DayInfo; 173 | updated_week_info: WeekOfYear; 174 | } 175 | export interface DocItem { 176 | rawMarkdown: string; 177 | formatedMarkdown: string; 178 | category: string; 179 | line: number; 180 | } 181 | 182 | export interface Pagination { 183 | title: string; 184 | pathname: string; 185 | } 186 | export interface PaginationInfo { 187 | prev: Pagination | undefined; 188 | next: Pagination | undefined; 189 | } 190 | export interface BuildOptions { 191 | paginationText: string; 192 | paginationHtml: string; 193 | dbMeta: DBMeta; 194 | dbIndex: DBIndex; 195 | } 196 | export interface RepoMeta { 197 | name: string; 198 | description: string; 199 | url: string; 200 | default_branch: string; 201 | language: string | undefined; 202 | stargazers_count: number; 203 | subscribers_count: number; 204 | forks_count: number; 205 | tags: string[]; 206 | created_at: string; 207 | updated_at: string; 208 | checked_at: string; 209 | } 210 | 211 | export interface ItemsJson { 212 | items: Record; 213 | } 214 | 215 | export interface ParsedFilename { 216 | name: string; 217 | ext: string; 218 | type: string; 219 | } 220 | export interface FileMeta { 221 | sha1: string; 222 | checked_at: string; 223 | created_at: string; 224 | updated_at: string; 225 | meta_created_at: string; 226 | } 227 | 228 | export interface FileMetaWithSource extends FileMeta { 229 | sourceIdentifier: string; 230 | filepath: string; 231 | } 232 | 233 | export interface DbMetaSource { 234 | files: Record; 235 | meta: RepoMeta; 236 | created_at: string; 237 | updated_at: string; 238 | } 239 | export interface DBMeta { 240 | sources: Record; 241 | checked_at: string; 242 | } 243 | export interface IndexItem { 244 | t: number; 245 | d: number; 246 | w: number; 247 | } 248 | export type DBIndex = Record; 249 | export interface Author { 250 | url: string; 251 | name: string; 252 | avatar?: string; 253 | } 254 | 255 | export interface FeedItem { 256 | id: string; 257 | image?: string; 258 | url: string; 259 | _slug: string; 260 | _filepath: string; 261 | summary: string; 262 | date_published: string; 263 | date_modified: string; 264 | tags?: string[]; 265 | authors?: Author[]; 266 | title: string; 267 | _short_title?: string; 268 | author?: Author; 269 | content_text?: string; 270 | content_html: string; 271 | } 272 | 273 | export interface BaseFeed { 274 | version: string; 275 | icon: string; 276 | favicon: string; 277 | language: string; 278 | } 279 | export interface FeedInfo extends BaseFeed { 280 | title: string; 281 | _site_title: string; 282 | _seo_title: string; 283 | description: string; 284 | home_page_url: string; 285 | feed_url: string; 286 | } 287 | export interface Feed extends FeedInfo { 288 | items: FeedItem[]; 289 | } 290 | -------------------------------------------------------------------------------- /parser/markdown/heading.ts: -------------------------------------------------------------------------------- 1 | import { 2 | DocItem, 3 | ExpiredValue, 4 | FileInfo, 5 | ParseOptions, 6 | } from "../../interface.ts"; 7 | import { 8 | Content, 9 | fromMarkdown, 10 | gfm, 11 | gfmFromMarkdown, 12 | gfmToMarkdown, 13 | Link, 14 | remarkInlineLinks, 15 | Root, 16 | toMarkdown, 17 | visit, 18 | } from "../../deps.ts"; 19 | import { childrenToRoot, promiseLimit } from "../../util.ts"; 20 | import log from "../../log.ts"; 21 | import formatMarkdownItem from "../../format-markdown-item.ts"; 22 | import { uglyFormatItemIdentifier } from "./util.ts"; 23 | export default function ( 24 | content: string, 25 | fileInfo: FileInfo, 26 | dbCachedStars: Record, 27 | ): Promise { 28 | const sourceConfig = fileInfo.sourceConfig; 29 | const fileConfig = sourceConfig.files[fileInfo.filepath]; 30 | const options = fileConfig.options; 31 | const isParseCategory = options.is_parse_category === undefined 32 | ? true 33 | : options.is_parse_category; 34 | const items: DocItem[] = []; 35 | const tree = fromMarkdown(content, "utf8", { 36 | extensions: [gfm()], 37 | mdastExtensions: [gfmFromMarkdown()], 38 | }); 39 | // @ts-ignore: remarkInlineLinks is not typed 40 | remarkInlineLinks()(tree); 41 | 42 | let index = 0; 43 | let currentLevel = 0; 44 | let currentSubCategory = ""; 45 | let currentCategory = ""; 46 | let lowestHeadingLevel = 3; 47 | // first check valided sections 48 | let isReachedValidSection = false; 49 | const validSections: Content[] = []; 50 | for (const rootNode of tree.children) { 51 | if (!isReachedValidSection) { 52 | // check is valid now 53 | if ( 54 | rootNode.type === "heading" && 55 | rootNode.depth === options.max_heading_level 56 | ) { 57 | isReachedValidSection = true; 58 | } else { 59 | continue; 60 | } 61 | } 62 | if (rootNode.type === "heading") { 63 | currentLevel = rootNode.depth; 64 | if ( 65 | currentLevel > lowestHeadingLevel 66 | ) { 67 | lowestHeadingLevel = currentLevel; 68 | } 69 | validSections.push(rootNode); 70 | } else if (rootNode.type === "list") { 71 | // check if all links is author link 72 | // if so, it's a table of content 73 | // ignore it 74 | let internalLinkCount = 0; 75 | let externalLinkCount = 0; 76 | visit(childrenToRoot(rootNode.children), "link", (node) => { 77 | if (!node.url.startsWith("#")) { 78 | internalLinkCount++; 79 | } else { 80 | externalLinkCount++; 81 | } 82 | }); 83 | // for fix some repo's toc include a little external links 84 | // we still treat it as toc if internal link count is more than 80% 85 | // for example: https://github.com/EbookFoundation/free-programming-books/blob/main/books/free-programming-books-langs.md#bootstrap 86 | if ( 87 | externalLinkCount === 0 || 88 | (internalLinkCount > 10 && externalLinkCount < 2) 89 | ) { 90 | validSections.push(rootNode); 91 | } 92 | } else if (rootNode.type !== "thematicBreak") { 93 | validSections.push(rootNode); 94 | } 95 | } 96 | const min_heading_level = options.min_heading_level || lowestHeadingLevel; 97 | const max_heading_level = options.max_heading_level || 2; 98 | const heading_level = options.heading_level || 3; 99 | const funcs: (() => Promise)[] = []; 100 | let tempItemSections: Content[] = []; 101 | for (const rootNode of validSections) { 102 | if (rootNode.type === "heading" && rootNode.depth <= heading_level) { 103 | currentLevel = rootNode.depth; 104 | if (currentLevel === heading_level) { 105 | // yes this is item start 106 | if (tempItemSections.length > 0) { 107 | const item = childrenToRoot(tempItemSections); 108 | let category = ""; 109 | if (currentCategory) { 110 | category = currentCategory.trim().replace(/\n/g, " "); 111 | } 112 | if (currentSubCategory) { 113 | if (category) { 114 | category += " / "; 115 | } 116 | category += currentSubCategory.trim().replace(/\n/g, " "); 117 | } 118 | const line = 119 | tempItemSections[tempItemSections.length - 1].position!.end 120 | .line; 121 | const itemIdentifier = uglyFormatItemIdentifier(fileInfo, item); 122 | const fn = () => { 123 | return formatMarkdownItem(item, fileInfo, dbCachedStars).then( 124 | (formatedItem) => { 125 | return { 126 | formatedMarkdown: toMarkdown(formatedItem, { 127 | extensions: [gfmToMarkdown()], 128 | }).trim(), 129 | rawMarkdown: itemIdentifier, 130 | category: isParseCategory ? category : "", 131 | line, 132 | }; 133 | }, 134 | ); 135 | }; 136 | funcs.push(fn); 137 | } 138 | 139 | tempItemSections = [rootNode]; 140 | } 141 | if ( 142 | currentLevel < min_heading_level && currentLevel >= max_heading_level 143 | ) { 144 | currentCategory = toMarkdown(childrenToRoot(rootNode.children), { 145 | extensions: [gfmToMarkdown()], 146 | }); 147 | } else if (currentLevel === min_heading_level) { 148 | currentSubCategory = toMarkdown(childrenToRoot(rootNode.children), { 149 | extensions: [gfmToMarkdown()], 150 | }); 151 | } 152 | } else { 153 | tempItemSections.push(rootNode); 154 | } 155 | } 156 | 157 | // add last item 158 | if (tempItemSections.length > 1) { 159 | const item = childrenToRoot(tempItemSections); 160 | let category = ""; 161 | // TODO category issue 162 | if (currentCategory) { 163 | category = currentCategory.trim().replace(/\n/g, " "); 164 | } 165 | if (currentSubCategory) { 166 | if (category) { 167 | category += " / "; 168 | } 169 | category += currentSubCategory.trim().replace(/\n/g, " "); 170 | } 171 | const line = tempItemSections[tempItemSections.length - 1].position!.end 172 | .line; 173 | const itemIdentifier = uglyFormatItemIdentifier(fileInfo, item); 174 | const fn = () => { 175 | return formatMarkdownItem(item, fileInfo, dbCachedStars).then( 176 | (formatedItem) => { 177 | return { 178 | formatedMarkdown: toMarkdown(formatedItem, { 179 | extensions: [gfmToMarkdown()], 180 | }).trim(), 181 | rawMarkdown: itemIdentifier, 182 | category: isParseCategory ? category : "", 183 | line: line, 184 | }; 185 | }, 186 | ); 187 | }; 188 | funcs.push(fn); 189 | } 190 | return promiseLimit(funcs); 191 | } 192 | 193 | function uglyRemoveAutoGeneratedMarkdown( 194 | fileInfo: FileInfo, 195 | item: Root, 196 | ): Root { 197 | const sourceConfig = fileInfo.sourceConfig; 198 | const fileConfig = sourceConfig.files[fileInfo.filepath]; 199 | const sourceIdentifier = sourceConfig.identifier; 200 | if (sourceIdentifier === "stefanbuck/awesome-browser-extensions-for-github") { 201 | // remove the last part 202 | const children = item.children; 203 | return { 204 | type: "root", 205 | children: children.slice(0, children.length - 1), 206 | }; 207 | } else { 208 | return item; 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /example/repo-meta.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 54346799, 3 | "node_id": "MDEwOlJlcG9zaXRvcnk1NDM0Njc5OQ==", 4 | "name": "public-apis", 5 | "full_name": "public-apis/public-apis", 6 | "private": false, 7 | "owner": { 8 | "login": "public-apis", 9 | "id": 51121562, 10 | "node_id": "MDEyOk9yZ2FuaXphdGlvbjUxMTIxNTYy", 11 | "avatar_url": "https://avatars.githubusercontent.com/u/51121562?v=4", 12 | "gravatar_id": "", 13 | "url": "https://api.github.com/users/public-apis", 14 | "html_url": "https://github.com/public-apis", 15 | "followers_url": "https://api.github.com/users/public-apis/followers", 16 | "following_url": "https://api.github.com/users/public-apis/following{/other_user}", 17 | "gists_url": "https://api.github.com/users/public-apis/gists{/gist_id}", 18 | "starred_url": "https://api.github.com/users/public-apis/starred{/owner}{/repo}", 19 | "subscriptions_url": "https://api.github.com/users/public-apis/subscriptions", 20 | "organizations_url": "https://api.github.com/users/public-apis/orgs", 21 | "repos_url": "https://api.github.com/users/public-apis/repos", 22 | "events_url": "https://api.github.com/users/public-apis/events{/privacy}", 23 | "received_events_url": "https://api.github.com/users/public-apis/received_events", 24 | "type": "Organization", 25 | "site_admin": false 26 | }, 27 | "html_url": "https://github.com/public-apis/public-apis", 28 | "description": "A collective list of free APIs", 29 | "fork": false, 30 | "url": "https://api.github.com/repos/public-apis/public-apis", 31 | "forks_url": "https://api.github.com/repos/public-apis/public-apis/forks", 32 | "keys_url": "https://api.github.com/repos/public-apis/public-apis/keys{/key_id}", 33 | "collaborators_url": "https://api.github.com/repos/public-apis/public-apis/collaborators{/collaborator}", 34 | "teams_url": "https://api.github.com/repos/public-apis/public-apis/teams", 35 | "hooks_url": "https://api.github.com/repos/public-apis/public-apis/hooks", 36 | "issue_events_url": "https://api.github.com/repos/public-apis/public-apis/issues/events{/number}", 37 | "events_url": "https://api.github.com/repos/public-apis/public-apis/events", 38 | "assignees_url": "https://api.github.com/repos/public-apis/public-apis/assignees{/user}", 39 | "branches_url": "https://api.github.com/repos/public-apis/public-apis/branches{/branch}", 40 | "tags_url": "https://api.github.com/repos/public-apis/public-apis/tags", 41 | "blobs_url": "https://api.github.com/repos/public-apis/public-apis/git/blobs{/sha}", 42 | "git_tags_url": "https://api.github.com/repos/public-apis/public-apis/git/tags{/sha}", 43 | "git_refs_url": "https://api.github.com/repos/public-apis/public-apis/git/refs{/sha}", 44 | "trees_url": "https://api.github.com/repos/public-apis/public-apis/git/trees{/sha}", 45 | "statuses_url": "https://api.github.com/repos/public-apis/public-apis/statuses/{sha}", 46 | "languages_url": "https://api.github.com/repos/public-apis/public-apis/languages", 47 | "stargazers_url": "https://api.github.com/repos/public-apis/public-apis/stargazers", 48 | "contributors_url": "https://api.github.com/repos/public-apis/public-apis/contributors", 49 | "subscribers_url": "https://api.github.com/repos/public-apis/public-apis/subscribers", 50 | "subscription_url": "https://api.github.com/repos/public-apis/public-apis/subscription", 51 | "commits_url": "https://api.github.com/repos/public-apis/public-apis/commits{/sha}", 52 | "git_commits_url": "https://api.github.com/repos/public-apis/public-apis/git/commits{/sha}", 53 | "comments_url": "https://api.github.com/repos/public-apis/public-apis/comments{/number}", 54 | "issue_comment_url": "https://api.github.com/repos/public-apis/public-apis/issues/comments{/number}", 55 | "contents_url": "https://api.github.com/repos/public-apis/public-apis/contents/{+path}", 56 | "compare_url": "https://api.github.com/repos/public-apis/public-apis/compare/{base}...{head}", 57 | "merges_url": "https://api.github.com/repos/public-apis/public-apis/merges", 58 | "archive_url": "https://api.github.com/repos/public-apis/public-apis/{archive_format}{/ref}", 59 | "downloads_url": "https://api.github.com/repos/public-apis/public-apis/downloads", 60 | "issues_url": "https://api.github.com/repos/public-apis/public-apis/issues{/number}", 61 | "pulls_url": "https://api.github.com/repos/public-apis/public-apis/pulls{/number}", 62 | "milestones_url": "https://api.github.com/repos/public-apis/public-apis/milestones{/number}", 63 | "notifications_url": "https://api.github.com/repos/public-apis/public-apis/notifications{?since,all,participating}", 64 | "labels_url": "https://api.github.com/repos/public-apis/public-apis/labels{/name}", 65 | "releases_url": "https://api.github.com/repos/public-apis/public-apis/releases{/id}", 66 | "deployments_url": "https://api.github.com/repos/public-apis/public-apis/deployments", 67 | "created_at": "2016-03-20T23:49:42Z", 68 | "updated_at": "2022-10-01T01:27:15Z", 69 | "pushed_at": "2022-09-26T16:04:55Z", 70 | "git_url": "git://github.com/public-apis/public-apis.git", 71 | "ssh_url": "git@github.com:public-apis/public-apis.git", 72 | "clone_url": "https://github.com/public-apis/public-apis.git", 73 | "svn_url": "https://github.com/public-apis/public-apis", 74 | "homepage": "http://public-apis.org", 75 | "size": 5031, 76 | "stargazers_count": 210463, 77 | "watchers_count": 210463, 78 | "language": "Python", 79 | "has_issues": true, 80 | "has_projects": false, 81 | "has_downloads": true, 82 | "has_wiki": false, 83 | "has_pages": false, 84 | "forks_count": 24096, 85 | "mirror_url": null, 86 | "archived": false, 87 | "disabled": false, 88 | "open_issues_count": 61, 89 | "license": { 90 | "key": "mit", 91 | "name": "MIT License", 92 | "spdx_id": "MIT", 93 | "url": "https://api.github.com/licenses/mit", 94 | "node_id": "MDc6TGljZW5zZTEz" 95 | }, 96 | "allow_forking": true, 97 | "is_template": false, 98 | "web_commit_signoff_required": false, 99 | "topics": [ 100 | "api", 101 | "apis", 102 | "dataset", 103 | "development", 104 | "free", 105 | "list", 106 | "lists", 107 | "open-source", 108 | "public", 109 | "public-api", 110 | "public-apis", 111 | "resources", 112 | "software" 113 | ], 114 | "visibility": "public", 115 | "forks": 24096, 116 | "open_issues": 61, 117 | "watchers": 210463, 118 | "default_branch": "master", 119 | "permissions": { 120 | "admin": false, 121 | "maintain": false, 122 | "push": false, 123 | "triage": false, 124 | "pull": true 125 | }, 126 | "temp_clone_token": "", 127 | "organization": { 128 | "login": "public-apis", 129 | "id": 51121562, 130 | "node_id": "MDEyOk9yZ2FuaXphdGlvbjUxMTIxNTYy", 131 | "avatar_url": "https://avatars.githubusercontent.com/u/51121562?v=4", 132 | "gravatar_id": "", 133 | "url": "https://api.github.com/users/public-apis", 134 | "html_url": "https://github.com/public-apis", 135 | "followers_url": "https://api.github.com/users/public-apis/followers", 136 | "following_url": "https://api.github.com/users/public-apis/following{/other_user}", 137 | "gists_url": "https://api.github.com/users/public-apis/gists{/gist_id}", 138 | "starred_url": "https://api.github.com/users/public-apis/starred{/owner}{/repo}", 139 | "subscriptions_url": "https://api.github.com/users/public-apis/subscriptions", 140 | "organizations_url": "https://api.github.com/users/public-apis/orgs", 141 | "repos_url": "https://api.github.com/users/public-apis/repos", 142 | "events_url": "https://api.github.com/users/public-apis/events{/privacy}", 143 | "received_events_url": "https://api.github.com/users/public-apis/received_events", 144 | "type": "Organization", 145 | "site_admin": false 146 | }, 147 | "network_count": 24096, 148 | "subscribers_count": 3683 149 | } 150 | -------------------------------------------------------------------------------- /format-markdown-item.ts: -------------------------------------------------------------------------------- 1 | import { DocItem, ExpiredValue, FileInfo } from "./interface.ts"; 2 | import { Content, Link, pLimit, Root, toMarkdown, visit } from "./deps.ts"; 3 | import { 4 | childrenToMarkdown, 5 | childrenToRoot, 6 | getDomain, 7 | gotGithubStar, 8 | isMock, 9 | promiseLimit, 10 | } from "./util.ts"; 11 | import log from "./log.ts"; 12 | const GithubSpecialOwner = [ 13 | "marketplace", 14 | "help", 15 | "blog", 16 | "about", 17 | "explore", 18 | "topics", 19 | "issues", 20 | "pulls", 21 | "notifications", 22 | "settings", 23 | "new", 24 | "organizations", 25 | "repositories", 26 | "packages", 27 | "people", 28 | "dashboard", 29 | "projects", 30 | "stars", 31 | "gists", 32 | "security", 33 | "marketplace", 34 | "pricing", 35 | "customer-stories", 36 | "nonprofit", 37 | "education", 38 | "nonprofit", 39 | "education", 40 | "enterprise", 41 | "login", 42 | "join", 43 | "watching", 44 | "new", 45 | "integrations", 46 | "marketplace", 47 | "pricing", 48 | "customer-stories", 49 | "nonprofit", 50 | "education", 51 | "nonprofit", 52 | "education", 53 | "enterprise", 54 | "login", 55 | "join", 56 | "watching", 57 | "new", 58 | "integrations", 59 | "marketplace", 60 | "pricing", 61 | "customer-stories", 62 | "nonprofit", 63 | "education", 64 | "nonprofit", 65 | "education", 66 | "enterprise", 67 | "login", 68 | "join", 69 | "watching", 70 | "new", 71 | "integrations", 72 | "marketplace", 73 | "pricing", 74 | "customer-stories", 75 | "nonprofit", 76 | "education", 77 | "nonprofit", 78 | "education", 79 | "enterprise", 80 | "login", 81 | "join", 82 | "watching", 83 | "new", 84 | "integrations", 85 | "marketplace", 86 | "pricing", 87 | "customer-stories", 88 | "nonprofit", 89 | "education", 90 | "nonprofit", 91 | "education", 92 | "enterprise", 93 | "login", 94 | "join", 95 | "watching", 96 | "new", 97 | "integrations", 98 | "marketplace", 99 | "pricing", 100 | "customer-stories", 101 | "nonprofit", 102 | "education", 103 | "nonprofit", 104 | "education", 105 | "enterprise", 106 | "login", 107 | "join", 108 | "watching", 109 | "new", 110 | "integrations", 111 | "marketplace", 112 | "pricing", 113 | "customer-stories", 114 | "nonprofit", 115 | "education", 116 | "nonprofit", 117 | "education", 118 | "enterprise", 119 | "login", 120 | "join", 121 | "watching", 122 | "new", 123 | "integrations", 124 | "marketplace", 125 | "pricing", 126 | "customer-stories", 127 | "nonprofit", 128 | "education", 129 | "nonprofit", 130 | "education", 131 | "enterprise", 132 | "login", 133 | "join", 134 | "watching", 135 | "new", 136 | "integrations", 137 | "marketplace", 138 | "pricing", 139 | "customer-stories", 140 | "nonprofit", 141 | "education", 142 | "nonprofit", 143 | "education", 144 | "enterprise", 145 | "login", 146 | "join", 147 | "watching", 148 | "new", 149 | "integrations", 150 | "marketplace", 151 | "pricing", 152 | "customer-stories", 153 | "nonprofit", 154 | "education", 155 | "nonprofit", 156 | "education", 157 | "enterprise", 158 | "login", 159 | "join", 160 | "watching", 161 | "new", 162 | "integrations", 163 | "marketplace", 164 | "pricing", 165 | "features", 166 | ]; 167 | export interface MatchedNode { 168 | node: Link; 169 | meta: Record; 170 | } 171 | export default async function formatItemMarkdown( 172 | item: Content | Root, 173 | fileInfo: FileInfo, 174 | dbCachedStars: Record, 175 | ): Promise { 176 | const sourceConfig = fileInfo.sourceConfig; 177 | const filepath = fileInfo.filepath; 178 | const fileConfig = sourceConfig.files[filepath]; 179 | const sourceMeta = fileInfo.sourceMeta; 180 | const repoMeta = sourceMeta.meta; 181 | const repoUrl = repoMeta.url; 182 | const defaultBranch = repoMeta.default_branch; 183 | const { options } = fileConfig; 184 | // get all github link, and add badge 185 | const matchedNodes: MatchedNode[] = []; 186 | visit(item, (node) => { 187 | if (node.type === "html") { 188 | if (node.value.includes(" { 192 | const url = p1; 193 | let formated = p1; 194 | if (url.startsWith("http")) { 195 | // do nothing 196 | } else if (url.startsWith("/")) { 197 | formated = `${repoUrl}/raw/${defaultBranch}${url}`; 198 | } else { 199 | formated = `${repoUrl}/raw/${defaultBranch}/${url}`; 200 | } 201 | const urlObj = new URL(formated); 202 | if (urlObj.hostname === "github.com") { 203 | formated = formated.replace("/blob/", "/raw/"); 204 | } 205 | return `src="${formated}"`; 206 | }); 207 | } 208 | } 209 | if ( 210 | node.type === "link" && 211 | (node.url.startsWith("http:") || node.url.startsWith("https:")) 212 | ) { 213 | const url = node.url; 214 | try { 215 | const urlObj = new URL(url); 216 | if ( 217 | urlObj.hostname === "github.com" && node.children && 218 | node.children.length > 0 && node.children[0].type === "text" && 219 | !node.children[0].value.startsWith(![]) 220 | ) { 221 | // disable white list pathname 222 | const pathname = urlObj.pathname; 223 | const pathArr = pathname.split("/"); 224 | const owner = pathArr[1]; 225 | const repo = pathArr[2]; 226 | 227 | if (owner && repo && !GithubSpecialOwner.includes(owner)) { 228 | matchedNodes.push({ 229 | node, 230 | meta: { 231 | owner, 232 | repo, 233 | }, 234 | }); 235 | } 236 | } 237 | } catch (e) { 238 | log.debug("url parse error", url, e); 239 | } 240 | } else if ( 241 | node.type === "link" && !node.url.startsWith("#") && 242 | !node.url.includes("://") 243 | ) { 244 | // transform relative link to absolute link 245 | const url = node.url; 246 | if (url.startsWith("/")) { 247 | node.url = `${repoUrl}/blob/${defaultBranch}${url}`; 248 | } else { 249 | node.url = `${repoUrl}/blob/${defaultBranch}/${filepath}/${url}`; 250 | } 251 | } else if (node.type === "image" && !node.url.startsWith("http")) { 252 | const url = node.url; 253 | if (url.startsWith("/")) { 254 | node.url = `${repoUrl}/raw/${defaultBranch}${url}`; 255 | } else { 256 | node.url = `${repoUrl}/raw/${defaultBranch}/${url}`; 257 | } 258 | } 259 | // check is there is blob, replace to raw 260 | if (node.type === "image" && node.url.includes("blob")) { 261 | const urlObj = new URL(node.url); 262 | if (urlObj.hostname === "github.com") { 263 | node.url = node.url.replace("/blob/", "/raw/"); 264 | } 265 | } 266 | }); 267 | if (!isMock()) { 268 | const limit = pLimit(30); 269 | await Promise.all( 270 | matchedNodes.map((matched) => { 271 | const { owner, repo } = matched.meta; 272 | const node = matched.node; 273 | return limit(() => 274 | gotGithubStar(owner, repo, dbCachedStars).then((star: string) => { 275 | if (star) { 276 | const badge = ` (⭐${star})`; 277 | node.children = [ 278 | ...node.children, 279 | { 280 | type: "text", 281 | value: badge, 282 | }, 283 | ]; 284 | } 285 | }).catch((_e) => { 286 | // ignore error 287 | }) 288 | ); 289 | }), 290 | ); 291 | } 292 | return item; 293 | } 294 | -------------------------------------------------------------------------------- /example/public-apis-simple.md: -------------------------------------------------------------------------------- 1 |
2 |

Public APIs

3 | A collective list of free APIs for use in software and web development 4 |
5 | 6 |
7 | 8 |
9 | Status 10 |
11 | 12 | Number of Categories 13 | 14 | 15 | Number of APIs 16 | 17 |
18 | 19 | Tests of push and pull 20 | 21 | 22 | Validate links 23 | 24 | 25 | Tests of validate package 26 | 27 |
28 | 29 |
30 | 31 |
32 | The Project 33 |
34 | Contributing Guide • 35 | API for this project • 36 | Issues • 37 | Pull Requests • 38 | License 39 | 40 |
41 | 42 |
43 | Alternative sites for the project (unofficials) 44 |
45 | Free APIs • 46 | Dev Resources • 47 | Public APIs Site • 48 | Apihouse • 49 | Collective APIs 50 |
51 | 52 |
53 | 54 | --- 55 | 56 |
57 | 58 | 88 | 89 |
90 | 91 | --- 92 | 93 |
94 | 95 | ## Index 96 | 97 | - [Animals](#animals) 98 | - [Anime](#anime) 99 | - [Anti-Malware](#anti-malware) 100 | - [Art & Design](#art--design) 101 | - [Authentication & Authorization](#authentication--authorization) 102 | - [Blockchain](#blockchain) 103 | - [Books](#books) 104 | - [Business](#business) 105 | - [Calendar](#calendar) 106 | - [Cloud Storage & File Sharing](#cloud-storage--file-sharing) 107 | - [Continuous Integration](#continuous-integration) 108 | - [Cryptocurrency](#cryptocurrency) 109 | - [Currency Exchange](#currency-exchange) 110 | - [Data Validation](#data-validation) 111 | - [Development](#development) 112 | - [Dictionaries](#dictionaries) 113 | - [Documents & Productivity](#documents--productivity) 114 | - [Email](#email) 115 | - [Entertainment](#entertainment) 116 | - [Environment](#environment) 117 | - [Events](#events) 118 | - [Finance](#finance) 119 | - [Food & Drink](#food--drink) 120 | - [Games & Comics](#games--comics) 121 | - [Geocoding](#geocoding) 122 | - [Government](#government) 123 | - [Health](#health) 124 | - [Jobs](#jobs) 125 | - [Machine Learning](#machine-learning) 126 | - [Music](#music) 127 | - [News](#news) 128 | - [Open Data](#open-data) 129 | - [Open Source Projects](#open-source-projects) 130 | - [Patent](#patent) 131 | - [Personality](#personality) 132 | - [Phone](#phone) 133 | - [Photography](#photography) 134 | - [Programming](#programming) 135 | - [Science & Math](#science--math) 136 | - [Security](#security) 137 | - [Shopping](#shopping) 138 | - [Social](#social) 139 | - [Sports & Fitness](#sports--fitness) 140 | - [Test Data](#test-data) 141 | - [Text Analysis](#text-analysis) 142 | - [Tracking](#tracking) 143 | - [Transportation](#transportation) 144 | - [URL Shorteners](#url-shorteners) 145 | - [Vehicle](#vehicle) 146 | - [Video](#video) 147 | - [Weather](#weather) 148 | 149 | ### Animals 150 | 151 | | API | Description | Auth | HTTPS | CORS | 152 | | ---------------------------------------------------------------- | ---------------------------------------- | -------- | ----- | ------- | 153 | | [AdoptAPet](https://www.adoptapet.com/public/apis/pet_list.html) | Resource to help get pets adopted | `apiKey` | Yes | Yes | 154 | | [Axolotl](https://theaxolotlapi.netlify.app/) | Collection of axolotl pictures and facts | No | Yes | No | 155 | | [xeno-canto](https://xeno-canto.org/explore/api) | Bird recordings | No | Yes | Unknown | 156 | | [Zoo Animals](https://zoo-animal-api.herokuapp.com/) | Facts and pictures of zoo animals | No | Yes | Yes | 157 | 158 | **[⬆ Back to Index](#index)** 159 | 160 | ### Anime 161 | 162 | | API | Description | Auth | HTTPS | CORS | 163 | | -------------------------------------------------------- | ------------------------------------------------------------------------ | -------- | ----- | ------- | 164 | | [AniAPI](https://aniapi.com/docs/) | Anime discovery, streaming & syncing with trackers | `OAuth` | Yes | Yes | 165 | | [AniDB](https://wiki.anidb.net/HTTP_API_Definition) | Anime Database | `apiKey` | No | Unknown | 166 | | [AniList](https://github.com/AniList/ApiV2-GraphQL-Docs) | Anime discovery & tracking | `OAuth` | Yes | Unknown | 167 | | [Waifu.im](https://waifu.im/docs) | Get waifu pictures from an archive of over 4000 images and multiple tags | No | Yes | Yes | 168 | | [Waifu.pics](https://waifu.pics/docs) | Image sharing platform for anime images | No | Yes | No | 169 | 170 | **[⬆ Back to Index](#index)** 171 | -------------------------------------------------------------------------------- /db.ts: -------------------------------------------------------------------------------- 1 | import { 2 | fromMarkdown, 3 | gfm, 4 | gfmFromMarkdown, 5 | gfmToMarkdown, 6 | render, 7 | toMarkdown, 8 | } from "./deps.ts"; 9 | import { 10 | DayInfo, 11 | DBIndex, 12 | DBMeta, 13 | ExpiredValue, 14 | File, 15 | FileInfo, 16 | Item, 17 | WeekOfYear, 18 | } from "./interface.ts"; 19 | import log from "./log.ts"; 20 | import formatMarkdownItem from "./format-markdown-item.ts"; 21 | import { 22 | getDayNumber, 23 | getDbContentHtmlPath, 24 | getDbContentPath, 25 | getDbItemsPath, 26 | getWeekNumber, 27 | parseDayInfo, 28 | parseWeekInfo, 29 | readJSONFile, 30 | readTextFile, 31 | writeJSONFile, 32 | writeTextFile, 33 | } from "./util.ts"; 34 | export type StringOrNumber = string | number; 35 | export function getFile( 36 | sourceIdentifier: string, 37 | filepath: string, 38 | ): Promise { 39 | const fileDbPath = getDbContentPath(sourceIdentifier, filepath); 40 | return readTextFile(fileDbPath); 41 | } 42 | export function getHtmlFile( 43 | sourceIdentifier: string, 44 | filepath: string, 45 | ): Promise { 46 | const fileDbPath = getDbContentHtmlPath(sourceIdentifier, filepath); 47 | return readTextFile(fileDbPath); 48 | } 49 | export async function updateFile( 50 | fileInfo: FileInfo, 51 | content: string, 52 | stars: Record, 53 | ) { 54 | const file = fileInfo.filepath; 55 | const sourceConfig = fileInfo.sourceConfig; 56 | const sourceIdentifier = sourceConfig.identifier; 57 | // check items length 58 | const tree = fromMarkdown(content, "utf8", { 59 | extensions: [gfm()], 60 | mdastExtensions: [gfmFromMarkdown()], 61 | }); 62 | 63 | // format link etc. 64 | const overviewMarkdownTree = await formatMarkdownItem(tree, fileInfo, stars); 65 | const overviewMarkdownContent = toMarkdown( 66 | overviewMarkdownTree, 67 | { 68 | extensions: [gfmToMarkdown()], 69 | }, 70 | ); 71 | const dbContentPath = getDbContentPath(sourceIdentifier, file); 72 | await writeTextFile(dbContentPath, overviewMarkdownContent); 73 | // also write html 74 | const dbContentHtmlPath = getDbContentHtmlPath(sourceIdentifier, file); 75 | const htmlContent = render(overviewMarkdownContent, { 76 | allowIframes: true, 77 | }); 78 | await writeTextFile(dbContentHtmlPath, htmlContent); 79 | } 80 | export async function updateItems( 81 | fileInfo: FileInfo, 82 | items: Record, 83 | dbIndex: DBIndex, 84 | ) { 85 | const file = fileInfo.filepath; 86 | const sourceConfig = fileInfo.sourceConfig; 87 | const sourceIdentifier = sourceConfig.identifier; 88 | const sourceCategory = sourceConfig.category; 89 | const itemKeys = Object.keys(items); 90 | if (itemKeys.length === 0) { 91 | return; 92 | } 93 | const dbItemsPath = getDbItemsPath( 94 | fileInfo.sourceConfig.identifier, 95 | fileInfo.filepath, 96 | ); 97 | await writeJSONFile(dbItemsPath, items); 98 | 99 | // write to index 100 | // delete old index 101 | const keys = Object.keys(dbIndex); 102 | for (const key of keys) { 103 | if (key.startsWith(sourceIdentifier + ":")) { 104 | delete dbIndex[key]; 105 | } 106 | } 107 | for (const itemKey of itemKeys) { 108 | const item = items[itemKey]; 109 | dbIndex[`${sourceIdentifier}:${fileInfo.filepath}:${item.sha1}`] = { 110 | t: new Date(item.updated_at).getTime(), 111 | d: item.updated_day, 112 | w: item.updated_week, 113 | }; 114 | } 115 | } 116 | 117 | export interface UpdatedItemsParam { 118 | since_date: Date; 119 | source_identifiers?: string[]; 120 | } 121 | export interface ItemsResult { 122 | items: Record; 123 | since_id: number; 124 | has_next: boolean; 125 | } 126 | export function getItems( 127 | sourceIdentifier: string, 128 | file: string, 129 | ): Promise> { 130 | const dbItemsPath = getDbItemsPath(sourceIdentifier, file); 131 | return readJSONFile(dbItemsPath) as Promise>; 132 | } 133 | export async function getItemsByDays( 134 | days: number[], 135 | dbIndex: DBIndex, 136 | isDay: boolean, 137 | ): Promise> { 138 | const keys = Object.keys(dbIndex); 139 | const items: Record = {}; 140 | const indexKey = isDay ? "d" : "w"; 141 | const todos: Record = {}; 142 | for (const key of keys) { 143 | const item = dbIndex[key]; 144 | 145 | if (days.includes(item[indexKey])) { 146 | const arr = key.split(":"); 147 | const sourceIdentifier = arr[0]; 148 | const file = arr[1]; 149 | const sha1 = arr[2]; 150 | const dbItemsPath = getDbItemsPath(sourceIdentifier, file); 151 | if (!todos[dbItemsPath]) { 152 | todos[dbItemsPath] = []; 153 | } 154 | todos[dbItemsPath].push(sha1); 155 | } 156 | } 157 | const todoKeys = Object.keys(todos); 158 | const promises: Promise[] = []; 159 | for (const todoKey of todoKeys) { 160 | promises.push(readJSONFile(todoKey)); 161 | } 162 | let rIndex = 0; 163 | const results = await Promise.all(promises) as unknown as Record< 164 | string, 165 | Item 166 | >[]; 167 | for (const result of results) { 168 | for (const sha1 of todos[todoKeys[rIndex]]) { 169 | if (!result[sha1]) { 170 | throw new Error(`sha1 ${sha1} not found in ${todoKeys[rIndex]}`); 171 | } 172 | items[sha1] = result[sha1]; 173 | } 174 | rIndex++; 175 | } 176 | return items; 177 | } 178 | export async function getDayItems( 179 | dayNumber: number, 180 | dbIndex: DBIndex, 181 | isDay: boolean, 182 | ): Promise> { 183 | const keys = Object.keys(dbIndex); 184 | const items: Record = {}; 185 | const indexKey = isDay ? "d" : "w"; 186 | 187 | const todos: Record = {}; 188 | for (const key of keys) { 189 | const item = dbIndex[key]; 190 | if (item[indexKey] === dayNumber) { 191 | const arr = key.split(":"); 192 | const sourceIdentifier = arr[0]; 193 | const file = arr[1]; 194 | const sha1 = arr[2]; 195 | const dbItemsPath = getDbItemsPath(sourceIdentifier, file); 196 | if (!todos[dbItemsPath]) { 197 | todos[dbItemsPath] = []; 198 | } 199 | todos[dbItemsPath].push(sha1); 200 | } 201 | } 202 | 203 | const todoKeys = Object.keys(todos); 204 | const promises: Promise[] = []; 205 | for (const todoKey of todoKeys) { 206 | promises.push(readJSONFile(todoKey)); 207 | } 208 | let rIndex = 0; 209 | const results = await Promise.all(promises) as unknown as Record< 210 | string, 211 | Item 212 | >[]; 213 | for (const result of results) { 214 | for (const sha1 of todos[todoKeys[rIndex]]) { 215 | if (!result[sha1]) { 216 | throw new Error(`sha1 ${sha1} not found in ${todoKeys[rIndex]}`); 217 | } 218 | items[sha1] = result[sha1]; 219 | } 220 | rIndex++; 221 | } 222 | return items; 223 | } 224 | export function getUpdatedFiles( 225 | options: UpdatedItemsParam, 226 | dbIndex: DBIndex, 227 | ): File[] { 228 | const filesSet: Set = new Set(); 229 | const keys = Object.keys(dbIndex); 230 | for (const key of keys) { 231 | const item = dbIndex[key]; 232 | const arr = key.split(":"); 233 | const sourceIdentifier = arr[0]; 234 | const file = arr[1]; 235 | if (options.since_date) { 236 | if (item.t < options.since_date?.getTime()) { 237 | continue; 238 | } 239 | } 240 | if (options.source_identifiers && options.source_identifiers.length > 0) { 241 | if (!options.source_identifiers.includes(sourceIdentifier)) { 242 | continue; 243 | } 244 | } 245 | filesSet.add(`${sourceIdentifier}:${file}`); 246 | } 247 | const files: File[] = []; 248 | 249 | for (const file of filesSet) { 250 | const arr = file.split(":"); 251 | const sourceIdentifier = arr[0]; 252 | const filepath = arr[1]; 253 | files.push({ 254 | source_identifier: sourceIdentifier, 255 | file: filepath, 256 | }); 257 | } 258 | 259 | return files; 260 | } 261 | export function getUpdatedDays( 262 | dbIndex: DBIndex, 263 | options: UpdatedItemsParam, 264 | isDay: boolean, 265 | ): (DayInfo | WeekOfYear)[] { 266 | const days: (DayInfo | WeekOfYear)[] = []; 267 | const daysSet: Set = new Set(); 268 | const keys = Object.keys(dbIndex); 269 | const indexKey = isDay ? "d" : "w"; 270 | for (const key of keys) { 271 | const item = dbIndex[key]; 272 | const arr = key.split(":"); 273 | const sourceIdentifier = arr[0]; 274 | const file = arr[1]; 275 | const sha1 = arr[2]; 276 | if (options.since_date) { 277 | if (item.t < options.since_date?.getTime()) { 278 | continue; 279 | } 280 | } 281 | if (options.source_identifiers && options.source_identifiers.length > 0) { 282 | if (!options.source_identifiers.includes(sourceIdentifier)) { 283 | continue; 284 | } 285 | } 286 | daysSet.add(item[indexKey]); 287 | } 288 | return Array.from(daysSet).sort((a, b) => b - a).map((day) => { 289 | if (isDay) { 290 | return parseDayInfo(day); 291 | } else { 292 | return parseWeekInfo(day); 293 | } 294 | }); 295 | } 296 | -------------------------------------------------------------------------------- /fetch-sources.ts: -------------------------------------------------------------------------------- 1 | import { 2 | getDayNumber, 3 | getDbCachedStars, 4 | getDbIndex, 5 | getDbMeta, 6 | getWeekNumber, 7 | sha1, 8 | writeDbCachedStars, 9 | writeDbIndex, 10 | writeDbMeta, 11 | writeJSONFile, 12 | } from "./util.ts"; 13 | import parser from "./parser/mod.ts"; 14 | import log from "./log.ts"; 15 | import { 16 | FileInfo, 17 | Item, 18 | ParsedItemsFilePath, 19 | RepoMetaOverride, 20 | RunOptions, 21 | } from "./interface.ts"; 22 | import initItems from "./init-items.ts"; 23 | import Github from "./adapters/github.ts"; 24 | import { getItems, updateFile, updateItems } from "./db.ts"; 25 | import renderMarkdown from "./render-markdown.ts"; 26 | export default async function (options: RunOptions) { 27 | const force = options.forceFetch; 28 | const isRebuild = options.rebuild; 29 | const config = options.config; 30 | const file_min_updated_hours = config.file_min_updated_hours; 31 | const sourcesMap = config.sources; 32 | let sourceIdentifiers = options.sourceIdentifiers; 33 | let isSpecificSource = true; 34 | if (sourceIdentifiers.length === 0) { 35 | isSpecificSource = false; 36 | sourceIdentifiers = Object.keys(sourcesMap); 37 | } 38 | // limit 39 | const limit = options.limit; 40 | if (limit && limit > 0) { 41 | sourceIdentifiers = sourceIdentifiers.slice(0, limit); 42 | } 43 | const dbMeta = await getDbMeta(); 44 | const dbIndex = await getDbIndex(); 45 | const dbCachedStars = await getDbCachedStars(); 46 | const dbSources = dbMeta.sources; 47 | 48 | const invalidFiles: ParsedItemsFilePath[] = []; 49 | let sourceIndex = 0; 50 | 51 | try { 52 | for (const sourceIdentifier of sourceIdentifiers) { 53 | sourceIndex++; 54 | log.info( 55 | `[${sourceIndex}/${sourceIdentifiers.length}] Fetching source: ${sourceIdentifier}`, 56 | ); 57 | const source = sourcesMap[sourceIdentifier]; 58 | const files = source.files; 59 | 60 | if (!dbSources[sourceIdentifier] || (isSpecificSource && isRebuild)) { 61 | // need to init source 62 | await initItems(source, options, dbMeta, dbIndex, dbCachedStars); 63 | continue; 64 | } else { 65 | // check is all files is init 66 | const dbSource = dbSources[sourceIdentifier]; 67 | const dbFiles = dbSource.files; 68 | const dbFileKeys = Object.keys(dbFiles); 69 | const isAllFilesInit = Object.keys(files).every((file) => { 70 | return dbFileKeys.includes(file); 71 | }); 72 | if (!isAllFilesInit) { 73 | // need to init source 74 | await initItems(source, options, dbMeta, dbIndex, dbCachedStars); 75 | continue; 76 | } 77 | } 78 | 79 | const dbSource = dbSources[sourceIdentifier]; 80 | const dbFiles = dbSource.files; 81 | const api = new Github(source); 82 | const fileKeys = Object.keys(files); 83 | let fileIndex = 0; 84 | // get file content and save it to raw data path 85 | for (const file of fileKeys) { 86 | fileIndex++; 87 | const dbFileMeta = dbFiles[file]; 88 | let isRebuild = false; 89 | 90 | if (dbFileMeta) { 91 | const dbFileMetaUpdatedAt = new Date(dbFileMeta.updated_at); 92 | if (dbFileMetaUpdatedAt.getTime() === 0) { 93 | log.info( 94 | `[${fileIndex}/${fileKeys.length}] ${source.identifier}/${file} is parsed failed, try to rebuild it.`, 95 | ); 96 | isRebuild = true; 97 | } 98 | } 99 | 100 | if (!dbFileMeta) { 101 | // reinit items 102 | isRebuild = true; 103 | } 104 | 105 | if (isRebuild) { 106 | await initItems(source, options, dbMeta, dbIndex, dbCachedStars); 107 | break; 108 | } 109 | 110 | // check is updated 111 | 112 | const dbFileUpdated = new Date(dbFileMeta.checked_at); 113 | 114 | const now = new Date(); 115 | const diff = now.getTime() - dbFileUpdated.getTime(); 116 | 117 | if (!force && (diff / 1000 / 60 / 60) < file_min_updated_hours) { 118 | // add max number function 119 | // not updated 120 | log.info( 121 | `${fileIndex}/${fileKeys.length}${sourceIdentifier}/${file} updated less than ${file_min_updated_hours} hours, skip`, 122 | ); 123 | continue; 124 | } else if (force) { 125 | log.info( 126 | `${sourceIdentifier}/${file} updated less than ${file_min_updated_hours} hours, force update`, 127 | ); 128 | } 129 | log.info( 130 | `${sourceIndex}/${sourceIdentifiers.length} try updating ${sourceIdentifier}/${file}`, 131 | ); 132 | const content = await api.getConent(file, source.default_branch); 133 | const contentSha1 = await sha1(content); 134 | const dbFileSha1 = dbFileMeta.sha1; 135 | log.debug( 136 | "dbFileSha1", 137 | dbFileSha1, 138 | "latest file contentSha1", 139 | contentSha1, 140 | ); 141 | 142 | if (dbFileSha1 === contentSha1 && !force) { 143 | log.info(`${file} is up to date, cause sha1 is same`); 144 | // update checked_at 145 | dbFileMeta.checked_at = new Date().toISOString(); 146 | continue; 147 | } else { 148 | let items: Record = {}; 149 | try { 150 | items = await getItems(sourceIdentifier, file); 151 | } catch (e) { 152 | log.warn(`get items error`, e); 153 | // try to reinit 154 | await initItems(source, options, dbMeta, dbIndex, dbCachedStars); 155 | continue; 156 | } 157 | const fileInfo: FileInfo = { 158 | sourceConfig: source, 159 | filepath: file, 160 | sourceMeta: dbSource, 161 | }; 162 | 163 | const docItems = await parser(content, fileInfo, dbCachedStars); 164 | //compare updated items 165 | const newItems: Record = {}; 166 | let newCount = 0; 167 | let totalCount = 0; 168 | let fileUpdatedAt = new Date(0); 169 | 170 | for (const docItem of docItems) { 171 | const itemSha1 = await sha1(docItem.rawMarkdown); 172 | totalCount++; 173 | // check markdown 174 | if (items[itemSha1]) { 175 | // it's a old item, 176 | // stay the same 177 | newItems[itemSha1] = { 178 | source_identifier: sourceIdentifier, 179 | file, 180 | sha1: itemSha1, 181 | markdown: docItem.formatedMarkdown, 182 | html: renderMarkdown(docItem.formatedMarkdown), 183 | category: docItem.category, 184 | category_html: renderMarkdown(docItem.category), 185 | updated_at: items[itemSha1].updated_at, 186 | checked_at: now.toISOString(), 187 | updated_day: items[itemSha1].updated_day, 188 | updated_week: items[itemSha1].updated_week, 189 | }; 190 | if (new Date(items[itemSha1].updated_at) > fileUpdatedAt) { 191 | fileUpdatedAt = new Date(items[itemSha1].updated_at); 192 | } 193 | } else { 194 | newCount++; 195 | const now = new Date(); 196 | // yes 197 | // this is a new item 198 | // add it to items 199 | newItems[itemSha1] = { 200 | source_identifier: sourceIdentifier, 201 | file, 202 | sha1: itemSha1, 203 | markdown: docItem.formatedMarkdown, 204 | html: renderMarkdown(docItem.formatedMarkdown), 205 | category: docItem.category, 206 | category_html: renderMarkdown(docItem.category), 207 | updated_at: now.toISOString(), 208 | checked_at: now.toISOString(), 209 | updated_day: getDayNumber(now), 210 | updated_week: getWeekNumber(now), 211 | }; 212 | if (now > fileUpdatedAt) { 213 | fileUpdatedAt = now; 214 | } 215 | } 216 | } 217 | 218 | await updateFile(fileInfo, content, dbCachedStars); 219 | await updateItems(fileInfo, newItems, dbIndex); 220 | 221 | dbFiles[file] = { 222 | ...dbFiles[file], 223 | updated_at: fileUpdatedAt.toISOString(), 224 | checked_at: now.toISOString(), 225 | sha1: contentSha1, 226 | }; 227 | log.info( 228 | `${sourceIndex}/${sourceIdentifiers.length} ${sourceIdentifier}/${file} updated, ${newCount} new items, ${totalCount} total items`, 229 | ); 230 | if (totalCount < 10) { 231 | invalidFiles.push({ 232 | sourceIdentifier, 233 | originalFilepath: file, 234 | }); 235 | } 236 | // if total count is 0, print it`` 237 | // also update repoMeta 238 | 239 | const metaOverrides: RepoMetaOverride = {}; 240 | if (source.default_branch) { 241 | metaOverrides.default_branch = source.default_branch; 242 | } 243 | const meta = await api.getRepoMeta(metaOverrides); 244 | dbSource.meta = meta; 245 | dbMeta.sources[sourceIdentifier].meta = { 246 | ...dbSource.meta, 247 | ...meta, 248 | }; 249 | } 250 | } 251 | dbMeta.sources[sourceIdentifier].files = dbFiles; 252 | dbMeta.sources[sourceIdentifier].updated_at = new Date().toISOString(); 253 | } 254 | // write to dbMeta 255 | await writeDbMeta(dbMeta); 256 | await writeDbIndex(dbIndex); 257 | await writeDbCachedStars(dbCachedStars); 258 | } catch (e) { 259 | // write to dbMeta 260 | await writeDbMeta(dbMeta); 261 | await writeDbIndex(dbIndex); 262 | await writeDbCachedStars(dbCachedStars); 263 | throw e; 264 | } 265 | if (invalidFiles.length > 0) { 266 | log.error(`Some files is invalid, please check it manually`); 267 | log.error(invalidFiles); 268 | await writeJSONFile("temp-invalid-files.json", invalidFiles); 269 | } 270 | } 271 | -------------------------------------------------------------------------------- /example/data/1-raw/EbookFoundation/free-programming-books/books/markdownlist_free-programming-books-zh.md: -------------------------------------------------------------------------------- 1 | ## 目录 2 | 3 | - [语言无关](#语言无关) 4 | - [版本控制](#版本控制) 5 | - [编程艺术](#编程艺术) 6 | - [编辑器](#编辑器) 7 | - [编译原理](#编译原理) 8 | - [操作系统](#操作系统) 9 | - [程序员杂谈](#程序员杂谈) 10 | - [大数据](#大数据) 11 | - [分布式系统](#分布式系统) 12 | - [管理和监控](#管理和监控) 13 | - [函数式概念](#函数式概念) 14 | - [计算机图形学](#计算机图形学) 15 | - [其它](#其它) 16 | - [软件开发方法](#软件开发方法) 17 | - [设计模式](#设计模式) 18 | - [数据库](#数据库) 19 | - [项目相关](#项目相关) 20 | - [在线教育](#在线教育) 21 | - [正则表达式](#正则表达式) 22 | - [智能系统](#智能系统) 23 | - [IDE](#ide) 24 | - [Web](#web) 25 | - [WEB 服务器](#web服务器) 26 | - [语言相关](#语言相关) 27 | - [Android](#android) 28 | - [Assembly](#assembly) 29 | - [AWK](#awk) 30 | - [C](#c) 31 | - [C#](#csharp) 32 | - [C++](#cpp) 33 | - [CoffeeScript](#coffeescript) 34 | - [Dart](#dart) 35 | - [Elasticsearch](#elasticsearch) 36 | - [Elixir](#elixir) 37 | - [Erlang](#erlang) 38 | - [Fortran](#fortran) 39 | - [Golang](#golang) 40 | - [Haskell](#haskell) 41 | - [HTML and CSS](#html-and-css) 42 | - [HTTP](#http) 43 | - [iOS](#ios) 44 | - [Java](#java) 45 | - [JavaScript](#javascript) 46 | - [AngularJS](#angularjs) 47 | - [Backbone.js](#backbonejs) 48 | - [D3.js](#d3js) 49 | - [Electron.js](#electronjs) 50 | - [ExtJS](#extjs) 51 | - [jQuery](#jquery) 52 | - [Node.js](#nodejs) 53 | - [React.js](#reactjs) 54 | - [Vue.js](#vuejs) 55 | - [Zepto.js](#zeptojs) 56 | - [LaTeX](#latex) 57 | - [LISP](#lisp) 58 | - [Lua](#lua) 59 | - [Markdown](#markdown) 60 | - [MySQL](#mysql) 61 | - [NoSQL](#nosql) 62 | - [Perl](#perl) 63 | - [PHP](#php) 64 | - [Laravel](#laravel) 65 | - [Symfony](#symfony) 66 | - [PostgreSQL](#postgresql) 67 | - [Python](#python) 68 | - [Django](#django) 69 | - [R](#r) 70 | - [reStructuredText](#restructuredtext) 71 | - [Ruby](#ruby) 72 | - [Rust](#rust) 73 | - [Scala](#scala) 74 | - [Scheme](#scheme) 75 | - [Scratch](#scratch) 76 | - [Shell](#shell) 77 | - [Swift](#swift) 78 | - [TypeScript](#typescript) 79 | - [Angular](#angular) 80 | - [Deno](#deno) 81 | - [VBA](#vba-microsoft-visual-basic-applications) 82 | - [Vim](#vim) 83 | - [Visual Prolog](#visual-prolog) 84 | 85 | ## 语言无关 86 | 87 | ### 版本控制 88 | 89 | - [沉浸式学 Git](https://web.archive.org/web/20191004044726/http://igit.linuxtoy.org:80/index.html) 90 | - [猴子都能懂的 GIT 入门](http://backlogtool.com/git-guide/cn/) 91 | - [学习 Git 分支](https://learngitbranching.js.org) 92 | - [Git - 简易指南](http://rogerdudler.github.io/git-guide/index.zh.html) 93 | - [Git 参考手册](http://gitref.justjavac.com) 94 | - [Git-Cheat-Sheet](https://github.com/flyhigher139/Git-Cheat-Sheet) - flyhigher139 95 | - [Git Community Book 中文版](http://gitbook.liuhui998.com) 96 | - [git-flow 备忘清单](http://danielkummer.github.io/git-flow-cheatsheet/index.zh_CN.html) 97 | - [Git magic](http://www-cs-students.stanford.edu/~blynn/gitmagic/intl/zh_cn/) 98 | - [Git Magic](http://www-cs-students.stanford.edu/~blynn/gitmagic/intl/zh_cn/) 99 | - [Git 教程](http://www.liaoxuefeng.com/wiki/0013739516305929606dd18361248578c67b8067c8c017b000) - 廖雪峰 100 | - [Github 帮助文档](https://github.com/waylau/github-help) 101 | - [GitHub 秘籍](https://snowdream86.gitbooks.io/github-cheat-sheet/content/zh/) 102 | - [Got GitHub](https://github.com/gotgit/gotgithub) 103 | - [GotGitHub](http://www.worldhello.net/gotgithub/index.html) 104 | - [HgInit (中文版)](https://zh-hginit.readthedocs.io/en/latest/) 105 | - [Mercurial 使用教程](https://www.mercurial-scm.org/wiki/ChineseTutorial) 106 | - [Pro Git](https://git-scm.com/book/zh/v2) 107 | - [Pro Git 第二版 中文版](https://bingohuang.gitbooks.io/progit2/content) - Bingo Huang 108 | - [svn 手册](http://svnbook.red-bean.com/nightly/zh/index.html) 109 | 110 | ### 编程艺术 111 | 112 | - [编程入门指南](http://www.kancloud.cn/kancloud/intro-to-prog/52592) 113 | - [程序员编程艺术](https://github.com/julycoding/The-Art-Of-Programming-by-July) 114 | - [每个程序员都应该了解的内存知识 (第一部分)](http://www.oschina.net/translate/what-every-programmer-should-know-about-memory-part1) 115 | 116 | ### 编辑器 117 | 118 | - [所需即所获：像 IDE 一样使用 vim](https://github.com/yangyangwithgnu/use_vim_as_ide) 119 | - [exvim--vim 改良成 IDE 项目](http://exvim.github.io/docs-zh/intro/) 120 | - [Vim 中文文档](https://github.com/vimcn/vimcdoc) 121 | 122 | ### 编译原理 123 | 124 | - [《计算机程序的结构和解释》公开课 翻译项目](https://github.com/DeathKing/Learning-SICP) 125 | 126 | ### 操作系统 127 | 128 | - [开源世界旅行手册](http://i.linuxtoy.org/docs/guide/index.html) 129 | - [理解 Linux 进程](https://github.com/tobegit3hub/understand_linux_process) 130 | - [命令行的艺术](https://github.com/jlevy/the-art-of-command-line/blob/master/README-zh.md) 131 | - [鸟哥的 Linux 私房菜 服务器架设篇](http://cn.linux.vbird.org/linux_server/) 132 | - [鸟哥的 Linux 私房菜 基础学习篇](http://cn.linux.vbird.org/linux_basic/linux_basic.php) 133 | - [嵌入式 Linux 知识库 (eLinux.org 中文版)](https://tinylab.gitbooks.io/elinux/content/zh/) 134 | - [Docker — 从入门到实践](https://github.com/yeasy/docker_practice) 135 | - [Docker 入门实战](http://yuedu.baidu.com/ebook/d817967416fc700abb68fca1) 136 | - [Docker 中文指南](https://github.com/widuu/chinese_docker) 137 | - [FreeBSD 使用手册](http://www.freebsd.org/doc/zh_CN.UTF-8/books/handbook/) 138 | - [Linux 构建指南](http://works.jinbuguo.com/lfs/lfs62/index.html) 139 | - [Linux 系统高级编程](http://sourceforge.net/projects/elpi/) 140 | - [Linux Documentation (中文版)](https://tinylab.gitbooks.io/linux-doc/content/zh-cn/) 141 | - [Linux Guide for Complete Beginners](http://happypeter.github.io/LGCB/book/) 142 | - [Linux 工具快速教程](https://github.com/me115/linuxtools_rst) 143 | - [Mac 开发配置手册](https://aaaaaashu.gitbooks.io/mac-dev-setup/content/) 144 | - [Operating Systems: Three Easy Pieces](http://pages.cs.wisc.edu/~remzi/OSTEP/) 145 | - [The Linux Command Line](http://billie66.github.io/TLCL/index.html) 146 | - [Ubuntu 参考手册](http://wiki.ubuntu.org.cn/UbuntuManual) 147 | - [uCore Lab: Operating System Course in Tsinghua University](https://www.gitbook.com/book/objectkuan/ucore-docs/details) 148 | - [UNIX TOOLBOX](https://web.archive.org/web/20210812021003/cb.vu/unixtoolbox_zh_CN.xhtml) _(:card_file_box: archived)_ 149 | 150 | ### 程序员杂谈 151 | 152 | - [程序员的自我修养](http://www.kancloud.cn/kancloud/a-programmer-prepares) 153 | 154 | ### 大数据 155 | 156 | - [面向程序员的数据挖掘指南](http://dataminingguide.books.yourtion.com) 157 | - [数据挖掘中经典的算法实现和详细的注释](https://github.com/linyiqun/DataMiningAlgorithm) 158 | - [Spark 编程指南简体中文版](https://aiyanbo.gitbooks.io/spark-programming-guide-zh-cn/content/) 159 | 160 | ### 分布式系统 161 | 162 | - [走向分布式](http://dcaoyuan.github.io/papers/pdfs/Scalability.pdf) (PDF) 163 | 164 | ### 管理和监控 165 | 166 | - [ElasticSearch 权威指南](https://www.gitbook.com/book/fuxiaopang/learnelasticsearch/details) 167 | - [Elasticsearch 权威指南（中文版）](https://web.archive.org/web/20200415002735/https://es.xiaoleilu.com/) _(:card_file_box: archived)_ 168 | - [ELKstack 中文指南](http://kibana.logstash.es) 169 | - [Logstash 最佳实践](https://github.com/chenryn/logstash-best-practice-cn) 170 | - [Mastering Elasticsearch(中文版)](http://udn.yyuap.com/doc/mastering-elasticsearch/) 171 | - [Puppet 2.7 Cookbook 中文版](https://www.gitbook.com/book/wizardforcel/puppet-27-cookbook/details) 172 | 173 | ### 函数式概念 174 | 175 | - [傻瓜函数编程](https://github.com/justinyhuang/Functional-Programming-For-The-Rest-of-Us-Cn) 176 | 177 | ### 计算机图形学 178 | 179 | - [LearnOpenGL CN](https://learnopengl-cn.github.io) 180 | - [OpenGL 教程](https://github.com/zilongshanren/opengl-tutorials) 181 | 182 | ### 其它 183 | 184 | - [深入理解并行编程](http://ifeve.com/perfbook/) 185 | - [SAN 管理入门系列](https://community.emc.com/docs/DOC-16067) 186 | - [Sketch 中文手册](http://sketchcn.com/sketch-chinese-user-manual.html#introduce) 187 | 188 | ### 软件开发方法 189 | 190 | - [傻瓜函数编程](https://github.com/justinyhuang/Functional-Programming-For-The-Rest-of-Us-Cn) (《Functional Programming For The Rest of Us》中文版) 191 | - [硝烟中的 Scrum 和 XP](http://www.infoq.com/cn/minibooks/scrum-xp-from-the-trenches) 192 | 193 | ### 设计模式 194 | 195 | - [深入设计模式](https://refactoringguru.cn/design-patterns) 196 | - [史上最全设计模式导学目录](http://blog.csdn.net/lovelion/article/details/17517213) 197 | - [图说设计模式](https://github.com/me115/design_patterns) 198 | 199 | ### 数据库 200 | 201 | 202 | 203 | ### 项目相关 204 | 205 | - [编码规范](https://github.com/ecomfe/spec) 206 | - [开源软件架构](http://www.ituring.com.cn/book/1143) 207 | - [让开发自动化系列专栏](https://wizardforcel.gitbooks.io/ibm-j-ap) 208 | - [追求代码质量](https://wizardforcel.gitbooks.io/ibm-j-cq) 209 | - [GNU make 指南](http://docs.huihoo.com/gnu/linux/gmake.html) 210 | - [Gradle 2 用户指南](https://github.com/waylau/Gradle-2-User-Guide) 211 | - [Gradle 中文使用文档](http://yuedu.baidu.com/ebook/f23af265998fcc22bcd10da2) 212 | - [Joel 谈软件]() 213 | - [selenium 中文文档](https://einverne.gitbook.io/selenium-doc/) 214 | 215 | ### 在线教育 216 | 217 | - [51CTO 学院](http://edu.51cto.com) 218 | - [黑马程序员](http://yun.itheima.com) 219 | - [汇智网](http://www.hubwiz.com) 220 | - [极客学院](http://www.jikexueyuan.com) 221 | - [计蒜客](http://www.jisuanke.com) 222 | - [慕课网](http://www.imooc.com/course/list) 223 | - [Codecademy](https://www.codecademy.com/?locale_code=zh) 224 | - [CodeSchool](https://www.codeschool.com) 225 | - [Coursera](https://www.coursera.org/courses?orderby=upcoming&lngs=zh) 226 | - [Learn X in Y minutes](https://learnxinyminutes.com) 227 | - [shiyanlou](https://www.shiyanlou.com) 228 | - [TeamTreeHouse](https://teamtreehouse.com) 229 | - [Udacity](https://www.udacity.com) 230 | - [xuetangX](https://www.xuetangx.com) 231 | 232 | ### 正则表达式 233 | 234 | - [正则表达式-菜鸟教程](http://www.runoob.com/regexp/regexp-tutorial.html) 235 | - [正则表达式 30 分钟入门教程](https://web.archive.org/web/20161119141236/http://deerchao.net:80/tutorials/regex/regex.htm) 236 | 237 | ### 智能系统 238 | 239 | - [一步步搭建物联网系统](https://github.com/phodal/designiot) 240 | 241 | ### IDE 242 | 243 | - [IntelliJ IDEA 简体中文专题教程](https://github.com/judasn/IntelliJ-IDEA-Tutorial) 244 | 245 | ### Web 246 | 247 | - [3 Web Designs in 3 Weeks](https://www.gitbook.com/book/juntao/3-web-designs-in-3-weeks/details) 248 | - [关于浏览器和网络的 20 项须知](http://www.20thingsilearned.com/zh-CN/home) 249 | - [浏览器开发工具的秘密](http://jinlong.github.io/2013/08/29/devtoolsecrets/) 250 | - [前端代码规范 及 最佳实践](http://coderlmn.github.io/code-standards/) 251 | - [前端开发体系建设日记](https://github.com/fouber/blog/issues/2) 252 | - [前端资源分享（二）](https://github.com/hacke2/hacke2.github.io/issues/3) 253 | - [前端资源分享（一）](https://github.com/hacke2/hacke2.github.io/issues/1) 254 | - [一站式学习 Wireshark](https://community.emc.com/thread/194901) 255 | - [移动前端开发收藏夹](https://github.com/hoosin/mobile-web-favorites) 256 | - [移动 Web 前端知识库](https://github.com/AlloyTeam/Mars) 257 | - [正则表达式 30 分钟入门教程](http://deerchao.net/tutorials/regex/regex.htm) 258 | - [Chrome 开发者工具中文手册](https://github.com/CN-Chrome-DevTools/CN-Chrome-DevTools) 259 | - [Chrome 扩展及应用开发](http://www.ituring.com.cn/minibook/950) 260 | - [Chrome 扩展开发文档](http://open.chrome.360.cn/extension_dev/overview.html) 261 | - [Growth: 全栈增长工程师指南](https://github.com/phodal/growth-ebook) 262 | - [Grunt 中文文档](http://www.gruntjs.net) 263 | - [Gulp 入门指南](https://github.com/nimojs/gulp-book) 264 | - [gulp 中文文档](http://www.gulpjs.com.cn/docs/) 265 | - [HTTP 接口设计指北](https://github.com/bolasblack/http-api-guide) 266 | - [HTTP/2.0 中文翻译](http://yuedu.baidu.com/ebook/478d1a62376baf1ffc4fad99?pn=1) 267 | - [http2 讲解](https://www.gitbook.com/book/ye11ow/http2-explained/details) 268 | - [JSON 风格指南](https://github.com/darcyliu/google-styleguide/blob/master/JSONStyleGuide.md) 269 | - [Wireshark 用户手册](https://web.archive.org/web/20200415002730/http://man.lupaworld.com/content/network/wireshark/index.html) 270 | 271 | ### WEB 服务器 272 | 273 | - [Apache 中文手册](http://works.jinbuguo.com/apache/menu22/index.html) 274 | - [Nginx 教程从入门到精通](http://www.ttlsa.com/nginx/nginx-stu-pdf/) - 运维生存时间 (PDF) 275 | - [Nginx 开发从入门到精通](http://tengine.taobao.org/book/index.html) - 淘宝团队 276 | 277 | ## 语言相关 278 | 279 | ### Android 280 | 281 | - [Android Note(开发过程中积累的知识点)](https://github.com/CharonChui/AndroidNote) 282 | - [Android6.0 新特性详解](http://leanote.com/blog/post/561658f938f41126b2000298) 283 | - [Android 开发技术前线(android-tech-frontier)](https://github.com/bboyfeiyu/android-tech-frontier) 284 | - [Google Android 官方培训课程中文版](http://hukai.me/android-training-course-in-chinese/index.html) 285 | - Google Material Design 正體中文版 ([译本一](https://wcc723.gitbooks.io/google_design_translate/content/style-icons.html)，[译本二](https://github.com/1sters/material_design_zh)) 286 | - [Material Design 中文版](http://wiki.jikexueyuan.com/project/material-design/) 287 | - [Point-of-Android](https://github.com/FX-Max/Point-of-Android) 288 | 289 | ### Assembly 290 | 291 | - 逆向工程权威指南 《Reverse Engineering for Beginners》 - Dennis Yurichev, Antiy Labs, Archer 292 | - [逆向工程权威指南 《Reverse Engineering for Beginners》 Vol.1](https://beginners.re/RE4B-CN-vol1.pdf) - Dennis Yurichev, Antiy Labs, Archer (PDF) 293 | - [逆向工程权威指南 《Reverse Engineering for Beginners》 Vol.2](https://beginners.re/RE4B-CN-vol2.pdf) - Dennis Yurichev, Antiy Labs, Archer (PDF) 294 | - [C/C++面向 WebAssembly 编程](https://github.com/3dgen/cppwasm-book/tree/master/zh) - Ending, Chai Shushan (HTML, [:package: examples](https://github.com/3dgen/cppwasm-book/tree/master/examples)) 295 | 296 | ### AWK 297 | 298 | - [awk 程序设计语言](https://github.com/wuzhouhui/awk) 299 | - [awk 中文指南](http://awk.readthedocs.org/en/latest/index.html) 300 | 301 | ### C 302 | 303 | - [新概念 C 语言教程](https://github.com/limingth/NCCL) 304 | - [Beej's Guide to Network Programming 簡體中文版](https://beej-zhtw-gitbook.netdpi.net) - Brian "Beej Jorgensen" Hall, 廖亚伦译 305 | - [C 语言常见问题集](http://c-faq-chn.sourceforge.net/ccfaq/ccfaq.html) 306 | - [Linux C 编程一站式学习](https://web.archive.org/web/20210514225440/http://docs.linuxtone.org/ebooks/C&CPP/c/) _(:card_file_box: archived)_ 307 | 308 | ### C\# 309 | 310 | - [精通 C#(第 6 版)](http://book.douban.com/subject/24827879/) 311 | 312 | ### C++ 313 | 314 | - [100 个 gcc 小技巧](https://github.com/hellogcc/100-gcc-tips/blob/master/src/index.md) 315 | - [100 个 gdb 小技巧](https://github.com/hellogcc/100-gdb-tips/blob/master/src/index.md) 316 | - [简单易懂的 C 魔法](https://web.archive.org/web/20210413213859/http://www.nowamagic.net/librarys/books/contents/c) _(:card_file_box: archived)_ 317 | - [像计算机科学家一样思考（C++版)](http://www.ituring.com.cn/book/1203) (《How To Think Like a Computer Scientist: C++ Version》中文版) 318 | - [C 语言编程透视](https://tinylab.gitbooks.io/cbook/content/) 319 | - [C/C++ Primer](https://github.com/andycai/cprimer) - andycai 320 | - [C++ 并发编程指南](https://github.com/forhappy/Cplusplus-Concurrency-In-Practice) 321 | - [C++ FAQ LITE(中文版)](http://www.sunistudio.com/cppfaq/) 322 | - [C++ Primer 5th Answers](https://github.com/Mooophy/Cpp-Primer) 323 | - [C++ Template 进阶指南](https://github.com/wuye9036/CppTemplateTutorial) 324 | - [CGDB 中文手册](https://github.com/leeyiw/cgdb-manual-in-chinese) 325 | - [Cmake 实践](https://web.archive.org/web/20170615174144/http://sewm.pku.edu.cn/src/paradise/reference/CMake%20Practice.pdf) (PDF) 326 | - [GNU make 指南](http://docs.huihoo.com/gnu/linux/gmake.html) 327 | - [Google C++ 风格指南](http://zh-google-styleguide.readthedocs.org/en/latest/google-cpp-styleguide/contents/) 328 | - [ZMQ 指南](https://github.com/anjuke/zguide-cn) 329 | 330 | ### CoffeeScript 331 | 332 | - [CoffeeScript 编程风格指南](https://github.com/elrrrrrrr/coffeescript-style-guide/blob/master/README-ZH.md) 333 | - [CoffeeScript 编码风格指南](https://github.com/geekplux/coffeescript-style-guide) 334 | - [CoffeeScript 中文](http://coffee-script.org) 335 | 336 | ### Dart 337 | 338 | - [Dart 语言导览](https://web.archive.org/web/20200415002731/dart.lidian.info/wiki/Language_Tour) _(:card_file_box: archived)_ 339 | 340 | ### Elasticsearch 341 | 342 | - [Elasticsearch 权威指南](https://github.com/looly/elasticsearch-definitive-guide-cn) （《Elasticsearch the definitive guide》中文版） 343 | - [Mastering Elasticsearch(中文版)](http://udn.yyuap.com/doc/mastering-elasticsearch/) 344 | 345 | ### Elixir 346 | 347 | - [Elixir 编程语言教程](https://elixirschool.com/zh-hans) (Elixir School) 348 | - [Elixir Getting Started 中文翻译](https://github.com/Ljzn/ElixrGettingStartedChinese) 349 | - [Elixir 元编程与 DSL 中文翻译](https://github.com/Ljzn/MetaProgrammingInElixirChinese) 350 | - [Phoenix 框架中文文档](https://mydearxym.gitbooks.io/phoenix-doc-in-chinese/content/) 351 | 352 | ### Erlang 353 | 354 | - [Erlang 并发编程](https://github.com/liancheng/cpie-cn) (《Concurrent Programming in Erlang (Part I)》中文版) 355 | 356 | ### Fortran 357 | 358 | - [Fortran77 和 90/95 编程入门](http://micro.ustc.edu.cn/Fortran/ZJDing/) 359 | 360 | ### Golang 361 | 362 | - [深入解析 Go](https://tiancaiamao.gitbooks.io/go-internals/content/zh) - tiancaiamao 363 | - [学习 Go 语言](http://mikespook.com/learning-go/) 364 | - [Go 编程基础](https://github.com/Unknwon/go-fundamental-programming) 365 | - [Go 官方文档翻译](https://github.com/golang-china/golangdoc.translations) 366 | - [Go 简易教程](https://github.com/songleo/the-little-go-book_ZH_CN) - Song Song Li (《[The Little Go Book](https://github.com/karlseguin/the-little-go-book) - Karl Seguin》中文版) 367 | - [Go 命令教程](https://github.com/hyper-carrot/go_command_tutorial) 368 | - [Go 入门指南](https://github.com/Unknwon/the-way-to-go_ZH_CN) (《The Way to Go》中文版) 369 | - [Go 语法树入门](https://github.com/chai2010/go-ast-book) 370 | - [Go 语言标准库](https://github.com/polaris1119/The-Golang-Standard-Library-by-Example) 371 | - [Go 语言高级编程（Advanced Go Programming）](https://github.com/chai2010/advanced-go-programming-book) 372 | - [Go 语言设计与实现](https://draveness.me/golang) - draveness 373 | - [Go 语言实战笔记](https://github.com/rujews/go-in-action-notes) 374 | - [Go 指南](https://tour.go-zh.org/list) (《A Tour of Go》中文版) 375 | - [Go Web 编程](https://github.com/astaxie/build-web-application-with-golang) 376 | - [Go 实战开发](https://github.com/astaxie/go-best-practice) 377 | - [Go 语言博客实践](https://github.com/achun/Go-Blog-In-Action) 378 | - [Java 程序员的 Golang 入门指南](http://blog.csdn.net/dc_726/article/details/46565241) 379 | - [Network programming with Go 中文翻译版本](https://github.com/astaxie/NPWG_zh) 380 | - [Revel 框架手册](https://web.archive.org/web/20190610030938/https://gorevel.cn/docs/manual/index.html) _(:card_file_box: archived)_ 381 | - [The Little Go Book 繁體中文翻譯版](https://github.com/kevingo/the-little-go-book) - Karl Seguin, KevinGo, Jie Peng ([HTML](https://kevingo.gitbooks.io/the-little-go-book/)) 382 | 383 | ### Groovy 384 | 385 | - [Groovy 教程](https://www.w3cschool.cn/groovy) - W3Cschool 386 | 387 | ### Haskell 388 | 389 | - [Haskell 趣学指南](https://learnyouahaskell.mno2.org) 390 | - [Real World Haskell 中文版](http://cnhaskell.com) 391 | 392 | ### HTML and CSS 393 | 394 | - [前端代码规范](http://alloyteam.github.io/CodeGuide/) - 腾讯 AlloyTeam 团队 395 | - [通用 CSS 笔记、建议与指导](https://github.com/chadluo/CSS-Guidelines/blob/master/README.md) 396 | - [学习 CSS 布局](http://zh.learnlayout.com) 397 | - [Bootstrap 4 繁體中文手冊](https://bootstrap.hexschool.com) - 六角學院 398 | - [Bootstrap 5 繁體中文手冊](https://bootstrap5.hexschool.com) - 六角學院 399 | - [CSS3 Tutorial 《CSS3 教程》](https://github.com/waylau/css3-tutorial) 400 | - [CSS 参考手册](http://css.doyoe.com) 401 | - [Emmet 文档](http://yanxyz.github.io/emmet-docs/) 402 | - [HTML5 教程](http://www.w3school.com.cn/html5/index.asp) 403 | - [HTML 和 CSS 编码规范](http://codeguide.bootcss.com) 404 | - [Sass Guidelines 中文](http://sass-guidelin.es/zh/) 405 | 406 | ### iOS 407 | 408 | - [网易斯坦福大学公开课：iOS 7 应用开发字幕文件](https://github.com/jkyin/Subtitle) 409 | - [Apple Watch 开发初探](http://nilsun.github.io/apple-watch/) 410 | - [Google Objective-C Style Guide 中文版](http://zh-google-styleguide.readthedocs.org/en/latest/google-objc-styleguide/) 411 | - [iOS7 人机界面指南](http://isux.tencent.com/ios-human-interface-guidelines-ui-design-basics-ios7.html) 412 | - [iOS 开发 60 分钟入门](https://github.com/qinjx/30min_guides/blob/master/ios.md) 413 | - [iPhone 6 屏幕揭秘](http://wileam.com/iphone-6-screen-cn/) 414 | 415 | ### Java 416 | 417 | - [阿里巴巴 Java 开发手册]() (PDF) 418 | - [用 jersey 构建 REST 服务](https://github.com/waylau/RestDemo) 419 | - [Activiti 5.x 用户指南](https://github.com/waylau/activiti-5.x-user-guide) 420 | - [Apache MINA 2 用户指南](https://github.com/waylau/apache-mina-2.x-user-guide) 421 | - [Apache Shiro 用户指南](https://github.com/waylau/apache-shiro-1.2.x-reference) 422 | - [Google Java 编程风格指南](http://hawstein.com/2014/01/20/google-java-style/) 423 | - [H2 Database 教程](https://github.com/waylau/h2-database-doc) 424 | - [Java 编程思想](https://java.quanke.name) - quanke 425 | - [Java 编码规范](https://github.com/waylau/java-code-conventions) 426 | - [Java Servlet 3.1 规范](https://github.com/waylau/servlet-3.1-specification) 427 | - [Jersey 2.x 用户指南](https://github.com/waylau/Jersey-2.x-User-Guide) 428 | - [JSSE 参考指南](https://github.com/waylau/jsse-reference-guide) 429 | - [MyBatis 中文文档](http://mybatis.github.io/mybatis-3/zh/index.html) 430 | - [Netty 4.x 用户指南](https://github.com/waylau/netty-4-user-guide) 431 | - [Netty 实战(精髓)](https://github.com/waylau/essential-netty-in-action) 432 | - [Nutz-book Nutz 烹调向导](http://nutzbook.wendal.net) 433 | - [Nutz 文档](https://nutzam.com/core/nutz_preface.html) 434 | - [REST 实战](https://github.com/waylau/rest-in-action) 435 | - [Spring Boot 参考指南](https://github.com/qibaoguang/Spring-Boot-Reference-Guide) (:construction: _翻译中_) 436 | - [Spring Framework 4.x 参考文档](https://github.com/waylau/spring-framework-4-reference) 437 | 438 | ### JavaScript 439 | 440 | - [命名函数表达式探秘](http://justjavac.com/named-function-expressions-demystified.html) - kangax、为之漫笔(翻译) (原始地址无法打开，所以此处地址为 justjavac 博客上的备份) 441 | - [你不知道的 JavaScript](https://github.com/getify/You-Dont-Know-JS/tree/1ed-zh-CN) 442 | - [深入理解 JavaScript 系列](http://www.cnblogs.com/TomXu/archive/2011/12/15/2288411.html) 443 | - [现代 JavaScript 教程](https://zh.javascript.info) - Ilya Kantor 444 | - [学用 JavaScript 设计模式](http://www.oschina.net/translate/learning-javascript-design-patterns) - 开源中国 445 | - [Airbnb JavaScript 规范](https://github.com/adamlu/javascript-style-guide) 446 | - [ECMAScript 6 入门](http://es6.ruanyifeng.com) - 阮一峰 447 | - [Google JavaScript 代码风格指南](https://web.archive.org/web/20200415002735/bq69.com/blog/articles/script/868/google-javascript-style-guide.html) _(:card_file_box: archived)_ 448 | - [JavaScript 标准参考教程（alpha）](http://javascript.ruanyifeng.com) 449 | - [javascript 的 12 个怪癖](https://github.com/justjavac/12-javascript-quirks) 450 | - [JavaScript 秘密花园](http://bonsaiden.github.io/JavaScript-Garden/zh/) 451 | - [《JavaScript 模式》](https://github.com/jayli/javascript-patterns) (《JavaScript patterns》译本) 452 | - [JavaScript 原理](https://web.archive.org/web/20170112164945/http://typeof.net/s/jsmech/) 453 | - [JavaScript Promise 迷你书](http://liubin.github.io/promises-book/) 454 | - [JavaScript 编程指南](http://pij.robinqu.me) ([源码](https://github.com/RobinQu/Programing-In-Javascript)) 455 | - [JavaScript 核心概念及实践](http://icodeit.org/jsccp/) (PDF) 456 | 457 | #### AngularJS 458 | 459 | > :information_source: See also … [Angular](#angular) 460 | 461 | - [构建自己的 AngularJS](https://github.com/xufei/Make-Your-Own-AngularJS/blob/master/01.md) - Xu Fei (HTML) 462 | - [在 Windows 环境下用 Yeoman 构建 AngularJS 项目](http://www.waylau.com/build-angularjs-app-with-yeoman-in-windows/) - Way Lau (HTML) 463 | - [AngularJS 入门教程](https://github.com/zensh/AngularjsTutorial_cn) - Yan Qing, Hou Zhenyu, 速冻沙漠 (HTML) (:card_file_box: _archived_) 464 | - [AngularJS 最佳实践和风格指南](https://github.com/mgechev/angularjs-style-guide/blob/master/README-zh-cn.md) - Minko Gechev, Xuefeng Zhu, Shintaro Kaneko et al. (HTML) 465 | 466 | #### Backbone.js 467 | 468 | - [Backbone.js 入门教程](http://www.the5fire.com/backbone-js-tutorials-pdf-download.html) (PDF) 469 | - [Backbone.js 入门教程第二版](https://github.com/the5fire/backbonejs-learning-note) 470 | - [Backbone.js 中文文档](https://web.archive.org/web/20200916085144/https://www.html.cn/doc/backbone/) _(:card_file_box: archived)_ 471 | 472 | #### D3.js 473 | 474 | - [楚狂人的 D3 教程](http://www.cnblogs.com/winleisure/tag/D3.js/) 475 | - [官方 API 文档](https://github.com/mbostock/d3/wiki/API--%E4%B8%AD%E6%96%87%E6%89%8B%E5%86%8C) 476 | - [张天旭的 D3 教程](http://blog.csdn.net/zhang__tianxu/article/category/1623437) 477 | - [Learning D3.JS](http://d3.decembercafe.org) - 十二月咖啡馆 478 | 479 | #### Electron.js 480 | 481 | - [Electron 中文文档](https://wizardforcel.gitbooks.io/electron-doc/content) - WizardForcel 482 | - [Electron 中文文档](https://www.w3cschool.cn/electronmanual) - W3Cschool 483 | 484 | #### ExtJS 485 | 486 | - [Ext4.1.0 中文文档](http://extjs-doc-cn.github.io/ext4api/) 487 | 488 | #### jQuery 489 | 490 | - [简单易懂的 JQuery 魔法](https://web.archive.org/web/20201127045453/http://www.nowamagic.net/librarys/books/contents/jquery) _(:card_file_box: archived)_ 491 | - [How to write jQuery plugin](http://i5ting.github.io/How-to-write-jQuery-plugin/build/jquery.plugin.html) 492 | 493 | #### Node.js 494 | 495 | - [七天学会 NodeJS](http://nqdeng.github.io/7-days-nodejs/) - 阿里团队 496 | - [使用 Express + MongoDB 搭建多人博客](https://github.com/nswbmw/N-blog) 497 | - [express.js 中文文档](http://expressjs.jser.us) 498 | - [Express 框架](http://javascript.ruanyifeng.com/nodejs/express.html) 499 | - [koa 中文文档](https://github.com/guo-yu/koa-guide) 500 | - [Learn You The Node.js For Much Win! (中文版)](https://www.npmjs.com/package/learnyounode-zh-cn) 501 | - [Node debug 三法三例](http://i5ting.github.io/node-debug-tutorial/) 502 | - [Node.js 包教不包会](https://github.com/alsotang/node-lessons) 503 | - [Node.js Fullstack《從零到一的進撃》](https://github.com/jollen/nodejs-fullstack-lessons) 504 | - [Node 入门](http://www.nodebeginner.org/index-zh-cn.html) 505 | - [Nodejs Wiki Book](https://github.com/nodejs-tw/nodejs-wiki-book) (繁体中文) 506 | - [nodejs 中文文档](https://www.gitbook.com/book/0532/nodejs/details) 507 | - [The NodeJS 中文文档](https://www.gitbook.com/book/0532/nodejs/details) - 社区翻译 508 | 509 | #### React.js 510 | 511 | - [Learn React & Webpack by building the Hacker News front page](https://github.com/theJian/build-a-hn-front-page) 512 | - [React-Bits 中文文档](https://github.com/hateonion/react-bits-CN) 513 | - [React Native 中文文档(含最新 Android 内容)](http://wiki.jikexueyuan.com/project/react-native/) 514 | - [React webpack-cookbook](https://github.com/fakefish/react-webpack-cookbook) 515 | - [React.js 入门教程](http://fraserxu.me/intro-to-react/) 516 | - [React.js 中文文档](https://discountry.github.io/react/) 517 | 518 | #### Vue.js 519 | 520 | - [Vue.js 中文文档](https://cn.vuejs.org/v2/guide/) 521 | - [Vue3.0 学习教程与实战案例](https://vue3.chengpeiquan.com) - chengpeiquan 522 | 523 | #### Zepto.js 524 | 525 | - [Zepto.js 中文文档](https://web.archive.org/web/20210303025214/https://www.css88.com/doc/zeptojs_api/) _(:card_file_box: archived)_ 526 | 527 | ### LaTeX 528 | 529 | - [大家來學 LaTeX](https://github.com/49951331/graduate-project-102pj/blob/master/docs/latex123.pdf) (PDF) 530 | - [一份不太简短的 LaTeX2ε 介绍](http://ctan.org/pkg/lshort-zh-cn) 531 | - [LaTeX 笔记](http://www.dralpha.com/zh/tech/tech.htm) 532 | 533 | ### LISP 534 | 535 | - [ANSI Common Lisp 中文翻译版](http://acl.readthedocs.org/en/latest/) 536 | - [Common Lisp 高级编程技术](http://www.ituring.com.cn/minibook/862) (《On Lisp》中文版) 537 | 538 | ### Lua 539 | 540 | - [Lua 5.3 参考手册](https://www.runoob.com/manual/lua53doc/) 541 | 542 | ### Markdown 543 | 544 | - [献给写作者的 Markdown 新手指南](http://www.jianshu.com/p/q81RER) 545 | - [Markdown 語法說明](https://markdown.tw) 546 | 547 | ### MySQL 548 | 549 | - [21 分钟 MySQL 入门教程](http://www.cnblogs.com/mr-wid/archive/2013/05/09/3068229.html) 550 | - [MySQL 索引背后的数据结构及算法原理](http://blog.codinglabs.org/articles/theory-of-mysql-index.html) 551 | 552 | ### NoSQL 553 | 554 | - [带有详细注释的 Redis 2.6 代码](https://github.com/huangz1990/annotated_redis_source) 555 | - [带有详细注释的 Redis 3.0 代码](https://github.com/huangz1990/redis-3.0-annotated) 556 | - [Disque 使用教程](http://disque.huangz.me) 557 | - [Redis 命令参考](http://redisdoc.com) 558 | - [Redis 设计与实现](http://redisbook.com) 559 | - [The Little MongoDB Book](https://github.com/justinyhuang/the-little-mongodb-book-cn/blob/master/mongodb.md) 560 | - [The Little Redis Book](https://github.com/JasonLai256/the-little-redis-book/blob/master/cn/redis.md) 561 | 562 | ### Perl 563 | 564 | - [Master Perl Today](https://github.com/fayland/chinese-perl-book) 565 | - [Perl 5 教程](https://web.archive.org/web/20150326073235/http://net.pku.edu.cn/~yhf/tutorial/perl/perl.html) 566 | - [Perl 教程](http://www.yiibai.com/perl) 567 | 568 | ### PHP 569 | 570 | - [深入理解 PHP 内核](http://www.php-internals.com/book/) 571 | - [CodeIgniter 使用手冊](https://web.archive.org/web/20210624143822/https://codeigniter.org.tw/userguide3/) _(:card_file_box: archived)_ 572 | - [Composer 中文文档](http://docs.phpcomposer.com) 573 | - [Phalcon7 中文文档](https://web.archive.org/web/20220330065727/myleftstudio.com/) _(:card_file_box: archived)_ 574 | - [PHP 之道](http://wulijun.github.io/php-the-right-way/) 575 | - [PHP 标准规范中文版](https://psr.phphub.org) 576 | - [PHP 中文手册](http://php.net/manual/zh/) 577 | - [Yii2 中文文档](http://www.yiichina.com/doc/guide/2.0) 578 | 579 | #### Laravel 580 | 581 | - [Laravel 5.4 中文文档](http://d.laravel-china.org/docs/5.4) 582 | - [Laravel 6 中文文档](https://learnku.com/docs/laravel/6.x) 583 | - [Laravel 7 中文文档](https://learnku.com/docs/laravel/7.x) 584 | - [Laravel 8 中文文档](https://learnku.com/docs/laravel/8.x) 585 | 586 | #### Symfony 587 | 588 | - [Symfony 2 实例教程](https://wusuopu.gitbooks.io/symfony2_tutorial/content) 589 | - [Symfony 5 快速开发](https://web.archive.org/web/20210812222957/symfony.com/doc/current/the-fast-track/zh_CN/index.html) _(:card_file_box: archived)_ 590 | 591 | ### PostgreSQL 592 | 593 | - [PostgreSQL 8.2.3 中文文档](http://works.jinbuguo.com/postgresql/menu823/index.html) 594 | - [PostgreSQL 9.3.1 中文文档](http://www.postgres.cn/docs/9.3/index.html) 595 | - [PostgreSQL 9.4.4 中文文档](http://www.postgres.cn/docs/9.4/index.html) 596 | - [PostgreSQL 9.5.3 中文文档](http://www.postgres.cn/docs/9.5/index.html) 597 | - [PostgreSQL 9.6.0 中文文档](http://www.postgres.cn/docs/9.6/index.html) 598 | 599 | ### Python 600 | 601 | - [简明 Python 教程](https://web.archive.org/web/20200822010330/https://bop.mol.uno/) - Swaroop C H、沈洁元(翻译)、漠伦(翻译) _(:card_file_box: archived)_ 602 | - [人生苦短，我用 python](https://www.cnblogs.com/derek1184405959/p/8579428.html) - (内含丰富的笔记以及各类教程) 603 | - [深入 Python 3](https://github.com/jiechic/diveintopython3) 604 | - [Matplotlib 3.0.3 中文文档](http://www.osgeo.cn/matplotlib/) - (Online) 605 | - [Numpy 1.16 中文文档](http://www.osgeo.cn/numpy/) - (Online) 606 | - [Python 3 文档(简体中文) 3.2.2 documentation](http://docspy3zh.readthedocs.org/en/latest/) 607 | - [Python 3.8.0a3 中文文档](http://www.osgeo.cn/cpython/) - (目前在线最全的中文文档了，Online) 608 | - [Python 中文学习大本营](http://www.pythondoc.com) 609 | - [Python 最佳实践指南](https://pythonguidecn.readthedocs.io/zh/latest/) 610 | - [Python Cookbook 第三版](http://python3-cookbook.readthedocs.io/zh_CN/latest/) - David Beazley、Brian K.Jones、熊能(翻译) 611 | - [Python 教程 - 廖雪峰的官方网站](http://www.liaoxuefeng.com/wiki/0014316089557264a6b348958f449949df42a6d3a2e542c000) 612 | - [Python 进阶](https://interpy.eastlakeside.com) - eastlakeside 613 | - [Python 之旅](https://web.archive.org/web/20191217091745/http://funhacks.net/explore-python/) - Ethan _(:card_file_box: archived)_ 614 | - [Tornado 6.1 中文文档](http://www.osgeo.cn/tornado/) - (网络上其他的都是较旧版本的，Online) 615 | 616 | #### Django 617 | 618 | - [Django 1.11.6 中文文档](https://www.yiyibooks.cn/xx/Django_1.11.6/index.html) 619 | - [Django 2.2.1 中文文档](http://www.osgeo.cn/django/) - (这个很新，也很全，Online) 620 | - [Django 搭建个人博客教程 (2.1)](https://www.dusaiphoto.com/article/detail/2) - (杜赛) (HTML) 621 | - [Django book 2.0](http://djangobook.py3k.cn/2.0/) 622 | - [Django Girls 教程 (1.11)](https://tutorial.djangogirls.org/zh/) (HTML) 623 | 624 | ### R 625 | 626 | - [153 分钟学会 R](http://cran.r-project.org/doc/contrib/Liu-FAQ.pdf) (PDF) 627 | - [统计学与 R 读书笔记](http://cran.r-project.org/doc/contrib/Xu-Statistics_and_R.pdf) (PDF) 628 | - [用 R 构建 Shiny 应用程序](https://web.archive.org/web/20200220023703/yanping.me/shiny-tutorial/) (《Building 'Shiny' Applications with R》中文版) _(:card_file_box: archived)_ 629 | - [R 导论](http://cran.r-project.org/doc/contrib/Ding-R-intro_cn.pdf) (《An Introduction to R》中文版) (PDF) 630 | 631 | ### reStructuredText 632 | 633 | - [reStructuredText 入门](http://www.pythondoc.com/sphinx/rest.html) 634 | 635 | ### Ruby 636 | 637 | - [笨方法学 Ruby](http://lrthw.github.io) 638 | - [Rails 风格指南](https://github.com/JuanitoFatas/rails-style-guide/blob/master/README-zhCN.md) 639 | - [Ruby 风格指南](https://github.com/JuanitoFatas/ruby-style-guide/blob/master/README-zhCN.md) 640 | - [Ruby on Rails 实战圣经](https://ihower.tw/rails4/) 641 | - [Ruby on Rails 指南](https://ruby-china.github.io/rails-guides/) 642 | - [Sinatra](http://www.sinatrarb.com/intro-zh.html) 643 | 644 | ### Rust 645 | 646 | - [通过例子学习 Rust](https://github.com/rustcc/rust-by-example/) 647 | - [Rust 官方教程](https://github.com/KaiserY/rust-book-chinese) 648 | - [Rust 语言学习笔记](https://github.com/photino/rust-notes) 649 | - [RustPrimer](https://github.com/rustcc/RustPrimer) 650 | - [Tour of Rust](https://tourofrust.com/00_zh-cn.html) 651 | 652 | ### Scala 653 | 654 | - [Effective Scala](http://twitter.github.io/effectivescala/index-cn.html) 655 | - [Scala 初学者指南](https://www.gitbook.com/book/windor/beginners-guide-to-scala/details) (《The Neophyte's Guide to Scala》中文版) 656 | - [Scala 课堂](http://twitter.github.io/scala_school/zh_cn/index.html) (Twitter 的 Scala 中文教程) 657 | 658 | ### Scheme 659 | 660 | - [Scheme 入门教程](http://deathking.github.io/yast-cn/) (《Yet Another Scheme Tutorial》中文版) 661 | 662 | ### Scratch 663 | 664 | - [创意计算课程指南](http://cccgchinese.strikingly.com) 665 | 666 | ### Shell 667 | 668 | - [Shell 编程范例](https://tinylab.gitbooks.io/shellbook/content) - 泰晓科技 669 | - [Shell 编程基础](http://wiki.ubuntu.org.cn/Shell%E7%BC%96%E7%A8%8B%E5%9F%BA%E7%A1%80) 670 | - [Shell 脚本编程 30 分钟入门](https://github.com/qinjx/30min_guides/blob/master/shell.md) 671 | - [shell-book](http://me.52fhy.com/shell-book/) 672 | - [The Linux Command Line 中文版](http://billie66.github.io/TLCL/book/) 673 | 674 | ### Swift 675 | 676 | - [《The Swift Programming Language》中文版](https://www.gitbook.com/book/numbbbbb/-the-swift-programming-language-/details) 677 | 678 | ### TypeScript 679 | 680 | - [TypeScript 教程](https://www.runoob.com/typescript/ts-tutorial.html) - runoob (HTML) 681 | - [TypeScript 入门教程](https://www.runoob.com/w3cnote/getting-started-with-typescript.html) - runoob (HTML) 682 | - [TypeScript 中文网](https://www.tslang.cn) (HTML) 683 | - [TypeScript Deep Dive 中文版](https://github.com/jkchao/typescript-book-chinese) - 三毛 (HTML) 684 | - [TypeScript Handbook（中文版）](https://www.runoob.com/manual/gitbook/TypeScript/_book/) - Patrick Zhong (HTML) 685 | 686 | #### Angular 687 | 688 | > :information_source: See also … [AngularJS](#angularjs) 689 | 690 | - [Angular 文档简介](https://angular.cn/docs) - Wang Zhicheng, Ye Zhimin, Yang Lin et al. (HTML) 691 | - [Angular Material 组件库](https://material.angular.cn) - Wang Zhicheng, Ye Zhimin, Yang Lin et al. (HTML) 692 | - [Angular Tutorial (教程：英雄之旅)](https://angular.cn/tutorial) - Wang Zhicheng, Ye Zhimin, Yang Lin et al. (HTML) 693 | 694 | #### Deno 695 | 696 | - [Deno 钻研之术](https://deno-tutorial.js.org) 697 | - [Deno 进阶开发笔记](https://chenshenhai.com/deno_note) - 大深海 698 | 699 | ### VBA (Microsoft Visual Basic Applications) 700 | 701 | - [简明 Excel VBA](https://github.com/Youchien/concise-excel-vba) 702 | 703 | ### Vim 704 | 705 | - [大家來學 VIM](http://www.study-area.org/tips/vim/index.html) 706 | 707 | ### Visual Prolog 708 | 709 | - [Visual Prolog 7 边练边学](http://wiki.visual-prolog.com/index.php?title=Visual_Prolog_for_Tyros_in_Chinese) 710 | - [Visual Prolog 7 初学指南](http://wiki.visual-prolog.com/index.php?title=A_Beginners_Guide_to_Visual_Prolog_in_Chinese) 711 | --------------------------------------------------------------------------------