├── CHANGELOG.txt ├── packages ├── backend │ ├── src │ │ ├── README.md │ │ ├── Components │ │ │ ├── Monitor │ │ │ │ ├── Router │ │ │ │ │ ├── index.ts │ │ │ │ │ └── monitorRouter.ts │ │ │ │ ├── Types │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── monitorSymbols.ts │ │ │ │ │ └── monitorTypes.ts │ │ │ │ └── Service │ │ │ │ │ └── monitorApi.ts │ │ │ ├── Account │ │ │ │ ├── Router │ │ │ │ │ └── index.ts │ │ │ │ ├── Service │ │ │ │ │ ├── index.ts │ │ │ │ │ └── accountApi.ts │ │ │ │ └── Types │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── accountSymbols.ts │ │ │ │ │ └── accountTypes.ts │ │ │ ├── Post │ │ │ │ ├── Router │ │ │ │ │ └── index.ts │ │ │ │ ├── Service │ │ │ │ │ ├── index.ts │ │ │ │ │ └── postApi.ts │ │ │ │ ├── Types │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── postSymbols.ts │ │ │ │ │ └── postTypes.ts │ │ │ │ └── DAL │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── postMigration.ts │ │ │ │ │ ├── postSchema.ts │ │ │ │ │ └── postDAL.ts │ │ │ ├── User │ │ │ │ ├── Router │ │ │ │ │ ├── index.ts │ │ │ │ │ └── userRouter.ts │ │ │ │ ├── Service │ │ │ │ │ ├── index.ts │ │ │ │ │ └── userApi.ts │ │ │ │ ├── DAL │ │ │ │ │ ├── userMigration.ts │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── userSchema.ts │ │ │ │ │ └── userDAL.ts │ │ │ │ └── Types │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── userSymbols.ts │ │ │ │ │ └── userTypes.ts │ │ │ ├── Comment │ │ │ │ ├── Router │ │ │ │ │ └── index.ts │ │ │ │ ├── Types │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── commentSymbols.ts │ │ │ │ │ └── commentTypes.ts │ │ │ │ ├── Service │ │ │ │ │ ├── index.ts │ │ │ │ │ └── commentApi.ts │ │ │ │ └── DAL │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── commentMigration.ts │ │ │ │ │ ├── commentSchema.ts │ │ │ │ │ └── commentDAL.ts │ │ │ ├── Image │ │ │ │ ├── Service │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── imageApi.ts │ │ │ │ │ └── imageService.ts │ │ │ │ └── Types │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── imageSymbols.ts │ │ │ │ │ └── imageTypes.ts │ │ │ ├── Video │ │ │ │ ├── Service │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── videoApi.ts │ │ │ │ │ └── videoService.ts │ │ │ │ └── Types │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── videoSymbols.ts │ │ │ │ │ └── videoTypes.ts │ │ │ ├── SubComment │ │ │ │ ├── Router │ │ │ │ │ └── index.ts │ │ │ │ ├── Service │ │ │ │ │ ├── index.ts │ │ │ │ │ └── subCommentApi.ts │ │ │ │ ├── Types │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── subCommentSymbols.ts │ │ │ │ │ └── subCommentTypes.ts │ │ │ │ └── DAL │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── subCommentMigration.ts │ │ │ │ │ ├── subCommentSchema.ts │ │ │ │ │ └── subCommentDAL.ts │ │ │ ├── RepostComment │ │ │ │ ├── Router │ │ │ │ │ └── index.ts │ │ │ │ ├── Service │ │ │ │ │ ├── index.ts │ │ │ │ │ └── repostCommentApi.ts │ │ │ │ ├── Types │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── repostCommentSymbols.ts │ │ │ │ │ └── repostCommentTypes.ts │ │ │ │ └── DAL │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── repostCommentMigration.ts │ │ │ │ │ ├── repostCommentSchema.ts │ │ │ │ │ └── repostCommentDAL.ts │ │ │ └── Base │ │ │ │ └── baseTypes.ts │ │ ├── Logger │ │ │ ├── index.ts │ │ │ └── logger.ts │ │ ├── Jobs │ │ │ ├── Queue │ │ │ │ ├── index.ts │ │ │ │ ├── timeWindow.ts │ │ │ │ └── queue.ts │ │ │ └── Scheduler │ │ │ │ ├── index.ts │ │ │ │ └── scheduler.ts │ │ ├── Utility │ │ │ ├── cookie │ │ │ │ ├── index.ts │ │ │ │ └── cookie.ts │ │ │ ├── extractHtml │ │ │ │ ├── index.ts │ │ │ │ └── extractHtml.ts │ │ │ ├── urlParse │ │ │ │ ├── index.ts │ │ │ │ └── getLastSegment.ts │ │ │ ├── hideString │ │ │ │ ├── index.ts │ │ │ │ └── hideString.ts │ │ │ ├── generateQuery │ │ │ │ ├── index.ts │ │ │ │ └── generateQuery.ts │ │ │ ├── parsePostId │ │ │ │ ├── index.ts │ │ │ │ └── parsePostId.ts │ │ │ ├── json │ │ │ │ ├── index.ts │ │ │ │ ├── json.ts │ │ │ │ └── jsonStream.ts │ │ │ ├── showProgress │ │ │ │ ├── index.ts │ │ │ │ └── progress.ts │ │ │ ├── readCredential │ │ │ │ ├── index.ts │ │ │ │ └── readCredential.ts │ │ │ ├── parseFollowersCount │ │ │ │ └── parseFollowersCount.ts │ │ │ └── migrate │ │ │ │ └── migrate.ts │ │ ├── Error │ │ │ ├── ErrorHandler │ │ │ │ ├── index.ts │ │ │ │ └── errorHandler.ts │ │ │ └── ErrorClass │ │ │ │ ├── index.ts │ │ │ │ ├── NotImplementedError.ts │ │ │ │ ├── DatabaseError.ts │ │ │ │ ├── ResourceError.ts │ │ │ │ ├── BaseError.ts │ │ │ │ ├── ServerError.ts │ │ │ │ ├── ConflictError.ts │ │ │ │ ├── NotFoundError.ts │ │ │ │ └── BadRequestError.ts │ │ ├── index.ts │ │ ├── Config │ │ │ ├── index.ts │ │ │ ├── httpCode.ts │ │ │ ├── paths.ts │ │ │ ├── constants.ts │ │ │ └── axios.ts │ │ └── Loaders │ │ │ ├── initFolders.ts │ │ │ ├── rxdb.ts │ │ │ └── express.ts │ ├── nodemon.json │ └── package.json ├── frontend │ ├── .env │ ├── src │ │ ├── Component │ │ │ ├── CookieBox │ │ │ │ ├── index.ts │ │ │ │ └── CookieBox.module.scss │ │ │ ├── AccountModal │ │ │ │ ├── index.ts │ │ │ │ ├── accountModal.module.scss │ │ │ │ └── accountModal.tsx │ │ │ ├── MonitorIntro │ │ │ │ ├── index.ts │ │ │ │ └── MonitorIntro.tsx │ │ │ ├── MonitorPopover │ │ │ │ ├── index.tsx │ │ │ │ └── monitorPopover.module.scss │ │ │ ├── PostCard │ │ │ │ ├── index.ts │ │ │ │ ├── cardItems.module.scss │ │ │ │ ├── index.module.scss │ │ │ │ └── PhotosPreviewer.tsx │ │ │ ├── CommentList │ │ │ │ ├── index.tsx │ │ │ │ └── CommentList.module.scss │ │ │ ├── SavePostModal │ │ │ │ ├── index.ts │ │ │ │ └── SavePostModal.tsx │ │ │ ├── SubCommentList │ │ │ │ ├── index.ts │ │ │ │ └── SubCommentList.module.scss │ │ │ ├── RepostCommentList │ │ │ │ └── index.ts │ │ │ ├── AppSider │ │ │ │ ├── Collapser.module.scss │ │ │ │ ├── AppSider.module.scss │ │ │ │ ├── Collapser.tsx │ │ │ │ └── AppSider.tsx │ │ │ ├── DatePicker │ │ │ │ └── DatePicker.ts │ │ │ ├── PostQuery │ │ │ │ └── PostQuery.module.scss │ │ │ ├── ExportModal │ │ │ │ └── ExportModal.tsx │ │ │ ├── AppHeader │ │ │ │ └── AppHeader.module.scss │ │ │ ├── DebounceSelect │ │ │ │ └── DebounceSelect.jsx │ │ │ └── ImportModal │ │ │ │ └── ImportModal.tsx │ │ ├── Utility │ │ │ ├── timeFormat │ │ │ │ ├── index.ts │ │ │ │ └── timeFormat.ts │ │ │ ├── route │ │ │ │ ├── index.ts │ │ │ │ ├── useQuery.ts │ │ │ │ ├── generateQueryString.ts │ │ │ │ └── getRouteState.ts │ │ │ └── parseUrl │ │ │ │ ├── index.ts │ │ │ │ ├── getImageUrl.ts │ │ │ │ └── getVideoUrl.ts │ │ ├── react-app-env.d.ts │ │ ├── Pages │ │ │ ├── Home │ │ │ │ ├── index.ts │ │ │ │ └── Home.module.scss │ │ │ ├── PostContent │ │ │ │ ├── index.ts │ │ │ │ └── PostContent.tsx │ │ │ └── CommentContent │ │ │ │ ├── index.ts │ │ │ │ └── CommentContent.tsx │ │ ├── App.module.scss │ │ ├── Store │ │ │ ├── actions │ │ │ │ ├── index.ts │ │ │ │ ├── account.ts │ │ │ │ └── routeState.ts │ │ │ ├── index.ts │ │ │ ├── reducers │ │ │ │ ├── index.ts │ │ │ │ ├── account.ts │ │ │ │ └── routeState.ts │ │ │ ├── actionTypes.ts │ │ │ └── store.ts │ │ ├── Routes │ │ │ ├── index.ts │ │ │ └── app.ts │ │ ├── Api │ │ │ ├── config.ts │ │ │ ├── index.ts │ │ │ ├── user.ts │ │ │ ├── import.ts │ │ │ ├── account.ts │ │ │ ├── comment.ts │ │ │ ├── post.ts │ │ │ └── monitor.ts │ │ ├── setupTests.ts │ │ ├── App.test.tsx │ │ ├── index.css │ │ ├── index.tsx │ │ ├── App.tsx │ │ ├── types.ts │ │ └── logo.svg │ ├── public │ │ ├── robots.txt │ │ ├── favicon.ico │ │ ├── logo192.png │ │ ├── logo512.png │ │ ├── manifest.json │ │ └── index.html │ ├── tsconfig.json │ ├── package.json │ └── README.md ├── extension-chrome │ ├── src │ │ ├── react-app-env.d.ts │ │ ├── App.css │ │ ├── setupTests.ts │ │ ├── App.test.tsx │ │ ├── index.css │ │ ├── App.tsx │ │ ├── reportWebVitals.ts │ │ ├── index.tsx │ │ ├── Components │ │ │ ├── Crawler │ │ │ │ └── Crawler.tsx │ │ │ └── Server │ │ │ │ └── Server.tsx │ │ └── logo.svg │ ├── public │ │ ├── robots.txt │ │ ├── favicon.ico │ │ ├── logo192.png │ │ ├── logo512.png │ │ ├── manifest.json │ │ └── index.html │ ├── .gitignore │ ├── tsconfig.json │ ├── package.json │ └── README.md └── documentation │ ├── docs │ ├── .vuepress │ │ ├── components │ │ │ ├── OtherComponent.vue │ │ │ ├── Foo │ │ │ │ └── Bar.vue │ │ │ └── demo-component.vue │ │ ├── styles │ │ │ ├── index.styl │ │ │ └── palette.styl │ │ ├── public │ │ │ ├── simple-workflow.svg:Zone.Identifier │ │ │ └── logo.svg │ │ └── enhanceApp.js │ ├── guide │ │ ├── debug.png │ │ ├── headbook.png │ │ ├── build.md │ │ ├── disclaimer.md │ │ ├── publish.md │ │ ├── video.md │ │ ├── user.md │ │ ├── account.md │ │ ├── monitor.md │ │ └── get-start.md │ ├── zh │ │ ├── guide │ │ │ ├── debug.png │ │ │ ├── disclaimer.md │ │ │ ├── headbook.png │ │ │ ├── build.md │ │ │ ├── publish.md │ │ │ ├── video.md │ │ │ ├── user.md │ │ │ ├── account.md │ │ │ ├── monitor.md │ │ │ ├── README.md │ │ │ └── get-start.md │ │ └── index.md │ └── index.md │ ├── .npmignore │ └── package.json ├── .vscode ├── settings.json └── launch.json ├── .prettierrc ├── lerna.json ├── .dockerignore ├── .gitignore ├── Dockerfile ├── .github └── workflows │ ├── publish.yml_backup │ └── deploy-doc.yml_backup ├── README_zh-CN.md ├── package.json ├── LICENSE └── README.md /CHANGELOG.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/backend/src/README.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Monitor/Router/index.ts: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/frontend/.env: -------------------------------------------------------------------------------- 1 | DANGEROUSLY_DISABLE_HOST_CHECK=true -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "prettier.requireConfig": true 3 | } -------------------------------------------------------------------------------- /packages/backend/src/Logger/index.ts: -------------------------------------------------------------------------------- 1 | export * from './logger'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Jobs/Queue/index.ts: -------------------------------------------------------------------------------- 1 | export * from './queue'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Jobs/Scheduler/index.ts: -------------------------------------------------------------------------------- 1 | export * from './scheduler'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/cookie/index.ts: -------------------------------------------------------------------------------- 1 | export * from './cookie'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/extractHtml/index.ts: -------------------------------------------------------------------------------- 1 | export * from './extractHtml' -------------------------------------------------------------------------------- /packages/backend/src/Utility/urlParse/index.ts: -------------------------------------------------------------------------------- 1 | export * from './getLastSegment' -------------------------------------------------------------------------------- /packages/backend/src/Utility/hideString/index.ts: -------------------------------------------------------------------------------- 1 | export * from './hideString'; 2 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/CookieBox/index.ts: -------------------------------------------------------------------------------- 1 | export * from './CookieBox'; 2 | -------------------------------------------------------------------------------- /packages/frontend/src/Utility/timeFormat/index.ts: -------------------------------------------------------------------------------- 1 | export * from './timeFormat'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Account/Router/index.ts: -------------------------------------------------------------------------------- 1 | export * from './accountRouter' -------------------------------------------------------------------------------- /packages/backend/src/Components/Post/Router/index.ts: -------------------------------------------------------------------------------- 1 | export * from './postRouter'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Post/Service/index.ts: -------------------------------------------------------------------------------- 1 | export * from './postService'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Components/User/Router/index.ts: -------------------------------------------------------------------------------- 1 | export * from './userRouter'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Components/User/Service/index.ts: -------------------------------------------------------------------------------- 1 | export * from './userService'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Error/ErrorHandler/index.ts: -------------------------------------------------------------------------------- 1 | export * from './errorHandler'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/generateQuery/index.ts: -------------------------------------------------------------------------------- 1 | export * from './generateQuery'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/parsePostId/index.ts: -------------------------------------------------------------------------------- 1 | export * from './parsePostId'; 2 | -------------------------------------------------------------------------------- /packages/frontend/src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Comment/Router/index.ts: -------------------------------------------------------------------------------- 1 | export * from './commentRouter'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Image/Service/index.ts: -------------------------------------------------------------------------------- 1 | export * from './imageService'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Video/Service/index.ts: -------------------------------------------------------------------------------- 1 | export * from './videoService'; 2 | -------------------------------------------------------------------------------- /packages/extension-chrome/src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/AccountModal/index.ts: -------------------------------------------------------------------------------- 1 | export * from './accountModal'; 2 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/MonitorIntro/index.ts: -------------------------------------------------------------------------------- 1 | export * from './MonitorIntro'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Account/Service/index.ts: -------------------------------------------------------------------------------- 1 | export * from './accountService'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Components/SubComment/Router/index.ts: -------------------------------------------------------------------------------- 1 | export * from './subCommentRouter'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/index.ts: -------------------------------------------------------------------------------- 1 | import { loader } from './Loaders/loader'; 2 | 3 | loader(); 4 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/MonitorPopover/index.tsx: -------------------------------------------------------------------------------- 1 | export * from './MonitorPopover'; 2 | -------------------------------------------------------------------------------- /packages/frontend/src/Pages/Home/index.ts: -------------------------------------------------------------------------------- 1 | import Home from './Home'; 2 | 3 | export { Home }; 4 | -------------------------------------------------------------------------------- /packages/backend/src/Components/SubComment/Service/index.ts: -------------------------------------------------------------------------------- 1 | export * from './subCommentService'; 2 | -------------------------------------------------------------------------------- /packages/frontend/src/App.module.scss: -------------------------------------------------------------------------------- 1 | .inner-layout { 2 | min-height: calc(100vh - 65px); 3 | } 4 | -------------------------------------------------------------------------------- /packages/backend/src/Components/RepostComment/Router/index.ts: -------------------------------------------------------------------------------- 1 | export * from './repostCommentRouter'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Components/RepostComment/Service/index.ts: -------------------------------------------------------------------------------- 1 | export * from './repostCommentService'; 2 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/json/index.ts: -------------------------------------------------------------------------------- 1 | export * from './json'; 2 | export * from './jsonStream'; 3 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/PostCard/index.ts: -------------------------------------------------------------------------------- 1 | import PostCard from './PostCard'; 2 | export { PostCard }; 3 | -------------------------------------------------------------------------------- /packages/frontend/src/Store/actions/index.ts: -------------------------------------------------------------------------------- 1 | export * from './routeState'; 2 | export * from './account'; 3 | -------------------------------------------------------------------------------- /packages/backend/src/Components/User/DAL/userMigration.ts: -------------------------------------------------------------------------------- 1 | const userMigration = {}; 2 | export { userMigration }; 3 | -------------------------------------------------------------------------------- /packages/frontend/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Image/Types/index.ts: -------------------------------------------------------------------------------- 1 | export * from './imageTypes'; 2 | export * from './imageSymbols'; 3 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Monitor/Types/index.ts: -------------------------------------------------------------------------------- 1 | export * from './monitorTypes' 2 | export * from './monitorSymbols' -------------------------------------------------------------------------------- /packages/backend/src/Components/Post/Types/index.ts: -------------------------------------------------------------------------------- 1 | export * from './postTypes'; 2 | export * from './postSymbols'; 3 | -------------------------------------------------------------------------------- /packages/backend/src/Components/User/Types/index.ts: -------------------------------------------------------------------------------- 1 | export * from './userSymbols'; 2 | export * from './userTypes'; 3 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Video/Types/index.ts: -------------------------------------------------------------------------------- 1 | export * from './videoSymbols'; 2 | export * from './videoTypes'; 3 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/PostCard/cardItems.module.scss: -------------------------------------------------------------------------------- 1 | .card-item-icon-selected { 2 | color: #1890ff; 3 | } 4 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Account/Types/index.ts: -------------------------------------------------------------------------------- 1 | export * from './accountSymbols'; 2 | export * from './accountTypes'; 3 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Comment/Types/index.ts: -------------------------------------------------------------------------------- 1 | export * from './commentTypes'; 2 | export * from './commentSymbols'; 3 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/showProgress/index.ts: -------------------------------------------------------------------------------- 1 | import showProgress from './progress'; 2 | 3 | export { showProgress }; 4 | -------------------------------------------------------------------------------- /packages/extension-chrome/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /packages/frontend/src/Pages/PostContent/index.ts: -------------------------------------------------------------------------------- 1 | import PostContent from './PostContent'; 2 | 3 | export { PostContent }; 4 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "all", 3 | "tabWidth": 2, 4 | "arrowParens": "always", 5 | "singleQuote": true 6 | } 7 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Comment/Service/index.ts: -------------------------------------------------------------------------------- 1 | export * from './commentService'; 2 | export * from './commentCrawler'; 3 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/CommentList/index.tsx: -------------------------------------------------------------------------------- 1 | import CommentList from './CommentList'; 2 | 3 | export { CommentList }; 4 | -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": [ 3 | "packages/*", 4 | "./" 5 | ], 6 | "version": "1.3.0", 7 | "npmClient": "npm" 8 | } 9 | -------------------------------------------------------------------------------- /packages/backend/src/Components/SubComment/Types/index.ts: -------------------------------------------------------------------------------- 1 | export * from './subCommentSymbols'; 2 | export * from './subCommentTypes'; 3 | -------------------------------------------------------------------------------- /packages/frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Combo819/social-media-archiver/HEAD/packages/frontend/public/favicon.ico -------------------------------------------------------------------------------- /packages/frontend/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Combo819/social-media-archiver/HEAD/packages/frontend/public/logo192.png -------------------------------------------------------------------------------- /packages/frontend/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Combo819/social-media-archiver/HEAD/packages/frontend/public/logo512.png -------------------------------------------------------------------------------- /packages/frontend/src/Component/SavePostModal/index.ts: -------------------------------------------------------------------------------- 1 | import SavePostModal from './SavePostModal'; 2 | 3 | export { SavePostModal }; 4 | -------------------------------------------------------------------------------- /packages/frontend/src/Pages/CommentContent/index.ts: -------------------------------------------------------------------------------- 1 | import CommentContent from './CommentContent'; 2 | 3 | export { CommentContent }; 4 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/SubCommentList/index.ts: -------------------------------------------------------------------------------- 1 | import SubCommentList from './SubCommentList'; 2 | 3 | export { SubCommentList }; 4 | -------------------------------------------------------------------------------- /packages/frontend/src/Routes/index.ts: -------------------------------------------------------------------------------- 1 | import routes, { Route } from './app'; 2 | 3 | export { routes }; 4 | export type RouteType = Route; 5 | -------------------------------------------------------------------------------- /packages/backend/src/Components/RepostComment/Types/index.ts: -------------------------------------------------------------------------------- 1 | export * from './repostCommentSymbols'; 2 | export * from './repostCommentTypes'; 3 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Post/DAL/index.ts: -------------------------------------------------------------------------------- 1 | export * from './postDAL'; 2 | export * from './postSchema'; 3 | export * from './postMigration'; 4 | -------------------------------------------------------------------------------- /packages/backend/src/Components/User/DAL/index.ts: -------------------------------------------------------------------------------- 1 | export * from './userDAL'; 2 | export * from './userSchema'; 3 | export * from './userMigration'; 4 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/readCredential/index.ts: -------------------------------------------------------------------------------- 1 | import { getCredentialFile } from './readCredential'; 2 | 3 | export { getCredentialFile }; 4 | -------------------------------------------------------------------------------- /packages/documentation/docs/.vuepress/components/OtherComponent.vue: -------------------------------------------------------------------------------- 1 | 4 | -------------------------------------------------------------------------------- /packages/documentation/docs/guide/debug.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Combo819/social-media-archiver/HEAD/packages/documentation/docs/guide/debug.png -------------------------------------------------------------------------------- /packages/frontend/src/Component/RepostCommentList/index.ts: -------------------------------------------------------------------------------- 1 | import RepostCommentList from './RepostCommentList'; 2 | export { RepostCommentList }; 3 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Post/DAL/postMigration.ts: -------------------------------------------------------------------------------- 1 | import dayjs from 'dayjs'; 2 | 3 | const postMigration = {}; 4 | 5 | export { postMigration }; 6 | -------------------------------------------------------------------------------- /packages/documentation/docs/guide/headbook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Combo819/social-media-archiver/HEAD/packages/documentation/docs/guide/headbook.png -------------------------------------------------------------------------------- /packages/documentation/docs/zh/guide/debug.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Combo819/social-media-archiver/HEAD/packages/documentation/docs/zh/guide/debug.png -------------------------------------------------------------------------------- /packages/documentation/docs/zh/guide/disclaimer.md: -------------------------------------------------------------------------------- 1 | # 免责声明 2 | 本项目采用MIT协议。自行承担使用本项目所引起的任何风险。 3 | 请遵守各平台API的条款和条件。如有必要,请及时删除下载的内容。 4 | 本项目用于学习与交流,请勿公开部署。 -------------------------------------------------------------------------------- /packages/extension-chrome/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Combo819/social-media-archiver/HEAD/packages/extension-chrome/public/favicon.ico -------------------------------------------------------------------------------- /packages/extension-chrome/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Combo819/social-media-archiver/HEAD/packages/extension-chrome/public/logo192.png -------------------------------------------------------------------------------- /packages/extension-chrome/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Combo819/social-media-archiver/HEAD/packages/extension-chrome/public/logo512.png -------------------------------------------------------------------------------- /packages/frontend/src/Utility/route/index.ts: -------------------------------------------------------------------------------- 1 | export * from './generateQueryString'; 2 | export * from './getRouteState'; 3 | export * from './useQuery'; 4 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Comment/DAL/index.ts: -------------------------------------------------------------------------------- 1 | export * from './commentDAL'; 2 | export * from './commentSchema'; 3 | export * from './commentMigration'; 4 | -------------------------------------------------------------------------------- /packages/backend/src/Config/index.ts: -------------------------------------------------------------------------------- 1 | export * from './constants'; 2 | export * from './axios'; 3 | export * from './httpCode'; 4 | export * from './paths'; 5 | -------------------------------------------------------------------------------- /packages/documentation/docs/zh/guide/headbook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Combo819/social-media-archiver/HEAD/packages/documentation/docs/zh/guide/headbook.png -------------------------------------------------------------------------------- /packages/documentation/docs/zh/guide/build.md: -------------------------------------------------------------------------------- 1 | # 打包 2 | 完成代码之后,就可以为Linux、Mac和Windows构建二进制可执行文件了。 3 | 在项目根目录下,运行: 4 | ```bash 5 | npm run dist 6 | ``` 7 | 打包后的文件将会在 `dist` 目录中。 -------------------------------------------------------------------------------- /packages/backend/src/Components/Comment/DAL/commentMigration.ts: -------------------------------------------------------------------------------- 1 | import dayjs from 'dayjs'; 2 | 3 | const commentMigration = { 4 | }; 5 | 6 | export { commentMigration }; 7 | -------------------------------------------------------------------------------- /packages/backend/src/Components/SubComment/DAL/index.ts: -------------------------------------------------------------------------------- 1 | export * from './subCommentDAL'; 2 | export * from './subCommentMigration'; 3 | export * from './subCommentSchema'; 4 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Video/Types/videoSymbols.ts: -------------------------------------------------------------------------------- 1 | const VIDEO_IOC_SYMBOLS = { 2 | IVideoService: Symbol('IVideoService'), 3 | }; 4 | export { VIDEO_IOC_SYMBOLS }; 5 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/CommentList/CommentList.module.scss: -------------------------------------------------------------------------------- 1 | .comment-list-pagination { 2 | margin-top: 16px; 3 | margin-bottom: 16px; 4 | float: right; 5 | } 6 | -------------------------------------------------------------------------------- /packages/frontend/src/Store/index.ts: -------------------------------------------------------------------------------- 1 | import { RootState as RootStateType } from './reducers'; 2 | export * from './actions'; 3 | 4 | export type RootState = RootStateType; 5 | -------------------------------------------------------------------------------- /packages/backend/src/Components/SubComment/DAL/subCommentMigration.ts: -------------------------------------------------------------------------------- 1 | import dayjs from 'dayjs'; 2 | 3 | const subCommentMigration = {}; 4 | 5 | export { subCommentMigration }; 6 | -------------------------------------------------------------------------------- /packages/backend/src/Components/RepostComment/DAL/index.ts: -------------------------------------------------------------------------------- 1 | export * from './repostCommentDAL'; 2 | export * from './repostCommentSchema'; 3 | export * from './repostCommentMigration'; 4 | -------------------------------------------------------------------------------- /packages/frontend/src/Api/config.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | 3 | const BASE_URL = ''; 4 | 5 | axios.defaults.baseURL = BASE_URL + '/api'; 6 | 7 | export { axios, BASE_URL }; 8 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/SubCommentList/SubCommentList.module.scss: -------------------------------------------------------------------------------- 1 | .sub-comment-list-pagination { 2 | margin-top: 16px; 3 | margin-bottom: 16px; 4 | float: right; 5 | } 6 | -------------------------------------------------------------------------------- /packages/frontend/src/Utility/parseUrl/index.ts: -------------------------------------------------------------------------------- 1 | import { getImageUrl } from './getImageUrl'; 2 | import { getVideoUrl } from './getVideoUrl'; 3 | export { getImageUrl, getVideoUrl }; 4 | -------------------------------------------------------------------------------- /packages/extension-chrome/src/App.css: -------------------------------------------------------------------------------- 1 | @import '~antd/dist/antd.css'; 2 | .App { 3 | text-align: center; 4 | width: 400px; 5 | height: 300px; 6 | padding: 20px 30px; 7 | } 8 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Account/Types/accountSymbols.ts: -------------------------------------------------------------------------------- 1 | const ACCOUNT_IOC_SYMBOLS = { 2 | IAccountService: Symbol('IAccountService'), 3 | }; 4 | 5 | export { ACCOUNT_IOC_SYMBOLS }; 6 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Monitor/Types/monitorSymbols.ts: -------------------------------------------------------------------------------- 1 | const MONITOR_IOC_SYMBOLS = { 2 | IMonitorService: Symbol.for('IMonitorService'), 3 | }; 4 | 5 | export { MONITOR_IOC_SYMBOLS }; -------------------------------------------------------------------------------- /packages/backend/src/Components/RepostComment/DAL/repostCommentMigration.ts: -------------------------------------------------------------------------------- 1 | import dayjs from 'dayjs'; 2 | 3 | const repostCommentMigration = {}; 4 | 5 | export { repostCommentMigration }; 6 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Image/Types/imageSymbols.ts: -------------------------------------------------------------------------------- 1 | const IMAGE_IOC_SYMBOLS = { 2 | ImageServiceInterface: Symbol('ImageServiceInterface'), 3 | }; 4 | 5 | export { IMAGE_IOC_SYMBOLS }; 6 | -------------------------------------------------------------------------------- /packages/frontend/src/Pages/Home/Home.module.scss: -------------------------------------------------------------------------------- 1 | .home-card{ 2 | margin-top: 16px; 3 | } 4 | 5 | .home-pagination{ 6 | margin-top: 16px; 7 | margin-bottom: 16px; 8 | float: right; 9 | } -------------------------------------------------------------------------------- /packages/frontend/src/Api/index.ts: -------------------------------------------------------------------------------- 1 | export * from './post'; 2 | export * from './comment'; 3 | export * from './monitor'; 4 | export * from './user'; 5 | export * from './account'; 6 | export * from './import' -------------------------------------------------------------------------------- /packages/backend/src/Components/User/Types/userSymbols.ts: -------------------------------------------------------------------------------- 1 | const USER_IOC_SYMBOLS = { 2 | IUserService: Symbol('IUserService'), 3 | IUserDAL: Symbol('IUserDAL'), 4 | }; 5 | 6 | export { USER_IOC_SYMBOLS }; 7 | -------------------------------------------------------------------------------- /packages/documentation/.npmignore: -------------------------------------------------------------------------------- 1 | pids 2 | logs 3 | node_modules 4 | npm-debug.log 5 | coverage/ 6 | run 7 | dist 8 | .DS_Store 9 | .nyc_output 10 | .basement 11 | config.local.js 12 | basement_dist 13 | -------------------------------------------------------------------------------- /packages/backend/nodemon.json: -------------------------------------------------------------------------------- 1 | { 2 | "watch": ["src"], 3 | "ext": ".ts,.js", 4 | "ignore": [], 5 | "exec": "ts-node ./src/index.ts", 6 | "env": { 7 | "NODE_ENV": "development" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /packages/frontend/src/Utility/route/useQuery.ts: -------------------------------------------------------------------------------- 1 | import { useLocation } from 'react-router-dom'; 2 | 3 | function useQuery() { 4 | return new URLSearchParams(useLocation().search); 5 | } 6 | 7 | export { useQuery }; 8 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /.vscode 3 | credential.json 4 | /static 5 | /storage 6 | /*-rxdb-* 7 | /credential* 8 | **/web 9 | /dist 10 | /copyScript.js 11 | /rxdb 12 | /src/images 13 | **/node_modules 14 | **/.vscode 15 | -------------------------------------------------------------------------------- /packages/backend/src/Config/httpCode.ts: -------------------------------------------------------------------------------- 1 | const HTTP_STATUS_CODE = { 2 | OK: 200, 3 | BAD_REQUEST: 400, 4 | NOT_FOUND: 404, 5 | INTERNAL_SERVER: 500, 6 | CONFLICT: 409, 7 | }; 8 | 9 | export { HTTP_STATUS_CODE }; 10 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/MonitorPopover/monitorPopover.module.scss: -------------------------------------------------------------------------------- 1 | .card { 2 | .remove-icon { 3 | color: #ff4d4f; 4 | cursor: pointer; 5 | } 6 | .validate-icon{ 7 | cursor: pointer; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /packages/documentation/docs/zh/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | home: true 3 | heroImage: /logo.svg 4 | tagline: 一个Node.js模板,可存档任何社交媒体的帖子 5 | actionText: 快速开始 → 6 | actionLink: /zh/guide/ 7 | features: null 8 | footer: Made by with ❤️ 9 | --- 10 | -------------------------------------------------------------------------------- /packages/frontend/src/Utility/parseUrl/getImageUrl.ts: -------------------------------------------------------------------------------- 1 | import { BASE_URL } from '../../Api/config'; 2 | const getImageUrl = (fileName: string): string => { 3 | return `${BASE_URL}/images/${fileName}`; 4 | }; 5 | 6 | export { getImageUrl }; 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /static 3 | **/build 4 | **/credential* 5 | **/dist 6 | /rxdb 7 | **/log 8 | **/node_modules 9 | **/storage 10 | 11 | npm-debug.log* 12 | yarn-debug.log* 13 | yarn-error.log* 14 | lerna-debug.log* 15 | .next 16 | out -------------------------------------------------------------------------------- /packages/backend/src/Components/Post/Types/postSymbols.ts: -------------------------------------------------------------------------------- 1 | const POST_IOC_SYMBOLS = { 2 | IPostService: Symbol('IPostService'), 3 | IPostDAL: Symbol('IPostDAL'), 4 | IPostCrawler: Symbol('IPostCrawler'), 5 | }; 6 | 7 | export { POST_IOC_SYMBOLS }; 8 | -------------------------------------------------------------------------------- /packages/frontend/src/Utility/parseUrl/getVideoUrl.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import { BASE_URL } from '../../Api/config'; 3 | const getVideoUrl = (fileName:string): string => { 4 | return `${BASE_URL}/videos/${fileName}`; 5 | }; 6 | 7 | export { getVideoUrl }; 8 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Video/Types/videoTypes.ts: -------------------------------------------------------------------------------- 1 | type ParamsQueue = { 2 | url: string; 3 | staticPath: string; 4 | }; 5 | 6 | interface IVideoService { 7 | downloadVideo: (videoUrl: string) => void; 8 | } 9 | 10 | export { ParamsQueue, IVideoService }; 11 | -------------------------------------------------------------------------------- /packages/documentation/docs/.vuepress/styles/index.styl: -------------------------------------------------------------------------------- 1 | /** 2 | * Custom Styles here. 3 | * 4 | * ref:https://v1.vuepress.vuejs.org/config/#index-styl 5 | */ 6 | 7 | .home .hero img 8 | max-width 450px!important 9 | 10 | p>img 11 | display block 12 | margin auto -------------------------------------------------------------------------------- /packages/frontend/src/Component/AccountModal/accountModal.module.scss: -------------------------------------------------------------------------------- 1 | .profile{ 2 | .avatar{ 3 | display: inline-block; 4 | } 5 | .username{ 6 | margin-left: 20px; 7 | display: inline-block; 8 | } 9 | margin-bottom: 20px; 10 | } -------------------------------------------------------------------------------- /packages/frontend/src/Component/PostCard/index.module.scss: -------------------------------------------------------------------------------- 1 | .deleteButton{ 2 | float: right; 3 | cursor: pointer; 4 | color: rgba(0, 0, 0, 0.45); 5 | } 6 | 7 | .deleteButton:hover{ 8 | color:#ff4d4f 9 | } 10 | 11 | .time{ 12 | color: #00000073; 13 | } -------------------------------------------------------------------------------- /packages/backend/src/Components/Comment/Types/commentSymbols.ts: -------------------------------------------------------------------------------- 1 | const COMMENT_IOC_SYMBOLS = { 2 | ICommentService: Symbol('ICommentService'), 3 | ICommentDAL: Symbol('ICommentDAL'), 4 | ICommentCrawler: Symbol('ICommentCrawler'), 5 | }; 6 | 7 | export { COMMENT_IOC_SYMBOLS }; 8 | -------------------------------------------------------------------------------- /packages/documentation/docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | home: true 3 | heroImage: /logo.svg 4 | tagline: A Node.js template to be implemented to archive post from any social media. 5 | actionText: Quick Start → 6 | actionLink: /guide/ 7 | features: null 8 | footer: Made by with ❤️ 9 | --- 10 | -------------------------------------------------------------------------------- /packages/documentation/docs/.vuepress/styles/palette.styl: -------------------------------------------------------------------------------- 1 | /** 2 | * Custom palette here. 3 | * 4 | * ref:https://v1.vuepress.vuejs.org/zh/config/#palette-styl 5 | */ 6 | 7 | $accentColor = #3eaf7c 8 | $textColor = #2c3e50 9 | $borderColor = #eaecef 10 | $codeBgColor = #282c34 11 | -------------------------------------------------------------------------------- /packages/documentation/docs/guide/build.md: -------------------------------------------------------------------------------- 1 | # Build 2 | After you have completed the code, you can build binary executable files for Linux, Mac, and Windows. 3 | In project root directory, run: 4 | ```bash 5 | npm run dist 6 | ``` 7 | Then, archived files for each platform will be created in `dist` directory. -------------------------------------------------------------------------------- /packages/extension-chrome/src/setupTests.ts: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom'; 6 | -------------------------------------------------------------------------------- /packages/backend/src/Error/ErrorClass/index.ts: -------------------------------------------------------------------------------- 1 | export * from './BadRequestError'; 2 | export * from './DatabaseError'; 3 | export * from './NotFoundError'; 4 | export * from './ResourceError'; 5 | export * from './ServerError'; 6 | export * from './ConflictError'; 7 | export * from './NotImplementedError'; 8 | -------------------------------------------------------------------------------- /packages/frontend/src/setupTests.ts: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom/extend-expect'; 6 | -------------------------------------------------------------------------------- /packages/backend/src/Components/SubComment/Types/subCommentSymbols.ts: -------------------------------------------------------------------------------- 1 | const SUB_COMMENT_IOC_SYMBOLS = { 2 | ISubCommentDAL: Symbol('ISubCommentDAL'), 3 | ISubCommentService: Symbol('ISubCommentService'), 4 | ISubCommentCrawler: Symbol('ISubCommentCrawler'), 5 | }; 6 | 7 | export { SUB_COMMENT_IOC_SYMBOLS }; 8 | -------------------------------------------------------------------------------- /packages/documentation/docs/.vuepress/components/Foo/Bar.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 16 | -------------------------------------------------------------------------------- /packages/frontend/src/Api/user.ts: -------------------------------------------------------------------------------- 1 | import { axios } from './config'; 2 | import { AxiosPromise } from 'axios'; 3 | 4 | function getUsersByNameApi(username: string): AxiosPromise { 5 | return axios({ 6 | url: '/user', 7 | params: { username }, 8 | }); 9 | } 10 | 11 | export { getUsersByNameApi }; 12 | -------------------------------------------------------------------------------- /packages/frontend/src/Store/reducers/index.ts: -------------------------------------------------------------------------------- 1 | import { combineReducers } from 'redux'; 2 | import { account } from './account'; 3 | import { routeState } from './routeState'; 4 | const rootReducer = combineReducers({ routeState, account }); 5 | export type RootState = ReturnType; 6 | export { rootReducer }; 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # run `npm run build && cd frontend&&npm run build` before build the image 2 | # docker 3 | FROM node:12.18.1 4 | WORKDIR /app 5 | COPY ["package.json", "package-lock.json*", "./"] 6 | RUN npm install --production 7 | COPY ./build ./build 8 | COPY ./frontend/build/ ./frontend/build 9 | CMD [ "node","/app/build/index.js" ] -------------------------------------------------------------------------------- /packages/documentation/docs/.vuepress/components/demo-component.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 16 | -------------------------------------------------------------------------------- /packages/frontend/src/Store/actions/account.ts: -------------------------------------------------------------------------------- 1 | import { IUser } from '../../types'; 2 | import { ACCOUNT } from '../actionTypes'; 3 | 4 | const accountCreators = { 5 | setAccount: (payload: IUser | null) => ({ 6 | type: ACCOUNT.SET_ACCOUNT, 7 | payload, 8 | }), 9 | }; 10 | 11 | export { accountCreators }; 12 | -------------------------------------------------------------------------------- /packages/extension-chrome/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "manifest_version": 2, 3 | "name": "social-media-archiver", 4 | "description": "archive a post", 5 | "version": "1.0.0", 6 | "browser_action": { 7 | "default_popup": "index.html" 8 | }, 9 | "permissions": ["http://*/","https://*/","storage"] 10 | } 11 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Image/Types/imageTypes.ts: -------------------------------------------------------------------------------- 1 | type ParamsQueue = { 2 | url: string; 3 | staticPath: string; 4 | }; 5 | 6 | interface ImageServiceInterface { 7 | downloadImage: ( 8 | imageUrl: string, 9 | priority?: number, 10 | ) => void; 11 | } 12 | 13 | export { ParamsQueue, ImageServiceInterface }; 14 | -------------------------------------------------------------------------------- /packages/backend/src/Components/RepostComment/Types/repostCommentSymbols.ts: -------------------------------------------------------------------------------- 1 | const REPOST_COMMENT_IOC_SYMBOLS = { 2 | IRepostCommentDAL: Symbol('IRepostCommentDAL'), 3 | IRepostCommentService: Symbol('IRepostCommentService'), 4 | IRepostCommentCrawler: Symbol('IRepostCommentCrawler'), 5 | }; 6 | 7 | export { REPOST_COMMENT_IOC_SYMBOLS }; 8 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Image/Service/imageApi.ts: -------------------------------------------------------------------------------- 1 | import { downloadAxios } from '../../../Config'; 2 | import { AxiosPromise } from 'axios'; 3 | 4 | function downloadImageApi(url: string): AxiosPromise { 5 | return downloadAxios({ 6 | url, 7 | responseType: 'stream', 8 | }); 9 | } 10 | 11 | export { downloadImageApi }; 12 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Video/Service/videoApi.ts: -------------------------------------------------------------------------------- 1 | import { downloadAxios } from '../../../Config'; 2 | import { AxiosPromise } from 'axios'; 3 | 4 | function downloadVideoApi(url: string): AxiosPromise { 5 | return downloadAxios({ 6 | url, 7 | responseType: 'stream', 8 | }); 9 | } 10 | 11 | export { downloadVideoApi }; 12 | -------------------------------------------------------------------------------- /packages/extension-chrome/src/App.test.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { render, screen } from '@testing-library/react'; 3 | import App from './App'; 4 | 5 | test('renders learn react link', () => { 6 | render(); 7 | const linkElement = screen.getByText(/learn react/i); 8 | expect(linkElement).toBeInTheDocument(); 9 | }); 10 | -------------------------------------------------------------------------------- /packages/frontend/src/App.test.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { render } from '@testing-library/react'; 3 | import App from './App'; 4 | 5 | test('renders learn react link', () => { 6 | const { getByText } = render(); 7 | const linkElement = getByText(/learn react/i); 8 | expect(linkElement).toBeInTheDocument(); 9 | }); 10 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/AppSider/Collapser.module.scss: -------------------------------------------------------------------------------- 1 | .collapser { 2 | right: 0; 3 | position: absolute; 4 | margin-right: -20px; 5 | margin-top: 10px; 6 | background-color: #1890ff; 7 | width: 25px; 8 | padding-left: 3px; 9 | padding-bottom: 5px; 10 | border-radius: 3px; 11 | cursor: pointer; 12 | display: inline; 13 | } 14 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Account/Service/accountApi.ts: -------------------------------------------------------------------------------- 1 | import { AxiosPromise } from 'axios'; 2 | import { axios, BASE_URL } from '../../../Config'; 3 | 4 | function getLoginStatusApi(cookie: string): any { 5 | return { 6 | data: { 7 | login: false, 8 | uid: '', 9 | }, 10 | }; 11 | } 12 | 13 | export { getLoginStatusApi }; 14 | -------------------------------------------------------------------------------- /packages/documentation/docs/guide/disclaimer.md: -------------------------------------------------------------------------------- 1 | # Disclaimer 2 | The social media archiver is under MIT license. Use it at your own risk. 3 | Please obey and respect the platform API terms and conditions. Delete the content in time if necessary. 4 | The social media archiver project is mainly for educational purposes. Using it as public service is not recommended. -------------------------------------------------------------------------------- /packages/frontend/src/Component/DatePicker/DatePicker.ts: -------------------------------------------------------------------------------- 1 | import { Dayjs } from 'dayjs'; 2 | import dayjsGenerateConfig from 'rc-picker/es/generate/dayjs'; 3 | import generatePicker from 'antd/es/date-picker/generatePicker'; 4 | import 'antd/es/date-picker/style/index'; 5 | 6 | const DatePicker = generatePicker(dayjsGenerateConfig); 7 | 8 | export { DatePicker }; 9 | -------------------------------------------------------------------------------- /packages/frontend/src/Store/actionTypes.ts: -------------------------------------------------------------------------------- 1 | const ROUTE_STATE = { 2 | HOME: { 3 | SET_HOME: 'ROUTE_STATE.HOME.SET_HOME', 4 | }, 5 | POST_CONTENT: { 6 | SET_POST_CONTENT: 'ROUTE_STATE.POST_CONTENT_SET_POST_CONTENT', 7 | }, 8 | }; 9 | 10 | const ACCOUNT = { 11 | SET_ACCOUNT: 'ACCOUNT.SET_ACCOUNT', 12 | }; 13 | export { ROUTE_STATE, ACCOUNT }; 14 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Post/Service/postApi.ts: -------------------------------------------------------------------------------- 1 | import { crawlerAxios } from '../../../Config'; 2 | import { AxiosPromise } from 'axios'; 3 | import { NotImplementedError } from '../../../Error/ErrorClass'; 4 | function getPostApi(postId: string): AxiosPromise { 5 | throw new NotImplementedError('getPostApi is not implemented'); 6 | } 7 | 8 | export { getPostApi }; 9 | -------------------------------------------------------------------------------- /packages/backend/src/Error/ErrorClass/NotImplementedError.ts: -------------------------------------------------------------------------------- 1 | import { BaseError } from './BaseError'; 2 | 3 | class NotImplementedError extends BaseError { 4 | constructor(message: string) { 5 | super(message); 6 | this.name = 'NotImplementedError'; 7 | Object.setPrototypeOf(this, NotImplementedError.prototype); 8 | } 9 | } 10 | 11 | export { NotImplementedError }; 12 | -------------------------------------------------------------------------------- /packages/frontend/src/Api/import.ts: -------------------------------------------------------------------------------- 1 | import { axios } from './config'; 2 | 3 | export function importData( 4 | data: FormData, 5 | type: 'post' | 'user' | 'comment' | 'subComment' | 'repostComment', 6 | ) { 7 | return axios({ 8 | method: 'post', 9 | url: `/${type}/import`, 10 | data: data, 11 | headers: { 'Content-Type': 'multipart/form-data' }, 12 | }); 13 | } 14 | -------------------------------------------------------------------------------- /packages/documentation/docs/.vuepress/public/simple-workflow.svg:Zone.Identifier: -------------------------------------------------------------------------------- 1 | [ZoneTransfer] 2 | ZoneId=3 3 | ReferrerUrl=https://app.diagrams.net/#Wb!hxRjnCx2_kuZ83BK28Fpd-Q3VU6TdUNLqCib8fv4Ik_WLzst7_BoT5CbbhvxELA3%2F01DY2LBUNHX5APAUKOR5EIOSCIZOQBKQNF 4 | HostUrl=https://app.diagrams.net/#Wb!hxRjnCx2_kuZ83BK28Fpd-Q3VU6TdUNLqCib8fv4Ik_WLzst7_BoT5CbbhvxELA3%2F01DY2LBUNHX5APAUKOR5EIOSCIZOQBKQNF 5 | -------------------------------------------------------------------------------- /packages/backend/src/Components/SubComment/Service/subCommentApi.ts: -------------------------------------------------------------------------------- 1 | import { crawlerAxios } from '../../../Config'; 2 | import { AxiosPromise } from 'axios'; 3 | import { NotImplementedError } from '../../../Error/ErrorClass'; 4 | 5 | function getSubCommentApi(): AxiosPromise { 6 | throw new NotImplementedError('getSubCommentApi is not implemented'); 7 | } 8 | 9 | export { getSubCommentApi }; 10 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/urlParse/getLastSegment.ts: -------------------------------------------------------------------------------- 1 | import { URL } from 'url'; 2 | 3 | function getUrlLastSegment(url: string) { 4 | const urlObj = new URL(url); 5 | const pathname = urlObj.pathname; 6 | const pathnameArray = pathname.split('/'); 7 | const lastSegment = pathnameArray[pathnameArray.length - 1]; 8 | return lastSegment; 9 | } 10 | 11 | export { getUrlLastSegment }; 12 | -------------------------------------------------------------------------------- /packages/backend/src/Error/ErrorClass/DatabaseError.ts: -------------------------------------------------------------------------------- 1 | import { HTTP_STATUS_CODE } from '../../Config/httpCode'; 2 | import { BaseError } from './BaseError'; 3 | 4 | class DatabaseError extends BaseError { 5 | constructor(message: string) { 6 | super(message); 7 | this.name = 'DatabaseError'; 8 | Object.setPrototypeOf(this, DatabaseError.prototype); 9 | } 10 | } 11 | 12 | export { DatabaseError }; 13 | -------------------------------------------------------------------------------- /packages/backend/src/Error/ErrorClass/ResourceError.ts: -------------------------------------------------------------------------------- 1 | import { BaseError } from './BaseError'; 2 | 3 | /** 4 | * failed to request resource from external service 5 | */ 6 | class ResourceError extends BaseError { 7 | constructor(message: string) { 8 | super(message); 9 | this.name = 'ResourceError'; 10 | Object.setPrototypeOf(this, ResourceError.prototype); 11 | } 12 | } 13 | 14 | export { ResourceError }; 15 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/parseFollowersCount/parseFollowersCount.ts: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | 3 | export const parseFollowersCount = (countStr: string) => { 4 | if(_.isNumber(countStr)) return countStr; 5 | let base: number = 1; 6 | if (countStr.includes('万')) { 7 | base = 10 ** 4; 8 | } 9 | if (countStr.includes('亿')) { 10 | base = 10 ** 8; 11 | } 12 | 13 | return parseFloat(countStr) * base; 14 | }; 15 | -------------------------------------------------------------------------------- /packages/documentation/docs/zh/guide/publish.md: -------------------------------------------------------------------------------- 1 | # 发布 2 | ## 改变远程仓库 3 | 如果你没有fork这个GitHub仓库,你需要自己在GitHub再创建一个仓库,然后改变remote origin 4 | ``` 5 | git remote set-url origin YOUR_GITHUB_REPO_URL 6 | ``` 7 | 8 | 你可以发布源代码和二进制可执行文件。 9 | 1. 将`.github/workflows/publish.yml_backup`重命名为`.github/workflows/publish.yml` 10 | 2. (可选)在`CHANGELOG.txt`中添加变更日志 11 | 3. 运行`lerna version`,这个命令会创建一个新的tag并推送到Github仓库,这将触发 Github Action`publish`发布版本。 12 | -------------------------------------------------------------------------------- /packages/frontend/src/Store/store.ts: -------------------------------------------------------------------------------- 1 | import { createStore, compose } from 'redux'; 2 | import { rootReducer } from './reducers'; 3 | 4 | declare global { 5 | interface Window { 6 | __REDUX_DEVTOOLS_EXTENSION_COMPOSE__?: typeof compose; 7 | } 8 | } 9 | 10 | const composeEnhancers = window.__REDUX_DEVTOOLS_EXTENSION_COMPOSE__ || compose; 11 | const store = createStore(rootReducer, composeEnhancers()); 12 | export { store }; 13 | -------------------------------------------------------------------------------- /packages/backend/src/Components/RepostComment/Service/repostCommentApi.ts: -------------------------------------------------------------------------------- 1 | import { crawlerAxios } from '../../../Config'; 2 | import { AxiosPromise } from 'axios'; 3 | import { NotImplementedError } from '../../../Error/ErrorClass'; 4 | 5 | /** 6 | * 7 | */ 8 | function getRepostCommentApi(): AxiosPromise { 9 | throw new NotImplementedError('getRepostCommentApi is not implemented'); 10 | } 11 | 12 | export { getRepostCommentApi }; 13 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/CookieBox/CookieBox.module.scss: -------------------------------------------------------------------------------- 1 | .cookie-box { 2 | .form-item { 3 | /* margin-bottom: 0; */ 4 | textarea { 5 | resize: none; 6 | } 7 | p { 8 | height: 150px; 9 | border: 1px dashed rgb(180, 180, 180); 10 | } 11 | } 12 | .action-div { 13 | float: right; 14 | 15 | .edit-icon{ 16 | position: relative; 17 | top: -30px; 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Monitor/Service/monitorApi.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import { URL, URLSearchParams } from 'url'; 3 | import { crawlerAxios } from '../../../Config'; 4 | import { CollectionTypes, MonitorCollection } from '../Types'; 5 | 6 | const collectionHandlers: { 7 | [key in CollectionTypes]?: ( 8 | collection: MonitorCollection, 9 | ) => Promise; 10 | } = {}; 11 | 12 | export { collectionHandlers }; 13 | -------------------------------------------------------------------------------- /packages/extension-chrome/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /packages/backend/src/Error/ErrorClass/BaseError.ts: -------------------------------------------------------------------------------- 1 | import { HTTP_STATUS_CODE } from '../../Config/httpCode'; 2 | 3 | class BaseError extends Error { 4 | isOperational: boolean; 5 | httpStatusCode: number; 6 | constructor(message: string) { 7 | super(message); 8 | this.name = 'BaseError'; 9 | this.isOperational = false; 10 | this.httpStatusCode = HTTP_STATUS_CODE.INTERNAL_SERVER; 11 | } 12 | } 13 | 14 | export { BaseError }; 15 | -------------------------------------------------------------------------------- /packages/backend/src/Error/ErrorClass/ServerError.ts: -------------------------------------------------------------------------------- 1 | import { HTTP_STATUS_CODE } from '../../Config/httpCode'; 2 | import { BaseError } from './BaseError'; 3 | 4 | class ServerError extends BaseError { 5 | constructor(message: string) { 6 | super(message); 7 | this.name = 'ServerError'; 8 | this.httpStatusCode = HTTP_STATUS_CODE.INTERNAL_SERVER; 9 | Object.setPrototypeOf(this, ServerError.prototype); 10 | } 11 | } 12 | 13 | export { ServerError }; 14 | -------------------------------------------------------------------------------- /packages/extension-chrome/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/PostQuery/PostQuery.module.scss: -------------------------------------------------------------------------------- 1 | .form { 2 | :global { 3 | .ant-form-item-label { 4 | text-align: left; 5 | } 6 | } 7 | overflow: hidden; 8 | .search-button { 9 | float: right; 10 | } 11 | } 12 | 13 | .date-picker-dropdown { 14 | @media screen and (max-width: 560px) { 15 | width: 300px; 16 | :global { 17 | .ant-picker-panels { 18 | display: block; 19 | } 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Account/Types/accountTypes.ts: -------------------------------------------------------------------------------- 1 | import { IUser } from '../../User/Types'; 2 | 3 | interface IAccountService { 4 | mode: 'cookie' | 'non-cookie'; 5 | init: () => Promise; 6 | getMode: () => 'cookie' | 'non-cookie'; 7 | getCookie: () => string; 8 | getUserProfile(): IUser | null; 9 | setCookie(cookie: string): Promise; 10 | validateCookie(cookie: string): Promise; 11 | } 12 | 13 | export { IAccountService }; 14 | -------------------------------------------------------------------------------- /packages/backend/src/Error/ErrorClass/ConflictError.ts: -------------------------------------------------------------------------------- 1 | import { HTTP_STATUS_CODE } from '../../Config/httpCode'; 2 | import { BaseError } from './BaseError'; 3 | 4 | class ConflictError extends BaseError { 5 | constructor(message: string) { 6 | super(message); 7 | this.name = 'ConflictError'; 8 | this.httpStatusCode = HTTP_STATUS_CODE.CONFLICT; 9 | Object.setPrototypeOf(this, ConflictError.prototype); 10 | } 11 | } 12 | 13 | export { ConflictError }; 14 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Comment/Service/commentApi.ts: -------------------------------------------------------------------------------- 1 | import { crawlerAxios } from '../../../Config'; 2 | import { AxiosPromise } from 'axios'; 3 | import { NotImplementedError } from '../../../Error/ErrorClass'; 4 | 5 | /** 6 | * get a batch of comments 7 | */ 8 | function getCommentApi(/* params here */): AxiosPromise { 9 | /* axios config here */ 10 | throw new NotImplementedError('getCommentApi is not implemented'); 11 | } 12 | 13 | export { getCommentApi }; 14 | -------------------------------------------------------------------------------- /packages/documentation/docs/.vuepress/enhanceApp.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Client app enhancement file. 3 | * 4 | * https://v1.vuepress.vuejs.org/guide/basic-config.html#app-level-enhancements 5 | */ 6 | 7 | export default ({ 8 | Vue, // the version of Vue being used in the VuePress app 9 | options, // the options for the root Vue instance 10 | router, // the router instance for the app 11 | siteData // site metadata 12 | }) => { 13 | // ...apply enhancements for the site. 14 | } 15 | -------------------------------------------------------------------------------- /packages/frontend/src/index.css: -------------------------------------------------------------------------------- 1 | @import '~antd/dist/antd.css'; 2 | 3 | body { 4 | margin: 0; 5 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 6 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 7 | sans-serif; 8 | -webkit-font-smoothing: antialiased; 9 | -moz-osx-font-smoothing: grayscale; 10 | } 11 | 12 | code { 13 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 14 | monospace; 15 | } 16 | -------------------------------------------------------------------------------- /packages/extension-chrome/src/App.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import { Server } from './Components/Server/Server'; 3 | import { Crawler } from './Components/Crawler/Crawler'; 4 | import './App.css'; 5 | 6 | function App() { 7 | const [server, setServer] = useState(''); 8 | return ( 9 |
10 | 11 | 12 |
13 | ); 14 | } 15 | 16 | export default App; 17 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/showProgress/progress.ts: -------------------------------------------------------------------------------- 1 | import ProgressBar from 'progress'; 2 | 3 | function showProgress(data: any, totalLength: string, fileName: string) { 4 | const progressBar = new ProgressBar('-> downloading [:bar] :percent :etas', { 5 | width: 40, 6 | complete: '=', 7 | incomplete: ' ', 8 | renderThrottle: 1, 9 | total: parseInt(totalLength), 10 | }); 11 | data.on('data', (chunk: any) => progressBar.tick(chunk.length)); 12 | } 13 | 14 | export default showProgress; 15 | -------------------------------------------------------------------------------- /packages/frontend/src/Store/actions/routeState.ts: -------------------------------------------------------------------------------- 1 | import { ROUTE_STATE } from '../actionTypes'; 2 | import { HomeState, PostContentState } from '../reducers/routeState'; 3 | 4 | const routeStateCreators = { 5 | setHomeRoute: (payload: HomeState) => { 6 | return { type: ROUTE_STATE.HOME.SET_HOME, payload }; 7 | }, 8 | setPostContentState: (payload: PostContentState) => { 9 | return { type: ROUTE_STATE.POST_CONTENT.SET_POST_CONTENT, payload }; 10 | }, 11 | }; 12 | 13 | export { routeStateCreators }; 14 | -------------------------------------------------------------------------------- /packages/extension-chrome/src/reportWebVitals.ts: -------------------------------------------------------------------------------- 1 | import { ReportHandler } from 'web-vitals'; 2 | 3 | const reportWebVitals = (onPerfEntry?: ReportHandler) => { 4 | if (onPerfEntry && onPerfEntry instanceof Function) { 5 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 6 | getCLS(onPerfEntry); 7 | getFID(onPerfEntry); 8 | getFCP(onPerfEntry); 9 | getLCP(onPerfEntry); 10 | getTTFB(onPerfEntry); 11 | }); 12 | } 13 | }; 14 | 15 | export default reportWebVitals; 16 | -------------------------------------------------------------------------------- /packages/backend/src/Error/ErrorClass/NotFoundError.ts: -------------------------------------------------------------------------------- 1 | import { HTTP_STATUS_CODE } from '../../Config/httpCode'; 2 | import { BaseError } from './BaseError'; 3 | 4 | class NotFoundError extends BaseError { 5 | isOperational: boolean; 6 | constructor(message: string) { 7 | super(message); 8 | this.name = 'NotFoundError'; 9 | this.httpStatusCode = HTTP_STATUS_CODE.NOT_FOUND; 10 | this.isOperational = true; 11 | Object.setPrototypeOf(this, NotFoundError.prototype); 12 | } 13 | } 14 | 15 | export { NotFoundError }; 16 | -------------------------------------------------------------------------------- /packages/backend/src/Jobs/Scheduler/scheduler.ts: -------------------------------------------------------------------------------- 1 | import { CronJob } from 'cron'; 2 | 3 | /** 4 | * 5 | * @param cronCommand the function to be executed in the cron 6 | * @param interval the interval of the cronjob in seconds 7 | * @returns 8 | */ 9 | const startCronJob = (cronCommand: () => any, interval: number) => { 10 | const job = new CronJob( 11 | `*/${interval} * * * * *`, 12 | cronCommand, 13 | null, 14 | true, 15 | 'America/New_York', 16 | ); 17 | 18 | return job; 19 | }; 20 | 21 | export { startCronJob }; 22 | -------------------------------------------------------------------------------- /packages/backend/src/Error/ErrorClass/BadRequestError.ts: -------------------------------------------------------------------------------- 1 | import { HTTP_STATUS_CODE } from '../../Config/httpCode'; 2 | import { BaseError } from './BaseError'; 3 | 4 | class BadRequestError extends BaseError { 5 | isOperational: boolean; 6 | constructor(message: string) { 7 | super(message); 8 | this.name = 'BadRequestError'; 9 | this.isOperational = true; 10 | this.httpStatusCode = HTTP_STATUS_CODE.BAD_REQUEST; 11 | Object.setPrototypeOf(this, BadRequestError.prototype); 12 | } 13 | } 14 | 15 | export { BadRequestError }; 16 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/hideString/hideString.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | 3 | /** 4 | * hide part of the sensitive string 5 | * @param str sensitive string 6 | * @param keep keep the n trailing digits 7 | * @returns a partially hidden string 8 | */ 9 | const hideString = (str: string, keep: number) => { 10 | const len: number = str.length; 11 | if (len <= keep) { 12 | return '*'.repeat(len); 13 | } 14 | const showString: string = str.slice(len - keep); 15 | return '*'.repeat(len - keep) + showString; 16 | }; 17 | 18 | export { hideString }; 19 | -------------------------------------------------------------------------------- /packages/frontend/src/Store/reducers/account.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import { IUser } from '../../types'; 3 | import { ACCOUNT } from '../actionTypes'; 4 | 5 | const initState: IUser | null = null; 6 | 7 | function account( 8 | state = initState, 9 | action: { type: string; payload: IUser | null }, 10 | ) { 11 | const { type, payload } = action; 12 | switch (type) { 13 | case ACCOUNT.SET_ACCOUNT: { 14 | return _.cloneDeep(payload); 15 | } 16 | default: { 17 | return state; 18 | } 19 | } 20 | } 21 | 22 | export { account }; 23 | -------------------------------------------------------------------------------- /packages/backend/src/Loaders/initFolders.ts: -------------------------------------------------------------------------------- 1 | import { staticPath, rxdbBasePath, logFolderPath } from '../Config'; 2 | import { logger } from '../Logger'; 3 | const fs = require('fs'); 4 | 5 | function initFolders() { 6 | if (!fs.existsSync(staticPath)) { 7 | logger.info('creating folder ' + staticPath); 8 | fs.mkdirSync(staticPath, { recursive: true }); 9 | } 10 | 11 | if (!fs.existsSync(rxdbBasePath)) { 12 | logger.info('creating folder ' + rxdbBasePath); 13 | fs.mkdirSync(rxdbBasePath, { recursive: true }); 14 | } 15 | } 16 | 17 | export { initFolders }; 18 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/parsePostId/parsePostId.ts: -------------------------------------------------------------------------------- 1 | import isUrl from 'is-url'; 2 | import cheerio from 'cheerio'; 3 | import { URL } from 'url'; 4 | import _ from 'lodash'; 5 | import { BadRequestError, NotImplementedError } from '../../Error/ErrorClass'; 6 | 7 | /** 8 | * get post id from url 9 | * @param urlStr possible post url 10 | * @returns the post id, if it's not a valid post url, return empty string "" 11 | */ 12 | export async function parsePostId(urlStr: string): Promise { 13 | throw new NotImplementedError('parsePostId not implemented yet'); 14 | } 15 | -------------------------------------------------------------------------------- /packages/documentation/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@social-media-archiver/documentation", 3 | "version": "0.0.1", 4 | "description": "Social Media Archiver is a Node.js template to be implemented to archive post from any social media.", 5 | "main": "index.js", 6 | "authors": { 7 | "name": "", 8 | "email": "" 9 | }, 10 | "repository": "/@social-media-archiver/documentation", 11 | "scripts": { 12 | "start": "vuepress dev docs", 13 | "build": "vuepress build docs" 14 | }, 15 | "license": "MIT", 16 | "devDependencies": { 17 | "vuepress": "^1.5.3" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/AppSider/AppSider.module.scss: -------------------------------------------------------------------------------- 1 | .app-sider-fix { 2 | overflow: visible; 3 | height: 100vh; 4 | position: fixed; 5 | display: block !important; 6 | padding-top: 85px; 7 | margin-top: -65px; 8 | left: 0; 9 | z-index: 2; 10 | box-shadow: 5px 0px 15px rgb(153, 153, 153); 11 | .sider-content { 12 | overflow: hidden !important; 13 | width: 100%; 14 | padding: 0 10px; 15 | } 16 | 17 | } 18 | 19 | .search-button { 20 | border-radius: 3px; 21 | margin-left: 20px; 22 | position: fixed; 23 | margin-top: 25px; 24 | z-index: 20; 25 | } 26 | -------------------------------------------------------------------------------- /packages/extension-chrome/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import './index.css'; 4 | import App from './App'; 5 | import reportWebVitals from './reportWebVitals'; 6 | 7 | ReactDOM.render( 8 | 9 | 10 | , 11 | document.getElementById('root'), 12 | ); 13 | 14 | // If you want to start measuring performance in your app, pass a function 15 | // to log results (for example: reportWebVitals(console.log)) 16 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 17 | reportWebVitals(); 18 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml_backup: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - "v*.*.*" 6 | 7 | jobs: 8 | publish: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v2 13 | - name: pack 14 | run: npm run dist 15 | - name: Release 16 | uses: softprops/action-gh-release@v1 17 | with: 18 | body_path: CHANGELOG.txt 19 | files: | 20 | ./dist/linux.tar.gz 21 | ./dist/macos.tar.gz 22 | ./dist/win.zip 23 | ./dist/extension-chrome.zip 24 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/json/json.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs-extra'; 2 | 3 | const readJson = async (path: string) => { 4 | if (!fs.existsSync(path)) { 5 | return {}; 6 | } 7 | try { 8 | const json: object = await fs.readJSON( 9 | path, 10 | ); 11 | return json; 12 | } catch (error) { 13 | return {}; 14 | } 15 | }; 16 | 17 | const writeJson = async ( 18 | path: string, 19 | newJsonObj: object, 20 | ) => { 21 | try { 22 | fs.writeJSON(path, newJsonObj); 23 | return true; 24 | } catch (error) { 25 | return false; 26 | } 27 | }; 28 | 29 | export { readJson, writeJson }; 30 | -------------------------------------------------------------------------------- /packages/frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "esModuleInterop": true, 12 | "allowSyntheticDefaultImports": true, 13 | "strict": true, 14 | "forceConsistentCasingInFileNames": true, 15 | "module": "esnext", 16 | "moduleResolution": "node", 17 | "resolveJsonModule": true, 18 | "isolatedModules": true, 19 | "noEmit": true, 20 | "jsx": "react" 21 | }, 22 | "include": [ 23 | "src" 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /packages/frontend/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /packages/documentation/docs/guide/publish.md: -------------------------------------------------------------------------------- 1 | # Publish 2 | 3 | ## Change Remote 4 | You need to create a Github repository if you didn't fork the repository. And then change the origin remote 5 | ``` 6 | git remote set-url origin YOUR_GITHUB_REPO_URL 7 | ``` 8 | ## Publish 9 | You can make a release of the source code and the binary executable files. 10 | 1. rename `.github/workflows/publish.yml_backup` to `.github/workflows/publish.yml` 11 | 2. (Optional) add changelog in `CHANGELOG.txt` 12 | 3. run `lerna version`, this command creates a new tag and push to remote, which will trigger the Github Action `publish` to publish the release. 13 | -------------------------------------------------------------------------------- /packages/frontend/src/Utility/timeFormat/timeFormat.ts: -------------------------------------------------------------------------------- 1 | import dayjs from 'dayjs'; 2 | import _ from 'lodash'; 3 | const convertToUnix = (obj: any) => { 4 | const newObj = _.cloneDeep(obj); 5 | for (const key of Object.keys(newObj)) { 6 | if (dayjs.isDayjs(newObj[key])) { 7 | newObj[key] = newObj[key].valueOf(); 8 | } 9 | if (_.isArray(newObj[key])) { 10 | for (let i: number = 0; i < newObj[key].length; i++) { 11 | if (dayjs.isDayjs(newObj[key][i])) { 12 | newObj[key][i] = newObj[key][i].valueOf(); 13 | } 14 | } 15 | } 16 | } 17 | return newObj; 18 | }; 19 | 20 | export { convertToUnix }; 21 | -------------------------------------------------------------------------------- /packages/documentation/docs/zh/guide/video.md: -------------------------------------------------------------------------------- 1 | # Video 2 | Video组件可以下载视频媒体文件。 它接受视频文件 url 并将视频文件作为静态资源下载到本地文件系统。 但是,某些平台中的视频不是静态视频文件。 视频是逐段下载和播放的。 在这种情况下,你需要自己重写Video组件。 3 | 在`packages/backend/src/Components/Video/Service/videoService.ts`中: 4 | 你需要重写 `then` 函数中的代码。 你可以使用第三方库,如 `youtube-dl-exec` 或手动编写自己的代码。 5 | ```typescript 6 | private videoQueueFunc(queueParams: ParamsQueue): Promise { 7 | const { url, staticPath } = queueParams; 8 | return new Promise((resolve, reject) => { 9 | downloadVideoApi(url) 10 | .then((res) => { 11 | // override the download code here 12 | }) 13 | .catch(reject); 14 | }); 15 | } 16 | ``` -------------------------------------------------------------------------------- /packages/frontend/src/Routes/app.ts: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, ComponentClass } from 'react'; 2 | import { Home } from '../Pages/Home'; 3 | import { PostContent } from '../Pages/PostContent'; 4 | import { CommentContent } from '../Pages/CommentContent'; 5 | 6 | export interface Route { 7 | path: string; 8 | component: FunctionComponent | ComponentClass; 9 | exact?: boolean; 10 | } 11 | 12 | const routes: Route[] = [ 13 | { path: '/post/:postId', component: PostContent }, 14 | { path: '/comment/:commentId', component: CommentContent }, 15 | { 16 | path: '/', 17 | component: Home, 18 | exact: true, 19 | }, 20 | ]; 21 | 22 | export default routes; 23 | -------------------------------------------------------------------------------- /packages/extension-chrome/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "esModuleInterop": true, 12 | "allowSyntheticDefaultImports": true, 13 | "strict": true, 14 | "forceConsistentCasingInFileNames": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "module": "esnext", 17 | "moduleResolution": "node", 18 | "resolveJsonModule": true, 19 | "isolatedModules": true, 20 | "noEmit": true, 21 | "jsx": "react-jsx" 22 | }, 23 | "include": [ 24 | "src" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/migrate/migrate.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | /** 3 | * migrate old data to the latest version 4 | * @param oldInfos 5 | * @param version 6 | * @param migrationStrategy 7 | * @returns 8 | */ 9 | function migrate( 10 | oldInfos: any[], 11 | version: number, 12 | migrationStrategy: { [key: number]: (oldInfo: any) => any }, 13 | ): Info[] { 14 | const migrateFunc = _.get(migrationStrategy, `${version + 1}`); 15 | let newInfos = oldInfos; 16 | if (_.isFunction(migrateFunc)) { 17 | newInfos = oldInfos.map(migrateFunc); 18 | return migrate(newInfos, version + 1, migrationStrategy); 19 | } 20 | return newInfos; 21 | } 22 | 23 | export { migrate }; 24 | -------------------------------------------------------------------------------- /packages/backend/src/Config/paths.ts: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | 3 | const staticPath = path.resolve(process.cwd(), './storage', 'static'); 4 | 5 | const credentialJsonPath = path.resolve(process.cwd(), './', 'credential.json'); 6 | 7 | const rxdbBasePath = path.resolve(process.cwd(), 'storage', 'rxdb'); 8 | 9 | const rxdbPath = path.resolve(rxdbBasePath, 'postcrawler'); 10 | 11 | const logFolderPath = path.resolve(process.cwd(), 'log'); 12 | const collectionPath = path.resolve( 13 | process.cwd(), 14 | 'storage', 15 | 'collection.json', 16 | ); 17 | 18 | export { 19 | staticPath, 20 | credentialJsonPath, 21 | rxdbBasePath, 22 | rxdbPath, 23 | logFolderPath, 24 | collectionPath, 25 | }; 26 | -------------------------------------------------------------------------------- /packages/backend/src/Error/ErrorHandler/errorHandler.ts: -------------------------------------------------------------------------------- 1 | import { Response } from 'express'; 2 | import { logger } from '../../Logger'; 3 | import { BaseError } from '../ErrorClass/BaseError'; 4 | 5 | class ErrorHandler { 6 | async handleError(err: Error | BaseError, res?: Response) { 7 | if (res) { 8 | this.httpErrorHandler(err, res); 9 | } 10 | this.logError(err); 11 | } 12 | 13 | async httpErrorHandler(err: Error | BaseError, res: Response) { 14 | if (err instanceof BaseError) { 15 | res.status(err.httpStatusCode || 500).send(err.message); 16 | } else { 17 | res.status(500).send(err.message); 18 | } 19 | } 20 | 21 | async logError(err: Error | BaseError) { 22 | logger.error(err); 23 | } 24 | } 25 | 26 | export { ErrorHandler }; 27 | -------------------------------------------------------------------------------- /packages/backend/src/Components/RepostComment/DAL/repostCommentSchema.ts: -------------------------------------------------------------------------------- 1 | import { RxJsonSchema } from 'rxdb'; 2 | import { IRepostComment } from '../Types'; 3 | 4 | export const version = 0; 5 | export const repostCommentSchema: RxJsonSchema = { 6 | title: 'repostComment schema', 7 | version: version, 8 | description: 'repostComment schema', 9 | type: 'object', 10 | properties: { 11 | id: { type: 'string', primary: true }, 12 | content: { type: 'string' }, 13 | createTime: { type: 'number' }, 14 | user: { type: 'string', ref: 'user' }, 15 | repostedId: { type: 'string', ref: 'post' }, //the parent post id 16 | saveTime: { type: 'number' }, 17 | }, 18 | required: ['id', 'content', 'user', 'createTime', 'repostedId', 'saveTime'], 19 | indexes: ['repostedId', 'user'], 20 | }; 21 | -------------------------------------------------------------------------------- /packages/frontend/src/Utility/route/generateQueryString.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | 3 | function generateQueryString(values: any) { 4 | let validValues: { [key: string]: any } = {}; 5 | Object.keys(values).forEach((key: string) => { 6 | const value = values[key]; 7 | if (value === undefined || value === null) { 8 | return false; 9 | } 10 | if (value.length !== 0) { 11 | validValues[key] = value; 12 | } 13 | }); 14 | 15 | const params = Object.keys(validValues) 16 | .map( 17 | (key) => 18 | `${key}=${encodeURIComponent( 19 | _.isObject(validValues[key]) 20 | ? JSON.stringify(validValues[key]) 21 | : validValues[key], 22 | )}`, 23 | ) 24 | .join('&'); 25 | return params; 26 | } 27 | 28 | export { generateQueryString }; 29 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/AppSider/Collapser.tsx: -------------------------------------------------------------------------------- 1 | import React, { Dispatch, SetStateAction } from 'react'; 2 | import { RightOutlined, LeftOutlined } from '@ant-design/icons'; 3 | import styles from './Collapser.module.scss'; 4 | type CollapserProps = { 5 | collapsed: boolean; 6 | setCollapsed: Dispatch>; 7 | }; 8 | 9 | function Collapser(props: CollapserProps) { 10 | const { collapsed, setCollapsed } = props; 11 | return ( 12 |
{ 14 | setCollapsed(!collapsed); 15 | }} 16 | className={styles['collapser']} 17 | > 18 | {collapsed ? ( 19 | 20 | ) : ( 21 | 22 | )} 23 |
24 | ); 25 | } 26 | 27 | export { Collapser }; 28 | -------------------------------------------------------------------------------- /packages/frontend/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import './index.css'; 4 | import App from './App'; 5 | import * as serviceWorker from './serviceWorker'; 6 | import { BrowserRouter as Router } from 'react-router-dom'; 7 | import { Provider } from 'react-redux'; 8 | import { store } from './Store/store'; 9 | ReactDOM.render( 10 | 11 | 12 | 13 | 14 | 15 | 16 | , 17 | document.getElementById('root'), 18 | ); 19 | 20 | // If you want your app to work offline and load faster, you can change 21 | // unregister() to register() below. Note this comes with some pitfalls. 22 | // Learn more about service workers: https://bit.ly/CRA-PWA 23 | serviceWorker.unregister(); 24 | -------------------------------------------------------------------------------- /packages/backend/src/Config/constants.ts: -------------------------------------------------------------------------------- 1 | const BASE_URL: string = ''; 2 | 3 | const Q_CONCURRENCY: number = 1; 4 | const MAX_ITEM_WINDOW: number = 6; 5 | 6 | //small number has higher priority 7 | const Q_PRIORITY = { 8 | CRAWLER_POST: 1, 9 | DOWNLOAD_VIDEO: 2, 10 | DOWNLOAD_POST_IMAGE: 2, 11 | FETCH_MESSAGE: 2, 12 | FETCH_COLLECTION: 2, 13 | SEND_MESSAGE: 2, //send message is not implemented 14 | CRAWLER_COMMENT: 3, 15 | CRAWLER_SUB_COMMENT: 3, 16 | CRAWLER_REPOST_COMMENT: 3, 17 | DOWNLOAD_IMAGE: 3, //download other images, like avatar 18 | }; 19 | 20 | const MAX_MONITOR_COLLECTION = 5; 21 | 22 | const PORT = 5000; 23 | 24 | const MONITOR_INTERVAL: number = 120; //in second 25 | 26 | export { 27 | BASE_URL, 28 | Q_CONCURRENCY, 29 | Q_PRIORITY, 30 | MAX_MONITOR_COLLECTION, 31 | PORT, 32 | MONITOR_INTERVAL, 33 | MAX_ITEM_WINDOW, 34 | }; 35 | -------------------------------------------------------------------------------- /packages/backend/src/Logger/logger.ts: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import pino from 'pino'; 3 | import { logFolderPath } from '../Config/paths'; 4 | import pretty from 'pino-pretty'; 5 | 6 | const logFilePath: string = path.resolve( 7 | logFolderPath, 8 | 'social-media-archiver.log', 9 | ); 10 | const fs = require('fs'); 11 | if (!fs.existsSync(logFolderPath)) { 12 | console.log('creating log folder ' + logFolderPath); 13 | fs.mkdirSync(logFolderPath, { recursive: true }); 14 | } 15 | 16 | const streams: any[] = [{ stream: fs.createWriteStream(logFilePath) }]; 17 | 18 | if (process.env.NODE_ENV === 'development') { 19 | streams.push({ 20 | stream: pretty({}), 21 | }); 22 | } 23 | 24 | var logger = pino( 25 | { 26 | level: 'debug', // this MUST be set at the lowest level of the 27 | // destinations 28 | }, 29 | pino.multistream(streams, {}), 30 | ); 31 | 32 | export { logger }; 33 | -------------------------------------------------------------------------------- /packages/backend/src/Components/SubComment/DAL/subCommentSchema.ts: -------------------------------------------------------------------------------- 1 | import { RxJsonSchema } from 'rxdb'; 2 | import { ISubComment } from '../Types'; 3 | 4 | export const version = 0; 5 | export const subCommentSchema: RxJsonSchema = { 6 | title: 'subcomment schema', 7 | version: version, 8 | description: 'subcomment schema', 9 | type: 'object', 10 | properties: { 11 | id: { type: 'string', primary: true }, 12 | floorNumber: { type: 'number' }, 13 | content: { type: 'string' }, 14 | upvotesCount: { type: 'number' }, 15 | createTime: { type: 'number' }, 16 | user: { type: 'string', ref: 'user' }, 17 | commentId: { type: 'string', ref: 'comment' }, 18 | saveTime: { type: 'number' }, 19 | replyTo: { type: 'string', ref: 'user' }, 20 | }, 21 | required: ['id', 'user', 'createTime', 'commentId', 'saveTime', 'content'], 22 | indexes: ['commentId', 'user'], 23 | }; 24 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/cookie/cookie.ts: -------------------------------------------------------------------------------- 1 | const parseCookie = (str: string) => 2 | str 3 | .split(';') 4 | .map((v) => v.split('=')) 5 | .reduce((acc: { [property: string]: string }, v: string[]) => { 6 | acc[decodeURIComponent(v[0].trim())] = decodeURIComponent(v[1].trim()); 7 | return acc; 8 | }, {}); 9 | 10 | const serializeSingleCookie = (name: string, val: string) => 11 | `${encodeURIComponent(name)}=${encodeURIComponent(val)}`; 12 | 13 | const serializeCookies = (obj: { [property: string]: string }) => { 14 | return Object.keys(obj) 15 | .map((key) => serializeSingleCookie(key, obj[key])) 16 | .join('; '); 17 | }; 18 | 19 | const modifyCookie = (cookie: string, key: string, value: string) => { 20 | const parsedCookie = parseCookie(cookie); 21 | parsedCookie[key] = value; 22 | return serializeCookies(parsedCookie); 23 | }; 24 | 25 | export { modifyCookie }; 26 | -------------------------------------------------------------------------------- /packages/frontend/src/Api/account.ts: -------------------------------------------------------------------------------- 1 | import { AxiosPromise } from 'axios'; 2 | import { IUser } from '../types'; 3 | import { axios } from './config'; 4 | 5 | function getAccountProfileApi(): AxiosPromise<{ result: IUser }> { 6 | return axios({ 7 | url: '/account/profile', 8 | }); 9 | } 10 | 11 | function getCookieApi(): AxiosPromise<{ result: string }> { 12 | return axios({ 13 | url: '/account/cookie', 14 | }); 15 | } 16 | 17 | function validateCookieApi(cookie: string): AxiosPromise<{ result: boolean }> { 18 | return axios({ 19 | url: '/account/cookie/validate', 20 | method: 'post', 21 | data: { cookie }, 22 | }); 23 | } 24 | 25 | function setCookieApi(cookie: string): AxiosPromise { 26 | return axios({ 27 | url: '/account/cookie', 28 | method: 'post', 29 | data: { cookie }, 30 | }); 31 | } 32 | export { getAccountProfileApi, getCookieApi, validateCookieApi, setCookieApi }; 33 | -------------------------------------------------------------------------------- /packages/backend/src/Components/User/DAL/userSchema.ts: -------------------------------------------------------------------------------- 1 | import { RxJsonSchema } from 'rxdb'; 2 | import { IUser } from '../Types'; 3 | 4 | const version = 0; 5 | const userSchema: RxJsonSchema = { 6 | title: 'userSchema', 7 | version: version, 8 | description: 'user schema', 9 | type: 'object', 10 | properties: { 11 | id: { type: 'string', primary: true }, 12 | username: { type: 'string' }, // the name shown on posts 13 | gender: { type: 'string' }, 14 | followersCount: { type: 'number' }, 15 | followingCount: { type: 'number' }, 16 | image: { 17 | type: 'object', 18 | properties: { name: { type: 'string' }, originUrl: { type: 'string' } }, 19 | }, // a larger profile image, 20 | profileUrl: { type: 'string' }, 21 | }, 22 | required: ['id', 'gender', 'followersCount', 'followingCount', 'image'], 23 | indexes: ['username'], 24 | }; 25 | 26 | export { userSchema, version }; 27 | -------------------------------------------------------------------------------- /packages/documentation/docs/guide/video.md: -------------------------------------------------------------------------------- 1 | # Video 2 | The video component can download video media file. It accepts a video file url and download the video file as a static resource to the local file system. However, videos in some of the platforms are not static video files. The video is download and played segment by segment. In this scenario, you have to override the video component. 3 | In `packages/backend/src/Components/Video/Service/videoService.ts`: 4 | You can override the code inside `then` function. You can use third-party library like `youtube-dl-exec` or write your own code manually. 5 | ```typescript 6 | private videoQueueFunc(queueParams: ParamsQueue): Promise { 7 | const { url, staticPath } = queueParams; 8 | return new Promise((resolve, reject) => { 9 | downloadVideoApi(url) 10 | .then((res) => { 11 | // override the download code here 12 | }) 13 | .catch(reject); 14 | }); 15 | } 16 | ``` -------------------------------------------------------------------------------- /.github/workflows/deploy-doc.yml_backup: -------------------------------------------------------------------------------- 1 | # .github/workflows/netlify.yml 2 | name: Build and Deploy to Netlify 3 | on: 4 | push: 5 | pull_request: 6 | jobs: 7 | build: 8 | runs-on: ubuntu-18.04 9 | steps: 10 | - uses: actions/checkout@v2 11 | 12 | # ( Build to ./dist or other directory... ) 13 | - name: Build 14 | run: cd packages/documentation && npm ci&&npm run build 15 | - name: Deploy to Netlify 16 | uses: nwtgck/actions-netlify@v1.2.2 17 | with: 18 | publish-dir: 'packages/documentation/public' 19 | production-branch: doc-init 20 | deploy-message: 'Deploy from GitHub Actions' 21 | enable-pull-request-comment: false 22 | enable-commit-comment: true 23 | overwrites-pull-request-comment: true 24 | env: 25 | NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} 26 | NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} 27 | timeout-minutes: 1 28 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/ExportModal/ExportModal.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | import { Modal } from 'antd'; 3 | 4 | type Props = { 5 | visible: boolean; 6 | setVisible: (visible: boolean) => void; 7 | }; 8 | export function ExportModal(props: Props) { 9 | const { visible, setVisible } = props; 10 | return ( 11 | setVisible(false)}> 12 | 29 | 30 | ); 31 | } 32 | -------------------------------------------------------------------------------- /README_zh-CN.md: -------------------------------------------------------------------------------- 1 | [English](./README.md) | 简体中文 2 |

3 | 4 | 5 |

Social Media Archiver

6 | 7 |

8 | 9 | [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-908a85?logo=gitpod)](https://gitpod.io/from-referrer/) 10 | 11 | Social Media Archiver 是一个 Node.js 模板,只需写少许平台相关代码,便能存档该平台的帖子(Post)。
12 | 文档: https://social-media-archiver.ml/zh/
13 | 演示: Social Media Archiver 社交平台存档器
14 | ## 快速开始 15 | 1. 克隆仓库 (也可以直接点击上面的 Gitpod 按钮) 16 | ``` 17 | git clone https://github.com/Combo819/social-media-archiver.git 18 | ``` 19 | 2. 安装Lerna 20 | ``` 21 | npm install -g lerna 22 | ``` 23 | 3. 安装依赖 24 | ``` 25 | cd social-media-archiver 26 | lerna bootstrap 27 | ``` 28 | 4. 启动 29 | ``` 30 | npm run start 31 | ``` 32 | 更多细节请查阅文档 -------------------------------------------------------------------------------- /packages/backend/src/Components/User/Service/userApi.ts: -------------------------------------------------------------------------------- 1 | import { AxiosPromise } from 'axios'; 2 | import { crawlerAxios } from '../../../Config'; 3 | import { NotImplementedError } from '../../../Error/ErrorClass'; 4 | 5 | /** 6 | * this should return a promise resolved to be the userRaw object 7 | * to meet the format of userService.transformUserResponse parameter. 8 | * If not, transform it to the userRaw object in then and return it. 9 | * @param userId the user id 10 | */ 11 | function getUserInfoByIdApi(userId: string): Promise { 12 | throw new NotImplementedError('getUserInfoByIdApi is not implemented'); 13 | } 14 | 15 | /** 16 | * get the user id by the user name from the platform server 17 | * @param username unique username 18 | * @returns the promise resolved to the user id 19 | */ 20 | function getUserIdByUsernameApi(username: string): Promise { 21 | throw new NotImplementedError('getUserIdByUsernameApi is not implemented'); 22 | } 23 | 24 | export { getUserInfoByIdApi, getUserIdByUsernameApi }; 25 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Monitor/Types/monitorTypes.ts: -------------------------------------------------------------------------------- 1 | //defined the collection types like 'favorite' | 'watchLater' | 'chat' | 'history' | 'upvote' 2 | type CollectionTypes = 'favorite' | 'chat'; 3 | 4 | type MonitorCollection = { 5 | url: string; 6 | type: CollectionTypes; 7 | postIds: Set; 8 | [key: string]: any; 9 | }; 10 | 11 | interface IMonitorService { 12 | //interface for collection service 13 | loadFromJson(): void; 14 | getHandler( 15 | collectionType: CollectionTypes, 16 | ): (collection: MonitorCollection) => Promise; 17 | proceed(): void; 18 | add(collectionUrl: string, collectionType: CollectionTypes): boolean; 19 | remove(collectionUrl: string): boolean; 20 | validate( 21 | collectionUrl: string, 22 | collectionType: CollectionTypes, 23 | ): Promise; 24 | getCollectionTypes(): CollectionTypes[]; 25 | getMonitorCollections(): MonitorCollection[]; 26 | getMaxCollectionSize(): number; 27 | } 28 | 29 | export { CollectionTypes, IMonitorService, MonitorCollection }; 30 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/AppHeader/AppHeader.module.scss: -------------------------------------------------------------------------------- 1 | .app-header { 2 | background-color: #fff; 3 | box-shadow: 0px 5px 15px rgb(196, 196, 196); 4 | z-index: 3; 5 | /* .save { 6 | margin-right: 100px; 7 | @media screen and (max-width: 420px) { 8 | margin-right: 30px; 9 | } 10 | } */ 11 | .monitor-div { 12 | .monitor { 13 | cursor: pointer; 14 | } 15 | .monitor-disabled { 16 | cursor: not-allowed; 17 | } 18 | .question { 19 | cursor: pointer; 20 | font-size: 14px; 21 | color: rgb(102, 102, 102); 22 | margin-left: -10px; 23 | } 24 | } 25 | .account { 26 | float: right; 27 | .icon { 28 | cursor: pointer; 29 | font-size: 20px; 30 | } 31 | } 32 | .dropdown { 33 | display: none; 34 | } 35 | .space { 36 | display: inline-flex; 37 | } 38 | 39 | @media screen and (max-width: 550px) { 40 | .dropdown { 41 | display: inline-block; 42 | } 43 | .space { 44 | display: none; 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "version": "1.0.1", 4 | "name": "social-media-archiver", 5 | "devDependencies": { 6 | "@commitlint/cli": "^13.2.1", 7 | "@commitlint/config-conventional": "^13.2.0", 8 | "cz-conventional-changelog": "^3.3.0", 9 | "git-cz": "^4.8.0", 10 | "lerna": "^4.0.0", 11 | "prettier": "2.4.1" 12 | }, 13 | "scripts": { 14 | "start": "lerna run --parallel --scope @social-media-archiver/backend --scope @social-media-archiver/frontend start", 15 | "dist": "npx lerna bootstrap && npx lerna run --scope @social-media-archiver/backend dist", 16 | "commit": "git-cz", 17 | "commitmsg": "validate-commit-msg", 18 | "bootstrap": "lerna bootstrap", 19 | "frontend":"lerna run --parallel --scope @social-media-archiver/frontend start", 20 | "backend":"lerna run --parallel --scope @social-media-archiver/backend start", 21 | "documentation": "lerna run --parallel --scope @social-media-archiver/documentation start" 22 | }, 23 | "config": { 24 | "commitizen": { 25 | "path": "./node_modules/cz-conventional-changelog" 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Combo819 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | English | [简体中文](./README_zh-CN.md) 2 | 3 |

4 | 5 | 6 |

Social Media Archiver

7 | 8 |

9 | 10 | [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-908a85?logo=gitpod)](https://gitpod.io/from-referrer/)
11 | Social Media Archiver is a Node.js template to be implemented to archive post from any social media.
12 | Documentation: https://social-media-archiver.ml/
13 | Demo: Social Media Archiver
14 | ## Getting Started 15 | 1. Clone the repository (or click on the Gitpod button above) 16 | ``` 17 | git clone https://github.com/Combo819/social-media-archiver.git 18 | ``` 19 | 2. Install Lerna 20 | ``` 21 | npm install -g lerna 22 | ``` 23 | 3. install dependencies 24 | ``` 25 | cd social-media-archiver 26 | lerna bootstrap 27 | ``` 28 | 4. start 29 | ``` 30 | npm run start 31 | ``` 32 | See more detail on Documentation -------------------------------------------------------------------------------- /packages/backend/src/Utility/extractHtml/extractHtml.ts: -------------------------------------------------------------------------------- 1 | import cheerio from 'cheerio'; 2 | import _ from 'lodash'; 3 | import { getUrlLastSegment } from '../urlParse'; 4 | 5 | /** 6 | * extract the target html from the clean html, replace the image src to the local path. 7 | * return the target html and the image original urls 8 | * @param cleanHtml 9 | * @returns 10 | */ 11 | const extractHtml = ( 12 | cleanHtml: string, 13 | imageUrlSkip?: (url: string) => boolean, 14 | ): { html: string; embedImages: string[] } => { 15 | const $ = cheerio.load(cleanHtml); 16 | const embedImages: string[] = []; 17 | $('img').each(function (index: number, element: any) { 18 | const oldSrc = $(element).attr('src'); 19 | if (oldSrc) { 20 | if (_.isFunction(imageUrlSkip) && imageUrlSkip(oldSrc)) return true; // skip this image 21 | const fileName: string = getUrlLastSegment(oldSrc); 22 | const newSrc = '/images/' + fileName; 23 | $(element).attr('src', newSrc); 24 | embedImages.push(oldSrc); 25 | } 26 | }); 27 | return { 28 | html: $.html(), 29 | embedImages, 30 | }; 31 | }; 32 | 33 | export { extractHtml }; 34 | -------------------------------------------------------------------------------- /packages/frontend/src/App.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Layout, BackTop } from 'antd'; 3 | import { Switch, Route } from 'react-router-dom'; 4 | import { routes } from './Routes'; 5 | 6 | import { UpCircleOutlined } from '@ant-design/icons'; 7 | import { AppHeader } from './Component/AppHeader/AppHeader'; 8 | import { AppSider } from './Component/AppSider/AppSider'; 9 | import styles from './App.module.scss'; 10 | 11 | const { Content, Sider } = Layout; 12 | function App(): JSX.Element { 13 | return ( 14 | 15 | 16 | 17 | 18 | {routes.map((item): React.ReactNode => { 19 | return ( 20 | ( 23 | 24 | )} 25 | > 26 | ); 27 | })} 28 | 29 | 30 | 31 | 34 | 35 | 36 | ); 37 | } 38 | 39 | export default App; 40 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Comment/DAL/commentSchema.ts: -------------------------------------------------------------------------------- 1 | import { RxJsonSchema } from 'rxdb'; 2 | import { IComment } from '../Types'; 3 | 4 | const version = 0; 5 | const commentSchema: RxJsonSchema = { 6 | title: 'userSchema', 7 | version: version, 8 | description: 'user schema', 9 | type: 'object', 10 | properties: { 11 | id: { type: 'string', primary: true }, 12 | floorNumber: { type: 'number' }, 13 | content: { type: 'string' }, 14 | subCommentsCount: { type: 'number' }, 15 | upvotesCount: { type: 'number' }, 16 | createTime: { type: 'number' }, 17 | postId: { type: 'string', ref: 'post' }, 18 | user: { type: 'string', ref: 'user' }, 19 | subComments: { 20 | type: 'array', 21 | ref: 'subcomment', 22 | items: { type: 'string' }, 23 | }, 24 | image: { 25 | type: 'object', 26 | properties: { name: { type: 'string' }, originUrl: { type: 'string' } }, 27 | }, 28 | saveTime: { type: 'number' }, 29 | replyTo: { type: 'string', ref: 'user' }, 30 | }, 31 | required: ['id', 'content', 'user', 'createTime', 'postId', 'saveTime'], 32 | indexes: ['user', 'postId'], 33 | }; 34 | 35 | export { commentSchema, version }; 36 | -------------------------------------------------------------------------------- /packages/backend/src/Components/User/Types/userTypes.ts: -------------------------------------------------------------------------------- 1 | import { RxCollection, RxDocument } from 'rxdb'; 2 | import { IBaseDAL, IBaseService } from '../../Base/baseTypes'; 3 | /** 4 | * Interface to model the User Schema for TypeScript. 5 | */ 6 | type IUser = { 7 | id: string; // The user's unique ID. 8 | username: string; // The user's username. 9 | profileUrl?: string; // The user's home URL. 10 | gender: string; // same as the platform 11 | followersCount: number; 12 | followingCount: number; 13 | image: { name: string; originUrl: string }; // for the user's profile image 14 | }; 15 | 16 | interface IUserService extends IBaseService { 17 | getUserByName: (name: string) => Promise; 18 | fetchUser: (userId: string) => Promise; 19 | getUserIdByName: (username: string) => Promise; 20 | countUserByName(name: string): Promise; 21 | transformUserResponse(userRaw: any): IUser; 22 | } 23 | 24 | interface IUserDAL extends IBaseDAL {} 25 | 26 | type UserDocument = RxDocument; 27 | type UserCollection = RxCollection; 28 | 29 | export { IUser, IUserService, UserDocument, UserCollection, IUserDAL }; 30 | -------------------------------------------------------------------------------- /packages/documentation/docs/zh/guide/user.md: -------------------------------------------------------------------------------- 1 | # User 2 | 3 | ## 转换 4 | 5 | 在 `packages/backend/src/Components/User/Service/userService.ts` 6 | 7 | ```typescript 8 | transformUserResponse(userRaw: any): IUser { 9 | throw new NotImplementedError( 10 | `transformUserResponse in UserService is not implemented`, 11 | ); 12 | } 13 | ``` 14 | 15 | 该函数将接收 userRaw 对象。 你得将 userRaw 对象转换为`IUser`对象。 见`packages/backend/src/Components/User/Types/userTypes.ts` 16 | 17 | ## (可选) API 18 | 19 | 在大多数情况下,你不需要通过 API 请求来获取 user 信息,因为 user 信息通常会附加到其他请求中。 20 | 在`packages/backend/src/Components/User/Service/userApi.ts`中, 21 | 22 | ```typescript 23 | function getUserInfoByIdApi(userId: string): Promise { 24 | throw new NotImplementedError('getUserInfoByIdApi is not implemented'); 25 | } 26 | ``` 27 | 28 | 这函数需要返回一个 resolve 为 userRaw 对象的 promise,以满足 `userService.transformUserResponse` 参数的格式。 29 | 如果请求的结果不是 userRaw,则在 then 中将其转换为 userRaw 对象并返回。 30 | 31 | ```typescript 32 | function getUserInfoByIdApi(userId: string): Promise { 33 | return crawlerAxios({ 34 | url: `/api/user/${id}`, 35 | }).then((res) => { 36 | const userRaw = res.data; // depends on the response format 37 | return userRaw; 38 | }); 39 | } 40 | ``` 41 | -------------------------------------------------------------------------------- /packages/frontend/src/Api/comment.ts: -------------------------------------------------------------------------------- 1 | import { axios } from './config'; 2 | import { AxiosPromise } from 'axios'; 3 | import { IComment } from '../types'; 4 | /** 5 | * get comments from postId 6 | * @param postId the comments are under this postId 7 | * @param page 8 | * @param pageSize 9 | */ 10 | function getCommentsApi( 11 | postId: string, 12 | queryString: string, 13 | ): AxiosPromise<{ comments: IComment[]; totalNumber: number }> { 14 | return axios({ 15 | url: `/comment${queryString}`, 16 | params: { postId }, 17 | }); 18 | } 19 | 20 | function getRepostCommentsApi(repostedId: string, queryString: string) { 21 | return axios({ 22 | url: `/repostComment${queryString}`, 23 | params: { repostedId }, 24 | }); 25 | } 26 | 27 | function getSingleCommentApi( 28 | commentId: string, 29 | ): AxiosPromise<{ result: IComment }> { 30 | return axios({ 31 | url: `/comment/${commentId}`, 32 | }); 33 | } 34 | 35 | function getSubCommentsApi(commentId: string, queryString: string) { 36 | return axios({ 37 | url: `/subComment${queryString}`, 38 | params: { commentId }, 39 | }); 40 | } 41 | 42 | export { 43 | getCommentsApi, 44 | getSingleCommentApi, 45 | getSubCommentsApi, 46 | getRepostCommentsApi, 47 | }; 48 | -------------------------------------------------------------------------------- /packages/frontend/src/Api/post.ts: -------------------------------------------------------------------------------- 1 | import { axios } from './config'; 2 | import { AxiosPromise } from 'axios'; 3 | import { IPost } from '../types'; 4 | function getPostsApi( 5 | queryString: string, 6 | ): AxiosPromise<{ post: IPost[]; totalNumber: number }> { 7 | return axios({ 8 | url: `/post${queryString}`, 9 | }); 10 | } 11 | 12 | /** 13 | * get a single post with comments by postId 14 | * @param postId the comments are under this postId 15 | * @param page page of comments 16 | * @param pageSize comments each page 17 | */ 18 | function getSinglePostApi( 19 | postId: string, 20 | page: number, 21 | pageSize: number, 22 | ): AxiosPromise<{ post: IPost; totalNumber: number }> { 23 | return axios({ 24 | url: `/post/${postId}`, 25 | params: { page: page - 1, pageSize }, 26 | }); 27 | } 28 | 29 | function savePostApi(postIdUrl: string): AxiosPromise { 30 | return axios({ 31 | method: 'post', 32 | url: `/post`, 33 | data: { postIdUrl }, 34 | }); 35 | } 36 | 37 | function deletePostApi(postId: string): AxiosPromise<{ result: boolean }> { 38 | return axios({ 39 | method: 'delete', 40 | url: `/post/${postId}`, 41 | }); 42 | } 43 | 44 | export { getPostsApi, getSinglePostApi, savePostApi, deletePostApi }; 45 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Base/baseTypes.ts: -------------------------------------------------------------------------------- 1 | import { MangoQuery, MangoQuerySelector } from 'rxdb'; 2 | 3 | interface IBaseDAL { 4 | findOneById(id: string): Promise; 5 | upsert(info: Info): Promise; 6 | count(selector: MangoQuerySelector): Promise; 7 | query(queryObj: MangoQuery): Promise; 8 | remove(id: string): Promise; 9 | bulkInsert(infos: Info[]): Promise<{ success: any[]; error: any[] }>; 10 | exportData(): Promise; 11 | getVersion(): number; 12 | } 13 | 14 | interface IBaseService { 15 | startCrawling(id: string): void; 16 | save(info: Info): Promise; 17 | getOneByIdPopulated(id: string): Promise; 18 | queryPopulated(queryObj: MangoQuery): Promise; 19 | 20 | importData( 21 | filePath: string, 22 | version: number, 23 | batchSize: number, 24 | ): Promise; 25 | exportData(): Promise; 26 | count(selector: MangoQuerySelector): Promise; 27 | getVersion(): number; 28 | } 29 | 30 | interface IBaseCrawler { 31 | lazyInject: () => void; 32 | startCrawling: (id: string) => void; 33 | } 34 | 35 | export { IBaseDAL, IBaseService, IBaseCrawler }; 36 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/readCredential/readCredential.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import { credentialJsonPath } from '../../Config'; 3 | import { logger } from '../../Logger'; 4 | const fs = require('fs'); 5 | 6 | interface ParsedConfigs { 7 | cookie: string; 8 | users: string[]; 9 | } 10 | async function getCredentialFile() { 11 | let cookie: string = ''; 12 | let users: string[] = []; 13 | if (!fs.existsSync(credentialJsonPath)) { 14 | logger.info(`the config json file ${credentialJsonPath} doesn't exist`); 15 | return { cookie, users }; 16 | } 17 | const rawData: string = fs.readFileSync(credentialJsonPath).toString('utf-8'); 18 | const parsedConfigs: ParsedConfigs = JSON.parse(rawData); 19 | cookie = parsedConfigs.cookie; 20 | users = parsedConfigs.users; 21 | if (typeof cookie !== 'string') { 22 | logger.info( 23 | `the cookie doesn't exist or is not string in ${credentialJsonPath}`, 24 | ); 25 | } 26 | if (!_.isUndefined(users) && !_.isArray(users)) { 27 | logger.info( 28 | `the users in ${credentialJsonPath} should be empty or an array of string`, 29 | ); 30 | } 31 | if (_.isUndefined(users)) { 32 | users = []; 33 | } 34 | return { cookie, users }; 35 | } 36 | 37 | export { getCredentialFile }; 38 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Attach to Chrome", 9 | "port": 9222, 10 | "request": "attach", 11 | "type": "pwa-chrome", 12 | "webRoot": "${workspaceFolder}/frontend", 13 | "url": "http://localhost:3000/*" 14 | }, 15 | { 16 | "name": "Debug Backend", 17 | "type": "node", 18 | "request": "launch", 19 | "args": ["${workspaceRoot}/packages/backend/src"], 20 | "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], 21 | "sourceMaps": true, 22 | "cwd": "${workspaceRoot}/packages/backend", 23 | "protocol": "inspector", 24 | "outputCapture": "std", 25 | "env": { 26 | "NODE_ENV": "development" 27 | } 28 | }, 29 | 30 | { 31 | "name": "Current TS File", 32 | "type": "node", 33 | "request": "launch", 34 | "args": ["${relativeFile}"], 35 | "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], 36 | "sourceMaps": true, 37 | "cwd": "${workspaceRoot}", 38 | "protocol": "inspector" 39 | } 40 | ] 41 | } 42 | -------------------------------------------------------------------------------- /packages/backend/src/Jobs/Queue/timeWindow.ts: -------------------------------------------------------------------------------- 1 | import { AsyncPriorityQueue } from 'async'; 2 | 3 | export class TimeWindow { 4 | private q: AsyncPriorityQueue; 5 | private processingItems: number; 6 | private timeLength: number; 7 | private maxItems: number; 8 | /** 9 | * requesting flow controller. limit the items than can be proceeded in a time window 10 | * @param q the queue 11 | * @param timeLength the time window length, in s 12 | * @param maxItems the max number of items can be proceeded in the time window 13 | */ 14 | constructor( 15 | q: AsyncPriorityQueue, 16 | timeLength: number, 17 | maxItems: number, 18 | ) { 19 | this.q = q; 20 | this.processingItems = 0; 21 | this.timeLength = timeLength * 1000; 22 | this.maxItems = maxItems; 23 | } 24 | 25 | execute(): void { 26 | this.increase(); 27 | setTimeout(() => { 28 | this.decrease(); 29 | }, this.timeLength); 30 | } 31 | private increase(): void { 32 | this.processingItems = this.processingItems + 1; 33 | if (this.processingItems > this.maxItems) { 34 | this.q.pause(); 35 | } 36 | } 37 | private decrease(): void { 38 | this.processingItems = this.processingItems - 1; 39 | if (this.processingItems <= this.maxItems) { 40 | this.q.resume(); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /packages/documentation/docs/zh/guide/account.md: -------------------------------------------------------------------------------- 1 | # Account 2 | 3 | account 组件用于管理 cookie。 从前端提交的 cookie 会在 account 组件中处理。 4 | 5 | ## 校验(可选) 6 | 7 | 在前端发送 cookie 之前,它会验证 cookie 是否可用。 8 | 在`packages/backend/src/Components/Account/Service/accountApi.ts` 9 | 10 | ```TypeScript 11 | function getLoginStatusApi(cookie: string): AxiosPromise { 12 | return { 13 | data:{ 14 | login:false, 15 | uid:'', 16 | } 17 | }; 18 | } 19 | ``` 20 | 21 | `getLoginStatusApi` 将返回一个 login status 和一个用户 ID,表明 cookie 属于哪个用户。 默认情况下,它总是返回 `false` 和 `''`,但前端可以选择“add cookie anyway”。 你可以覆盖此函数以使用真实 API 验证 cookie。 22 | 23 | ## 附带cookie(可选) 24 | 25 | 在`packages/backend/src/Config/axios.ts` 26 | 默认情况下,如果设置了 cookie,所有对平台的 API 请求都会被拦截并加上 cookie。 27 | 28 | ```TypeScript 29 | if (accountService.getMode() === 'cookie') { 30 | request.headers['cookie'] = accountService.getCookie(); 31 | } 32 | ``` 33 | 34 | 如果平台使用其他方式授权,你可以重写此函数。 35 | 例如,如果平台使用 `Authorization` header,你可以重写此函数以添加 `Authorization` header。 36 | 37 | ```typescript 38 | if (accountService.getMode() === 'cookie') { 39 | request.headers['Authorization'] = `Bearer ${accountService.getCookie()}`; 40 | } 41 | ``` 42 | 43 | 其他某些平台可能会使用`api_key`来授权。 44 | 45 | ```typescript 46 | if (accountService.getMode() === 'cookie') { 47 | request.params['api_key'] = accountService.getCookie(); 48 | } 49 | ``` 50 | -------------------------------------------------------------------------------- /packages/frontend/src/Store/reducers/routeState.ts: -------------------------------------------------------------------------------- 1 | import { ROUTE_STATE } from '../actionTypes'; 2 | import _ from 'lodash'; 3 | 4 | type HomeState = { 5 | queryString: string; 6 | id: string; 7 | }; 8 | 9 | type PostContentState = { 10 | queryString: string; 11 | id: string; 12 | }; 13 | 14 | type RouteState = { 15 | home: HomeState; 16 | postContent: PostContentState; 17 | }; 18 | 19 | const initState: RouteState = { 20 | home: { 21 | queryString: '', 22 | id: '', 23 | }, 24 | postContent: { 25 | queryString: '', 26 | id: '', 27 | }, 28 | }; 29 | 30 | function routeState( 31 | state = initState, 32 | action: { type: string; payload: HomeState | PostContentState }, 33 | ): RouteState { 34 | const { type, payload } = action; 35 | switch (type) { 36 | case ROUTE_STATE.HOME.SET_HOME: { 37 | const newState = _.cloneDeep(state); 38 | newState['home'] = _.cloneDeep(payload) as HomeState; 39 | return newState; 40 | } 41 | case ROUTE_STATE.POST_CONTENT.SET_POST_CONTENT: { 42 | const newState = _.cloneDeep(state); 43 | newState['postContent'] = _.cloneDeep(payload) as PostContentState; 44 | return newState; 45 | } 46 | default: { 47 | return state; 48 | } 49 | } 50 | } 51 | 52 | export { routeState }; 53 | 54 | export type { RouteState, HomeState, PostContentState }; 55 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/generateQuery/generateQuery.ts: -------------------------------------------------------------------------------- 1 | import { MangoQuery, MangoQuerySelector } from 'rxdb'; 2 | 3 | function generateQuery( 4 | page: number, 5 | pageSize: number, 6 | orderBy: string, 7 | orderType: 'asc' | 'desc', 8 | createdAt: number[], 9 | saveTime: number[], 10 | users: string[], 11 | text: string, 12 | ) { 13 | const selector: MangoQuerySelector = {}; 14 | const query: MangoQuery = { selector }; 15 | 16 | if (!page) { 17 | page = 0; 18 | } 19 | if (!pageSize) { 20 | pageSize = 10; 21 | } 22 | query.skip = page * pageSize; 23 | query.limit = pageSize; 24 | 25 | if (orderBy) { 26 | query.sort = [{ [orderBy]: orderType === 'asc' ? 'asc' : 'desc' }]; 27 | } 28 | 29 | if (createdAt.length === 1) { 30 | selector.createdAt = { $gt: createdAt[0] }; 31 | } 32 | if (createdAt.length === 2) { 33 | selector.createdAt = { $gte: createdAt[0], $lte: createdAt[1] }; 34 | } 35 | 36 | if (saveTime.length === 1) { 37 | selector.saveTime = { $gt: saveTime[0] }; 38 | } 39 | if (saveTime.length === 2) { 40 | selector.saveTime = { $gte: saveTime[0], $lte: saveTime[1] }; 41 | } 42 | 43 | if (users.length > 0) { 44 | selector.user = { $in: users }; 45 | } 46 | 47 | if (text) { 48 | selector.text = { $regex: text }; 49 | } 50 | 51 | return query; 52 | } 53 | 54 | export { generateQuery }; 55 | -------------------------------------------------------------------------------- /packages/extension-chrome/src/Components/Crawler/Crawler.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import { Form, Input, Button, message } from 'antd'; 3 | import axios from 'axios'; 4 | 5 | /** 6 | * 7 | * @param props 8 | * @returns 9 | */ 10 | 11 | export function Crawler(props: { server: string }) { 12 | const [form] = Form.useForm(); 13 | const [loading, setLoading] = useState(false); 14 | const onSubmit = async () => { 15 | console.log('submitting'); 16 | setLoading(true); 17 | try { 18 | const values = form.getFieldsValue(); 19 | const url = values.url; 20 | console.log(url, 'url'); 21 | await axios({ 22 | url: props.server + '/api/post', 23 | data: { postIdUrl: url }, 24 | method: 'post', 25 | }); 26 | 27 | message.success(`post ${url} backup is processing`); 28 | } catch (error) { 29 | message.error('Failed to backup'); 30 | alert(error); 31 | } finally { 32 | setLoading(false); 33 | } 34 | }; 35 | return ( 36 |
37 | 38 | 39 | 40 | 41 | 44 | 45 |
46 | ); 47 | } 48 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/PostCard/PhotosPreviewer.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Col, Row } from 'antd'; 3 | import { PhotoProvider, PhotoConsumer } from 'react-photo-view'; 4 | import { getImageUrl } from '../../Utility/parseUrl'; 5 | import 'react-photo-view/dist/index.css'; 6 | import { Image } from '../../types'; 7 | 8 | function PhotosPreviewer(props: { images: Image[][] }) { 9 | const { images } = props; 10 | return ( 11 | 12 | {images.map((item) => ( 13 | 14 | {item.map((image) => ( 15 | 16 | 21 | 30 | 31 | 32 | ))} 33 | 34 | ))} 35 | 36 | ); 37 | } 38 | 39 | export { PhotosPreviewer }; 40 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/MonitorIntro/MonitorIntro.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Modal } from 'antd'; 3 | 4 | function showMonitorIntro() { 5 | Modal.info({ 6 | icon: null, 7 | content: ( 8 |
9 |
10 |

What does monitor do?

11 |

12 | Monitor can monitor the collections. You can add the api url of a 13 | collection, like a upvote, watch later, favorite, and so on, as long 14 | as it's supported. Once you add the api url, the monitor module will 15 | request the api periodically, and catch the new added post in the 16 | collection, and then backup that post. 17 |

18 |

19 | First, select the collection type, and then copy the api url from 20 | the Chrome debug tool to the collection url input box. 21 |

22 |
23 |
24 |

什么是monitor?

25 |

26 | Monitor可以监控集合。你可以添加集合(比如点赞,收藏,稍后观看,喜爱等等,只要程序支持这种集合)的api 27 | url。添加后,monitor模块会定时请求api,当监测到有新的贴文被添加进这个集合后,本程序就会把这个新的贴文备份下来。 28 |

29 |

30 | 首先,选择collection 31 | type,然后复制Chrome调试工具中的api的url到collection url输入框中。 32 |

33 |
34 |
35 | ), 36 | }); 37 | } 38 | 39 | export { showMonitorIntro }; 40 | -------------------------------------------------------------------------------- /packages/backend/src/Components/RepostComment/Types/repostCommentTypes.ts: -------------------------------------------------------------------------------- 1 | import { RxCollection, RxDocument } from 'rxdb'; 2 | import { PostDocument } from '../../Post/Types/postTypes'; 3 | import { IBaseCrawler, IBaseDAL, IBaseService } from '../../Base/baseTypes'; 4 | import { IUser } from '../../User/Types/userTypes'; 5 | 6 | type IRepostComment = { 7 | id: string; 8 | content: string; // unicode and html 9 | user: string; 10 | createTime: number; 11 | repostedId: string; 12 | saveTime: number; 13 | }; 14 | 15 | type RepostCommentDocument = RxDocument; 16 | 17 | type RepostCommentCollection = RxCollection; 18 | 19 | type IRepostCommentPopulated = Omit & { user: IUser }; 20 | 21 | interface IRepostCommentDAL 22 | extends IBaseDAL {} 23 | 24 | interface IRepostCommentService 25 | extends IBaseService< 26 | IRepostComment, 27 | RepostCommentDocument, 28 | IRepostCommentPopulated 29 | > {} 30 | 31 | interface IRepostCommentCrawler extends IBaseCrawler {} 32 | 33 | type RepostCommentCrawlerParams = { 34 | postId: string; 35 | /* possible other properties */ 36 | }; 37 | 38 | export { 39 | IRepostComment, 40 | RepostCommentDocument, 41 | RepostCommentCollection, 42 | IRepostCommentService, 43 | IRepostCommentDAL, 44 | RepostCommentCrawlerParams, 45 | IRepostCommentPopulated, 46 | IRepostCommentCrawler, 47 | }; 48 | -------------------------------------------------------------------------------- /packages/extension-chrome/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@social-media-archiver/extension-chrome", 3 | "version": "1.3.0", 4 | "private": true, 5 | "dependencies": { 6 | "@testing-library/jest-dom": "^5.11.4", 7 | "@testing-library/react": "^11.1.0", 8 | "@testing-library/user-event": "^12.1.10", 9 | "@types/chrome": "^0.0.159", 10 | "@types/jest": "^26.0.15", 11 | "@types/node": "^12.0.0", 12 | "@types/react": "^17.0.0", 13 | "@types/react-dom": "^17.0.0", 14 | "antd": "^4.16.13", 15 | "axios": "^0.23.0", 16 | "cross-env": "^7.0.3", 17 | "react": "^17.0.2", 18 | "react-dom": "^17.0.2", 19 | "react-scripts": "4.0.3", 20 | "typescript": "^4.1.2", 21 | "web-vitals": "^1.0.1" 22 | }, 23 | "scripts": { 24 | "start": "react-scripts start", 25 | "build": "cross-env INLINE_RUNTIME_CHUNK=false react-scripts build", 26 | "test": "react-scripts test", 27 | "eject": "react-scripts eject" 28 | }, 29 | "eslintConfig": { 30 | "extends": [ 31 | "react-app", 32 | "react-app/jest" 33 | ] 34 | }, 35 | "browserslist": { 36 | "production": [ 37 | ">0.2%", 38 | "not dead", 39 | "not op_mini all" 40 | ], 41 | "development": [ 42 | "last 1 chrome version", 43 | "last 1 firefox version", 44 | "last 1 safari version" 45 | ] 46 | }, 47 | "devDependencies": { 48 | "cross-env": "^7.0.3", 49 | "prettier": "2.4.1" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /packages/backend/src/Utility/json/jsonStream.ts: -------------------------------------------------------------------------------- 1 | import Batch from 'stream-json/utils/Batch'; 2 | import StreamArray from 'stream-json/streamers/StreamArray'; 3 | import { chain } from 'stream-chain'; 4 | import { Writable } from 'stream'; 5 | import fs from 'fs'; 6 | 7 | /** 8 | * https://stackoverflow.com/questions/42896447/parse-large-json-file-in-nodejs-and-handle-each-object-independently 9 | * Read json stream from local file and pass a batch of records to the worker function 10 | * @param jsonPath the local json file path 11 | * @param batchSize 12 | * @param worker 13 | * @returns 14 | */ 15 | function processJsonAsStream( 16 | jsonPath: string, 17 | batchSize: number, 18 | worker: (value: any[]) => Promise, 19 | ) { 20 | return new Promise((res, rej) => { 21 | const processingStream = new Writable({ 22 | write(value: { key: string; value: T }[], encoding, callback) { 23 | //Save to mongo or do any other async actions 24 | worker(value).then(callback).catch(callback); 25 | }, 26 | //Don't skip this, as we need to operate with objects, not buffers 27 | objectMode: true, 28 | }); 29 | 30 | const pipeline = chain([ 31 | fs.createReadStream(jsonPath), 32 | StreamArray.withParser(), 33 | new Batch({ batchSize }), 34 | processingStream, 35 | ]); 36 | 37 | pipeline.on('end', res); 38 | 39 | pipeline.on('error', rej); 40 | }); 41 | } 42 | 43 | export { processJsonAsStream }; 44 | -------------------------------------------------------------------------------- /packages/documentation/docs/guide/user.md: -------------------------------------------------------------------------------- 1 | # User 2 | 3 | ## Transform 4 | 5 | in `packages/backend/src/Components/User/Service/userService.ts` 6 | 7 | ```typescript 8 | transformUserResponse(userRaw: any): IUser { 9 | throw new NotImplementedError( 10 | `transformUserResponse in UserService is not implemented`, 11 | ); 12 | } 13 | ``` 14 | 15 | The function wil take in the raw user object. You should transform the raw user object into a `IUser` object. see `packages/backend/src/Components/User/Types/userTypes.ts` 16 | 17 | ## (Optional) API 18 | 19 | In most cases, you will not need to make a API request to get the user information, since the user information is usually attached to the other response. 20 | in `packages/backend/src/Components/User/Service/userApi.ts`, 21 | 22 | ```typescript 23 | function getUserInfoByIdApi(userId: string): Promise { 24 | throw new NotImplementedError('getUserInfoByIdApi is not implemented'); 25 | } 26 | ``` 27 | 28 | this should return a promise resolved to be the `userRaw` object to meet the format of `userService.transformUserResponse` parameter. 29 | If not, transform it to the `userRaw` object in `then` and return it. 30 | 31 | ```typescript 32 | function getUserInfoByIdApi(userId: string): Promise { 33 | return crawlerAxios({ 34 | url: `/api/user/${id}`, 35 | }).then((res) => { 36 | const userRaw = res.data; // depends on the response format 37 | return userRaw; 38 | }); 39 | } 40 | ``` 41 | -------------------------------------------------------------------------------- /packages/frontend/src/Api/monitor.ts: -------------------------------------------------------------------------------- 1 | import { axios } from './config'; 2 | import { AxiosPromise } from 'axios'; 3 | 4 | type MonitorCollection = { 5 | url: string; 6 | type: string; 7 | }; 8 | 9 | function validateCollectionApi( 10 | collectionUrl: string, 11 | type: string, 12 | ): AxiosPromise { 13 | return axios({ 14 | url: '/monitor/validate', 15 | method: 'POST', 16 | data: { 17 | url: collectionUrl, 18 | type, 19 | }, 20 | }); 21 | } 22 | 23 | function addCollectionApi( 24 | collectionUrl: string, 25 | type: string, 26 | ): AxiosPromise { 27 | return axios({ 28 | url: '/monitor', 29 | method: 'POST', 30 | data: { 31 | url: collectionUrl, 32 | type, 33 | }, 34 | }); 35 | } 36 | 37 | function removeCollectionApi(collectionUrl: string): AxiosPromise { 38 | return axios({ 39 | url: '/monitor', 40 | method: 'DELETE', 41 | data: { 42 | url: collectionUrl, 43 | }, 44 | }); 45 | } 46 | 47 | function getCollectionsApi(): AxiosPromise { 48 | return axios({ 49 | method: 'GET', 50 | url: '/monitor', 51 | }); 52 | } 53 | 54 | function getCollectionTypesApi(): AxiosPromise { 55 | return axios({ 56 | method: 'GET', 57 | url: '/monitor/types', 58 | }); 59 | } 60 | 61 | export { 62 | validateCollectionApi, 63 | addCollectionApi, 64 | removeCollectionApi, 65 | getCollectionsApi, 66 | getCollectionTypesApi, 67 | }; 68 | -------------------------------------------------------------------------------- /packages/backend/src/Config/axios.ts: -------------------------------------------------------------------------------- 1 | import axios, { AxiosInstance } from 'axios'; 2 | import { 3 | IAccountService, 4 | ACCOUNT_IOC_SYMBOLS, 5 | } from '../Components/Account/Types'; 6 | import { BASE_URL } from './'; 7 | import { container } from './inversify.config'; 8 | 9 | interface Headers { 10 | 'User-Agent': String; 11 | } 12 | 13 | interface DownloadHeader { 14 | 'User-Agent': string; 15 | } 16 | 17 | const header: Headers = { 18 | 'User-Agent': 19 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36', 20 | }; 21 | 22 | const downloadHeader: DownloadHeader = { 23 | 'User-Agent': 24 | 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36', 25 | }; 26 | 27 | const crawlerAxios: AxiosInstance = axios.create({ 28 | baseURL: BASE_URL, 29 | //headers: header, 30 | }); 31 | 32 | const downloadAxios: AxiosInstance = axios.create({ 33 | //headers:downloadHeader, 34 | }); 35 | 36 | [crawlerAxios, downloadAxios].forEach((item: AxiosInstance) => { 37 | item.interceptors.request.use((request) => { 38 | const accountService = container.get( 39 | ACCOUNT_IOC_SYMBOLS.IAccountService, 40 | ); 41 | if (accountService.getMode() === 'cookie') { 42 | request.headers['cookie'] = accountService.getCookie(); 43 | } 44 | return request; 45 | }); 46 | }); 47 | 48 | export { crawlerAxios, downloadAxios, axios }; 49 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Post/DAL/postSchema.ts: -------------------------------------------------------------------------------- 1 | import { RxJsonSchema } from 'rxdb'; 2 | import { IPost } from '../Types'; 3 | 4 | const version: number = 0; 5 | 6 | const postSchema: RxJsonSchema = { 7 | title: 'post schema', 8 | version: version, 9 | description: 'post schema', 10 | type: 'object', 11 | properties: { 12 | id: { type: 'string', primary: true }, 13 | createTime: { type: 'number' }, 14 | content: { type: 'string' }, 15 | repostsCount: { type: 'number' }, 16 | commentsCount: { type: 'number' }, 17 | upvotesCount: { type: 'number' }, 18 | user: { type: 'string', ref: 'user' }, 19 | comments: { type: 'array', ref: 'comment', items: { type: 'string' } }, 20 | images: { 21 | type: 'array', 22 | items: { 23 | type: 'object', 24 | properties: { name: { type: 'string' }, originUrl: { type: 'string' } }, 25 | }, 26 | }, 27 | videos: { 28 | type: 'array', 29 | items: { 30 | type: 'object', 31 | properties: { name: { type: 'string' }, originUrl: { type: 'string' } }, 32 | }, 33 | }, 34 | saveTime: { type: 'number' }, 35 | repostingId: { type: 'string', ref: 'post' }, 36 | repostComments: { 37 | type: 'array', 38 | ref: 'repostcomment', 39 | items: { type: 'string' }, 40 | }, 41 | }, 42 | required: ['id', 'user',"content","createTime",'saveTime'], 43 | indexes: ['saveTime', 'createTime', 'user'], 44 | }; 45 | 46 | export { postSchema, version }; 47 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/DebounceSelect/DebounceSelect.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useRef, useMemo } from 'react'; 2 | 3 | import { Select, Spin } from 'antd'; 4 | import debounce from 'lodash/debounce'; 5 | 6 | /** 7 | * https://ant.design/components/select/#components-select-demo-select-users 8 | * the fetchApi should return format like {value:string,label:string} 9 | * @returns 10 | */ 11 | export function DebounceSelect({ fetchApi, debounceTimeout = 800, ...props }) { 12 | const [fetching, setFetching] = useState(false); 13 | const [options, setOptions] = useState([]); 14 | const fetchRef = useRef(0); 15 | const debounceFetcher = useMemo(() => { 16 | const loadOptions = (value) => { 17 | fetchRef.current += 1; 18 | const fetchId = fetchRef.current; 19 | setOptions([]); 20 | setFetching(true); 21 | fetchApi(value).then((newOptions) => { 22 | if (fetchId !== fetchRef.current) { 23 | // for fetch callback order 24 | return; 25 | } 26 | 27 | setOptions(newOptions); 28 | setFetching(false); 29 | }); 30 | }; 31 | 32 | return debounce(loadOptions, debounceTimeout); 33 | }, [fetchApi, debounceTimeout]); 34 | 35 | return ( 36 | 42 | 43 | 44 | 47 | 48 | 49 | 50 | ); 51 | } 52 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Video/Service/videoService.ts: -------------------------------------------------------------------------------- 1 | import { injectable } from 'inversify'; 2 | import 'reflect-metadata'; 3 | import { IVideoService, ParamsQueue } from '../Types'; 4 | import { downloadVideoApi } from './videoApi'; 5 | import fs from 'fs'; 6 | import path from 'path'; 7 | import { asyncPriorityQueuePush, QueueTask } from '../../../Jobs/Queue'; 8 | import { Q_PRIORITY, staticPath } from '../../../Config'; 9 | import { ResourceError } from '../../../Error/ErrorClass'; 10 | import { getUrlLastSegment } from '../../../Utility/urlParse'; 11 | 12 | @injectable() 13 | class VideoService implements IVideoService { 14 | downloadVideo(videoUrl: string) { 15 | const params: ParamsQueue = { 16 | url: videoUrl, 17 | staticPath, 18 | }; 19 | 20 | asyncPriorityQueuePush( 21 | this.videoQueueFunc, 22 | params, 23 | Q_PRIORITY.DOWNLOAD_VIDEO, 24 | ).catch((error) => { 25 | throw new ResourceError( 26 | `Failed to fetch video ${videoUrl}:${error.response?.status}`, 27 | ); 28 | }); 29 | } 30 | 31 | private videoQueueFunc(queueParams: ParamsQueue): Promise { 32 | const { url, staticPath } = queueParams; 33 | return new Promise((resolve, reject) => { 34 | downloadVideoApi(url) 35 | .then((res) => { 36 | const { data } = res; 37 | if (!fs.existsSync(path.resolve(staticPath, 'videos'))) { 38 | fs.mkdirSync(path.resolve(staticPath, 'videos')); 39 | } 40 | const writer = fs.createWriteStream( 41 | path.resolve(staticPath, 'videos', getUrlLastSegment(url)), 42 | ); 43 | data.pipe(writer); 44 | writer.on('finish', resolve); 45 | writer.on('error', reject); 46 | }) 47 | .catch(reject); 48 | }); 49 | } 50 | } 51 | 52 | export { VideoService }; 53 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Comment/Types/commentTypes.ts: -------------------------------------------------------------------------------- 1 | import { RxCollection, RxDocument, MangoQuerySelector } from 'rxdb'; 2 | import { IBaseDAL, IBaseService } from '../../Base/baseTypes'; 3 | import { IUser } from '../../User/Types/userTypes'; 4 | import { PostDocument } from '../../Post/Types/postTypes'; 5 | 6 | type IComment = { 7 | id: string; 8 | floorNumber?: number; 9 | content: string; // unicode and html 10 | subCommentsCount?: number; // the number of sub comments shown on client 11 | user: string; 12 | upvotesCount?: number; 13 | createTime: number; 14 | subComments?: string[]; 15 | image?: { name: string; originUrl: string }; 16 | postId: string; 17 | saveTime: number; 18 | replyTo?: string; 19 | }; 20 | 21 | type CommentDocument = RxDocument; 22 | 23 | type CommentCollection = RxCollection; 24 | 25 | type ICommentPopulated = Omit, 'replyTo'> & { 26 | user: IUser; 27 | } & { 28 | replyTo?: IUser; 29 | }; 30 | 31 | interface ICommentDAL extends IBaseDAL { 32 | addSubComments: ( 33 | subCommentIds: string[], 34 | commentDoc: CommentDocument, 35 | ) => Promise; 36 | } 37 | 38 | interface ICommentService 39 | extends IBaseService< 40 | IComment, 41 | CommentDocument, 42 | ICommentPopulated 43 | > { 44 | addSubComments: ( 45 | subCommentIds: string[], 46 | commentId: string, 47 | ) => Promise; 48 | } 49 | 50 | type CommentCrawlParams = { 51 | postId: string; 52 | /* possible other properties */ 53 | }; 54 | 55 | interface ICommentCrawler { 56 | startCrawling: (postId: string) => void; 57 | } 58 | 59 | export { 60 | IComment, 61 | CommentDocument, 62 | CommentCollection, 63 | ICommentService, 64 | ICommentDAL, 65 | CommentCrawlParams, 66 | ICommentPopulated, 67 | ICommentCrawler, 68 | }; 69 | -------------------------------------------------------------------------------- /packages/frontend/src/Utility/route/getRouteState.ts: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | import { HomeState, PostContentState } from '../../Store/reducers/routeState'; 3 | 4 | /** 5 | * generate the new home state based on the query string. You will update this returned state to redux 6 | * @param {string} search the query string in url 7 | * @param {HomeState} oldState old home state in redux 8 | * @param {string} postId 9 | * @returns {HomeState} 10 | */ 11 | const getNewHomeState = ( 12 | search: string, 13 | oldState: HomeState, 14 | postId: String, 15 | ): HomeState => { 16 | const searchParams = new URLSearchParams(search); 17 | const state: any = _.cloneDeep(oldState); 18 | searchParams.forEach((value: any, key: any) => { 19 | if (!value) { 20 | return false; 21 | } 22 | const strings = [ 23 | 'page', 24 | 'pageSize', 25 | 'content', 26 | 'orderBy', 27 | 'orderType', 28 | 'id', 29 | ]; 30 | let decodedValue = decodeURIComponent(value); 31 | const arr = ['users', 'createdAt', 'saveTime']; 32 | if (strings.includes(key)) { 33 | state[key] = decodedValue; 34 | } 35 | if (arr.includes(key)) { 36 | state[key] = JSON.parse(decodedValue); 37 | } 38 | }); 39 | state['id'] = postId; 40 | return state; 41 | }; 42 | 43 | const getNewPostContentState = ( 44 | search: string, 45 | oldState: PostContentState, 46 | postId: string, 47 | ) => { 48 | const searchParams = new URLSearchParams(search); 49 | const state: any = _.cloneDeep(oldState); 50 | searchParams.forEach((value: any, key: any) => { 51 | if (!value) { 52 | return false; 53 | } 54 | let decodedValue = decodeURIComponent(value); 55 | state[key] = decodedValue; 56 | }); 57 | state['id'] = postId; 58 | return state; 59 | }; 60 | 61 | export { getNewHomeState, getNewPostContentState }; 62 | -------------------------------------------------------------------------------- /packages/frontend/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 17 | 18 | 27 | React App 28 | 29 | 30 | 31 |
32 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /packages/extension-chrome/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 17 | 18 | 27 | React App 28 | 29 | 30 | 31 |
32 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Image/Service/imageService.ts: -------------------------------------------------------------------------------- 1 | import { injectable } from 'inversify'; 2 | import 'reflect-metadata'; 3 | import { ParamsQueue, ImageServiceInterface } from '../Types'; 4 | import { downloadImageApi } from './imageApi'; 5 | import fs from 'fs'; 6 | import path from 'path'; 7 | import { asyncPriorityQueuePush } from '../../../Jobs/Queue'; 8 | import { Q_PRIORITY, staticPath } from '../../../Config'; 9 | import { ResourceError } from '../../../Error/ErrorClass'; 10 | import { getUrlLastSegment } from '../../../Utility/urlParse'; 11 | 12 | @injectable() 13 | class ImageService implements ImageServiceInterface { 14 | downloadImage(imageUrl: string, priority?: number) { 15 | if (!priority) { 16 | priority = Q_PRIORITY.DOWNLOAD_IMAGE; 17 | } 18 | const params: ParamsQueue = { 19 | url: imageUrl, 20 | staticPath, 21 | }; 22 | 23 | asyncPriorityQueuePush(this.imageQueueFunc, params, priority).catch( 24 | (error) => { 25 | throw new ResourceError(`Failed to fetch image ${imageUrl}:${error}`); 26 | }, 27 | ); 28 | } 29 | private imageQueueFunc(queueParams: ParamsQueue): Promise { 30 | const { url, staticPath } = queueParams; 31 | return new Promise((resolve, reject) => { 32 | downloadImageApi(url) 33 | .then((res) => { 34 | const { data } = res; 35 | if (!fs.existsSync(path.resolve(staticPath, 'images'))) { 36 | fs.mkdirSync(path.resolve(staticPath, 'images')); 37 | } 38 | const writer = fs.createWriteStream( 39 | path.resolve(staticPath, 'images', getUrlLastSegment(url)), 40 | ); 41 | 42 | data.pipe(writer); 43 | writer.on('finish', resolve); 44 | writer.on('error', (err) => { 45 | reject(err); 46 | }); 47 | }) 48 | .catch((err) => { 49 | reject(err); 50 | }); 51 | }); 52 | } 53 | } 54 | 55 | export { ImageService }; 56 | -------------------------------------------------------------------------------- /packages/backend/src/Components/User/DAL/userDAL.ts: -------------------------------------------------------------------------------- 1 | import { 2 | IUserDAL, 3 | UserDocument, 4 | IUser, 5 | UserCollection, 6 | } from '../Types/userTypes'; 7 | import { database } from '../../../Loaders/rxdb'; 8 | import { MangoQuery } from 'rxdb'; 9 | import { injectable } from 'inversify'; 10 | import 'reflect-metadata'; 11 | import { NotImplementedError } from '../../../Error/ErrorClass'; 12 | import { DatabaseError } from '../../../Error/ErrorClass'; 13 | import { version } from './userSchema'; 14 | @injectable() 15 | class UserDAL implements IUserDAL { 16 | constructor() { 17 | if (!database) { 18 | throw new DatabaseError('Database is not connect'); 19 | } 20 | } 21 | async upsert(userInfo: IUser) { 22 | const userDoc: UserDocument = await database.user.atomicUpsert(userInfo); 23 | return userDoc; 24 | } 25 | 26 | async findOneById(userId: string): Promise { 27 | if (!userId) { 28 | return null; 29 | } 30 | const userCollection: UserCollection = database.user; 31 | const userDoc: UserDocument | null = await userCollection 32 | .findOne(userId) 33 | .exec(); 34 | return userDoc; 35 | } 36 | 37 | async query(queryObj: MangoQuery): Promise { 38 | const userDocs: UserDocument[] = await database.user.find(queryObj).exec(); 39 | return userDocs; 40 | } 41 | count(): never { 42 | throw new NotImplementedError('method is not implemented'); 43 | } 44 | remove(): never { 45 | throw new NotImplementedError('method is not implemented'); 46 | } 47 | 48 | async bulkInsert(infos: IUser[]) { 49 | const userCollection: UserCollection = database.user; 50 | return await userCollection.bulkInsert(infos); 51 | } 52 | 53 | getVersion() { 54 | return version; 55 | } 56 | 57 | exportData() { 58 | const userCollection: UserCollection = database.user; 59 | return userCollection.find().exec(); 60 | } 61 | } 62 | 63 | export { UserDAL }; 64 | -------------------------------------------------------------------------------- /packages/backend/src/Jobs/Queue/queue.ts: -------------------------------------------------------------------------------- 1 | import async, { AsyncPriorityQueue } from 'async'; 2 | import { MAX_ITEM_WINDOW, Q_CONCURRENCY } from '../../Config'; 3 | import { TimeWindow } from './timeWindow'; 4 | const { priorityQueue } = async; 5 | 6 | interface QueueTask { 7 | params: Params; 8 | handler(params: Params): Promise; 9 | } 10 | 11 | const worker = (task: QueueTask, callback: any): void => { 12 | timeWindow.execute(); 13 | const { params, handler } = task; 14 | handler(params) 15 | .then((res: any) => { 16 | callback(null, res); 17 | }) 18 | .catch((err: Error) => { 19 | callback(err); 20 | }); 21 | }; 22 | 23 | /** 24 | * https://caolan.github.io/async/v3/docs.html#priorityQueue 25 | */ 26 | const pq: AsyncPriorityQueue> = priorityQueue( 27 | worker, 28 | Q_CONCURRENCY, 29 | ); 30 | 31 | const timeWindow = new TimeWindow(pq, 30, MAX_ITEM_WINDOW); 32 | 33 | /** 34 | * push a task into the async priority queue 35 | * @param handler an async function to be executed in the priority queue 36 | * @param params the params object that the handler will take 37 | * @param priority 38 | * @returns a promise that will be resolved when the handler is resolved, or be rejected when the handler reject an error. 39 | */ 40 | function asyncPriorityQueuePush( 41 | handler: (params: Params) => Promise, 42 | params: Params, 43 | priority: number, 44 | ): Promise { 45 | return new Promise((resolve, reject) => { 46 | const task = { 47 | handler, 48 | params, 49 | }; 50 | pq.push(task, priority, (err: Error | null | undefined, result: any) => { 51 | // this callback function will be called after the `callback` in worker is called? 52 | if (err) { 53 | reject(err); 54 | } else { 55 | resolve(result); 56 | } 57 | }); 58 | }); 59 | } 60 | 61 | export { asyncPriorityQueuePush, QueueTask }; 62 | -------------------------------------------------------------------------------- /packages/frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@social-media-archiver/frontend", 3 | "version": "1.3.0", 4 | "private": true, 5 | "proxy": "http://localhost:5000", 6 | "dependencies": { 7 | "@testing-library/jest-dom": "^4.2.4", 8 | "@testing-library/react": "^9.5.0", 9 | "@testing-library/user-event": "^7.2.1", 10 | "@types/axios": "^0.14.0", 11 | "@types/jest": "^24.9.1", 12 | "@types/lodash": "^4.14.161", 13 | "@types/node": "^12.12.55", 14 | "@types/react": "^16.9.49", 15 | "@types/react-dom": "^16.9.8", 16 | "@types/react-html-parser": "^2.0.1", 17 | "@types/react-router-dom": "^5.1.5", 18 | "@typescript-eslint/eslint-plugin": "^4.0.1", 19 | "antd": "^4.6.2", 20 | "axios": "^0.20.0", 21 | "dayjs": "^1.10.6", 22 | "lodash": "^4.17.20", 23 | "react": "^16.13.1", 24 | "react-dom": "^16.13.1", 25 | "react-html-parser": "^2.0.2", 26 | "react-medium-image-zoom": "^4.3.1", 27 | "react-photo-view": "^0.5.2", 28 | "react-player": "^2.6.1", 29 | "react-redux": "^7.2.2", 30 | "react-router-dom": "^5.2.0", 31 | "react-scripts": "3.4.3", 32 | "react-use": "^17.2.4", 33 | "redux": "^4.0.5", 34 | "sass": "^1.37.5" 35 | }, 36 | "scripts": { 37 | "start": "cross-env CI=true react-scripts start", 38 | "build": "cross-env CI=false react-scripts build", 39 | "test": "react-scripts test", 40 | "eject": "react-scripts eject" 41 | }, 42 | "eslintConfig": { 43 | "extends": "react-app" 44 | }, 45 | "browserslist": { 46 | "production": [ 47 | ">0.2%", 48 | "not dead", 49 | "not op_mini all" 50 | ], 51 | "development": [ 52 | "last 1 chrome version", 53 | "last 1 firefox version", 54 | "last 1 safari version" 55 | ] 56 | }, 57 | "devDependencies": { 58 | "@types/react-redux": "^7.1.16", 59 | "@typescript-eslint/parser": "^4.0.1", 60 | "cross-env": "^7.0.3", 61 | "prettier": "2.4.1", 62 | "typescript": "^4.3.5" 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/SavePostModal/SavePostModal.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import { Modal, Form, Input, message, Tooltip } from 'antd'; 3 | import { savePostApi } from '../../Api'; 4 | import { QuestionCircleOutlined } from '@ant-design/icons'; 5 | interface SavePostModalProps { 6 | visible: boolean; 7 | closeModal: () => void; 8 | } 9 | 10 | export default function SavePostModal(props: SavePostModalProps) { 11 | const { visible, closeModal } = props; 12 | const [form] = Form.useForm(); 13 | const [loading, setLoading] = useState(false); 14 | return ( 15 | { 19 | closeModal(); 20 | form.resetFields(); 21 | }} 22 | onOk={async () => { 23 | setLoading(true); 24 | try { 25 | const value = await form.validateFields(); 26 | const response = await savePostApi(value.postIdUrl); 27 | setLoading(false); 28 | closeModal(); 29 | form.resetFields(); 30 | if (response?.data?.status === 'error') { 31 | message.error( 32 | response?.data?.message || 33 | `post "${value.postIdUrl}" doesn't exist or the token has expired`, 34 | ); 35 | } else { 36 | message.success(`Post ${value.postIdUrl} backup processing`); 37 | } 38 | } catch (err) { 39 | message.error('Error Network: Failed to save the post'); 40 | setLoading(false); 41 | } 42 | }} 43 | visible={visible} 44 | > 45 |
46 | 53 | 54 | 55 |
56 |
57 | ); 58 | } 59 | -------------------------------------------------------------------------------- /packages/documentation/docs/zh/guide/get-start.md: -------------------------------------------------------------------------------- 1 | # 开始 2 | 3 | ## 背景知识 4 | 5 | - Typescript 6 | - Node.js 7 | 8 | ## 依赖 9 | 10 | 1. Node.js 11 | 2. npm 12 | 3. [lerna](https://lerna.js.org/#getting-started): `npm i -g lerna` 13 | 14 | ## 安装 15 | 16 | 1. 克隆仓库 17 | 18 | ```shell 19 | git clone https://github.com/Combo819/social-media-archiver.git 20 | ``` 21 | 22 | 2. 安装依赖 23 | 24 | ```shell 25 | cd social-media-archiver 26 | lerna bootstrap 27 | ``` 28 | ## 配置 29 | 30 | In `packages/backend/src/Config/constants.ts`: 31 | 32 | ```typescript 33 | const BASE_URL: string = `PLATFORM_DOMAIN`; 34 | const Q_CONCURRENCY: number = 1; // number of concurrent API requests to the platform 35 | const MAX_ITEM_WINDOW: number = 6; // max number of API requests to the platform in a 30s window 36 | ``` 37 | 38 | ## 启动 39 | 40 | 同时启动前后端: 41 | 42 | ```shell 43 | npm run start 44 | ``` 45 | 46 | 默认情况下,前端将运行在端口`3000`上,后端将运行在端口`5000`上。 47 | 前端的 API 调用会被自动代理到端口`5000`。 48 | 因此,如果你正在进行前端开发,请确保后端运行在端口`5000`上。 49 | 当你看到 50 | 51 | ```log 52 | @social-media-archiver/backend: Development mode, Listening on port 5000 53 | ``` 54 | 55 | 说明后端正在运行了。 56 | 57 | ## 调试 58 | 59 | 要调试后端,首先停止正在运行的后端,然后在 VS Code 中打开调试面板, 60 | 在下拉菜单中选择 `Debug Backend` ,然后单击`Start Debugging`按钮。 61 | 62 | ![debug](./debug.png) 63 | 64 | 然后,你可以使用诸如断点之类的调试工具,调试消息将显示在Terminal右边的调试控制台中。 65 | 66 | ## 日志 67 | 68 | 在开发环境中,日志将显示在终端或调试控制台(对于VS Code调试模式)。还有一个日志文件在`packages/backend/log/social-media-archiver.log` 69 | 打包后,日志文件将会在 `log/social-media-archiver.log`. 70 | 你可以使用 [pino-pretty](https://github.com/pinojs/pino-pretty) 来格式化输出的日志. 71 | 72 | ## Commit 73 | 74 | 本项目采用 `git-cz`. 当你要commit到git时, `npm run commit`, 它会自动生成一个commit信息模板. 75 | 76 | ## 技术栈 77 | + 前端: [React](https://reactjs.org/) 78 | + UI: [Ant Design](https://ant.design/) 79 | + 服务: [Express](https://expressjs.com/) 80 | + Html处理: [cheerio](https://cheerio.js.org/) 81 | + 控制反转: [inversify](https://inversify.io/) 82 | + 流量控制: [async](https://caolan.github.io/async/v3/) 83 | + 数据库: [rxdb](https://rxdb.info/) 84 | + 打包: [pkg](https://github.com/vercel/pkg) 85 | -------------------------------------------------------------------------------- /packages/backend/src/Loaders/rxdb.ts: -------------------------------------------------------------------------------- 1 | import { createRxDatabase, RxDatabase, addRxPlugin } from 'rxdb'; 2 | import { commentSchema, commentMigration } from '../Components/Comment/DAL'; 3 | import { CommentCollection } from '../Components/Comment/Types'; 4 | import { 5 | repostCommentSchema, 6 | repostCommentMigration, 7 | } from '../Components/RepostComment/DAL'; 8 | import { RepostCommentCollection } from '../Components/RepostComment/Types'; 9 | import { 10 | subCommentMigration, 11 | subCommentSchema, 12 | } from '../Components/SubComment/DAL'; 13 | import { SubCommentCollection } from '../Components/SubComment/Types'; 14 | import { userMigration, userSchema } from '../Components/User/DAL'; 15 | import { UserCollection } from '../Components/User/Types'; 16 | import { postMigration, postSchema } from '../Components/Post/DAL'; 17 | import { PostCollection } from '../Components/Post/Types'; 18 | 19 | import { rxdbPath } from '../Config'; 20 | 21 | type DataBaseCollections = { 22 | user: UserCollection; 23 | post: PostCollection; 24 | comment: CommentCollection; 25 | subcomment: SubCommentCollection; 26 | repostcomment: RepostCommentCollection; 27 | }; 28 | 29 | type DatabaseType = RxDatabase; 30 | 31 | addRxPlugin(require('pouchdb-adapter-leveldb')); 32 | const leveldown = require('leveldown'); 33 | 34 | let database: DatabaseType; 35 | 36 | const connectDB = async () => { 37 | database = await createRxDatabase({ 38 | name: rxdbPath, 39 | adapter: leveldown, 40 | }); 41 | await database.addCollections({ 42 | post: { 43 | schema: postSchema, 44 | migrationStrategies: postMigration, 45 | }, 46 | user: { schema: userSchema, migrationStrategies: userMigration }, 47 | comment: { schema: commentSchema, migrationStrategies: commentMigration }, 48 | subcomment: { 49 | schema: subCommentSchema, 50 | migrationStrategies: subCommentMigration, 51 | }, 52 | repostcomment: { 53 | schema: repostCommentSchema, 54 | migrationStrategies: repostCommentMigration, 55 | }, 56 | }); 57 | 58 | return database; 59 | }; 60 | 61 | export { connectDB, database }; 62 | -------------------------------------------------------------------------------- /packages/documentation/docs/guide/monitor.md: -------------------------------------------------------------------------------- 1 | # Monitor 2 | 3 | The monitor component is used to monitor the collection of posts in platform. When a new post is added to the monitored collection, the monitor component will trigger the archiving of the post. 4 | 5 | ## Types 6 | 7 | First, we need to define the types of the collection. There are many different types of collections in different platforms. For example, favorite, watchLater, history, chat, like, etc. You should define the types in two different places. 8 | In `packages/backend/src/Components/Monitor/Types/monitorTypes.ts` 9 | 10 | ```typescript 11 | type CollectionTypes = 'favorite' | 'chat'; //override the types here 12 | ``` 13 | 14 | In `packages/backend/src/Components/Monitor/Service/monitorService.ts` 15 | 16 | ```typescript 17 | private collectionTypes: CollectionTypes[] = ['chat', 'favorite']; //override the types here, keep them the same as CollectionTypes 18 | ``` 19 | 20 | ## Handler 21 | 22 | Each type of collection has a handler. The handler will do things for all the collections of this specified type added by user. Basically, it just request the collection url, get the last posts, and archive the new posts. You should add handler in `collectionHandlers` in `packages/backend/src/Components/Monitor/Service/monitorApi.ts` 23 | For example: 24 | The platform has a collection type called `favorite`. And the favorite collection url has such format: `http://www.example.com/api/favorite/{favoriteId}`. Then in `collectionHandlers` you can define a handler for this type of collection. 25 | 26 | ```typescript 27 | const collectionHandlers: { 28 | [key in CollectionTypes]?: ( 29 | collection: MonitorCollection, 30 | ) => Promise; 31 | } = { 32 | favorite: async (collection: MonitorCollection) => { 33 | const url = collection.url; 34 | const id = getUrlLastSegment(url); 35 | 36 | const { data } = await crawlerAxios.get(`api/favorite/${id}`); 37 | return data.map((post: any) => String(post.id)); 38 | }, 39 | }; 40 | ``` 41 | 42 | The user will add some favorite url, and then the `favorite` handler will trigger archiving of the posts that newly added into the `favorite` collection. 43 | -------------------------------------------------------------------------------- /packages/frontend/README.md: -------------------------------------------------------------------------------- 1 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 2 | 3 | ## Available Scripts 4 | 5 | In the project directory, you can run: 6 | 7 | ### `npm start` 8 | 9 | Runs the app in the development mode.
10 | Open [http://localhost:3000](http://localhost:3000) to view it in the browser. 11 | 12 | The page will reload if you make edits.
13 | You will also see any lint errors in the console. 14 | 15 | ### `npm test` 16 | 17 | Launches the test runner in the interactive watch mode.
18 | See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. 19 | 20 | ### `npm run build` 21 | 22 | Builds the app for production to the `build` folder.
23 | It correctly bundles React in production mode and optimizes the build for the best performance. 24 | 25 | The build is minified and the filenames include the hashes.
26 | Your app is ready to be deployed! 27 | 28 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. 29 | 30 | ### `npm run eject` 31 | 32 | **Note: this is a one-way operation. Once you `eject`, you can’t go back!** 33 | 34 | If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. 35 | 36 | Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own. 37 | 38 | You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it. 39 | 40 | ## Learn More 41 | 42 | You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). 43 | 44 | To learn React, check out the [React documentation](https://reactjs.org/). 45 | -------------------------------------------------------------------------------- /packages/extension-chrome/README.md: -------------------------------------------------------------------------------- 1 | # Getting Started with Create React App 2 | 3 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 4 | 5 | ## Available Scripts 6 | 7 | In the project directory, you can run: 8 | 9 | ### `yarn start` 10 | 11 | Runs the app in the development mode.\ 12 | Open [http://localhost:3000](http://localhost:3000) to view it in the browser. 13 | 14 | The page will reload if you make edits.\ 15 | You will also see any lint errors in the console. 16 | 17 | ### `yarn test` 18 | 19 | Launches the test runner in the interactive watch mode.\ 20 | See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. 21 | 22 | ### `yarn build` 23 | 24 | Builds the app for production to the `build` folder.\ 25 | It correctly bundles React in production mode and optimizes the build for the best performance. 26 | 27 | The build is minified and the filenames include the hashes.\ 28 | Your app is ready to be deployed! 29 | 30 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. 31 | 32 | ### `yarn eject` 33 | 34 | **Note: this is a one-way operation. Once you `eject`, you can’t go back!** 35 | 36 | If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. 37 | 38 | Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own. 39 | 40 | You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it. 41 | 42 | ## Learn More 43 | 44 | You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). 45 | 46 | To learn React, check out the [React documentation](https://reactjs.org/). 47 | -------------------------------------------------------------------------------- /packages/backend/src/Components/SubComment/DAL/subCommentDAL.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ISubComment, 3 | ISubCommentDAL, 4 | SubCommentDocument, 5 | SubCommentCollection, 6 | } from '../Types/subCommentTypes'; 7 | import { database } from '../../../Loaders/rxdb'; 8 | import { MangoQuery, MangoQuerySelector } from 'rxdb'; 9 | import { injectable } from 'inversify'; 10 | import 'reflect-metadata'; 11 | import { version } from './subCommentSchema'; 12 | @injectable() 13 | class SubCommentDAL implements ISubCommentDAL { 14 | constructor() { 15 | if (!database) { 16 | throw new Error('database is not created!'); 17 | } 18 | } 19 | async upsert(subComment: ISubComment) { 20 | const subCommentCollection: SubCommentCollection = database.subcomment; 21 | const subCommentDoc: SubCommentDocument = 22 | await subCommentCollection.atomicUpsert(subComment); 23 | return subCommentDoc; 24 | } 25 | 26 | async query(queryObj: MangoQuery): Promise { 27 | const subCommentCollection: SubCommentCollection = database.subcomment; 28 | const subCommentDocs: SubCommentDocument[] = await subCommentCollection 29 | .find(queryObj) 30 | .exec(); 31 | return subCommentDocs; 32 | } 33 | 34 | async count(selector: MangoQuerySelector): Promise { 35 | const subCommentCollection: SubCommentCollection = database.subcomment; 36 | const totalNumber: number = ( 37 | await subCommentCollection.find({ selector }).exec() 38 | ).length; 39 | return totalNumber; 40 | } 41 | remove(): never { 42 | throw new Error('method is not implemented'); 43 | } 44 | 45 | async findOneById(subCommentId: string): Promise { 46 | if (!subCommentId) { 47 | return null; 48 | } 49 | const subCommentCollection: SubCommentCollection = database.subcomment; 50 | const subCommentDoc: SubCommentDocument | null = await subCommentCollection 51 | .findOne(subCommentId) 52 | .exec(); 53 | return subCommentDoc; 54 | } 55 | 56 | async bulkInsert(infos: ISubComment[]) { 57 | const subCommentCollection: SubCommentCollection = database.subcomment; 58 | return await subCommentCollection.bulkInsert(infos); 59 | } 60 | getVersion() { 61 | return version; 62 | } 63 | 64 | exportData() { 65 | const subCommentCollection: SubCommentCollection = database.subcomment; 66 | return subCommentCollection.find().exec(); 67 | } 68 | } 69 | 70 | export { SubCommentDAL }; 71 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Comment/DAL/commentDAL.ts: -------------------------------------------------------------------------------- 1 | import { 2 | CommentCollection, 3 | CommentDocument, 4 | IComment, 5 | ICommentDAL, 6 | } from '../Types/commentTypes'; 7 | import { database } from '../../../Loaders/rxdb'; 8 | import { MangoQuery, MangoQuerySelector } from 'rxdb'; 9 | import { inject, injectable } from 'inversify'; 10 | import 'reflect-metadata'; 11 | import { version } from './commentSchema'; 12 | 13 | @injectable() 14 | class CommentDAL implements ICommentDAL { 15 | constructor() { 16 | if (!database) { 17 | throw new Error('database is not created!'); 18 | } 19 | } 20 | async upsert(comment: IComment) { 21 | const commentDoc: CommentDocument = await database.comment.atomicUpsert( 22 | comment, 23 | ); 24 | return commentDoc; 25 | } 26 | 27 | async addSubComments(subCommentIds: string[], commentDoc: CommentDocument) { 28 | await commentDoc.update({ 29 | $addToSet: { subComments: { $each: subCommentIds } }, 30 | }); 31 | } 32 | 33 | async count(selector: MangoQuerySelector): Promise { 34 | const commentCollection: CommentCollection = database.comment; 35 | const totalNumber: number = ( 36 | await commentCollection.find({ selector }).exec() 37 | ).length; 38 | return totalNumber; 39 | } 40 | 41 | remove(): never { 42 | throw new Error('method is not implemented'); 43 | } 44 | 45 | async findOneById(commentId: string): Promise { 46 | if (!commentId) { 47 | //should check if commmentId is empty, otherwise it returns the first doc 48 | return null; 49 | } 50 | const commentCollection: CommentCollection = database.comment; 51 | const commentDoc: CommentDocument | null = await commentCollection 52 | .findOne(commentId) 53 | .exec(); 54 | return commentDoc; 55 | } 56 | async query(queryObj: MangoQuery): Promise { 57 | const commentCollection: CommentCollection = database.comment; 58 | const commentDoc: CommentDocument[] = await commentCollection 59 | .find(queryObj) 60 | .exec(); 61 | return commentDoc; 62 | } 63 | 64 | async bulkInsert(infos: IComment[]) { 65 | const commentCollection: CommentCollection = database.comment; 66 | return await commentCollection.bulkInsert(infos); 67 | } 68 | getVersion() { 69 | return version; 70 | } 71 | async exportData() { 72 | const commentCollection: CommentCollection = database.comment; 73 | return commentCollection.find().exec(); 74 | } 75 | } 76 | 77 | export { CommentDAL }; 78 | -------------------------------------------------------------------------------- /packages/extension-chrome/src/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/backend/src/Components/RepostComment/DAL/repostCommentDAL.ts: -------------------------------------------------------------------------------- 1 | import { 2 | IRepostCommentDAL, 3 | IRepostComment, 4 | RepostCommentDocument, 5 | RepostCommentCollection, 6 | } from '../Types/repostCommentTypes'; 7 | import { database } from '../../../Loaders/rxdb'; 8 | import { inject, injectable } from 'inversify'; 9 | import 'reflect-metadata'; 10 | import { MangoQuery, MangoQuerySelector } from 'rxdb'; 11 | import { version } from './repostCommentSchema'; 12 | @injectable() 13 | class RepostCommentDAL implements IRepostCommentDAL { 14 | constructor() { 15 | if (!database) { 16 | throw new Error('database is not created!'); 17 | } 18 | } 19 | 20 | async upsert(repostComment: IRepostComment) { 21 | const repostCommentDoc: RepostCommentDocument = 22 | await database.repostcomment.atomicUpsert(repostComment); 23 | return repostCommentDoc; 24 | } 25 | 26 | async count(selector: MangoQuerySelector): Promise { 27 | const repostCommentCollection: RepostCommentCollection = 28 | database.repostcomment; 29 | const totalNumber: number = ( 30 | await repostCommentCollection.find({ selector }).exec() 31 | ).length; 32 | return totalNumber; 33 | } 34 | remove(): never { 35 | throw new Error('method is not implemented'); 36 | } 37 | 38 | async findOneById( 39 | repostCommentId: string, 40 | ): Promise { 41 | if (!repostCommentId) { 42 | return null; 43 | } 44 | const repostCommentCollection: RepostCommentCollection = 45 | database.repostcomment; 46 | const repostDoc: RepostCommentDocument | null = 47 | await repostCommentCollection.findOne(repostCommentId).exec(); 48 | return repostDoc; 49 | } 50 | async query(queryObj: MangoQuery): Promise { 51 | const repostCommentCollection: RepostCommentCollection = 52 | database.repostcomment; 53 | const repostDocs: RepostCommentDocument[] = await repostCommentCollection 54 | .find(queryObj) 55 | .exec(); 56 | return repostDocs; 57 | } 58 | async bulkInsert(infos: IRepostComment[]) { 59 | const repostCommentCollection: RepostCommentCollection = 60 | database.repostcomment; 61 | return await repostCommentCollection.bulkInsert(infos); 62 | } 63 | 64 | getVersion() { 65 | return version; 66 | } 67 | 68 | async exportData() { 69 | const repostCommentCollection: RepostCommentCollection = 70 | database.repostcomment; 71 | return repostCommentCollection.find().exec(); 72 | } 73 | } 74 | 75 | export { RepostCommentDAL }; 76 | -------------------------------------------------------------------------------- /packages/backend/src/Components/User/Router/userRouter.ts: -------------------------------------------------------------------------------- 1 | import express, { Request, Response } from 'express'; 2 | import { container } from '../../../Config/inversify.config'; 3 | import asyncHandler from 'express-async-handler'; 4 | import { IUser, IUserService, USER_IOC_SYMBOLS } from '../Types'; 5 | import os from 'os'; 6 | import { BadRequestError } from '../../../Error/ErrorClass'; 7 | import path from 'path'; 8 | const multer = require('multer'); 9 | const upload = multer({ dest: os.tmpdir() }); 10 | const userRouter = express.Router(); 11 | 12 | userRouter.get( 13 | '/', 14 | asyncHandler(async (request: Request, response: Response) => { 15 | const username: string = request.query.username as string; 16 | const userService = container.get( 17 | USER_IOC_SYMBOLS.IUserService, 18 | ); 19 | const users: IUser[] = await userService.getUserByName(username); 20 | const totalNumber: number = await userService.countUserByName(username); 21 | 22 | response.send({ users, totalNumber }); 23 | }), 24 | ); 25 | 26 | userRouter.get( 27 | '/export', 28 | asyncHandler(async (request: Request, response: Response) => { 29 | const userService = container.get( 30 | USER_IOC_SYMBOLS.IUserService, 31 | ); 32 | const fileName: string = `user-v${userService.getVersion()}.json`; 33 | const data = JSON.stringify(await userService.exportData()); 34 | response.setHeader( 35 | 'Content-disposition', 36 | `attachment; filename=${fileName}`, 37 | ); 38 | response.setHeader('Content-type', 'application/json'); 39 | response.write(data, function (err) { 40 | if (err) { 41 | throw err; 42 | } 43 | response.end(); 44 | }); 45 | }), 46 | ); 47 | 48 | userRouter.post( 49 | '/import', 50 | upload.single('file'), 51 | asyncHandler(async (request: Request, response: Response) => { 52 | const batchSize: number = request.body.batchSize || 100; 53 | const version: number = request.body.version || 0; 54 | const userService = container.get( 55 | USER_IOC_SYMBOLS.IUserService, 56 | ); 57 | 58 | const file = (request as any).file; 59 | 60 | const uploadFolder = file?.destination; 61 | const fileName = file?.filename; 62 | if (!uploadFolder || !fileName) { 63 | throw new BadRequestError('Failed to upload file'); 64 | } 65 | const filePath = path.join(uploadFolder, fileName); 66 | userService.importData(filePath, version, batchSize); 67 | 68 | response.send('importing users'); 69 | }), 70 | ); 71 | 72 | export { userRouter }; 73 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Monitor/Router/monitorRouter.ts: -------------------------------------------------------------------------------- 1 | import asyncHandler from 'express-async-handler'; 2 | import express, { Request, Response } from 'express'; 3 | import { container } from '../../../Config/inversify.config'; 4 | import { ResourceError } from '../../../Error/ErrorClass'; 5 | import { 6 | CollectionTypes, 7 | IMonitorService, 8 | MONITOR_IOC_SYMBOLS, 9 | } from '../Types'; 10 | 11 | const monitorRouter = express.Router(); 12 | 13 | monitorRouter.get( 14 | '/', 15 | asyncHandler(async (req: Request, res: Response) => { 16 | const monitorService = container.get( 17 | MONITOR_IOC_SYMBOLS.IMonitorService, 18 | ); 19 | const monitors = await monitorService.getMonitorCollections(); 20 | res.send(monitors); 21 | }), 22 | ); 23 | 24 | monitorRouter.post( 25 | '/validate', 26 | asyncHandler(async (req: Request, res: Response) => { 27 | const { url, type }: { url: string; type: CollectionTypes } = req.body; 28 | const monitorService = container.get( 29 | MONITOR_IOC_SYMBOLS.IMonitorService, 30 | ); 31 | const isValid = await monitorService.validate(url, type); 32 | res.send(isValid); 33 | }), 34 | ); 35 | 36 | monitorRouter.post( 37 | '/', 38 | asyncHandler(async (req: Request, res: Response) => { 39 | const { url, type }: { url: string; type: CollectionTypes } = req.body; 40 | const monitorService = container.get( 41 | MONITOR_IOC_SYMBOLS.IMonitorService, 42 | ); 43 | const isAdded = await monitorService.add(url, type); 44 | res.send(isAdded); 45 | }), 46 | ); 47 | 48 | monitorRouter.delete( 49 | '/', 50 | asyncHandler(async (req: Request, res: Response) => { 51 | const { url }: { url: string } = req.body; 52 | const monitorService = container.get( 53 | MONITOR_IOC_SYMBOLS.IMonitorService, 54 | ); 55 | const isRemoved = await monitorService.remove(url); 56 | res.send(isRemoved); 57 | }), 58 | ); 59 | 60 | monitorRouter.get( 61 | '/max', 62 | asyncHandler((req: Request, res: Response) => { 63 | const monitorService = container.get( 64 | MONITOR_IOC_SYMBOLS.IMonitorService, 65 | ); 66 | const max = monitorService.getMaxCollectionSize(); 67 | res.send(max); 68 | }), 69 | ); 70 | 71 | monitorRouter.get('/types', (req: Request, res: Response) => { 72 | const monitorService = container.get( 73 | MONITOR_IOC_SYMBOLS.IMonitorService, 74 | ); 75 | const types = monitorService.getCollectionTypes(); 76 | res.send(types); 77 | }); 78 | 79 | export { monitorRouter }; 80 | -------------------------------------------------------------------------------- /packages/frontend/src/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /packages/frontend/src/Pages/PostContent/PostContent.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState } from 'react'; 2 | import { Col, Row, PageHeader } from 'antd'; 3 | import { PostCard } from '../../Component/PostCard'; 4 | import { useParams, useLocation, useHistory } from 'react-router-dom'; 5 | import { getSinglePostApi } from '../../Api'; 6 | import { CommentList } from '../../Component/CommentList'; 7 | import { IPost } from '../../types'; 8 | import { Switch, Route } from 'react-router-dom'; 9 | import { RepostCommentList } from '../../Component/RepostCommentList'; 10 | import { useSelector } from 'react-redux'; 11 | import { RootState } from '../../Store'; 12 | import { EnterOutlined } from '@ant-design/icons'; 13 | import _ from 'lodash'; 14 | 15 | function PostContent() { 16 | const { home } = useSelector((state: RootState) => state.routeState); 17 | function useQuery() { 18 | return new URLSearchParams(useLocation().search); 19 | } 20 | const history = useHistory(); 21 | const { postId } = useParams<{ postId: string }>(); 22 | const query = useQuery(); 23 | const [post, setPost] = useState({ comments: [] } as any); 24 | const [loading, setLoading] = useState(false); 25 | 26 | useEffect(() => { 27 | setLoading(true); 28 | getSinglePostApi( 29 | postId, 30 | parseInt(query.get('page') || '1'), 31 | parseInt(query.get('pageSize') || '10'), 32 | ) 33 | .then((res) => { 34 | const { post } = res.data; 35 | setPost(post); 36 | setLoading(false); 37 | }) 38 | .catch((err) => {}); 39 | }, [postId]); 40 | 41 | return ( 42 | <> 43 | 44 | 45 | } 47 | className="site-page-header" 48 | onBack={() => { 49 | history.push({ 50 | pathname: `/`, 51 | search: home.queryString, 52 | }); 53 | }} 54 | title="Back" 55 | subTitle="Back to post list" 56 | /> 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | ); 74 | } 75 | 76 | export default PostContent; 77 | -------------------------------------------------------------------------------- /packages/documentation/docs/guide/get-start.md: -------------------------------------------------------------------------------- 1 | # Get Started 2 | 3 | ## Background knowledge 4 | 5 | - Typescript 6 | - Node.js 7 | 8 | ## Prerequisites 9 | 10 | 1. Node.js 11 | 2. npm 12 | 3. [lerna](https://lerna.js.org/#getting-started): `npm i -g lerna` 13 | 14 | ## Setup 15 | 16 | 1. clone the repository 17 | 18 | ```shell 19 | git clone https://github.com/Combo819/social-media-archiver.git 20 | ``` 21 | 22 | 2. install dependencies 23 | ```shell 24 | cd social-media-archiver 25 | lerna bootstrap 26 | ``` 27 | 28 | ## Config 29 | 30 | In `packages/backend/src/Config/constants.ts`: 31 | 32 | ```typescript 33 | const BASE_URL: string = 'PLATFORM_DOMAIN'; 34 | const Q_CONCURRENCY: number = 1; // number of concurrent API requests to the platform 35 | const MAX_ITEM_WINDOW: number = 6; // max number of API requests to the platform in a 30s window 36 | ``` 37 | 38 | ## Run 39 | 40 | Start both the frontend and backend by running: 41 | 42 | ```shell 43 | npm run start 44 | ``` 45 | 46 | By default, the frontend runs on port `3000` and the backend runs on port `5000`. 47 | The API call from the frontend will be proxied to the port `5000`. 48 | Thus, if you are doing the frontend development, make sure the backend is running on port `5000`. 49 | When you see 50 | 51 | ```log 52 | @social-media-archiver/backend: Development mode, Listening on port 5000 53 | ``` 54 | 55 | Then the backend is running. 56 | 57 | ## Debug 58 | 59 | To debug the backend, stop the backend first, and open the debug panel in VS Code, 60 | select the `Debug Backend` in the dropdown, and click the `Start Debugging` button. 61 | 62 | ![debug](./debug.png) 63 | 64 | Then you can use the debug utilities like breakpoints and the debug message is displayed in the Debug Console right next to Terminal. 65 | 66 | ## Log 67 | 68 | In development mode, the log will be displayed in the terminal or the debug console(for VS Code debug mode). There is also a log file `packages/backend/log/social-media-archiver.log` 69 | In production mode, the log file is in `log/social-media-archiver.log`. 70 | You can use [pino-pretty](https://github.com/pinojs/pino-pretty) to pretty print the log file. 71 | 72 | ## Commit 73 | 74 | This repository use `git-cz`. Run `npm run commit` to commit the changes to the repository. It will generate a commit message template for you. 75 | 76 | ## Tech stack 77 | 78 | - Frontend: [React](https://reactjs.org/) 79 | - UI: [Ant Design](https://ant.design/) 80 | - Server: [Express](https://expressjs.com/) 81 | - Html Manipulation: [cheerio](https://cheerio.js.org/) 82 | - Inversion of Control: [inversify](https://inversify.io/) 83 | - Flow Control: [async](https://caolan.github.io/async/v3/) 84 | - Database: [rxdb](https://rxdb.info/) 85 | - Packaging: [pkg](https://github.com/vercel/pkg) 86 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/AccountModal/accountModal.tsx: -------------------------------------------------------------------------------- 1 | import { Modal, Avatar, Button, Tooltip, message } from 'antd'; 2 | import React, { useState } from 'react'; 3 | import { UserOutlined } from '@ant-design/icons'; 4 | import { IUser } from '../../types'; 5 | import { getImageUrl } from '../../Utility/parseUrl'; 6 | import styles from './accountModal.module.scss'; 7 | import { CookieBox } from '../CookieBox'; 8 | import { setCookieApi } from '../../Api'; 9 | import { useDispatch } from 'react-redux'; 10 | import { accountCreators } from '../../Store'; 11 | 12 | type AccountModalProps = { 13 | visible: boolean; 14 | account: IUser | null; 15 | setVisible: (value: boolean) => void; 16 | }; 17 | 18 | const getAvatar = (account: IUser | null) => { 19 | if (account) { 20 | return ; 21 | } else { 22 | return } />; 23 | } 24 | }; 25 | 26 | const getUsername = (account: IUser | null) => { 27 | if (account) { 28 | return ( 29 | {`@${ 30 | account && account.username 31 | }`} 32 | ); 33 | } else { 34 | return Not login; 35 | } 36 | }; 37 | 38 | export function AccountModal(props: AccountModalProps) { 39 | const { visible, account, setVisible } = props; 40 | const [loading, setLoading] = useState(false); 41 | const dispatch = useDispatch(); 42 | 43 | const onLogout = async () => { 44 | setLoading(true); 45 | try { 46 | const isCookieSet: boolean = await (await setCookieApi('')).data.result; 47 | 48 | if (isCookieSet) { 49 | message.success('logout succeeded'); 50 | dispatch(accountCreators.setAccount(null)); 51 | setVisible(false); 52 | } else { 53 | message.error('failed to logout'); 54 | } 55 | } catch (err) { 56 | message.error('failed to logout'); 57 | } finally { 58 | setLoading(false); 59 | } 60 | }; 61 | return ( 62 | setVisible(false)} 64 | footer={[ 65 | 66 | 74 | , 75 | ]} 76 | title="The Account to crawl web" 77 | visible={visible} 78 | > 79 |
80 |
{getAvatar(account)}
81 |
{getUsername(account)}
82 |
83 | 84 |
85 | ); 86 | } 87 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Post/Types/postTypes.ts: -------------------------------------------------------------------------------- 1 | import { RxCollection, RxDocument } from 'rxdb'; 2 | import { IUser, IUserService } from '../../User/Types/userTypes'; 3 | import { IBaseDAL, IBaseService, IBaseCrawler } from '../../Base/baseTypes'; 4 | type IPost = { 5 | id: string; // id of post 6 | createTime: number; // create time of post 7 | content: string; // content of post, usually a html string or plain string 8 | repostsCount?: number; // count of reposts 9 | commentsCount?: number; // count of comments 10 | upvotesCount?: number; // count of upvotes 11 | user: string; // user id of author 12 | comments?: string[]; // comments id of post 13 | images?: { name: string; originUrl: string }[]; // images of post, name usually is the last part of originUrl as A_LONG_HASH.jpg 14 | videos?: { name: string; originUrl: string }[]; // videos of post, name usually is the last part of originUrl as A_LONG_HASH.mp4 15 | saveTime: number; // save time of post 16 | repostingId?: string; // id of reposting post 17 | repostComments?: string[]; // the repost comments id of post 18 | }; 19 | 20 | type PostDocument = RxDocument; 21 | 22 | type PostCollection = RxCollection; 23 | 24 | interface IPostDAL extends IBaseDAL { 25 | addComments: (commentIds: string[], postDoc: PostDocument) => Promise; 26 | addRepostComments: ( 27 | repostCommentIds: string[], 28 | postDoc: PostDocument, 29 | ) => Promise; 30 | } 31 | 32 | type IPostPopulatedWithUser = Omit & { user?: IUser }; 33 | type IPostPopulated = IPostPopulatedWithUser & { 34 | reposting?: IPostPopulatedWithUser; 35 | }; 36 | 37 | type CrawlerPostParams = { 38 | postId: string; 39 | postService: IPostService; 40 | userService: IUserService; 41 | }; 42 | 43 | type CrawlerPostTask = { 44 | params: CrawlerPostParams; 45 | func: (params: CrawlerPostParams) => Promise; 46 | }; 47 | 48 | interface IPostService 49 | extends IBaseService { 50 | startCrawling(id: string): Promise; 51 | getPostById(id: string): Promise; 52 | populatePost(postDoc: PostDocument): Promise; 53 | addComments: (commentIds: string[], postId: string) => Promise; 54 | addRepostComments: ( 55 | repostCommentIds: string[], 56 | postId: string, 57 | ) => Promise; 58 | deleteDoc(id: string): Promise; 59 | } 60 | 61 | interface IPostCrawler extends IBaseCrawler{ 62 | startCrawling: (postId: string) => Promise; 63 | } 64 | 65 | export { 66 | IPost, 67 | PostDocument, 68 | PostCollection, 69 | IPostDAL, 70 | IPostService, 71 | IPostPopulated, 72 | CrawlerPostTask, 73 | CrawlerPostParams, 74 | IPostCrawler, 75 | }; 76 | -------------------------------------------------------------------------------- /packages/backend/src/Components/Post/DAL/postDAL.ts: -------------------------------------------------------------------------------- 1 | import { 2 | PostDocument, 3 | IPost, 4 | IPostDAL, 5 | PostCollection, 6 | } from '../Types/postTypes'; 7 | import { injectable } from 'inversify'; 8 | import 'reflect-metadata'; 9 | import { database } from '../../../Loaders/rxdb'; 10 | import { MangoQuery, MangoQuerySelector } from 'rxdb'; 11 | import { DatabaseError } from '../../../Error/ErrorClass'; 12 | import { version } from './postSchema'; 13 | @injectable() 14 | class PostDAL implements IPostDAL { 15 | constructor() { 16 | if (!database) { 17 | throw new DatabaseError('database is not created!'); 18 | } 19 | } 20 | async upsert(newPost: IPost) { 21 | const postDoc: PostDocument = await database.post.atomicUpsert(newPost); 22 | return postDoc; 23 | } 24 | 25 | async findOneById(postId: string): Promise { 26 | if (!postId) { 27 | return null; 28 | } 29 | const postDoc: PostDocument | null = await database.post 30 | .findOne(postId) 31 | .exec(); 32 | return postDoc; 33 | } 34 | 35 | async addComments(commentIds: string[], postDoc: PostDocument) { 36 | await postDoc.update({ 37 | $addToSet: { 38 | comments: { $each: commentIds }, 39 | }, 40 | }); 41 | } 42 | 43 | async addRepostComments(repostCommentIds: string[], postDoc: PostDocument) { 44 | await postDoc.update({ 45 | $addToSet: { 46 | repostComments: { $each: repostCommentIds }, 47 | }, 48 | }); 49 | } 50 | 51 | async remove(postId: string): Promise { 52 | const postDoc: PostDocument | null = await this.findOneById(postId); 53 | if (!postDoc) { 54 | return false; 55 | } else { 56 | const result: boolean = await postDoc.remove(); 57 | return result; 58 | } 59 | } 60 | 61 | async query(queryObj: MangoQuery): Promise { 62 | const postDocs: PostDocument[] = await database.post 63 | .find(queryObj) 64 | .exec(); 65 | return postDocs; 66 | } 67 | 68 | async count(selector: MangoQuerySelector) { 69 | const postCollection: PostCollection = database.post; 70 | // bad performance but I have no choice 71 | const postDocs: PostDocument[] = await postCollection 72 | .find({ selector }) 73 | .exec(); 74 | return postDocs.length; 75 | } 76 | 77 | async bulkInsert(infos: IPost[]): Promise<{ success: any[]; error: any[] }> { 78 | const postCollection: PostCollection = database.post; 79 | const result = await postCollection.bulkInsert(infos); 80 | return result; 81 | } 82 | 83 | async exportData() { 84 | const postCollection: PostCollection = database.post; 85 | return postCollection.find().exec(); 86 | } 87 | 88 | getVersion(): number { 89 | return version; 90 | } 91 | } 92 | 93 | export { PostDAL }; 94 | -------------------------------------------------------------------------------- /packages/backend/src/Loaders/express.ts: -------------------------------------------------------------------------------- 1 | import express, { NextFunction, Request, Response } from 'express'; 2 | import { staticPath, PORT } from '../Config'; 3 | import path from 'path'; 4 | import getPort from 'get-port'; 5 | import { ErrorHandler } from '../Error/ErrorHandler'; 6 | import { logger } from '../Logger'; 7 | import { monitorRouter } from '../Components/Monitor/Router/monitorRouter'; 8 | import { userRouter } from '../Components/User/Router'; 9 | import { commentRouter } from '../Components/Comment/Router'; 10 | import { repostCommentRouter } from '../Components/RepostComment/Router'; 11 | import { subCommentRouter } from '../Components/SubComment/Router'; 12 | import { postRouter } from '../Components/Post/Router'; 13 | import { accountRouter } from '../Components/Account/Router'; 14 | const open = require('open'); 15 | 16 | const expressLoader = (app: express.Application) => { 17 | app.use(express.urlencoded({ extended: true })); 18 | app.use(express.json()); 19 | 20 | app.use('/api/user', userRouter); 21 | app.use('/api/comment', commentRouter); 22 | app.use('/api/repostComment', repostCommentRouter); 23 | app.use('/api/subComment', subCommentRouter); 24 | app.use('/api/post', postRouter); 25 | app.use('/api/account', accountRouter); 26 | app.use('/api/monitor', monitorRouter); 27 | 28 | app.use(express.static(staticPath)); 29 | 30 | app.use( 31 | express.static(path.resolve(__dirname, '../../../', 'frontend', 'build')), 32 | ); 33 | 34 | // Error handling middleware, we delegate the handling to the centralized error handler 35 | app.use( 36 | async (err: Error, req: Request, res: Response, next: NextFunction) => { 37 | const errorHandler = new ErrorHandler(); 38 | await errorHandler.handleError(err, res); 39 | }, 40 | ); 41 | 42 | app.use('*', (request: Request, response: Response) => { 43 | response.sendFile( 44 | path.resolve(__dirname, '../../../', 'frontend', 'build', 'index.html'), 45 | ); 46 | }); 47 | 48 | getPort({ port: [PORT, PORT + 1, PORT + 2] }).then((res: number) => { 49 | const availblePort: number = res; 50 | app.listen(availblePort || 5000, () => { 51 | const listeningInfo: string = `listening on port ${ 52 | availblePort || 5000 53 | } \n`; 54 | logger.info(listeningInfo); 55 | try { 56 | if (process.env.NODE_ENV !== 'development') { 57 | open(`http://localhost:${availblePort}`); 58 | console.log(`Opening http://localhost:${availblePort}`); 59 | } else { 60 | console.log(`Development mode, Listening on port ${availblePort}`); 61 | } 62 | } catch (err) { 63 | console.log( 64 | `please open http://localhost:${availblePort} in your browser`, 65 | ); 66 | } 67 | }); 68 | }); 69 | }; 70 | 71 | export { expressLoader }; 72 | -------------------------------------------------------------------------------- /packages/frontend/src/Pages/CommentContent/CommentContent.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState } from 'react'; 2 | import { Col, Row, PageHeader, Card, Avatar } from 'antd'; 3 | import { LikeOutlined, EnterOutlined } from '@ant-design/icons'; 4 | import { useParams, useHistory } from 'react-router-dom'; 5 | import { getSingleCommentApi } from '../../Api'; 6 | import { SubCommentList } from '../../Component/SubCommentList'; 7 | import HtmlParser from 'react-html-parser'; 8 | import { getImageUrl } from '../../Utility/parseUrl'; 9 | import { useSelector } from 'react-redux'; 10 | import { RootState } from '../../Store'; 11 | import { IComment } from '../../types'; 12 | function CommentContent() { 13 | const { postContent } = useSelector((state: RootState) => state.routeState); 14 | const history = useHistory(); 15 | const { commentId } = useParams<{ commentId: string }>(); 16 | const [comment, SetComment] = useState(); 17 | const [loading, setLoading] = useState(false); 18 | 19 | useEffect(() => { 20 | setLoading(true); 21 | getSingleCommentApi(commentId) 22 | .then((res) => { 23 | const { result } = res.data; 24 | SetComment(result); 25 | setLoading(false); 26 | }) 27 | .catch((err) => {}); 28 | }, [commentId]); 29 | 30 | return ( 31 | <> 32 | 33 | 34 | } 36 | className="site-page-header" 37 | onBack={() => { 38 | history.push({ 39 | pathname: `/post/${comment?.postId}/comments`, 40 | search: postContent.queryString, 41 | }); 42 | }} 43 | title="Back" 44 | subTitle="Back to comment list" 45 | /> 46 | 47 | 48 | 49 | 50 | 53 | 57 | {comment?.upvotesCount} 58 | , 59 | ]} 60 | loading={loading} 61 | > 62 | 66 | } 67 | title={`@${comment?.user?.username}`} 68 | description={HtmlParser(comment?.content || '')} 69 | /> 70 | 71 | 72 | 73 | 74 | 75 | ); 76 | } 77 | 78 | export default CommentContent; 79 | -------------------------------------------------------------------------------- /packages/documentation/docs/.vuepress/public/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 8 | 16 | 29 | 30 | -------------------------------------------------------------------------------- /packages/backend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@social-media-archiver/backend", 3 | "version": "1.3.0", 4 | "description": "", 5 | "main": "src/index.ts", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "tsc": "tsc", 9 | "start": "nodemon", 10 | "build": "rimraf ./build &&npx tsc", 11 | "pkg": "pkg . --out-path=./dist/", 12 | "dist": "node ./pkgScript.js" 13 | }, 14 | "keywords": [], 15 | "author": "", 16 | "license": "ISC", 17 | "dependencies": { 18 | "@types/async": "^3.2.3", 19 | "@types/bluebird": "^3.5.32", 20 | "@types/cheerio": "^0.22.21", 21 | "@types/express": "^4.17.8", 22 | "@types/multer": "^1.4.7", 23 | "@types/progress": "^2.0.3", 24 | "async": "^3.2.0", 25 | "axios": "^0.20.0", 26 | "bluebird": "^3.7.2", 27 | "camelcase-keys": "^6.2.2", 28 | "cheerio": "^1.0.0-rc.3", 29 | "cron": "^1.8.2", 30 | "dayjs": "^1.10.6", 31 | "enquirer": "^2.3.6", 32 | "express": "^4.17.1", 33 | "express-async-handler": "^1.1.4", 34 | "form-data": "^3.0.0", 35 | "get-port": "^5.1.1", 36 | "inversify": "^5.1.1", 37 | "inversify-inject-decorators": "^3.1.0", 38 | "is-url": "^1.2.4", 39 | "leveldown": "^5.6.0", 40 | "lodash": "^4.17.20", 41 | "multer": "^1.4.3", 42 | "natives": "^1.1.6", 43 | "open": "^7.2.1", 44 | "pino": "^7.0.0-rc.2", 45 | "pino-pretty": "^7.1.0", 46 | "pouchdb-adapter-leveldb": "^7.2.2", 47 | "progress": "^2.0.3", 48 | "reflect-metadata": "^0.1.13", 49 | "rxdb": "^9.6.0", 50 | "rxjs": "^6.6.3", 51 | "sanitize-html": "^2.5.3", 52 | "sass": "^1.37.5", 53 | "stream-chain": "^2.2.4", 54 | "stream-json": "^1.7.2", 55 | "uuid": "^8.3.0" 56 | }, 57 | "devDependencies": { 58 | "@types/cron": "^1.7.3", 59 | "@types/fs-extra": "^9.0.12", 60 | "@types/is-url": "^1.2.28", 61 | "@types/lodash": "^4.14.161", 62 | "@types/luxon": "^1.27.1", 63 | "@types/node": "^14.6.2", 64 | "@types/puppeteer-core": "^2.0.0", 65 | "@types/readline-sync": "^1.4.3", 66 | "@types/sanitize-html": "^2.5.0", 67 | "@types/stream-json": "^1.7.1", 68 | "@types/uuid": "^8.3.0", 69 | "archiver": "^5.3.0", 70 | "cpy": "^8.1.1", 71 | "cross-env": "^7.0.3", 72 | "execa": "^5.0.0", 73 | "fs-extra": "^9.1.0", 74 | "nodemon": "^2.0.4", 75 | "pkg": "^4.4.9", 76 | "prettier": "2.4.1", 77 | "rimraf": "^3.0.2", 78 | "ts-loader": "^8.0.3", 79 | "ts-node": "^9.0.0", 80 | "typescript": "^4.3.5" 81 | }, 82 | "bin": "./build/index.js", 83 | "pkg": { 84 | "assets": [ 85 | "../frontend/build", 86 | "./node_modules/**/*" 87 | ], 88 | "options": [ 89 | "experimental-modules" 90 | ], 91 | "targets": [ 92 | "node12-win-x64", 93 | "node12-linux-x64", 94 | "node12-macos-x64" 95 | ] 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /packages/frontend/src/Component/ImportModal/ImportModal.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Button, Form, Input, Modal, Select, Upload } from 'antd'; 3 | import { UploadOutlined } from '@ant-design/icons'; 4 | import { importData } from '../../Api'; 5 | const { Option } = Select; 6 | type Props = { setImportModalVisible: Function; importModalVisible: boolean }; 7 | export function ImportModal({ 8 | setImportModalVisible, 9 | importModalVisible, 10 | }: Props) { 11 | const [form] = Form.useForm(); 12 | 13 | const onCancel = () => { 14 | setImportModalVisible(false); 15 | form.resetFields(); 16 | }; 17 | const onSubmit = () => { 18 | form.validateFields().then((values) => { 19 | const bodyFormData = new FormData(); 20 | bodyFormData.append('file', values.file.file); 21 | bodyFormData.append('version', values.version); 22 | 23 | const type: 24 | | 'post' 25 | | 'user' 26 | | 'comment' 27 | | 'subComment' 28 | | 'repostComment' = values.type; 29 | 30 | importData(bodyFormData, type).then(() => { 31 | setImportModalVisible(false); 32 | }); 33 | }); 34 | }; 35 | return ( 36 | Cancel, 41 | , 45 | ]} 46 | visible={importModalVisible} 47 | > 48 |
49 | 59 | 66 | 67 | 77 | false}> 78 | 79 | 80 | 81 | 91 | 92 | 93 |
94 |
95 | ); 96 | } 97 | --------------------------------------------------------------------------------