├── .nvmrc ├── .gitattributes ├── eslint.config.js ├── src ├── role.enum.ts ├── common │ ├── constants.ts │ ├── replace-domain.ts │ ├── slice-string.ts │ ├── convert-date.ts │ ├── get-english-topic.ts │ ├── feed-extract.ts │ ├── open-ai.ts │ ├── cloudflare-image.ts │ └── article-extract.ts ├── app.service.ts ├── dto │ ├── addWebsite.dto.ts │ ├── sign.dto.ts │ ├── updateWebsite.dto.ts │ ├── getArticleCount.dto.ts │ └── addArticle.dto.ts ├── main.ts ├── blog │ ├── website │ │ ├── website.interface.ts │ │ ├── website.service.spec.ts │ │ ├── website.controller.spec.ts │ │ ├── website.controller.ts │ │ └── website.service.ts │ ├── article │ │ ├── article.interface.ts │ │ ├── article.service.spec.ts │ │ ├── article.controller.spec.ts │ │ ├── article.controller.ts │ │ └── article.service.ts │ └── blog.module.ts ├── app.controller.ts ├── users │ ├── users.module.ts │ ├── users.service.ts │ └── users.service.spec.ts ├── pipe │ └── positiveInt.pipe.ts ├── auth │ ├── auth.service.spec.ts │ ├── auth.controller.spec.ts │ ├── auth.module.ts │ ├── auth.controller.ts │ ├── auth.guard.ts │ └── auth.service.ts ├── schemas │ ├── user.schema.ts │ ├── statistic.schema.ts │ ├── website.schema.ts │ └── article.schema.ts ├── app.controller.spec.ts ├── auto │ ├── auto.module.ts │ └── auto.service.ts └── app.module.ts ├── .gitignore ├── nest-cli.json ├── tsconfig.json ├── LICENSE ├── package.json └── README.md /.nvmrc: -------------------------------------------------------------------------------- 1 | 20.12.2 -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "extends": "eslint:recommended", 3 | } 4 | -------------------------------------------------------------------------------- /src/role.enum.ts: -------------------------------------------------------------------------------- 1 | export enum Role { 2 | User = "user", 3 | Admin = "admin", 4 | } 5 | -------------------------------------------------------------------------------- /src/common/constants.ts: -------------------------------------------------------------------------------- 1 | export const jwtConstants = "1LKLsD0jZK2wyj8yzBOXNI6XDshz9pJ4noYB/XAuSmQ="; 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .DS_Store 3 | node_modules/ 4 | docker-compose.yml 5 | dist/ 6 | .env.development 7 | .idea/ 8 | -------------------------------------------------------------------------------- /src/app.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from "@nestjs/common"; 2 | 3 | @Injectable() 4 | export class AppService { 5 | getHello(): string { 6 | return "Hello World!"; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/dto/addWebsite.dto.ts: -------------------------------------------------------------------------------- 1 | import { IsNotEmpty, IsString } from "class-validator"; 2 | 3 | export class AddWebsiteDto { 4 | @IsString() 5 | @IsNotEmpty() 6 | url: string; 7 | 8 | @IsString() 9 | @IsNotEmpty() 10 | name: string; 11 | } 12 | -------------------------------------------------------------------------------- /src/common/replace-domain.ts: -------------------------------------------------------------------------------- 1 | // 接收一个url和域名，将url中的域名替换为传入的新域名 2 | export default function replaceDomain(url: string, domain: string): string { 3 | const parsedUrl = new URL(url); 4 | parsedUrl.hostname = new URL(domain).hostname; 5 | return parsedUrl.href; 6 | } 7 | -------------------------------------------------------------------------------- /nest-cli.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/nest-cli", 3 | "collection": "@nestjs/schematics", 4 | "sourceRoot": "src", 5 | "compilerOptions": { 6 | "deleteOutDir": true, 7 | "builder": "swc", 8 | "typeCheck": true 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/dto/sign.dto.ts: -------------------------------------------------------------------------------- 1 | import { IsNotEmpty, IsString } from "class-validator"; 2 | 3 | export class SignDto { 4 | @IsNotEmpty() 5 | @IsString() 6 | username: string; 7 | 8 | @IsNotEmpty() 9 | @IsString() 10 | password: string; 11 | 12 | userId: number; 13 | } 14 | -------------------------------------------------------------------------------- /src/common/slice-string.ts: -------------------------------------------------------------------------------- 1 | export default function sliceString(str: string, bytes: number) { 2 | const buffer = Buffer.from(str, "utf8"); 3 | if (buffer.byteLength <= bytes) { 4 | return str; 5 | } 6 | const slice = buffer.subarray(0, bytes); 7 | return slice.toString("utf8"); 8 | } 9 | -------------------------------------------------------------------------------- /src/dto/updateWebsite.dto.ts: -------------------------------------------------------------------------------- 1 | import { IsUrl } from "class-validator"; 2 | 3 | export class UpdateWebsiteDto { 4 | @IsUrl() 5 | url: string; 6 | 7 | @IsUrl() 8 | rss: string; 9 | 10 | name: string; 11 | description: string; 12 | 13 | @IsUrl() 14 | cover: string; 15 | } 16 | -------------------------------------------------------------------------------- /src/common/convert-date.ts: -------------------------------------------------------------------------------- 1 | export default function convertDate(dateString: Date) { 2 | let date = new Date(dateString); 3 | 4 | // 如果转换的日期超过当前日期或者没有日期，则返回当前时间 5 | if (date.getTime() > Date.now() || isNaN(date.getTime())) { 6 | date = new Date(Date.now()); 7 | } 8 | 9 | return date; 10 | } 11 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | import { NestFactory } from "@nestjs/core"; 2 | import { AppModule } from "./app.module"; 3 | 4 | const port = process.env.PORT || 1994; 5 | 6 | async function bootstrap() { 7 | const app = await NestFactory.create(AppModule); 8 | await app.listen(port, "0.0.0.0"); 9 | } 10 | bootstrap(); 11 | -------------------------------------------------------------------------------- /src/blog/website/website.interface.ts: -------------------------------------------------------------------------------- 1 | import { Document } from "mongoose"; 2 | 3 | export interface Blog extends Document { 4 | url: string; 5 | name: string; 6 | description: string; 7 | cover: string; 8 | article_count: number; 9 | page_view: number; 10 | frequency: number; 11 | last_publish: Date; 12 | } 13 | -------------------------------------------------------------------------------- /src/app.controller.ts: -------------------------------------------------------------------------------- 1 | import { Controller, Get } from "@nestjs/common"; 2 | import { AppService } from "./app.service"; 3 | 4 | @Controller() 5 | export class AppController { 6 | constructor(private readonly appService: AppService) {} 7 | 8 | @Get() 9 | getHello(): string { 10 | return this.appService.getHello(); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/dto/getArticleCount.dto.ts: -------------------------------------------------------------------------------- 1 | import { IsDateString, IsNotEmpty, IsOptional, MaxDate } from "class-validator"; 2 | 3 | export class GetArticleCountDto { 4 | @IsNotEmpty() 5 | type: string; 6 | 7 | @IsOptional() 8 | topic: string; 9 | 10 | @IsOptional() 11 | @IsDateString() 12 | startAt: Date; 13 | 14 | // 截止日期不得超过现在 15 | @IsOptional() 16 | @IsDateString() 17 | @MaxDate(new Date()) 18 | endAt: Date; 19 | } 20 | -------------------------------------------------------------------------------- /src/users/users.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from "@nestjs/common"; 2 | import { UsersService } from "./users.service"; 3 | import { MongooseModule } from "@nestjs/mongoose"; 4 | import { UserSchema } from "../schemas/user.schema"; 5 | 6 | @Module({ 7 | imports: [MongooseModule.forFeature([{ name: "User", schema: UserSchema }])], 8 | providers: [UsersService], 9 | exports: [UsersService], 10 | }) 11 | export class UsersModule {} 12 | -------------------------------------------------------------------------------- /src/blog/article/article.interface.ts: -------------------------------------------------------------------------------- 1 | import mongoose, { Document } from "mongoose"; 2 | 3 | export interface Article extends Document { 4 | website_id: mongoose.Schema.Types.ObjectId; 5 | website: string; 6 | url: string; 7 | title: string; 8 | description: string; 9 | publish_date: Date; 10 | cover: string; 11 | isFeature: boolean; 12 | tags: string[]; 13 | page_view: number; 14 | content: string; 15 | isBlocked: boolean; 16 | } 17 | -------------------------------------------------------------------------------- /src/users/users.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from "@nestjs/common"; 2 | import { InjectModel } from "@nestjs/mongoose"; 3 | import { Model } from "mongoose"; 4 | 5 | export type User = any; 6 | 7 | @Injectable() 8 | export class UsersService { 9 | constructor(@InjectModel("User") private userModel: Model) {} 10 | 11 | async findOne(username: string): Promise { 12 | return this.userModel.find({ username: username }).exec(); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/pipe/positiveInt.pipe.ts: -------------------------------------------------------------------------------- 1 | import { ArgumentMetadata, Injectable, PipeTransform } from "@nestjs/common"; 2 | 3 | @Injectable() 4 | export class PositiveIntPipe implements PipeTransform { 5 | transform(value: any, metadata: ArgumentMetadata): any { 6 | if (metadata.type === "query") { 7 | if (metadata.data === "limit" && value < 1) { 8 | return 10; 9 | } 10 | if (metadata.data === "page" && value < 1) { 11 | return 1; 12 | } 13 | } 14 | return value; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/dto/addArticle.dto.ts: -------------------------------------------------------------------------------- 1 | import { IsNotEmpty, IsUrl } from "class-validator"; 2 | import mongoose from "mongoose"; 3 | 4 | export class AddArticleDto { 5 | @IsUrl() 6 | @IsNotEmpty() 7 | url: string; 8 | 9 | @IsNotEmpty() 10 | website_id: mongoose.Types.ObjectId; 11 | 12 | @IsUrl() 13 | website: string; 14 | 15 | author: string; 16 | 17 | @IsNotEmpty() 18 | title: string; 19 | 20 | description: string; 21 | 22 | publish_date: Date; 23 | 24 | cover: string; 25 | 26 | content: string; 27 | } 28 | -------------------------------------------------------------------------------- /src/auth/auth.service.spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from "@nestjs/testing"; 2 | import { AuthService } from "./auth.service"; 3 | 4 | describe("AuthService", () => { 5 | let service: AuthService; 6 | 7 | beforeEach(async () => { 8 | const module: TestingModule = await Test.createTestingModule({ 9 | providers: [AuthService], 10 | }).compile(); 11 | 12 | service = module.get(AuthService); 13 | }); 14 | 15 | it("should be defined", () => { 16 | expect(service).toBeDefined(); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /src/users/users.service.spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from "@nestjs/testing"; 2 | import { UsersService } from "./users.service"; 3 | 4 | describe("UsersService", () => { 5 | let service: UsersService; 6 | 7 | beforeEach(async () => { 8 | const module: TestingModule = await Test.createTestingModule({ 9 | providers: [UsersService], 10 | }).compile(); 11 | 12 | service = module.get(UsersService); 13 | }); 14 | 15 | it("should be defined", () => { 16 | expect(service).toBeDefined(); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /src/schemas/user.schema.ts: -------------------------------------------------------------------------------- 1 | import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; 2 | import { HydratedDocument } from "mongoose"; 3 | 4 | export type UserDocument = HydratedDocument; 5 | 6 | @Schema() 7 | export class User { 8 | @Prop() 9 | username: string; 10 | 11 | @Prop() 12 | password: string; 13 | 14 | @Prop() 15 | userId: number; 16 | 17 | // role分为admin和visitor两种 18 | @Prop({ 19 | enum: ["admin", "visitor"], 20 | }) 21 | role: string; 22 | } 23 | 24 | export const UserSchema = SchemaFactory.createForClass(User); 25 | -------------------------------------------------------------------------------- /src/blog/article/article.service.spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from "@nestjs/testing"; 2 | import { ArticleService } from "./article.service"; 3 | 4 | describe("ArticleService", () => { 5 | let service: ArticleService; 6 | 7 | beforeEach(async () => { 8 | const module: TestingModule = await Test.createTestingModule({ 9 | providers: [ArticleService], 10 | }).compile(); 11 | 12 | service = module.get(ArticleService); 13 | }); 14 | 15 | it("should be defined", () => { 16 | expect(service).toBeDefined(); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /src/blog/website/website.service.spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from "@nestjs/testing"; 2 | import { WebsiteService } from "./website.service"; 3 | 4 | describe("WebsiteService", () => { 5 | let service: WebsiteService; 6 | 7 | beforeEach(async () => { 8 | const module: TestingModule = await Test.createTestingModule({ 9 | providers: [WebsiteService], 10 | }).compile(); 11 | 12 | service = module.get(WebsiteService); 13 | }); 14 | 15 | it("should be defined", () => { 16 | expect(service).toBeDefined(); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /src/auth/auth.controller.spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from "@nestjs/testing"; 2 | import { AuthController } from "./auth.controller"; 3 | 4 | describe("AuthController", () => { 5 | let controller: AuthController; 6 | 7 | beforeEach(async () => { 8 | const module: TestingModule = await Test.createTestingModule({ 9 | controllers: [AuthController], 10 | }).compile(); 11 | 12 | controller = module.get(AuthController); 13 | }); 14 | 15 | it("should be defined", () => { 16 | expect(controller).toBeDefined(); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /src/common/get-english-topic.ts: -------------------------------------------------------------------------------- 1 | // 用于将中文的分类转换成英文 2 | export default function getEnglishTopic(chinese: string) { 3 | const topicList = new Map([ 4 | ["技术", "tech"], 5 | ["编程", "code"], 6 | ["社会", "society"], 7 | ["日记", "diary"], 8 | ["生活", "life"], 9 | ["政治", "politics"], 10 | ["职场", "career"], 11 | ["旅行", "travel"], 12 | ["人文社科", "culture"], 13 | ["学习", "education"], 14 | ["情感", "emotion"], 15 | ["综合", "others"], 16 | ]); 17 | 18 | if (!topicList.has(chinese)) { 19 | return "others"; 20 | } 21 | 22 | return topicList.get(chinese); 23 | } 24 | -------------------------------------------------------------------------------- /src/blog/article/article.controller.spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from "@nestjs/testing"; 2 | import { ArticleController } from "./article.controller"; 3 | 4 | describe("ArticleController", () => { 5 | let controller: ArticleController; 6 | 7 | beforeEach(async () => { 8 | const module: TestingModule = await Test.createTestingModule({ 9 | controllers: [ArticleController], 10 | }).compile(); 11 | 12 | controller = module.get(ArticleController); 13 | }); 14 | 15 | it("should be defined", () => { 16 | expect(controller).toBeDefined(); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /src/blog/website/website.controller.spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from "@nestjs/testing"; 2 | import { WebsiteController } from "./website.controller"; 3 | 4 | describe("WebsiteController", () => { 5 | let controller: WebsiteController; 6 | 7 | beforeEach(async () => { 8 | const module: TestingModule = await Test.createTestingModule({ 9 | controllers: [WebsiteController], 10 | }).compile(); 11 | 12 | controller = module.get(WebsiteController); 13 | }); 14 | 15 | it("should be defined", () => { 16 | expect(controller).toBeDefined(); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /src/schemas/statistic.schema.ts: -------------------------------------------------------------------------------- 1 | import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; 2 | import { HydratedDocument } from "mongoose"; 3 | 4 | export type StatisticDocument = HydratedDocument; 5 | 6 | @Schema() 7 | export class Statistic { 8 | @Prop({ 9 | required: true, 10 | }) 11 | date: Date; 12 | 13 | @Prop({ 14 | default: 0, 15 | }) 16 | website_count: number; 17 | 18 | @Prop({ 19 | default: 0, 20 | }) 21 | article_count: number; 22 | 23 | @Prop() 24 | inaccessible_article: number; 25 | } 26 | 27 | export const StatisticSchema = SchemaFactory.createForClass(Statistic); 28 | -------------------------------------------------------------------------------- /src/common/feed-extract.ts: -------------------------------------------------------------------------------- 1 | export default async function feedExtract(feedUrl: string) { 2 | try { 3 | // 请求https://vbwbtqsflwkgwxobfkhs.supabase.co/functions/v1/rss?url= 4 | let feed = await fetch(`${process.env.SUPABASE_EDGE_FUNCTION}/rss`, { 5 | method: "POST", 6 | headers: { 7 | "Content-Type": "application/json", 8 | "Authorization": `Bearer ${process.env.SUPABASE_ANON_KEY}` 9 | }, 10 | body: JSON.stringify({url: feedUrl}), 11 | }).then((res) => res.json()); 12 | return feed.entries.slice(0, 30); 13 | } catch (error) { 14 | throw new Error(`Error happen on extract ${feedUrl}: ${error}`); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "declaration": true, 5 | "removeComments": true, 6 | "emitDecoratorMetadata": true, 7 | "experimentalDecorators": true, 8 | "allowSyntheticDefaultImports": true, 9 | "target": "ES2022", 10 | "sourceMap": true, 11 | "outDir": "./dist", 12 | "baseUrl": "./", 13 | "paths": { 14 | "@/*": [ 15 | "src/*" 16 | ] 17 | } 18 | }, 19 | "incremental": true, 20 | "skipLibCheck": true, 21 | "strictNullChecks": false, 22 | "noImplicitAny": false, 23 | "strictBindCallApply": false, 24 | "forceConsistentCasingInFileNames": false, 25 | "noFallthroughCasesInSwitch": false 26 | } 27 | -------------------------------------------------------------------------------- /src/app.controller.spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from "@nestjs/testing"; 2 | import { AppController } from "./app.controller"; 3 | import { AppService } from "./app.service"; 4 | 5 | describe("AppController", () => { 6 | let appController: AppController; 7 | 8 | beforeEach(async () => { 9 | const app: TestingModule = await Test.createTestingModule({ 10 | controllers: [AppController], 11 | providers: [AppService], 12 | }).compile(); 13 | 14 | appController = app.get(AppController); 15 | }); 16 | 17 | describe("root", () => { 18 | it('should return "Hello World!"', () => { 19 | expect(appController.getHello()).toBe("Hello World!"); 20 | }); 21 | }); 22 | }); 23 | -------------------------------------------------------------------------------- /src/auto/auto.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from "@nestjs/common"; 2 | import { MongooseModule } from "@nestjs/mongoose"; 3 | import { BlogModule } from "@/blog/blog.module"; 4 | import { WebsiteSchema } from "@/schemas/website.schema"; 5 | import { AutoService } from "@/auto/auto.service"; 6 | import { ArticleSchema } from "@/schemas/article.schema"; 7 | import { StatisticSchema } from "@/schemas/statistic.schema"; 8 | 9 | @Module({ 10 | imports: [ 11 | MongooseModule.forFeature([ 12 | { name: "Website", schema: WebsiteSchema }, 13 | { name: "Article", schema: ArticleSchema }, 14 | { name: "Statistic", schema: StatisticSchema }, 15 | ]), 16 | BlogModule, 17 | ], 18 | providers: [AutoService], 19 | controllers: [], 20 | }) 21 | export class AutoModule {} 22 | -------------------------------------------------------------------------------- /src/auth/auth.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from "@nestjs/common"; 2 | import { AuthController } from "@/auth/auth.controller"; 3 | import { AuthService } from "@/auth/auth.service"; 4 | import { UsersModule } from "@/users/users.module"; 5 | import { JwtModule } from "@nestjs/jwt"; 6 | import { MongooseModule } from "@nestjs/mongoose"; 7 | import { UserSchema } from "@/schemas/user.schema"; 8 | import { jwtConstants } from "@/common/constants"; 9 | 10 | @Module({ 11 | imports: [ 12 | MongooseModule.forFeature([{ name: "User", schema: UserSchema }]), 13 | JwtModule.register({ 14 | global: true, 15 | secret: jwtConstants, 16 | signOptions: { expiresIn: "14d" }, 17 | }), 18 | UsersModule, 19 | ], 20 | controllers: [AuthController], 21 | providers: [AuthService], 22 | exports: [AuthService], 23 | }) 24 | export class AuthModule {} 25 | -------------------------------------------------------------------------------- /src/blog/blog.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from "@nestjs/common"; 2 | import { WebsiteController } from "@/blog/website/website.controller"; 3 | import { ArticleController } from "@/blog/article/article.controller"; 4 | import { WebsiteService } from "@/blog/website/website.service"; 5 | import { ArticleService } from "@/blog/article/article.service"; 6 | import { MongooseModule } from "@nestjs/mongoose"; 7 | import { WebsiteSchema } from "@/schemas/website.schema"; 8 | import { ArticleSchema } from "@/schemas/article.schema"; 9 | 10 | @Module({ 11 | imports: [ 12 | MongooseModule.forFeature([ 13 | { name: "Website", schema: WebsiteSchema }, 14 | { name: "Article", schema: ArticleSchema }, 15 | ]), 16 | ], 17 | controllers: [WebsiteController, ArticleController], 18 | providers: [WebsiteService, ArticleService], 19 | exports: [WebsiteService, ArticleService], 20 | }) 21 | export class BlogModule {} 22 | -------------------------------------------------------------------------------- /src/auth/auth.controller.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Body, 3 | Controller, 4 | Get, 5 | HttpCode, 6 | HttpStatus, 7 | Post, 8 | UseGuards, 9 | } from "@nestjs/common"; 10 | 11 | import { AuthGuard } from "@/auth/auth.guard"; 12 | import { AuthService } from "@/auth/auth.service"; 13 | import { SignDto } from "@/dto/sign.dto"; 14 | 15 | @Controller("auth") 16 | export class AuthController { 17 | constructor(private authService: AuthService) {} 18 | 19 | // /auth/login POST 20 | @HttpCode(HttpStatus.OK) 21 | @Post("login") 22 | signIn(@Body() signDto: SignDto) { 23 | return this.authService.signIn(signDto.username, signDto.password); 24 | } 25 | 26 | // /auth/signup POST 27 | @HttpCode(HttpStatus.CREATED) 28 | @Post("signup") 29 | signUp(@Body() signDto: SignDto) { 30 | return this.authService.signUp(signDto.username, signDto.password); 31 | } 32 | 33 | // /auth/validate GET 34 | // 检测access_token是否有效 35 | @UseGuards(AuthGuard) 36 | @HttpCode(HttpStatus.OK) 37 | @Get("validate") 38 | validate() { 39 | return this.authService.validate(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 李大毛 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/auth/auth.guard.ts: -------------------------------------------------------------------------------- 1 | import { 2 | CanActivate, 3 | ExecutionContext, 4 | Injectable, 5 | UnauthorizedException, 6 | } from "@nestjs/common"; 7 | import { JwtService } from "@nestjs/jwt"; 8 | import { Request } from "express"; 9 | 10 | @Injectable() 11 | export class AuthGuard implements CanActivate { 12 | constructor(private jwtService: JwtService) {} 13 | 14 | async canActivate(context: ExecutionContext): Promise { 15 | const request = context.switchToHttp().getRequest(); 16 | const token = this.extractTokenFromHeader(request); 17 | 18 | if (!token) { 19 | throw new UnauthorizedException(); 20 | } 21 | 22 | try { 23 | const payload = await this.jwtService.verifyAsync(token, { 24 | secret: process.env.AUTH_KEY, 25 | }); 26 | request["user"] = payload; 27 | } catch { 28 | throw new UnauthorizedException(); 29 | } 30 | 31 | return true; 32 | } 33 | 34 | private extractTokenFromHeader(request: Request): string | undefined { 35 | const [type, token] = request.headers.authorization?.split(" ") ?? []; 36 | return type === "Bearer" ? token : undefined; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/app.module.ts: -------------------------------------------------------------------------------- 1 | import {Module} from "@nestjs/common"; 2 | import {ConfigModule} from "@nestjs/config"; 3 | import {MongooseModule} from "@nestjs/mongoose"; 4 | import {AppController} from "@/app.controller"; 5 | import {AppService} from "@/app.service"; 6 | import {AuthModule} from "@/auth/auth.module"; 7 | import {BlogModule} from "@/blog/blog.module"; 8 | import {UsersModule} from "@/users/users.module"; 9 | import {ScheduleModule} from "@nestjs/schedule"; 10 | import {AutoModule} from "@/auto/auto.module"; 11 | import {CacheInterceptor, CacheModule} from "@nestjs/cache-manager"; 12 | import {APP_INTERCEPTOR} from "@nestjs/core"; 13 | 14 | @Module({ 15 | imports: [ 16 | ConfigModule.forRoot({ 17 | isGlobal: true, 18 | envFilePath: [".env.development", ".env.production"], 19 | }), 20 | CacheModule.register({ 21 | isGlobal: true, 22 | ttl: 14400 23 | }), 24 | MongooseModule.forRoot(process.env.MONGODB), 25 | AuthModule, 26 | ScheduleModule.forRoot(), 27 | UsersModule, 28 | BlogModule, 29 | AutoModule, 30 | ], 31 | controllers: [AppController], 32 | providers: [ 33 | AppService, 34 | { 35 | provide: APP_INTERCEPTOR, 36 | useClass: CacheInterceptor, 37 | } 38 | ], 39 | }) 40 | export class AppModule { 41 | } 42 | -------------------------------------------------------------------------------- /src/schemas/website.schema.ts: -------------------------------------------------------------------------------- 1 | import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; 2 | import { HydratedDocument } from "mongoose"; 3 | 4 | export type WebsiteDocument = HydratedDocument; 5 | 6 | @Schema() 7 | export class Website { 8 | @Prop({ 9 | required: true, 10 | unique: true, 11 | match: /^http(s)?:\/\/.+/, 12 | index: true, 13 | }) 14 | url: string; 15 | 16 | @Prop({ 17 | required: true, 18 | default: false, 19 | }) 20 | isDead: boolean; 21 | 22 | @Prop() 23 | rss: string; 24 | 25 | @Prop({ 26 | required: true, 27 | }) 28 | name: string; 29 | 30 | @Prop({ 31 | default: "", 32 | }) 33 | description: string; 34 | 35 | @Prop({ 36 | default: "", 37 | match: /^http(s)?:\/\/.+/, 38 | }) 39 | cover: string; 40 | 41 | @Prop({ 42 | default: 0, 43 | }) 44 | article_count: number; 45 | 46 | @Prop({ 47 | default: 0, 48 | }) 49 | page_view: number; 50 | 51 | @Prop({ default: Date.now }) 52 | last_publish: Date; 53 | 54 | @Prop({ 55 | default: Date.now, 56 | }) 57 | last_crawl: Date; 58 | 59 | @Prop({ 60 | default: 0, 61 | }) 62 | crawl_error: number; 63 | 64 | @Prop() 65 | categories: Map; 66 | } 67 | 68 | export const WebsiteSchema = SchemaFactory.createForClass(Website); 69 | -------------------------------------------------------------------------------- /src/schemas/article.schema.ts: -------------------------------------------------------------------------------- 1 | import { Prop, Schema, SchemaFactory, raw } from "@nestjs/mongoose"; 2 | import mongoose, { HydratedDocument } from "mongoose"; 3 | import { Website } from "./website.schema"; 4 | 5 | export type ArticleDocument = HydratedDocument
; 6 | 7 | @Schema() 8 | export class Article { 9 | @Prop({ 10 | type: mongoose.Schema.Types.ObjectId, 11 | ref: "Website", 12 | required: true, 13 | }) 14 | website_id: Website; 15 | 16 | @Prop() 17 | website: string; 18 | 19 | @Prop() 20 | author: string; 21 | 22 | @Prop({ 23 | required: true, 24 | unique: true, 25 | match: /^http(s)?:\/\/.+/, 26 | index: true, 27 | }) 28 | url: string; 29 | 30 | @Prop({ required: true }) 31 | title: string; 32 | 33 | @Prop({ default: "" }) 34 | description: string; 35 | 36 | @Prop() 37 | publish_date: Date; 38 | 39 | @Prop() 40 | cover: string; 41 | 42 | @Prop({ default: false }) 43 | isFeatured: boolean; 44 | 45 | @Prop({ default: [] }) 46 | tags: [String]; 47 | 48 | @Prop() 49 | topic: string; 50 | 51 | @Prop() 52 | abstract: string; 53 | 54 | @Prop({ 55 | default: 0, 56 | }) 57 | page_view: number; 58 | 59 | @Prop() 60 | content: string; 61 | 62 | @Prop({ default: false }) 63 | isBlocked: boolean; 64 | 65 | @Prop({ default: 0 }) 66 | crawl_error: number; 67 | } 68 | 69 | export const ArticleSchema = SchemaFactory.createForClass(Article); 70 | -------------------------------------------------------------------------------- /src/common/open-ai.ts: -------------------------------------------------------------------------------- 1 | import {Logger} from "@nestjs/common"; 2 | 3 | export default async function AIProcess(content: string) { 4 | const TOKEN = process.env.SUPABASE_ANON_KEY; 5 | const API = process.env.SUPABASE_EDGE_FUNCTION; 6 | const logger = new Logger(); 7 | 8 | const data = { 9 | system: "分析下方文本, 忽略其中可能存在的html标签，只关注内容本身，并做三件事: 1.根据文本内容, 根据文本内容选择最贴切的分类, 可选分类包括：{技术 | 编程 | 社会 | 情感 | 旅行 | 日记 |" + 10 | " 生活" + 11 | " |" + 12 | " 职场" + 13 | " |" + 14 | " 人文社科 |" + 15 | " 政治 |" + 16 | " 教育" + 17 | " | 综合}。请参考以下分类规则：与写代码相关的文章应该归于编程, 其他关于科技、硬件等内容归于技术。个人生活周期性的总结，不管是按天、按周还是按月的归为日记," + 18 | " 较长的、具有一定深度的内容归为生活。讨论与读书相关的应归为教育。与工作、个人发展相关的归为职场。; 2.根据文本内容, 生成1-5个标签; 3.根据文本内容, 生成一段120字以内的摘要," + 19 | " 确保摘要精炼并准确地传达文章主旨；如果下方文本很短不足以生成足够摘要信息，你可以只输出一句类似”作者去了东京并拍了照片”这样的摘要；如果文本更少连一句话摘要都无法生成，你就返回“文章长度过短无法生成摘要”。将结果按这个结构输出格式严密的JSON: {\"category\": \"{分类}\", \"tags\": [\"{标签1}\", \"{标签2}\"], \"abstract\": \"{概括}\"}", 20 | user: content, 21 | } 22 | try { 23 | const articleResponse = await fetch(`${API}/open-ai`, { 24 | method: "POST", 25 | headers: { 26 | "Content-Type": "text/plain", 27 | Accept: "text/plain", 28 | Authorization: `Bearer ${TOKEN}`, 29 | }, 30 | body: JSON.stringify(data), 31 | }); 32 | return await articleResponse.text(); // 解析响应数据为文本形式 33 | } catch { 34 | logger.error(`Error on AI process`); 35 | return null; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/common/cloudflare-image.ts: -------------------------------------------------------------------------------- 1 | import {Logger} from "@nestjs/common"; 2 | 3 | export default async function cloudflareImage(imgUrl: string, website: string) { 4 | const API_TOKEN = process.env.CLOUDFLARE_IMAGE_TOKEN; 5 | const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID; 6 | const URL = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/images/v1`; 7 | const DOMAIN = getDomain(website); 8 | const HASH = process.env.CLOUDFLARE_ACCOUNT_HASH; 9 | const logger = new Logger(); 10 | 11 | const formData = new FormData(); 12 | formData.append("url", imgUrl); 13 | formData.append( 14 | "metadata", 15 | JSON.stringify({ 16 | blog: DOMAIN, 17 | }), 18 | ); 19 | formData.append("requireSignedURLs", "false"); 20 | 21 | try { 22 | const response = await fetch(URL, { 23 | method: "POST", 24 | headers: { 25 | Authorization: `Bearer ${API_TOKEN}`, 26 | }, 27 | body: formData, 28 | }); 29 | 30 | if (response.status !== 200) { 31 | logger.error("Error uploading image to Cloudflare Image", response); 32 | return null; 33 | } 34 | 35 | const data = await response.json(); 36 | return `https://imagedelivery.net/${HASH}/${data.result.id}`; 37 | } catch (error) { 38 | logger.error("Error uploading image to Cloudflare Image", error); 39 | return null; 40 | } 41 | } 42 | 43 | function getDomain(website: string) { 44 | const parsedUrl = new URL(website); 45 | return parsedUrl.hostname; 46 | } 47 | -------------------------------------------------------------------------------- /src/auth/auth.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, UnauthorizedException } from "@nestjs/common"; 2 | import { JwtService } from "@nestjs/jwt"; 3 | import { InjectModel } from "@nestjs/mongoose"; 4 | import { Model } from "mongoose"; 5 | import { User } from "@/schemas/user.schema"; 6 | import bcrypt from "bcrypt"; 7 | 8 | @Injectable() 9 | export class AuthService { 10 | constructor( 11 | private jwtService: JwtService, 12 | @InjectModel("User") private userModel: Model, 13 | ) {} 14 | 15 | async signIn(username: string, password: string) { 16 | const user = await this.userModel.findOne({ username: username }).exec(); 17 | const hashPassword = user.password; 18 | const isMatch = await bcrypt.compare(password, hashPassword); 19 | if (!isMatch) throw new UnauthorizedException("Invalid password"); 20 | const payLoad = { username: user.username, sub: user.userId }; 21 | return { 22 | access_token: this.jwtService.sign(payLoad), 23 | }; 24 | } 25 | 26 | async signUp(username: string, pass: string) { 27 | const admin = await this.userModel.find(); 28 | if (admin.length > 0) { 29 | const error = { 30 | message: "Admin already exists", 31 | hasAdmin: true, 32 | }; 33 | throw new UnauthorizedException(error); 34 | } 35 | try { 36 | // 生成加盐的密码 37 | bcrypt.hash(pass, 10, async (err, hash) => { 38 | if (err) { 39 | console.error(err); 40 | throw new UnauthorizedException(); 41 | } 42 | const newUser = new this.userModel({ 43 | username: username, 44 | password: hash, 45 | userId: 1, 46 | }); 47 | await newUser.save(); 48 | }); 49 | return { 50 | message: "success", 51 | }; 52 | } catch (err) { 53 | console.error(err); 54 | throw new UnauthorizedException("Register failed"); 55 | } 56 | } 57 | 58 | // 检测access_token是否有效 59 | async validate() { 60 | return { 61 | message: "success", 62 | }; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/common/article-extract.ts: -------------------------------------------------------------------------------- 1 | import * as cheerio from "cheerio"; 2 | import cloudflareImage from "./cloudflare-image"; 3 | import AIProcess from "./open-ai"; 4 | import getEnglishTopic from "./get-english-topic"; 5 | import { Logger } from "@nestjs/common"; 6 | 7 | // 本函数用于从文章中提取出相应信息，包括标题、描述、内容、图片等。 8 | export default async function getArticleInfo( 9 | url: string, 10 | website: string, 11 | description: string, 12 | ) { 13 | let article = null; 14 | let cover = null; 15 | let abstract = null; 16 | let tags = null; 17 | let topic = null; 18 | let retries = 0; 19 | const logger = new Logger(); 20 | logger.debug(`Start extract article from ${url}`); 21 | 22 | while (!article && retries < 3) { 23 | try { 24 | article = await fetch(`${process.env.SUPABASE_EDGE_FUNCTION}/article`, { 25 | method: "POST", 26 | headers: { 27 | "Content-Type": "application/json", 28 | "Authorization": `Bearer ${process.env.SUPABASE_ANON_KEY}` 29 | }, 30 | body: JSON.stringify({url: url}), 31 | }).then((res) => res.json()) ; 32 | 33 | if (article.content) { 34 | const articleData = await AIProcess(article.content || description); 35 | 36 | if (articleData) { 37 | const articleJson = JSON.parse(articleData); 38 | // 获取文章摘要 39 | abstract = articleJson.abstract; 40 | 41 | // 获取文章标签 42 | tags = articleJson.tags; 43 | 44 | // 获取文章分类 45 | topic = getEnglishTopic(articleJson.category); 46 | } 47 | } 48 | 49 | if (article && article.image) { 50 | cover = await cloudflareImage(article.image, website); 51 | } 52 | logger.debug(`Successfully extract info from ${article.title}`); 53 | return { 54 | cover: cover || null, 55 | content: article.content || null, 56 | abstract: abstract, 57 | tags: tags, 58 | topic: topic, 59 | }; 60 | } catch (error) { 61 | logger.error(`Cannot extract from ${url}, ${error}`); 62 | await new Promise((resolve) => 63 | setTimeout( 64 | resolve, 65 | Math.floor(Math.random() * (3000 - 1000 + 1)) + 1000, 66 | ), 67 | ); 68 | retries++; 69 | } 70 | } 71 | return { 72 | cover: null, 73 | content: null, 74 | abstract: null, 75 | tags: null, 76 | topic: null, 77 | }; 78 | } 79 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "darmau-blog-backend", 3 | "version": "1.0.0", 4 | "description": "darmau.blog的后台项目", 5 | "author": "Darmau", 6 | "private": true, 7 | "license": "MIT", 8 | "scripts": { 9 | "build": "nest build", 10 | "dev": "nest start --watch", 11 | "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", 12 | "start": "nest start", 13 | "start:dev": "nest build --watch", 14 | "start:debug": "nest start --debug --watch", 15 | "start:prod": "node dist/main", 16 | "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", 17 | "test": "jest", 18 | "test:watch": "jest --watch", 19 | "test:cov": "jest --coverage", 20 | "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", 21 | "test:e2e": "jest --config ./test/jest-e2e.json" 22 | }, 23 | "dependencies": { 24 | "@nestjs/cache-manager": "^2.2.2", 25 | "@nestjs/common": "^10.3.8", 26 | "@nestjs/config": "^3.2.2", 27 | "@nestjs/core": "^10.3.8", 28 | "@nestjs/jwt": "^10.2.0", 29 | "@nestjs/mongoose": "^10.0.6", 30 | "@nestjs/platform-express": "^10.3.8", 31 | "@nestjs/schedule": "^4.0.2", 32 | "bcrypt": "^5.1.1", 33 | "cache-manager": "^5.5.1", 34 | "cheerio": "1.0.0-rc.12", 35 | "class-validator": "^0.14.1", 36 | "dotenv": "^16.4.5", 37 | "mongoose": "^8.3.2", 38 | "reflect-metadata": "^0.2.2", 39 | "rxjs": "^7.8.1" 40 | }, 41 | "devDependencies": { 42 | "@nestjs/cli": "^10.3.2", 43 | "@nestjs/schematics": "^10.1.1", 44 | "@nestjs/testing": "^10.3.8", 45 | "@swc/cli": "^0.3.12", 46 | "@swc/core": "^1.4.17", 47 | "@types/bcrypt": "^5.0.2", 48 | "@types/express": "^4.17.21", 49 | "@types/jest": "29.5.12", 50 | "@types/node": "20.12.7", 51 | "@types/supertest": "^6.0.2", 52 | "@typescript-eslint/eslint-plugin": "^7.7.1", 53 | "@typescript-eslint/parser": "^7.7.1", 54 | "eslint": "^9.1.1", 55 | "eslint-config-prettier": "^9.1.0", 56 | "eslint-plugin-prettier": "^5.1.3", 57 | "jest": "29.7.0", 58 | "prettier": "^3.2.5", 59 | "source-map-support": "^0.5.21", 60 | "supertest": "^7.0.0", 61 | "ts-jest": "29.1.2", 62 | "ts-loader": "^9.5.1", 63 | "ts-node": "^10.9.2", 64 | "tsconfig-paths": "4.2.0", 65 | "typescript": "^5.4.5" 66 | }, 67 | "jest": { 68 | "moduleFileExtensions": [ 69 | "js", 70 | "json", 71 | "ts" 72 | ], 73 | "rootDir": "src", 74 | "testRegex": ".*\\.spec\\.ts$", 75 | "transform": { 76 | "^.+\\.(t|j)s$": "ts-jest" 77 | }, 78 | "collectCoverageFrom": [ 79 | "**/*.(t|j)s" 80 | ], 81 | "coverageDirectory": "../coverage", 82 | "testEnvironment": "node" 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | Nest Logo 3 |

4 | 5 | [circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456 6 | [circleci-url]: https://circleci.com/gh/nestjs/nest 7 | 8 |

A progressive Node.js framework for building efficient and scalable server-side applications.

9 |

10 | NPM Version 11 | Package License 12 | NPM Downloads 13 | CircleCI 14 | Coverage 15 | Discord 16 | Backers on Open Collective 17 | Sponsors on Open Collective 18 | 19 | Support us 20 | 21 |

22 | 24 | 25 | This repository contains the backend for the [积薪](https://firewood.news) project, an independent blog navigation station. The backend is developed using Nest.js and is paired with MongoDB as the database. It can be easily deployed and run with Docker Compose. 26 | 27 | ## Features 28 | The main purpose of this backend is to provide a variety of APIs for the Firewood.news website, including: 29 | 30 | - Browsing, modifying, and deleting articles and blog data. 31 | - Periodically fetching article updates via scheduled tasks. 32 | - Summarizing website data and testing link accessibility. 33 | - Automatically categorizing articles, tagging them, and generating summaries during the fetching process. 34 | 35 | ## License 36 | This project is licensed under the MIT License. See the LICENSE file for details. -------------------------------------------------------------------------------- /src/auto/auto.service.ts: -------------------------------------------------------------------------------- 1 | import {Inject, Injectable, Logger} from "@nestjs/common"; 2 | import {InjectModel} from "@nestjs/mongoose"; 3 | import {Cron} from "@nestjs/schedule"; 4 | import {Model} from "mongoose"; 5 | import {ArticleService} from "@/blog/article/article.service"; 6 | import {WebsiteService} from "@/blog/website/website.service"; 7 | import {Article} from "@/schemas/article.schema"; 8 | import {Statistic} from "@/schemas/statistic.schema"; 9 | import {Website} from "@/schemas/website.schema"; 10 | import {CACHE_MANAGER} from "@nestjs/cache-manager"; 11 | import {Cache} from "cache-manager"; 12 | 13 | @Injectable() 14 | export class AutoService { 15 | private readonly logger = new Logger(AutoService.name); 16 | 17 | constructor( 18 | @InjectModel("Website") private websiteModel: Model, 19 | @InjectModel("Article") private articleModel: Model
, 20 | @InjectModel("Statistic") private statisticModel: Model, 21 | @Inject(CACHE_MANAGER) private cacheManager: Cache, 22 | private articleService: ArticleService, 23 | private websiteService: WebsiteService, 24 | ) { 25 | } 26 | 27 | // 获取所有website，分别将url传入updateArticlesByWebsite方法, 每4小时执行一次 28 | @Cron("0 0 0-16/4 * * *") 29 | async updateArticle() { 30 | const websites = await this.websiteModel.find().allowDiskUse(true); 31 | this.logger.log("Start update " + websites.length + " websites"); 32 | 33 | // 重置缓存 34 | await this.cacheManager.reset(); 35 | this.logger.log("Reset cache"); 36 | 37 | for (const website of websites) { 38 | try { 39 | await this.articleService.updateArticlesByWebsite(website.url); 40 | await this.websiteService.updatePageView(website._id.toString()); 41 | } catch (error) { 42 | this.logger.error( 43 | "Update websites failed at:" + website.url + "\n" + error.message, 44 | ); 45 | await this.websiteModel.findOneAndUpdate( 46 | {url: website.url}, 47 | {crawl_error: website.crawl_error + 1}, 48 | ); 49 | } 50 | } 51 | return this.logger.log("Auto update articles success"); 52 | } 53 | 54 | // 一个月进行一次的任务。遍历所有article，一次1000篇。检测是否可访问，不可访问的会删除。 55 | @Cron("0 0 19 * * *") 56 | async checkArticles() { 57 | // 获取当前日期 58 | const currentDate = new Date(); 59 | // 计算当前月份的第几天 60 | const currentDayOfMonth = currentDate.getDate(); 61 | // 计算这个月开始检查文章的偏移量 62 | const offset = (currentDayOfMonth - 1) * 1000; 63 | 64 | // 按publish_date从旧到新排序，设置每次查询的数量为1000，并根据偏移量查询 65 | const articles = await this.articleModel 66 | .find() 67 | .sort({publish_date: 1}) 68 | .limit(1000) 69 | .skip(offset) 70 | .allowDiskUse(true) 71 | .exec(); 72 | 73 | this.logger.log(`Start check articles from ${offset} to ${offset + 1000}`); 74 | 75 | for (const article of articles) { 76 | try { 77 | const res = await fetch(article.url, { 78 | method: "HEAD", 79 | redirect: "follow", 80 | }); 81 | if (!res.ok) { 82 | article.crawl_error += 1; 83 | await article.save(); 84 | // 如果错误次数大于3次，删除文章 85 | if (article.crawl_error > 3) { 86 | await this.articleModel.findByIdAndDelete(article._id); 87 | this.logger.warn( 88 | `Delete article ${article.title} in ${article.url}`, 89 | ); 90 | } else { 91 | this.logger.warn( 92 | `Failed to access ${article.url}, count: ${article.crawl_error}`, 93 | ); 94 | } 95 | } 96 | } catch (error) { 97 | article.crawl_error += 1; 98 | await article.save(); 99 | this.logger.error( 100 | `Failed to access ${article.url}, count: ${article.crawl_error}`, 101 | ); 102 | } 103 | } 104 | await this.cacheManager.reset(); 105 | return this.logger.log("Check articles success"); 106 | } 107 | 108 | // 每天凌晨1点执行一次，计算网站和文章的数量，写入数据库 109 | @Cron("0 0 17 * * *") 110 | async updateStatistics() { 111 | const date = new Date(); 112 | const websitesCount = await this.websiteModel.estimatedDocumentCount(); 113 | const articlesCount = await this.articleModel.estimatedDocumentCount(); 114 | const inaccessibleArticlesCount = await this.articleModel.countDocuments({ 115 | crawl_error: {$gte: 1}, 116 | }).exec(); 117 | 118 | const todayStatistic = new this.statisticModel({ 119 | date: date, 120 | website_count: websitesCount, 121 | article_count: articlesCount, 122 | inaccessible_article: inaccessibleArticlesCount, 123 | }); 124 | await todayStatistic.save(); 125 | await this.cacheManager.reset(); 126 | return this.logger.log("Update statistics success"); 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/blog/article/article.controller.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Body, 3 | Controller, 4 | Get, Inject, Logger, 5 | Post, 6 | Put, 7 | Query, 8 | UseGuards, 9 | UseInterceptors, 10 | } from "@nestjs/common"; 11 | import {AuthGuard} from "@/auth/auth.guard"; 12 | import {AddArticleDto} from "@/dto/addArticle.dto"; 13 | import {ArticleService} from "@/blog/article/article.service"; 14 | import {GetArticleCountDto} from "@/dto/getArticleCount.dto"; 15 | import {PositiveIntPipe} from "@/pipe/positiveInt.pipe"; 16 | import {CacheInterceptor} from "@nestjs/cache-manager"; 17 | import {CACHE_MANAGER} from "@nestjs/cache-manager"; 18 | import {Cache} from "cache-manager"; 19 | import {CacheTTL} from "@nestjs/common/cache"; 20 | 21 | @Controller("article") 22 | @UseInterceptors(CacheInterceptor) 23 | export class ArticleController { 24 | constructor( 25 | @Inject(CACHE_MANAGER) private cacheManager: Cache, 26 | private articleService: ArticleService 27 | ) { 28 | } 29 | 30 | private logger = new Logger('ArticleController'); 31 | 32 | // /article/latest?page=1&limit=10 33 | @Get("latest") 34 | async getAllUnblockedArticle( 35 | @Query("page", PositiveIntPipe) page: number = 1, 36 | @Query("limit", PositiveIntPipe) limit: number = 15, 37 | ) { 38 | return await this.articleService.getAllUnblockedArticle(page, limit); 39 | } 40 | 41 | // /article/all?page=1&limit=10 42 | @Get("all") 43 | @CacheTTL(15) 44 | async getAllArticle( 45 | @Query("page", PositiveIntPipe) page: number = 1, 46 | @Query("limit", PositiveIntPipe) limit: number = 15, 47 | ) { 48 | return await this.articleService.getAllArticle(page, limit); 49 | } 50 | 51 | // /article/featured?page=&limit=5 52 | @Get("featured") 53 | async getArticleByRecommend( 54 | @Query("page", PositiveIntPipe) page: number = 1, 55 | @Query("limit", PositiveIntPipe) limit: number = 15, 56 | ) { 57 | return await this.articleService.getArticleByRecommend(page, limit); 58 | } 59 | 60 | // 找出最新的指定分类的文章 61 | // /article/topic?topic=&page=1&limit=6 62 | @Get("topic") 63 | async getArticleByTopic( 64 | @Query("topic") topic: string, 65 | @Query("page", PositiveIntPipe) page: number = 1, 66 | @Query("limit", PositiveIntPipe) limit: number = 15, 67 | ) { 68 | return await this.articleService.getArticleByTopic(topic, page, limit); 69 | } 70 | 71 | // /article/count 72 | @Post("count") 73 | async getArticleCount(@Body() getArticleCount: GetArticleCountDto) { 74 | return await this.articleService.getArticleCount( 75 | getArticleCount.type, 76 | getArticleCount.topic, 77 | getArticleCount.startAt, 78 | getArticleCount.endAt, 79 | ); 80 | } 81 | 82 | // /article?website=https://darmau.design&page=1&limit=10 83 | @Get() 84 | async getArticleByBlog( 85 | @Query("website") url: string, 86 | @Query("page", PositiveIntPipe) page: number = 1, 87 | @Query("limit", PositiveIntPipe) limit: number = 15, 88 | ) { 89 | return await this.articleService.getArticleByBlog(url, page, limit); 90 | } 91 | 92 | @Get("article-count") 93 | async getArticleCountByBlog(@Query("id") id: string) { 94 | return await this.articleService.getArticleCountByBlog(id); 95 | } 96 | 97 | // /article/add POST 98 | @UseGuards(AuthGuard) 99 | @Post("add") 100 | async addArticle(@Body() addArticleDto: AddArticleDto) { 101 | return await this.articleService.addArticle( 102 | addArticleDto.url, 103 | addArticleDto.website_id, 104 | addArticleDto.website, 105 | addArticleDto.title, 106 | addArticleDto.description, 107 | addArticleDto.publish_date, 108 | addArticleDto.author, 109 | ); 110 | } 111 | 112 | // /article/block?id= 113 | @UseGuards(AuthGuard) 114 | @Put("block") 115 | async blockArticle(@Query("id") id: string) { 116 | await this.cacheManager.reset(); 117 | this.logger.debug('Cache reset'); 118 | return await this.articleService.blockArticle(id); 119 | } 120 | 121 | // /article/view?id= 122 | @Put("view") 123 | async addViewCount(@Query("id") id: string) { 124 | return await this.articleService.addPageView(id); 125 | } 126 | 127 | // /article/feature?id= PUT 128 | @UseGuards(AuthGuard) 129 | @Put("feature") 130 | async featureArticle(@Query("id") id: string) { 131 | await this.cacheManager.reset(); 132 | this.logger.debug('Cache reset'); 133 | return await this.articleService.setFeaturedArticle(id); 134 | } 135 | 136 | // /article/edit?id=&topic= PUT 137 | @UseGuards(AuthGuard) 138 | @Put("edit") 139 | async editArticleTopic( 140 | @Query("id") id: string, 141 | @Query("topic") topic: string, 142 | ) { 143 | return await this.articleService.editArticleTopic(id, topic); 144 | } 145 | 146 | // /article/hottest?limit=10 147 | // 获取一周范围内最热门的文章 148 | @Get("hottest") 149 | async getHotestArticle(@Query("limit", PositiveIntPipe) limit: number = 10) { 150 | return await this.articleService.getHotestArticle(limit); 151 | } 152 | 153 | // /article/random 154 | @Get("random") 155 | @CacheTTL(2) 156 | async getRandomArticle() { 157 | return await this.articleService.getRandomArticle(); 158 | } 159 | 160 | // /article/random-many 161 | @Get("random-many") 162 | @CacheTTL(900) 163 | async getManyRandomArticle() { 164 | return await this.articleService.getManyRandomArticle(); 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/blog/website/website.controller.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Body, 3 | Controller, 4 | Delete, 5 | Get, Inject, Logger, 6 | Post, 7 | Put, 8 | Query, 9 | UseGuards, 10 | UseInterceptors, 11 | } from "@nestjs/common"; 12 | import {WebsiteService} from "@/blog/website/website.service"; 13 | import {AddWebsiteDto} from "@/dto/addWebsite.dto"; 14 | import {UpdateWebsiteDto} from "@/dto/updateWebsite.dto"; 15 | import {AuthGuard} from "@/auth/auth.guard"; 16 | import {ArticleService} from "@/blog/article/article.service"; 17 | import {Website} from "@/schemas/website.schema"; 18 | import {PositiveIntPipe} from "@/pipe/positiveInt.pipe"; 19 | import {CacheInterceptor} from "@nestjs/cache-manager"; 20 | import {CACHE_MANAGER} from "@nestjs/cache-manager"; 21 | import {Cache} from "cache-manager"; 22 | import {CacheTTL} from "@nestjs/common/cache"; 23 | 24 | @Controller("website") 25 | @UseInterceptors(CacheInterceptor) 26 | export class WebsiteController { 27 | constructor( 28 | @Inject(CACHE_MANAGER) private cacheManager: Cache, 29 | private websiteService: WebsiteService, 30 | private articleService: ArticleService, 31 | ) { 32 | } 33 | 34 | private logger = new Logger('WebsiteController'); 35 | 36 | // /website/most-view?page=1&limit=10 37 | @Get("most-view") 38 | async getWebsiteByPageView( 39 | @Query("page", PositiveIntPipe) page: number = 1, 40 | @Query("limit", PositiveIntPipe) limit: number = 15, 41 | ) { 42 | return await this.websiteService.getWebsiteByPageView(page, limit); 43 | } 44 | 45 | // /website/latest?page=1&limit=15 46 | @Get("latest") 47 | async getWebsiteByLastPublish( 48 | @Query("page", PositiveIntPipe) page: number = 1, 49 | @Query("limit", PositiveIntPipe) limit: number = 15, 50 | ) { 51 | return await this.websiteService.getWebsiteByLastPublish(page, limit); 52 | } 53 | 54 | // /website/all?page=1&limit=15 55 | @Get("all") 56 | @CacheTTL(15) 57 | async getAllWebsite( 58 | @Query("page", PositiveIntPipe) page: number = 1, 59 | @Query("limit", PositiveIntPipe) limit: number = 15 60 | ) { 61 | return await this.websiteService.getAllWebsite(page, limit); 62 | } 63 | 64 | // /website/error?page=1&limit=15 65 | @Get("error") 66 | @CacheTTL(15) 67 | async getWebsiteByErrorCount( 68 | @Query("page", PositiveIntPipe) page: number = 1, 69 | @Query("limit", PositiveIntPipe) limit: number = 15 70 | ) { 71 | return await this.websiteService.getWebsiteByErrorCount(page, limit); 72 | } 73 | 74 | // /website/count 75 | @Get("count") 76 | async getWebsiteCount() { 77 | return await this.websiteService.getWebsiteCount(); 78 | } 79 | 80 | // /website?id= 81 | @Get() 82 | async getWebsiteById(@Query("id") id: string) { 83 | return await this.websiteService.getWebsiteById(id); 84 | } 85 | 86 | // /website/blog?url= 87 | // 根据域名查询网站信息 88 | @Get("blog") 89 | async getWebsiteByUrl(@Query("url") url: string) { 90 | // 将传入url之前添加协议名 91 | const httpsPrefix = "https://"; 92 | const httpPrefix = "http://"; 93 | const httpsUrl = httpsPrefix.concat(url); 94 | const httpUrl = httpPrefix.concat(url); 95 | // 获取网站信息 96 | const website = await this.websiteService.getWebsiteByUrl(httpsUrl); 97 | if (website) { 98 | return website; 99 | } else { 100 | return await this.websiteService.getWebsiteByUrl(httpUrl); 101 | } 102 | } 103 | 104 | // /website/add POST 105 | // 管理员增加网站 106 | @UseGuards(AuthGuard) 107 | @Post("add") 108 | async addWebsite(@Body() addWebsiteDto: AddWebsiteDto) { 109 | await this.cacheManager.reset(); 110 | this.logger.debug('Cache reset'); 111 | return await this.websiteService.addWebsite( 112 | addWebsiteDto.url, 113 | addWebsiteDto.name, 114 | ); 115 | } 116 | 117 | // /website?id= PUT 118 | // 管理员修改网站 119 | @UseGuards(AuthGuard) 120 | @Put() 121 | async updateWebsiteUrl( 122 | @Query("id") id: string, 123 | @Body() updateWebsiteDto: UpdateWebsiteDto, 124 | ) { 125 | await this.cacheManager.reset(); 126 | this.logger.debug('Cache reset'); 127 | return await this.websiteService.updateWebsite( 128 | id, 129 | updateWebsiteDto.url, 130 | updateWebsiteDto.rss, 131 | updateWebsiteDto.name, 132 | updateWebsiteDto.description, 133 | updateWebsiteDto.cover, 134 | ); 135 | } 136 | 137 | // /website?id= DELETE 138 | // 管理员删除网站 139 | @UseGuards(AuthGuard) 140 | @Delete() 141 | async deleteWebsite(@Query("id") id: string) { 142 | await this.cacheManager.reset(); 143 | this.logger.debug('Cache reset'); 144 | return await this.websiteService.deleteWebsite(id); 145 | } 146 | 147 | // /website/last-year?id= GET 148 | // 计算最近一年发布的文章数 149 | @Get("last-year") 150 | async getLastYearArticleCount(@Query("id") id: string) { 151 | return await this.websiteService.getLastYearArticleCount(id); 152 | } 153 | 154 | // 随机返回6个网站 155 | @Get("random") 156 | @CacheTTL(1) 157 | async getRandomWebsite(): Promise { 158 | return await this.websiteService.getRandomWebsite(); 159 | } 160 | 161 | // 手动更新开始抓取网站 162 | @Post("update") 163 | async updateWebsite(@Query("url") url: string) { 164 | await this.cacheManager.reset(); 165 | this.logger.debug('Cache reset'); 166 | return await this.articleService.updateArticlesByWebsite(url); 167 | } 168 | 169 | // 检测网站是否可访问 /website/check?url= 170 | @Get("check") 171 | async checkWebsite(@Query("url") url: string) { 172 | return await this.websiteService.checkWebsite(url); 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /src/blog/website/website.service.ts: -------------------------------------------------------------------------------- 1 | import {HttpException, Injectable, Logger} from "@nestjs/common"; 2 | import {InjectModel} from "@nestjs/mongoose"; 3 | import * as cheerio from "cheerio"; 4 | import {Model} from "mongoose"; 5 | import replaceDomain from "@/common/replace-domain"; 6 | import {Article} from "@/schemas/article.schema"; 7 | import {Website} from "@/schemas/website.schema"; 8 | 9 | @Injectable() 10 | export class WebsiteService { 11 | private readonly logger = new Logger(WebsiteService.name); 12 | 13 | constructor( 14 | @InjectModel("Website") private websiteModel: Model, 15 | @InjectModel("Article") private articleModel: Model
, 16 | ) { 17 | } 18 | 19 | // 根据网站总访问量，倒序排列，获取所有网站 20 | async getWebsiteByPageView( 21 | page: number, 22 | limit: number 23 | ): Promise { 24 | return await this.websiteModel 25 | .find({article_count: {$gt: 0}}) 26 | .sort({page_view: -1}) 27 | .skip((page - 1) * limit) 28 | .limit(limit) 29 | .allowDiskUse(true) 30 | .exec(); 31 | } 32 | 33 | // 根据最近更新时间，倒序排列，获取所有网站，根据传入的page和limit分页 34 | async getWebsiteByLastPublish( 35 | page: number, 36 | limit: number, 37 | ): Promise { 38 | return await this.websiteModel 39 | .find({article_count: {$gt: 0}}) 40 | .sort({last_publish: -1}) 41 | .skip((page - 1) * limit) 42 | .limit(limit) 43 | .allowDiskUse(true) 44 | .exec(); 45 | } 46 | 47 | // 根据抓取错误次数返回网站 48 | async getWebsiteByErrorCount( 49 | page: number, 50 | limit: number, 51 | ) { 52 | return await this.websiteModel 53 | .find() 54 | .sort({crawl_error: -1}) 55 | .skip((page - 1) * limit) 56 | .limit(limit) 57 | .allowDiskUse(true) 58 | .exec(); 59 | } 60 | 61 | // 返回所有文章 62 | async getAllWebsite( 63 | page: number, 64 | limit: number 65 | ): Promise { 66 | return await this.websiteModel 67 | .find() 68 | .sort({last_publish: -1}) 69 | .skip((page - 1) * limit) 70 | .limit(limit) 71 | .allowDiskUse(true) 72 | .exec(); 73 | } 74 | 75 | // 根据id获取指定网站信息 76 | async getWebsiteById(id: string): Promise { 77 | return await this.websiteModel.findById(id).exec(); 78 | } 79 | 80 | // 根据url获取网站信息 81 | async getWebsiteByUrl(url: string): Promise { 82 | const website = await this.websiteModel.findOne({url: url}).exec(); 83 | if (!website) { 84 | throw new HttpException("Website not found", 404); 85 | } 86 | return website; 87 | } 88 | 89 | // 管理员增加网站 90 | async addWebsite(url: string, name: string): Promise { 91 | if (url.endsWith("/")) { 92 | url = url.slice(0, -1); 93 | } 94 | const website = new this.websiteModel({url: url, name: name}); 95 | await website.save(); 96 | await this.updateWebsiteInfo(url); 97 | return await this.websiteModel.findOne({url: url}).exec(); 98 | } 99 | 100 | // 管理员修改网站信息 101 | async updateWebsite( 102 | id: string, 103 | url?: string, 104 | rss?: string, 105 | name?: string, 106 | description?: string, 107 | cover?: string, 108 | ): Promise { 109 | const website = await this.websiteModel.findById(id).exec(); 110 | if (rss) { 111 | website.rss = rss; 112 | } 113 | if (description) { 114 | website.description = description; 115 | } 116 | if (cover) { 117 | website.cover = cover; 118 | } 119 | // 修改文章内的网站名 120 | if (name) { 121 | website.name = name; 122 | const articles = await this.articleModel.find({website_id: id}).allowDiskUse(true).exec(); 123 | for (const article of articles) { 124 | await this.articleModel.findByIdAndUpdate(article._id, { 125 | author: name, 126 | }); 127 | } 128 | } 129 | // 修改文章中的url 130 | if (url) { 131 | website.url = url; 132 | const articles = await this.articleModel.find({website_id: id}).allowDiskUse(true).exec(); 133 | for (const article of articles) { 134 | await this.articleModel.findByIdAndUpdate(article._id, { 135 | url: replaceDomain(article.url, url), 136 | website: url, 137 | }); 138 | } 139 | } 140 | return await website.save(); 141 | } 142 | 143 | // 爬虫更新网站信息 144 | async updateWebsiteInfo(url: string): Promise { 145 | // 将相对域名转换为绝对域名 146 | function getAbsoluteUrl(domain: string, url: string) { 147 | if (url.startsWith("http")) { 148 | return url; 149 | } else { 150 | return `${domain}${url}`; 151 | } 152 | } 153 | 154 | // 发起Get请求，获取网页信息 155 | try { 156 | const response = await fetch(url); 157 | const html = await response.text(); 158 | 159 | // 使用cheerio解析网页信息 160 | const $ = cheerio.load(html); 161 | 162 | // 提取title、description、rss、favicon信息 163 | const description = $('head meta[name="description"]').attr("content"); 164 | const rss = 165 | $('head link[type="application/rss+xml"]').attr("href") || 166 | $('head link[type="application/atom+xml"]').attr("href"); 167 | const favIcon = $('head link[rel="icon"]').attr("href"); 168 | const website = await this.websiteModel.findOne({url: url}); 169 | 170 | website.cover = favIcon ? getAbsoluteUrl(url, favIcon) : null; 171 | website.description = description || "No description"; 172 | website.rss = rss ? getAbsoluteUrl(url, rss) : null; 173 | return await website.save(); 174 | } catch (err) { 175 | this.logger.error( 176 | `Failed to scrape data for website ${url} with error: ${err}`, 177 | ); 178 | } 179 | } 180 | 181 | // 管理员删除网站 182 | async deleteWebsite(id: string): Promise { 183 | // 删除网站 184 | await this.websiteModel.findByIdAndDelete(id); 185 | 186 | // 删除网站下的所有文章 187 | const articles = await this.articleModel.find({website_id: id}).allowDiskUse(true).exec(); 188 | for (const article of articles) { 189 | await this.articleModel.findByIdAndDelete(article._id); 190 | } 191 | return "Delete website successfully"; 192 | } 193 | 194 | // 遍历网站下的文章，计算访问量、分类以及最新发布时间 195 | async updatePageView(id: string): Promise { 196 | // 利用websiteId去article中查找website_id为websiteId的所有文章，并按发布时间倒序排列 197 | const articles = await this.articleModel 198 | .find({website_id: id}) 199 | .sort({publish_date: -1}) 200 | .allowDiskUse(true) 201 | .exec(); 202 | 203 | // 计算所有文章的page_view总和 204 | const pageView = articles.reduce( 205 | (totalPageView, article) => totalPageView + article.page_view, 206 | 0, 207 | ); 208 | 209 | this.logger.log(`Update page_view of website ${id} to ${pageView}`); 210 | 211 | // 统计文章的分类 212 | const articleCategories = new Map(); 213 | for (const article of articles) { 214 | const topic = article.topic || "未分类"; 215 | if (articleCategories.has(topic)) { 216 | articleCategories.set(topic, articleCategories.get(topic) + 1); 217 | } else { 218 | articleCategories.set(topic, 1); 219 | } 220 | } 221 | 222 | // 顺便更新最新文章发布时间 223 | const lastPublish = articles[0].publish_date; 224 | 225 | // 更新websiteModel中的page_view 226 | return await this.websiteModel 227 | .findByIdAndUpdate(id, { 228 | page_view: pageView, 229 | categories: articleCategories, 230 | last_publish: lastPublish, 231 | }) 232 | .exec(); 233 | } 234 | 235 | // 获取网站总数 236 | async getWebsiteCount(): Promise { 237 | return await this.websiteModel.find().countDocuments().exec(); 238 | } 239 | 240 | // 计算网站最近一年发布的文章 241 | async getLastYearArticleCount(id: string): Promise { 242 | if (!id) { 243 | throw new HttpException("Invalid website id", 400); 244 | } 245 | const oneYearAgo = new Date(); 246 | oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1); 247 | return this.articleModel.countDocuments({ 248 | website_id: id, 249 | publish_date: {$gte: oneYearAgo}, 250 | }); 251 | } 252 | 253 | // 随机抽取6个网站 254 | async getRandomWebsite(): Promise { 255 | return await this.websiteModel.aggregate([{$sample: {size: 6}}]).allowDiskUse(true).exec(); 256 | } 257 | 258 | // 检测网站是否可访问 259 | async checkWebsite(url: string): Promise { 260 | try { 261 | const response = await fetch(url); 262 | return response.ok; 263 | } catch (err) { 264 | return false; 265 | } 266 | } 267 | } 268 | -------------------------------------------------------------------------------- /src/blog/article/article.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, Logger } from "@nestjs/common"; 2 | import { InjectModel } from "@nestjs/mongoose"; 3 | import mongoose, { Model } from "mongoose"; 4 | import getArticleInfo from "@/common/article-extract"; 5 | import convertDate from "@/common/convert-date"; 6 | import { Article } from "@/schemas/article.schema"; 7 | import { Website } from "@/schemas/website.schema"; 8 | import feedExtract from "@/common/feed-extract"; 9 | 10 | @Injectable() 11 | export class ArticleService { 12 | constructor( 13 | @InjectModel("Article") private articleModel: Model
, 14 | @InjectModel("Website") private websiteModel: Model, 15 | ) {} 16 | 17 | private readonly logger = new Logger(ArticleService.name); 18 | 19 | // 根据最近发布时间，从最新到最旧，获取所有文章，排除被封禁的文章 20 | async getAllUnblockedArticle( 21 | page: number, 22 | limit: number, 23 | ): Promise { 24 | return await this.articleModel 25 | .find({ isBlocked: { $ne: true } }) 26 | .sort({ publish_date: -1 }) 27 | .skip((page - 1) * limit) 28 | .limit(limit) 29 | .allowDiskUse(true) 30 | .exec(); 31 | } 32 | 33 | // 根据最近发布时间，从最新到最旧，获取所有文章 34 | async getAllArticle(page: number, limit: number): Promise { 35 | return await this.articleModel 36 | .find() 37 | .sort({ publish_date: -1 }) 38 | .skip((page - 1) * limit) 39 | .limit(limit) 40 | .allowDiskUse(true) 41 | .exec(); 42 | } 43 | 44 | // 获取编辑推荐文章 45 | async getArticleByRecommend(page: number, limit: number): Promise { 46 | return await this.articleModel 47 | .find({ isFeatured: true, isBlocked: { $ne: true } }) 48 | .sort({ publish_date: -1 }) 49 | .skip((page - 1) * limit) 50 | .limit(limit) 51 | .allowDiskUse(true) 52 | .exec(); 53 | } 54 | 55 | // 获取一周内浏览量最高的文章 56 | async getHotestArticle(limit: number): Promise { 57 | const date = new Date(); 58 | date.setDate(date.getDate() - 7); 59 | return await this.articleModel 60 | .find({ publish_date: { $gte: date }, isBlocked: { $ne: true } }) 61 | .sort({ page_view: -1 }) 62 | .limit(limit) 63 | .allowDiskUse(true) 64 | .exec(); 65 | } 66 | 67 | // 随机获取1篇文章 68 | async getRandomArticle(): Promise { 69 | return await this.articleModel 70 | .aggregate([ 71 | { $match: { isBlocked: { $ne: true }, abstract: { $ne: null } } }, 72 | { $sample: { size: 1 } }, 73 | ]) 74 | .allowDiskUse(true) 75 | .exec(); 76 | }; 77 | 78 | async getManyRandomArticle(): Promise { 79 | // 最近3天的文章 80 | const date = new Date(); 81 | date.setDate(date.getDate() - 3); 82 | return await this.articleModel 83 | .aggregate([ 84 | { 85 | $match: { 86 | isBlocked: { $ne: true }, 87 | publish_date: { $gte: date }, 88 | isFeatured: { $ne: true }, 89 | }, 90 | }, 91 | { $sample: { size: 20 } }, 92 | ]) 93 | .allowDiskUse(true) 94 | .exec(); 95 | } 96 | 97 | // 获得指定分类的最新文章 98 | async getArticleByTopic( 99 | topic: string, 100 | page: number, 101 | limit: number, 102 | ): Promise { 103 | return await this.articleModel 104 | .find({ topic: topic, isBlocked: { $ne: true } }) 105 | .sort({ publish_date: -1 }) 106 | .skip((page - 1) * limit) 107 | .limit(limit) 108 | .allowDiskUse(true) 109 | .exec(); 110 | } 111 | 112 | // 获取指定博客所有文章 113 | async getArticleByBlog( 114 | website: string, 115 | page: number, 116 | limit: number, 117 | ): Promise { 118 | return await this.articleModel 119 | .find({ website: website, isBlocked: { $ne: true } }) 120 | .sort({ publish_date: -1 }) 121 | .skip((page - 1) * limit) 122 | .limit(limit) 123 | .allowDiskUse(true) 124 | .exec(); 125 | } 126 | 127 | // 新增文章 128 | async addArticle( 129 | url: string, 130 | website_id: mongoose.Types.ObjectId, 131 | website: string, 132 | title: string, 133 | description: string, 134 | publish_date: Date, 135 | author: string, 136 | ) { 137 | // 查询是否已存在该 article 138 | const existArticle = await this.articleModel.findOne({ url: url }).exec(); 139 | if (existArticle) { 140 | return { 141 | status: "EXIST", 142 | }; 143 | } 144 | 145 | try { 146 | const article = await getArticleInfo(url, website, description); 147 | this.logger.debug(`Start save article ${title}, publish at ${publish_date}`); 148 | const newArticle = new this.articleModel({ 149 | url: url, 150 | website_id: website_id, 151 | website: website, 152 | title: title, 153 | description: description, 154 | publish_date: publish_date, 155 | author: author, 156 | cover: article.cover, 157 | content: article.content, 158 | abstract: article.abstract, 159 | tags: article.tags, 160 | topic: article.topic, 161 | }); 162 | await newArticle.save(); 163 | } catch (error) { 164 | this.logger.error(`Error happen on extract or save: ${error}`); 165 | } 166 | 167 | await this.websiteModel.findByIdAndUpdate(website_id, { 168 | last_crawl: new Date(), 169 | }); 170 | return { 171 | status: "OK", 172 | }; 173 | } 174 | 175 | // 根据网站rss，获取网站最新文章，并传入addArticle方法 176 | async updateArticlesByWebsite(url: string): Promise { 177 | const website = await this.websiteModel.findOne({ url: url }).exec(); 178 | 179 | if (!website) { 180 | throw new Error("没有找到对应的网站信息"); 181 | } 182 | 183 | const websiteId = website._id; 184 | const websiteUrl = website.url; 185 | const author = website.name; 186 | const rss = website.rss; 187 | if (!rss) { 188 | throw new Error("该网站没有rss"); 189 | } 190 | 191 | // 从feed中提取文章信息，并找到content和summary 192 | const articlesFromFeed = await feedExtract(rss); 193 | 194 | for (const item of articlesFromFeed) { 195 | try { 196 | const feed = await this.addArticle( 197 | item.link, 198 | websiteId, 199 | websiteUrl, 200 | item.title, 201 | item.description, 202 | convertDate(item.published), 203 | author, 204 | ); 205 | if (feed.status === "EXIST") { 206 | break; 207 | } 208 | } catch { 209 | this.logger.error( 210 | `Add article ${item.title} of url ${item.link} failed`, 211 | ); 212 | } 213 | } 214 | 215 | // 更新网站文章数量 216 | const articleCount = await this.getArticleCountByWebsite(url); 217 | await this.websiteModel.findOneAndUpdate( 218 | { url: url }, 219 | { article_count: articleCount }, 220 | ); 221 | return this.websiteModel.findById(websiteId); 222 | } 223 | 224 | //切换某篇文章isFeatured的状态 225 | async setFeaturedArticle(id: string): Promise
{ 226 | const featuredArticle = await this.articleModel.findById(id).exec(); 227 | featuredArticle.isFeatured = !featuredArticle.isFeatured; 228 | return await featuredArticle.save(); 229 | } 230 | 231 | // 管理员封禁或解禁某篇文章 232 | async blockArticle(id: string): Promise
{ 233 | const blockedArticle = await this.articleModel.findById(id).exec(); 234 | blockedArticle.isBlocked = !blockedArticle.isBlocked; 235 | return await blockedArticle.save(); 236 | } 237 | 238 | //删除某篇文章 239 | async deleteArticle(id: string): Promise
{ 240 | return this.articleModel.findByIdAndDelete(id); 241 | } 242 | 243 | // 根据文章website字段，统计指定网站的文章数量 244 | async getArticleCountByWebsite(website: string): Promise { 245 | const blog = await this.websiteModel.findOne({ url: website }).exec(); 246 | return await this.articleModel 247 | .find({ website_id: blog._id }) 248 | .countDocuments() 249 | .exec(); 250 | } 251 | 252 | // 访问量记录，每次访问，访问量+1 253 | async addPageView(id: string): Promise { 254 | const article = await this.articleModel.findById(id).exec(); 255 | article.page_view += 1; 256 | await article.save(); 257 | return article.page_view; 258 | } 259 | 260 | // 获取文章总数 261 | async getArticleCount( 262 | type: string, 263 | topic?: string, 264 | startAt?: Date, 265 | endAt?: Date, 266 | ): Promise { 267 | switch (type) { 268 | case "all": 269 | return await this.articleModel.countDocuments().exec(); 270 | 271 | case "featured": 272 | return await this.articleModel 273 | .countDocuments({ isFeatured: true }) 274 | .exec(); 275 | 276 | case "topic": 277 | return await this.articleModel.countDocuments({ topic: topic }).exec(); 278 | 279 | case "date": 280 | return await this.articleModel 281 | .countDocuments({ publish_date: { $gte: startAt, $lte: endAt } }) 282 | .exec(); 283 | 284 | default: 285 | return await this.articleModel.countDocuments().exec(); 286 | } 287 | } 288 | 289 | // 修改文章分类 290 | async editArticleTopic(id: string, topic: string): Promise
{ 291 | const article = await this.articleModel.findById(id).exec(); 292 | article.topic = topic; 293 | return await article.save(); 294 | } 295 | 296 | async getArticleCountByBlog(id: string): Promise { 297 | return await this.articleModel 298 | .find({ website_id: id }) 299 | .countDocuments() 300 | .exec(); 301 | } 302 | } 303 | --------------------------------------------------------------------------------