├── .gitignore ├── output ├── logo-blue.png ├── logo-tech.png ├── logo-test.png ├── logo-custom.png └── logo-purple.png ├── src ├── services │ ├── interfaces │ │ └── workflow.interface.ts │ ├── weixin-aibench.workflow.ts │ ├── weixin-hellogithub.workflow.ts │ └── weixin-article.workflow.ts ├── utils │ ├── config │ │ ├── interfaces │ │ │ └── config-source.interface.ts │ │ ├── sources │ │ │ ├── env-config.source.ts │ │ │ └── db-config.source.ts │ │ └── config-manager.ts │ ├── common.ts │ ├── content-rank │ │ ├── content-ranker.test.ts │ │ └── content-ranker.ts │ ├── gen-image │ │ ├── text-logo.ts │ │ └── aliwanx2.1.image.ts │ ├── bark.notify.ts │ └── db │ │ └── mysql.db.ts ├── render │ ├── interfaces │ │ ├── template.interface.ts │ │ ├── aibench.template.ts │ │ └── aigithub.type.ts │ ├── weixin │ │ └── renderer.ts │ ├── test │ │ ├── test.weixin.template.ts │ │ ├── test.hellogithub.render.ts │ │ └── test.aibench.template.ts │ └── aibench │ │ └── renderer.ts ├── summarizer │ ├── interfaces │ │ └── summarizer.interface.ts │ ├── together-ai.summarizer.ts │ ├── deepseek-ai.summarizer.ts │ └── qianwen-ai.summarizer.ts ├── publishers │ ├── interfaces │ │ └── publisher.interface.ts │ └── weixin.publisher.ts ├── scrapers │ ├── interfaces │ │ └── scraper.interface.ts │ ├── twitter.scraper.ts │ ├── hellogithub.scraper.ts │ └── fireCrawl.scraper.ts ├── index.ts ├── test │ ├── hellogithub.scraper.test.ts │ ├── hellogithub.template.test.ts │ └── text-logo.test.ts ├── test.ts ├── prompts │ └── summarizer.prompt.ts ├── controllers │ └── cron.ts ├── api │ ├── xunfei.api.ts │ ├── deepseek.api.ts │ └── livebench.api.ts ├── templates │ ├── article.ejs │ ├── aibench-weixin.ejs │ ├── hellogithub-weixin.ejs │ └── aibench.ejs └── data-sources │ └── getCronSources.ts ├── tsconfig.json ├── .env.example ├── LICENSE ├── sql ├── config.sql └── cron_sources.sql ├── package.json ├── .github └── workflows │ └── deploy.yml ├── temp └── preview_weixin.html └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .env 3 | .vercel 4 | .devcontainer.json 5 | dist 6 | 7 | output -------------------------------------------------------------------------------- /output/logo-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuilenren/ai-trend-publish/main/output/logo-blue.png -------------------------------------------------------------------------------- /output/logo-tech.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuilenren/ai-trend-publish/main/output/logo-tech.png -------------------------------------------------------------------------------- /output/logo-test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuilenren/ai-trend-publish/main/output/logo-test.png -------------------------------------------------------------------------------- /output/logo-custom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuilenren/ai-trend-publish/main/output/logo-custom.png -------------------------------------------------------------------------------- /output/logo-purple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kuilenren/ai-trend-publish/main/output/logo-purple.png -------------------------------------------------------------------------------- /src/services/interfaces/workflow.interface.ts: -------------------------------------------------------------------------------- 1 | export interface Workflow { 2 | /** 3 | * 刷新工作流所需的资源和配置 4 | */ 5 | refresh(): Promise; 6 | 7 | /** 8 | * 执行工作流的主要处理逻辑 9 | */ 10 | process(): Promise; 11 | } 12 | -------------------------------------------------------------------------------- /src/utils/config/interfaces/config-source.interface.ts: -------------------------------------------------------------------------------- 1 | export interface IConfigSource { 2 | /** 3 | * 配置源的优先级，数字越小优先级越高 4 | */ 5 | priority: number; 6 | 7 | /** 8 | * 获取配置值 9 | * @param key 配置键 10 | * @returns 配置值的Promise，如果未找到返回null 11 | */ 12 | get(key: string): Promise; 13 | } 14 | -------------------------------------------------------------------------------- /src/render/interfaces/template.interface.ts: -------------------------------------------------------------------------------- 1 | export interface GeneratedTemplate { 2 | id: string; 3 | title: string; 4 | content: string; 5 | url: string; 6 | publishDate: string; 7 | metadata: Record; 8 | } 9 | 10 | export interface WeixinTemplate extends GeneratedTemplate { 11 | keywords: string[]; 12 | } 13 | -------------------------------------------------------------------------------- /src/render/interfaces/aibench.template.ts: -------------------------------------------------------------------------------- 1 | export interface ModelScore { 2 | name: string; 3 | score: number; 4 | reasoning?: number; 5 | coding?: number; 6 | math?: number; 7 | dataAnalysis?: number; 8 | language?: number; 9 | if?: number; 10 | } 11 | 12 | export interface CategoryData { 13 | name: string; 14 | icon: string; 15 | models: ModelScore[]; 16 | } 17 | 18 | export interface AIBenchTemplate { 19 | title: string; 20 | updateTime: string; 21 | categories: CategoryData[]; 22 | globalTop10: ModelScore[]; 23 | } 24 | -------------------------------------------------------------------------------- /src/summarizer/interfaces/summarizer.interface.ts: -------------------------------------------------------------------------------- 1 | export interface ContentSummarizer { 2 | // 验证配置是否完善 3 | validateConfig(): void; 4 | 5 | // 刷新配置 6 | refresh(): Promise; 7 | 8 | // 对内容进行摘要 9 | summarize(content: string, options?: Record): Promise; 10 | 11 | // 生成标题 12 | generateTitle( 13 | content: string, 14 | options?: Record 15 | ): Promise; 16 | } 17 | 18 | export interface Summary { 19 | title: string; 20 | content: string; 21 | keywords: string[]; 22 | score: number; 23 | } 24 | -------------------------------------------------------------------------------- /src/utils/config/sources/env-config.source.ts: -------------------------------------------------------------------------------- 1 | import { IConfigSource } from "../interfaces/config-source.interface"; 2 | 3 | export class EnvConfigSource implements IConfigSource { 4 | constructor(public priority: number = 100) {} 5 | 6 | async get(key: string): Promise { 7 | const value = process.env[key]; 8 | if (value === undefined) { 9 | return null; 10 | } 11 | 12 | try { 13 | // 尝试解析JSON格式的值 14 | return JSON.parse(value) as T; 15 | } catch { 16 | // 如果不是JSON格式，直接返回字符串值 17 | return value as unknown as T; 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/render/interfaces/aigithub.type.ts: -------------------------------------------------------------------------------- 1 | export interface AIGithubItem { 2 | itemId: string; 3 | author: string; 4 | title: string; 5 | } 6 | 7 | interface relatedUrl { 8 | url: string; 9 | title: string; 10 | } 11 | 12 | export interface AIGithubItemDetail extends AIGithubItem { 13 | name: string; 14 | 15 | url: string; 16 | description: string; 17 | language: string; 18 | totalStars: number; 19 | totalIssues: number; 20 | totalForks: number; 21 | contributors: number; 22 | lastWeekStars: number; 23 | tags: string[]; 24 | license: string; 25 | relatedUrls: relatedUrl[]; 26 | } 27 | -------------------------------------------------------------------------------- /src/publishers/interfaces/publisher.interface.ts: -------------------------------------------------------------------------------- 1 | export interface ContentPublisher { 2 | // 验证发布配置是否完善 3 | validateConfig(): void; 4 | // 刷新配置 5 | refresh(): Promise; 6 | 7 | // 上传图片到指定平台 8 | uploadImage(imageUrl: string): Promise; 9 | 10 | // 发布文章到指定平台 11 | publish(article: string, ...args: any[]): Promise; 12 | } 13 | 14 | export interface PublishResult { 15 | publishId: string; 16 | url?: string; 17 | status: PublishStatus; 18 | publishedAt: Date; 19 | platform: string; 20 | } 21 | 22 | export type PublishStatus = 23 | | "pending" 24 | | "published" 25 | | "failed" 26 | | "draft" 27 | | "scheduled"; 28 | -------------------------------------------------------------------------------- /src/scrapers/interfaces/scraper.interface.ts: -------------------------------------------------------------------------------- 1 | export interface ContentScraper { 2 | // 验证发布配置是否完善 3 | validateConfig(): void; 4 | 5 | // 刷新配置 6 | refresh(): Promise; 7 | 8 | // 抓取指定数据源的内容 9 | scrape(sourceId: string, options?: ScraperOptions): Promise; 10 | } 11 | 12 | export interface ScraperOptions { 13 | startDate?: Date; 14 | endDate?: Date; 15 | limit?: number; 16 | filters?: Record; 17 | } 18 | 19 | export interface ScrapedContent { 20 | id: string; 21 | title: string; 22 | content: string; 23 | url: string; 24 | publishDate: string; 25 | score: number; 26 | metadata: Record; 27 | } 28 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2017", // Specify ECMAScript target version 4 | "module": "commonjs", // Specify module code generation 5 | "outDir": "./dist", // Redirect output structure to the directory 6 | "rootDir": "./src", // Specify the root directory of input files 7 | "strict": true, // Enable all strict type-checking options 8 | "esModuleInterop": true, // Enables emit interoperability between CommonJS and ES Modules 9 | "skipLibCheck": true, // Skip type checking of declaration files 10 | "forceConsistentCasingInFileNames": true // Disallow inconsistently-cased references to the same file 11 | }, 12 | "include": ["src/**/*"], // Include all files in the src directory 13 | "exclude": ["node_modules", "dist"] // Exclude node_modules and dist directories 14 | } 15 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { startCronJobs } from "./controllers/cron"; 2 | import { ConfigManager } from "./utils/config/config-manager"; 3 | import { EnvConfigSource } from "./utils/config/sources/env-config.source"; 4 | import { DbConfigSource } from "./utils/config/sources/db-config.source"; 5 | import { MySQLDB } from "./utils/db/mysql.db"; 6 | 7 | async function bootstrap() { 8 | const configManager = ConfigManager.getInstance(); 9 | configManager.addSource(new EnvConfigSource()); 10 | 11 | const db = await MySQLDB.getInstance({ 12 | host: process.env.DB_HOST, 13 | port: Number(process.env.DB_PORT), 14 | user: process.env.DB_USER, 15 | password: process.env.DB_PASSWORD, 16 | database: process.env.DB_DATABASE, 17 | }); 18 | configManager.addSource(new DbConfigSource(db)); 19 | 20 | startCronJobs(); 21 | } 22 | 23 | bootstrap().catch(console.error); 24 | -------------------------------------------------------------------------------- /src/render/weixin/renderer.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | import ejs from "ejs"; 4 | import { WeixinTemplate } from "../interfaces/template.interface"; 5 | 6 | export class WeixinTemplateRenderer { 7 | private template: string; 8 | 9 | constructor() { 10 | // 读取模板文件 11 | const templatePath = path.join(__dirname, "../../templates/article.ejs"); 12 | this.template = fs.readFileSync(templatePath, "utf-8"); 13 | } 14 | 15 | /** 16 | * 渲染微信文章模板 17 | * @param articles 微信文章模板数组 18 | * @returns 渲染后的 HTML 19 | */ 20 | render(articles: WeixinTemplate[]): string { 21 | try { 22 | // 使用 EJS 渲染模板 23 | return ejs.render( 24 | this.template, 25 | { articles }, 26 | { 27 | rmWhitespace: true, 28 | } 29 | ); 30 | } catch (error) { 31 | console.error("模板渲染失败:", error); 32 | throw error; 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/hellogithub.scraper.test.ts: -------------------------------------------------------------------------------- 1 | import { HelloGithubScraper } from "../scrapers/hellogithub.scraper"; 2 | 3 | async function testScraper() { 4 | console.log("开始测试 HelloGithubScraper..."); 5 | const scraper = new HelloGithubScraper(); 6 | 7 | try { 8 | // 1. 获取热门仓库列表 9 | console.log("\n1. 获取热门仓库列表"); 10 | const hotItems = await scraper.getHotItems(1); 11 | console.log("热门仓库列表:"); 12 | console.log(JSON.stringify(hotItems, null, 2)); 13 | 14 | if (hotItems.length > 0) { 15 | // 2. 获取第一个仓库的详细信息 16 | console.log("\n2. 获取仓库详情"); 17 | const firstItem = hotItems[0]; 18 | console.log(`获取项目详情: ${firstItem.itemId}`); 19 | const result = await scraper.getItemDetail(firstItem.itemId); 20 | console.log("仓库详情:"); 21 | console.log(JSON.stringify(result, null, 2)); 22 | } 23 | } catch (error: any) { 24 | console.error("测试过程中发生错误:", error.message); 25 | } 26 | } 27 | 28 | // 运行测试 29 | testScraper(); 30 | -------------------------------------------------------------------------------- /src/utils/config/sources/db-config.source.ts: -------------------------------------------------------------------------------- 1 | import { MySQLDB } from "../../db/mysql.db"; 2 | import { IConfigSource } from "../interfaces/config-source.interface"; 3 | 4 | export class DbConfigSource implements IConfigSource { 5 | constructor( 6 | private db: MySQLDB, // 这里应该替换为你的实际数据库连接类型 7 | public priority: number = 10 8 | ) {} 9 | 10 | async get(key: string): Promise { 11 | try { 12 | const result = await this.db.queryOne( 13 | "SELECT `value` FROM `config` WHERE `key` = ?", 14 | [key] 15 | ); 16 | 17 | if (!result || result.length === 0) { 18 | return null; 19 | } 20 | 21 | const value = result.value; 22 | try { 23 | return JSON.parse(value) as T; 24 | } catch { 25 | return value as unknown as T; 26 | } 27 | } catch (error) { 28 | console.error(`Error fetching config from database: ${error}`); 29 | return null; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | 2 | #Craw API 3 | ## Required if monitoring web pages (https://www.firecrawl.dev/) 4 | FIRECRAWL_API_KEY=your_firecrawl_api_key 5 | 6 | # Required if monitoring Twitter/X trends (https://developer.x.com/) 7 | X_API_BEARER_TOKEN=your_twitter_api_bearer_token 8 | 9 | 10 | # Summarizer API 11 | ## DeepSeek AI Required from https://www.deepseek.com/ 12 | DEEPSEEK_API_KEY=your_deepseek_api_key 13 | 14 | ## image generation api, required from https://www.getimg.ai/ 15 | GETIMG_API_KEY=your_getimg_api_key 16 | 17 | # Required: API key from Together AI for trend analysis (https://www.together.ai/) 18 | TOGETHER_API_KEY=your_together_api_key 19 | 20 | 21 | # Public API 22 | ## weixin gongzhonghao 23 | WEIXIN_APP_ID=your_app_id 24 | WEIXIN_APP_SECRET=your_app_secret 25 | 26 | 27 | # Notification 28 | ## Bark Push Key 29 | BARK_URL=https://api.day.app/ 30 | 31 | 32 | # Database 33 | DB_HOST=your_db_host 34 | DB_PORT=your_db_port 35 | DB_USER=your_db_user 36 | DB_PASSWORD=your_db_password 37 | DB_DATABASE=your_db_database 38 | 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2025 Eric Ciarla 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | 9 | -------------------------------------------------------------------------------- /sql/config.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Navicat Premium Dump SQL 3 | 4 | Source Server : 107.173.152.50 5 | Source Server Type : MySQL 6 | Source Server Version : 80403 (8.4.3) 7 | Source Host : 107.173.152.50:3306 8 | Source Schema : trendfind 9 | 10 | Target Server Type : MySQL 11 | Target Server Version : 80403 (8.4.3) 12 | File Encoding : 65001 13 | 14 | Date: 27/02/2025 16:39:55 15 | */ 16 | 17 | SET NAMES utf8mb4; 18 | SET FOREIGN_KEY_CHECKS = 0; 19 | 20 | -- ---------------------------- 21 | -- Table structure for config 22 | -- ---------------------------- 23 | DROP TABLE IF EXISTS `config`; 24 | CREATE TABLE `config` ( 25 | `id` int NOT NULL AUTO_INCREMENT, 26 | `key` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, 27 | `value` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, 28 | PRIMARY KEY (`id`) USING BTREE 29 | ) ENGINE = InnoDB AUTO_INCREMENT = 11 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; 30 | 31 | SET FOREIGN_KEY_CHECKS = 1; 32 | -------------------------------------------------------------------------------- /src/test/hellogithub.template.test.ts: -------------------------------------------------------------------------------- 1 | import ejs from "ejs"; 2 | import fs from "fs"; 3 | import path from "path"; 4 | import { HelloGithubScraper } from "../scrapers/hellogithub.scraper"; 5 | 6 | async function testTemplate() { 7 | try { 8 | // 1. 获取数据 9 | const scraper = new HelloGithubScraper(); 10 | const hotItems = await scraper.getHotItems(1); 11 | const items = await Promise.all( 12 | hotItems.slice(0, 5).map((item) => scraper.getItemDetail(item.itemId)) 13 | ); 14 | 15 | // 2. 读取模板 16 | const templatePath = path.join( 17 | __dirname, 18 | "../templates/hellogithub-weixin.ejs" 19 | ); 20 | const template = fs.readFileSync(templatePath, "utf-8"); 21 | 22 | // 3. 渲染模板 23 | const html = ejs.render(template, { items }); 24 | 25 | // 4. 保存结果 26 | const outputPath = path.join( 27 | __dirname, 28 | "../../output/hellogithub-weixin.html" 29 | ); 30 | fs.writeFileSync(outputPath, html, "utf-8"); 31 | 32 | console.log("模板渲染成功！输出文件:", outputPath); 33 | } catch (error: any) { 34 | console.error("模板测试失败:", error.message); 35 | } 36 | } 37 | 38 | // 运行测试 39 | testTemplate(); 40 | -------------------------------------------------------------------------------- /sql/cron_sources.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Navicat Premium Dump SQL 3 | 4 | Source Server : 107.173.152.50 5 | Source Server Type : MySQL 6 | Source Server Version : 80403 (8.4.3) 7 | Source Host : 107.173.152.50:3306 8 | Source Schema : trendfind 9 | 10 | Target Server Type : MySQL 11 | Target Server Version : 80403 (8.4.3) 12 | File Encoding : 65001 13 | 14 | Date: 27/02/2025 16:40:12 15 | */ 16 | 17 | SET NAMES utf8mb4; 18 | SET FOREIGN_KEY_CHECKS = 0; 19 | 20 | -- ---------------------------- 21 | -- Table structure for cron_sources 22 | -- ---------------------------- 23 | DROP TABLE IF EXISTS `cron_sources`; 24 | CREATE TABLE `cron_sources` ( 25 | `id` int NOT NULL AUTO_INCREMENT, 26 | `NewsType` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, 27 | `NewsPlatform` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, 28 | `identifier` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL, 29 | PRIMARY KEY (`id`) USING BTREE 30 | ) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; 31 | 32 | SET FOREIGN_KEY_CHECKS = 1; 33 | -------------------------------------------------------------------------------- /src/utils/common.ts: -------------------------------------------------------------------------------- 1 | export function formatDate(dateString: string): string { 2 | // 尝试多种方式解析日期 3 | let date: Date; 4 | try { 5 | // 首先尝试直接解析 6 | date = new Date(dateString); 7 | 8 | // 检查是否为有效日期 9 | if (isNaN(date.getTime())) { 10 | // 尝试处理特殊格式 11 | if (dateString.includes("+")) { 12 | // 处理带时区的格式 13 | date = new Date(dateString.replace(/(\+\d{4})/, "UTC$1")); 14 | } else { 15 | // 尝试移除特殊字符后解析 16 | const cleanDate = dateString.replace(/[^\d\s:-]/g, ""); 17 | date = new Date(cleanDate); 18 | } 19 | } 20 | 21 | if (isNaN(date.getTime())) { 22 | throw new Error("Invalid date"); 23 | } 24 | } catch (error) { 25 | throw new Error(`Unable to parse date: ${dateString}`); 26 | } 27 | 28 | const year = date.getFullYear(); 29 | const month = (date.getMonth() + 1).toString().padStart(2, "0"); 30 | const day = date.getDate().toString().padStart(2, "0"); 31 | const hours = date.getHours().toString().padStart(2, "0"); 32 | const minutes = date.getMinutes().toString().padStart(2, "0"); 33 | const seconds = date.getSeconds().toString().padStart(2, "0"); 34 | 35 | return `${year}/${month}/${day} ${hours}:${minutes}:${seconds}`; 36 | } 37 | -------------------------------------------------------------------------------- /src/test.ts: -------------------------------------------------------------------------------- 1 | import { WeixinWorkflow } from "./services/weixin-article.workflow"; 2 | import { ConfigManager } from "./utils/config/config-manager"; 3 | import { EnvConfigSource } from "./utils/config/sources/env-config.source"; 4 | import { DbConfigSource } from "./utils/config/sources/db-config.source"; 5 | import { MySQLDB } from "./utils/db/mysql.db"; 6 | import { WeixinAIBenchWorkflow } from "./services/weixin-aibench.workflow"; 7 | import { WeixinHelloGithubWorkflow } from "./services/weixin-hellogithub.workflow"; 8 | async function bootstrap() { 9 | const configManager = ConfigManager.getInstance(); 10 | configManager.addSource(new EnvConfigSource()); 11 | 12 | const db = await MySQLDB.getInstance({ 13 | host: process.env.DB_HOST, 14 | port: Number(process.env.DB_PORT), 15 | user: process.env.DB_USER, 16 | password: process.env.DB_PASSWORD, 17 | database: process.env.DB_DATABASE, 18 | }); 19 | configManager.addSource(new DbConfigSource(db)); 20 | 21 | const weixinWorkflow = new WeixinWorkflow(); 22 | 23 | await weixinWorkflow.refresh(); 24 | await weixinWorkflow.process(); 25 | 26 | // const weixinAIBenchWorkflow = new WeixinAIBenchWorkflow(); 27 | // await weixinAIBenchWorkflow.refresh(); 28 | // await weixinAIBenchWorkflow.process(); 29 | 30 | // const weixinHelloGithubWorkflow = new WeixinHelloGithubWorkflow(); 31 | // await weixinHelloGithubWorkflow.refresh(); 32 | // await weixinHelloGithubWorkflow.process(); 33 | } 34 | 35 | bootstrap().catch(console.error); 36 | -------------------------------------------------------------------------------- /src/test/text-logo.test.ts: -------------------------------------------------------------------------------- 1 | import { TextLogoGenerator } from "../utils/gen-image/text-logo"; 2 | import path from "path"; 3 | 4 | async function testTextLogo() { 5 | try { 6 | // 蓝色渐变主题 7 | await TextLogoGenerator.saveToFile( 8 | { 9 | text: "大模型榜单", 10 | width: 1200, 11 | height: 400, 12 | fontSize: 160, 13 | backgroundColor: "#ffffff", 14 | gradientStart: "#1a73e8", 15 | gradientEnd: "#4285f4", 16 | }, 17 | path.join(__dirname, "../../output/logo-blue.png") 18 | ); 19 | 20 | // 深紫色渐变主题 21 | await TextLogoGenerator.saveToFile( 22 | { 23 | text: "大模型榜单", 24 | width: 1200, 25 | height: 400, 26 | fontSize: 160, 27 | backgroundColor: "#ffffff", 28 | gradientStart: "#6200ea", 29 | gradientEnd: "#9d46ff", 30 | }, 31 | path.join(__dirname, "../../output/logo-purple.png") 32 | ); 33 | 34 | // 科技蓝主题 35 | await TextLogoGenerator.saveToFile( 36 | { 37 | text: "大模型榜单", 38 | width: 1200, 39 | height: 400, 40 | fontSize: 160, 41 | backgroundColor: "#f8f9fa", 42 | gradientStart: "#0277bd", 43 | gradientEnd: "#039be5", 44 | }, 45 | path.join(__dirname, "../../output/logo-tech.png") 46 | ); 47 | 48 | console.log("Logo生成成功！请查看 output/ 目录下的三个不同主题的logo文件"); 49 | } catch (error) { 50 | console.error("Logo生成失败:", error); 51 | } 52 | } 53 | 54 | // 运行测试 55 | testTextLogo(); 56 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "trend-finder", 3 | "version": "1.0.0", 4 | "main": "src/index.ts", 5 | "scripts": { 6 | "start": "nodemon src/index.ts", 7 | "build": "tsc && npm run copy-templates", 8 | "test": "nodemon src/test.ts", 9 | "copy-templates": "copyfiles -u 1 \"src/templates/**/*\" dist" 10 | }, 11 | "author": "", 12 | "license": "ISC", 13 | "devDependencies": { 14 | "@types/cheerio": "^0.22.35", 15 | "@types/cli-progress": "^3.11.6", 16 | "@types/ejs": "^3.1.5", 17 | "@types/express": "^4.17.21", 18 | "@types/node": "^20.10.2", 19 | "@types/node-cron": "^3.0.11", 20 | "copyfiles": "^2.4.1", 21 | "nodemon": "^3.0.2", 22 | "rimraf": "^5.0.5", 23 | "ts-jest": "^29.2.5", 24 | "ts-node": "^10.9.1", 25 | "typescript": "^5.3.2" 26 | }, 27 | "dependencies": { 28 | "@mendable/firecrawl-js": "^1.8.1", 29 | "@supabase/supabase-js": "^2.46.1", 30 | "@types/sharp": "^0.31.1", 31 | "cheerio": "^1.0.0", 32 | "cli-progress": "^3.12.0", 33 | "dotenv": "^14.3.2", 34 | "ejs": "^3.1.10", 35 | "express": "^4.18.2", 36 | "express-async-handler": "^1.2.0", 37 | "firecrawl": "^1.7.2", 38 | "mysql2": "^3.12.0", 39 | "node-cron": "^3.0.3", 40 | "openai": "^4.85.4", 41 | "playht": "^0.13.0", 42 | "reflect-metadata": "^0.2.2", 43 | "resend": "^4.0.1-alpha.0", 44 | "sharp": "^0.33.5", 45 | "together-ai": "^0.9.0", 46 | "trend-finder": "file:", 47 | "ts-node-dev": "^2.0.0", 48 | "typeorm": "^0.3.20" 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/render/test/test.weixin.template.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | import { WeixinTemplateRenderer } from "../weixin/renderer"; 4 | import { WeixinTemplate } from "../interfaces/template.interface"; 5 | import { formatDate } from "../../utils/common"; 6 | 7 | // 生成示例HTML预览 8 | const previewArticles: WeixinTemplate[] = [ 9 | { 10 | id: "1", 11 | title: "人工智能发展最新突破：GPT-4展现多模态能力", 12 | content: `当你使用一个库时，它能够“即插即用”，这背后往往隐藏着一位工程师付出的巨大努力。编写高质量的技术文档是一项耗时且需要高度专业技能的工作。这些文档不仅包括了详细的API说明、示例代码和常见问题解答，还可能涵盖了一些最佳实践和性能优化建议。在软件开发领域，良好的文档可以显著提高开发效率，减少因理解错误导致的bug。对于开源项目来说，优质的文档更是吸引贡献者和用户的关键因素之一。很多工程师在完成核心功能开发后，会花费大量时间来完善相关文档，以确保其他开发者能够快速上手并充分利用该库的功能。这种对细节的关注和对用户体验的重视体现了工程师的专业精神。虽然编写文档的过程可能是枯燥乏味的，但其带来的长期收益却非常可观。因此，当下次你在享受某个库带来的便利时，请记得感谢那些默默无闻地为良好文档而努力工作的工程师们。`, 13 | url: "https://example.com/gpt4-breakthrough", 14 | publishDate: formatDate(new Date().toISOString()), 15 | keywords: ["GPT-4", "人工智能", "多模态", "OpenAI"], 16 | metadata: { 17 | author: "AI研究员", 18 | readTime: "5分钟", 19 | }, 20 | } 21 | ]; 22 | 23 | // 渲染并保存预览文件 24 | const renderer = new WeixinTemplateRenderer(); 25 | const html = renderer.render(previewArticles); 26 | 27 | // 确保temp目录存在 28 | const tempDir = path.join(__dirname, "../../../temp"); 29 | if (!fs.existsSync(tempDir)) { 30 | fs.mkdirSync(tempDir, { recursive: true }); 31 | } 32 | 33 | // 保存渲染结果 34 | const outputPath = path.join(tempDir, "preview_weixin.html"); 35 | fs.writeFileSync(outputPath, html, "utf-8"); 36 | console.log(`预览文件已生成：${outputPath}`); 37 | -------------------------------------------------------------------------------- /src/prompts/summarizer.prompt.ts: -------------------------------------------------------------------------------- 1 | export interface SummarizerPromptParams { 2 | content: string; 3 | language?: string; 4 | minLength?: number; 5 | maxLength?: number; 6 | } 7 | 8 | export const getSummarizerSystemPrompt = (): string => { 9 | return `你是一个专业的内容创作者和摘要生成器。你的任务是： 10 | 1. 理解原始内容的核心观点和背景 11 | 2. 基于原始内容进行优化，补充相关的背景信息、技术细节或实际应用场景 12 | 3. 确保优化后的内容准确、专业，并保持行文流畅 13 | 4. 生成一个专业的标题和3-5个关键词 14 | 5. 标题要求能够吸引读者，能够概括内容的核心观点，同时具有新闻性质，避免营销或广告语气 15 | 6. 生成一个0-100的分数，表示内容的重要性和价值，分数越高，表示内容越重要和有价值，同时越可能被读者关注，同时具有区分度，不应该分数很集中，精确到小数点后两位； 16 | 17 | 请只返回JSON格式数据，格式如下： 18 | { 19 | "title": "专业的标题", 20 | "content": "扩充和完善后的内容", 21 | "keywords": ["关键词1", "关键词2", "关键词3"], 22 | "score": 0-100 23 | }`; 24 | }; 25 | 26 | export const getSummarizerUserPrompt = ({ 27 | content, 28 | language = "中文", 29 | minLength = 300, 30 | maxLength = 800, 31 | }: SummarizerPromptParams): string => { 32 | return `请分析以下内容，在保持原意的基础上进行专业的扩充和完善，使用${language}，完善后的内容不少于${minLength}字，不超过${maxLength}字：\n\n${content}\n\n 33 | 要求： 34 | 1. 保持专业性，可以适当补充相关的技术细节、应用场景或行业背景； 35 | 2. 注意内容的连贯性和可读性； 36 | 3. 确保扩充的内容不需要无效信息，也不需要陈诉一些漂亮的话，而是需要有价值的信息； 37 | 4. 关键字的长度不超过4个字； 38 | 5. !!内容不要像是AIGC生成的，要像是一个人写的，不要出现"根据以上信息"、"根据以上内容"等字样，需要是新闻类型的； 39 | 6. 内容不要出现其他格式，例如markdown格式，而是纯文本； 40 | 7. 适当添加换行标记（），使内容更易读；`; 41 | }; 42 | 43 | export const getTitleSystemPrompt = (): string => { 44 | return `你是一个专业的内容创作者和标题生成器。你的任务是： 45 | 1. 从内容中提炼最核心、最有价值的信息 46 | 2. 生成一个引人注目且专业的标题 47 | 3. 标题要简洁明了，不超过64个字 48 | 4. 标题要准确反映内容的核心观点 49 | 5. 标题要具有新闻性质，避免营销或广告语气`; 50 | }; 51 | 52 | export const getTitleUserPrompt = ({ 53 | content, 54 | language = "中文" 55 | }: SummarizerPromptParams): string => { 56 | return `请为以下内容生成一个专业的标题，使用${language}：\n\n${content}\n\n`; 57 | }; -------------------------------------------------------------------------------- /src/controllers/cron.ts: -------------------------------------------------------------------------------- 1 | import cron from "node-cron"; 2 | import { WeixinWorkflow } from "../services/weixin-article.workflow"; 3 | import { Workflow } from "../services/interfaces/workflow.interface"; 4 | import { WeixinAIBenchWorkflow } from "../services/weixin-aibench.workflow"; 5 | import { WeixinHelloGithubWorkflow } from "../services/weixin-hellogithub.workflow"; 6 | 7 | // 工作流映射表，用于存储不同日期对应的工作流 8 | const workflowMap = new Map(); 9 | 10 | // 初始化工作流映射 11 | const initializeWorkflows = () => { 12 | // 周一的工作流 (1) 13 | workflowMap.set(1, new WeixinWorkflow()); 14 | // 其他日期的工作流可以在这里添加 15 | workflowMap.set(2, new WeixinAIBenchWorkflow()); // 周二 16 | // workflowMap.set(3, new AnotherWorkflow()); // 周三 17 | workflowMap.set(3, new WeixinHelloGithubWorkflow()); // 周三 18 | 19 | workflowMap.set(4, new WeixinWorkflow()); 20 | 21 | workflowMap.set(5, new WeixinWorkflow()); 22 | 23 | workflowMap.set(6, new WeixinWorkflow()); 24 | 25 | workflowMap.set(7, new WeixinWorkflow()); 26 | 27 | 28 | 29 | }; 30 | 31 | export const startCronJobs = async () => { 32 | console.log("初始化定时任务..."); 33 | initializeWorkflows(); 34 | 35 | // 每天凌晨3点执行 36 | cron.schedule( 37 | "0 3 * * *", 38 | async () => { 39 | const dayOfWeek = new Date().getDay(); // 0是周日，1-6是周一到周六 40 | const adjustedDay = dayOfWeek === 0 ? 7 : dayOfWeek; // 将周日的0转换为7 41 | 42 | const workflow = workflowMap.get(adjustedDay); 43 | if (workflow) { 44 | console.log(`开始执行周${adjustedDay}的工作流...`); 45 | try { 46 | await workflow.refresh(); 47 | await workflow.process(); 48 | } catch (error) { 49 | console.error(`工作流执行失败:`, error); 50 | } 51 | } else { 52 | console.log(`周${adjustedDay}没有配置对应的工作流`); 53 | } 54 | }, 55 | { 56 | timezone: "Asia/Shanghai", 57 | } 58 | ); 59 | }; 60 | -------------------------------------------------------------------------------- /src/utils/content-rank/content-ranker.test.ts: -------------------------------------------------------------------------------- 1 | import { ConfigManager } from "../config/config-manager"; 2 | import { DbConfigSource } from "../config/sources/db-config.source"; 3 | import { EnvConfigSource } from "../config/sources/env-config.source"; 4 | import { MySQLDB } from "../db/mysql.db"; 5 | import { ContentRanker } from "./content-ranker"; 6 | import dotenv from "dotenv"; 7 | 8 | // 加载环境变量 9 | dotenv.config(); 10 | 11 | async function main() { 12 | try { 13 | 14 | // 配置管理器 15 | const configManager = ConfigManager.getInstance(); 16 | configManager.addSource(new EnvConfigSource()); 17 | 18 | const db = await MySQLDB.getInstance({ 19 | host: process.env.DB_HOST, 20 | port: Number(process.env.DB_PORT), 21 | user: process.env.DB_USER, 22 | password: process.env.DB_PASSWORD, 23 | database: process.env.DB_DATABASE, 24 | }); 25 | configManager.addSource(new DbConfigSource(db)); 26 | 27 | 28 | 29 | 30 | const ranker = new ContentRanker({ 31 | provider: "deepseek", 32 | apiKey: await configManager.get("DEEPSEEK_API_KEY") as string, 33 | modelName: "deepseek-reasoner" 34 | }); 35 | 36 | // 测试数据 37 | const testContents = [ 38 | { 39 | id: "1", 40 | title: "GPT-4 Turbo发布：更强大的多模态能力和更低的价格", 41 | content: `OpenAI在开发者大会上发布了GPT-4 Turbo，这是GPT-4的最新版本。新版本带来了多项重要更新： 42 | 1. 支持更长的上下文窗口，从32K扩展到128K 43 | 2. 知识库更新到2023年4月 44 | 3. 更强大的多模态能力，可以理解和生成图像 45 | 4. API调用成本降低了3倍 46 | 5. 新增了JSON模式输出功能 47 | 这些更新将帮助开发者构建更强大的AI应用。`, 48 | publishDate: "2023-11-06", 49 | url: "https://example.com/gpt4-turbo", 50 | score: 0, 51 | metadata: {} 52 | }, 53 | { 54 | id: "2", 55 | title: "谷歌发布Gemini：超越GPT-4的多模态模型", 56 | content: `谷歌正式发布了其最新的AI模型Gemini，号称在多个基准测试中超越了GPT-4： 57 | 1. 在28个基准测试中的26个超越GPT-4 58 | 2. 原生支持多模态，可以同时处理文本、图像、音频和视频 59 | 3. 提供三个版本：Ultra、Pro和Nano 60 | 4. 已经整合到Bard和Pixel手机中 61 | 5. 开发者可以通过API访问 62 | 这标志着AI领域的新突破，将推动更多创新应用的出现。`, 63 | publishDate: "2023-12-07", 64 | url: "https://example.com/gemini", 65 | score: 0, 66 | metadata: {} 67 | } 68 | ]; 69 | 70 | console.log("开始评分测试..."); 71 | const results = await ranker.rankContents(testContents); 72 | console.log("评分结果:", results); 73 | 74 | } catch (error) { 75 | console.error("测试失败:", error); 76 | process.exit(1); 77 | } 78 | } 79 | 80 | // 运行测试 81 | main(); -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy to Production # 部署到生产环境 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | workflow_dispatch: # 允许手动触发工作流 8 | 9 | jobs: 10 | deploy: 11 | runs-on: ubuntu-latest # 在最新版本的 Ubuntu 运行器上运行 12 | 13 | steps: 14 | - uses: actions/checkout@v4 # 检出代码 15 | 16 | - name: Setup Node.js # 设置 Node.js 环境 17 | uses: actions/setup-node@v4 18 | with: 19 | node-version: "20" # 使用 Node.js 20 版本 20 | cache: "npm" # 启用 npm 依赖缓存 21 | 22 | - name: Install dependencies # 安装项目依赖 23 | run: npm ci 24 | 25 | - name: Build # 构建项目 26 | run: npm run build 27 | 28 | - name: Create .env file # 创建环境配置文件 29 | run: | 30 | cat > .env << EOL 31 | # 数据库配置 32 | DB_HOST=${{ secrets.DB_HOST }} 33 | DB_PORT=${{ secrets.DB_PORT }} 34 | DB_USER=${{ secrets.DB_USER }} 35 | DB_PASSWORD=${{ secrets.DB_PASSWORD }} 36 | DB_DATABASE=${{ secrets.DB_DATABASE }} 37 | EOL 38 | 39 | - name: Setup SSH # 设置 SSH 40 | run: | 41 | mkdir -p ~/.ssh 42 | echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa 43 | chmod 600 ~/.ssh/id_rsa 44 | ssh-keyscan "${{ secrets.SERVER_HOST }}" >> ~/.ssh/known_hosts 45 | 46 | - name: Test SSH Connection # 测试 SSH 连接 47 | run: | 48 | echo "Testing SSH connection..." 49 | echo "Server: ${{ secrets.SERVER_HOST }}" 50 | echo "User: ${{ secrets.SERVER_USER }}" 51 | ssh -i ~/.ssh/id_rsa "${{ secrets.SERVER_USER }}@${{ secrets.SERVER_HOST }}" "source ~/.nvm/nvm.sh && nvm use 22 && echo 'SSH connection successful'" 52 | 53 | - name: Package Files # 打包文件 54 | run: | 55 | tar -czf build.tar.gz dist package*.json .env 56 | 57 | - name: Upload to Server # 上传到服务器 58 | run: | 59 | scp -i ~/.ssh/id_rsa build.tar.gz "${{ secrets.SERVER_USER }}@${{ secrets.SERVER_HOST }}:~/app/" 60 | 61 | - name: Deploy on Server # 在服务器上部署 62 | run: | 63 | ssh -i ~/.ssh/id_rsa "${{ secrets.SERVER_USER }}@${{ secrets.SERVER_HOST }}" "cd ~/app && \ 64 | source ~/.nvm/nvm.sh && \ 65 | nvm use 22 && \ 66 | tar -xzf build.tar.gz && \ 67 | npm ci --production && \ 68 | pm2 delete trend-finder || true && \ 69 | pm2 start dist/index.js --name trend-finder && \ 70 | rm build.tar.gz" 71 | -------------------------------------------------------------------------------- /temp/preview_weixin.html: -------------------------------------------------------------------------------- 1 |
12 |
18 |
28 | 📑 目录 29 |
30 | 31 |
40 | 1. 41 | 42 | 人工智能发展最新突破：GPT-4展现多模态能力 43 | 44 |
45 | 46 |
47 | 48 |
49 |
59 | 人工智能发展最新突破：GPT-4展现多模态能力 60 |
61 |
71 | 📅 81 | 2025/02/25 16:21:06 82 |
83 |
95 | 96 |
当你使用一个库时，它能够“即插即用”，这背后往往隐藏着一位工程师付出的巨大努力。编写高质量的技术文档是一项耗时且需要高度专业技能的工作。这些文档不仅包括了详细的API说明、示例代码和常见问题解答，还可能涵盖了一些最佳实践和性能优化建议。
97 | 98 |
在软件开发领域，良好的文档可以显著提高开发效率，减少因理解错误导致的bug。对于开源项目来说，优质的文档更是吸引贡献者和用户的关键因素之一。很多工程师在完成核心功能开发后，会花费大量时间来完善相关文档，以确保其他开发者能够快速上手并充分利用该库的功能。
99 | 100 |
这种对细节的关注和对用户体验的重视体现了工程师的专业精神。虽然编写文档的过程可能是枯燥乏味的，但其带来的长期收益却非常可观。因此，当下次你在享受某个库带来的便利时，请记得感谢那些默默无闻地为良好文档而努力工作的工程师们。
101 | 102 |
103 |
104 | 105 |
-------------------------------------------------------------------------------- /src/utils/gen-image/text-logo.ts: -------------------------------------------------------------------------------- 1 | import sharp from "sharp"; 2 | 3 | export interface TextLogoOptions { 4 | text: string; 5 | width?: number; 6 | height?: number; 7 | fontSize?: number; 8 | backgroundColor?: string; 9 | textColor?: string; 10 | gradientStart?: string; 11 | gradientEnd?: string; 12 | } 13 | 14 | export class TextLogoGenerator { 15 | private static readonly DEFAULT_OPTIONS: Partial = { 16 | width: 1200, 17 | height: 400, 18 | fontSize: 160, 19 | backgroundColor: "#FFFFFF", 20 | textColor: "#1a73e8", 21 | gradientStart: "#1a73e8", 22 | gradientEnd: "#4285f4", 23 | }; 24 | 25 | public static async generate(options: TextLogoOptions): Promise { 26 | const finalOptions = { ...this.DEFAULT_OPTIONS, ...options }; 27 | const { 28 | width, 29 | height, 30 | text, 31 | fontSize, 32 | backgroundColor, 33 | gradientStart, 34 | gradientEnd, 35 | } = finalOptions; 36 | 37 | // 创建SVG文本 38 | const svg = ` 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 62 | ${text} 63 | 64 | 65 | `; 66 | 67 | // 使用sharp将SVG转换为图像 68 | return await sharp(Buffer.from(svg)) 69 | .resize(width!, height!) 70 | .png() 71 | .toBuffer(); 72 | } 73 | 74 | public static async saveToFile( 75 | options: TextLogoOptions, 76 | outputPath: string 77 | ): Promise { 78 | const buffer = await this.generate(options); 79 | await sharp(buffer).toFile(outputPath); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/render/test/test.hellogithub.render.ts: -------------------------------------------------------------------------------- 1 | import ejs from "ejs"; 2 | import fs from "fs"; 3 | import path from "path"; 4 | import { HelloGithubScraper } from "../../scrapers/hellogithub.scraper"; 5 | 6 | interface RenderOptions { 7 | title?: string; 8 | maxItems?: number; 9 | } 10 | 11 | class HelloGithubRenderer { 12 | private templatePath: string; 13 | private outputPath: string; 14 | 15 | constructor() { 16 | this.templatePath = path.join( 17 | __dirname, 18 | "../../templates/hellogithub-weixin.ejs" 19 | ); 20 | this.outputPath = path.join(__dirname, "../../../output"); 21 | } 22 | 23 | /** 24 | * 渲染 HelloGithub 内容 25 | * @param options 渲染选项 26 | */ 27 | async render(options: RenderOptions = {}) { 28 | const { title = "本周 AI 开源项目精选", maxItems = 5 } = options; 29 | 30 | try { 31 | // 1. 获取数据 32 | console.log("正在获取数据..."); 33 | const scraper = new HelloGithubScraper(); 34 | const hotItems = await scraper.getHotItems(1); 35 | const items = await Promise.all( 36 | hotItems.slice(0, maxItems).map(async (item) => { 37 | console.log(`正在获取项目详情: ${item.title}`); 38 | const detail = await scraper.getItemDetail(item.itemId); 39 | console.log( 40 | "项目相关链接:", 41 | JSON.stringify(detail.relatedUrls, null, 2) 42 | ); 43 | return detail; 44 | }) 45 | ); 46 | 47 | // 2. 读取并渲染模板 48 | console.log("正在渲染模板..."); 49 | const template = fs.readFileSync(this.templatePath, "utf-8"); 50 | const html = ejs.render(template, { 51 | items, 52 | title, 53 | renderDate: new Date().toISOString().split("T")[0], 54 | }); 55 | 56 | // 3. 确保输出目录存在 57 | if (!fs.existsSync(this.outputPath)) { 58 | fs.mkdirSync(this.outputPath, { recursive: true }); 59 | } 60 | 61 | // 4. 保存结果 62 | const fileName = `hellogithub-${ 63 | new Date().toISOString().split("T")[0] 64 | }.html`; 65 | const outputFilePath = path.join(this.outputPath, fileName); 66 | fs.writeFileSync(outputFilePath, html, "utf-8"); 67 | 68 | console.log("渲染完成！输出文件:", outputFilePath); 69 | return outputFilePath; 70 | } catch (error: any) { 71 | console.error("渲染失败:", error.message); 72 | throw error; 73 | } 74 | } 75 | } 76 | 77 | // 测试渲染功能 78 | async function testRender() { 79 | const renderer = new HelloGithubRenderer(); 80 | 81 | try { 82 | await renderer.render({ 83 | title: "🔥 本周 AI 开源项目精选", 84 | maxItems: 5, 85 | }); 86 | } catch (error: any) { 87 | console.error("测试失败:", error.message); 88 | } 89 | } 90 | 91 | // 如果直接运行此文件则执行测试 92 | if (require.main === module) { 93 | testRender(); 94 | } 95 | -------------------------------------------------------------------------------- /src/services/weixin-aibench.workflow.ts: -------------------------------------------------------------------------------- 1 | import { Workflow } from "./interfaces/workflow.interface"; 2 | import { LiveBenchAPI } from "../api/livebench.api"; 3 | import { ConfigManager } from "../utils/config/config-manager"; 4 | import { BarkNotifier } from "../utils/bark.notify"; 5 | import { AIBenchRenderer } from "../render/aibench/renderer"; 6 | import { WeixinPublisher } from "../publishers/weixin.publisher"; 7 | import path from "path"; 8 | 9 | export class WeixinAIBenchWorkflow implements Workflow { 10 | private liveBenchAPI: LiveBenchAPI; 11 | private renderer: AIBenchRenderer; 12 | private notify: BarkNotifier; 13 | private publisher: WeixinPublisher; 14 | 15 | constructor() { 16 | this.liveBenchAPI = new LiveBenchAPI(); 17 | this.renderer = new AIBenchRenderer(); 18 | this.notify = new BarkNotifier(); 19 | this.publisher = new WeixinPublisher(); 20 | } 21 | 22 | async refresh(): Promise { 23 | await this.notify.refresh(); 24 | await this.liveBenchAPI.refresh(); 25 | await this.publisher.refresh(); 26 | } 27 | 28 | async process(): Promise { 29 | try { 30 | // 获取所有模型的性能数据 31 | const modelData = await this.liveBenchAPI.getModelPerformance(); 32 | 33 | // 处理数据，清理模型名称中的多余空格 34 | const cleanedModelData: typeof modelData = Object.entries( 35 | modelData 36 | ).reduce((acc, [key, value]) => { 37 | acc[key.trim()] = { 38 | ...value, 39 | organization: value.organization?.trim(), 40 | }; 41 | return acc; 42 | }, {} as typeof modelData); 43 | 44 | // 使用 AIBenchRenderer 处理数据 45 | const templateData = AIBenchRenderer.render(cleanedModelData); 46 | 47 | // 渲染并保存文件 48 | const outputPath = path.join( 49 | "output", 50 | `aibench-${new Date().toISOString().split("T")[0]}.html` 51 | ); 52 | await this.renderer.renderToFile(templateData, outputPath); 53 | 54 | // 发布到微信公众号 55 | const title = `${new Date().toLocaleDateString()} AI模型性能榜单`; 56 | const mediaId = 57 | "SwCSRjrdGJNaWioRQUHzgF8cSi0Wuf1M6duNPIMX9ennpaMqttRXYwwXnZjmi6QI"; 58 | 59 | // 使用微信专用模板渲染内容 60 | const htmlContent = this.renderer.renderForWeixin(templateData); 61 | 62 | const publishResult = await this.publisher.publish( 63 | htmlContent, 64 | title, 65 | title, 66 | mediaId 67 | ); 68 | 69 | // 发送通知 70 | await this.notify.info( 71 | "AI Benchmark更新", 72 | `已生成并发布最新的AI模型性能榜单\n发布状态: ${publishResult.status}` 73 | ); 74 | } catch (error) { 75 | console.error("Error processing WeixinAIBenchWorkflow:", error); 76 | await this.notify.error( 77 | "AI Benchmark更新失败", 78 | `错误: ${error instanceof Error ? error.message : String(error)}` 79 | ); 80 | throw error; 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/utils/config/config-manager.ts: -------------------------------------------------------------------------------- 1 | import { IConfigSource } from "./interfaces/config-source.interface"; 2 | 3 | export class ConfigurationError extends Error { 4 | constructor(message: string) { 5 | super(message); 6 | this.name = "ConfigurationError"; 7 | } 8 | } 9 | 10 | interface RetryOptions { 11 | maxAttempts: number; 12 | delayMs: number; 13 | } 14 | 15 | export class ConfigManager { 16 | private static instance: ConfigManager; 17 | private configSources: IConfigSource[] = []; 18 | private defaultRetryOptions: RetryOptions = { 19 | maxAttempts: 3, 20 | delayMs: 1000, 21 | }; 22 | 23 | private constructor() {} 24 | 25 | public static getInstance(): ConfigManager { 26 | if (!ConfigManager.instance) { 27 | ConfigManager.instance = new ConfigManager(); 28 | } 29 | return ConfigManager.instance; 30 | } 31 | 32 | /** 33 | * 添加配置源 34 | * @param source 配置源实例 35 | */ 36 | public addSource(source: IConfigSource): void { 37 | this.configSources.push(source); 38 | // 按优先级排序（升序，数字越小优先级越高） 39 | this.configSources.sort((a, b) => a.priority - b.priority); 40 | } 41 | 42 | private async delay(ms: number): Promise { 43 | return new Promise(resolve => setTimeout(resolve, ms)); 44 | } 45 | 46 | private async getWithRetry( 47 | source: IConfigSource, 48 | key: string, 49 | options: RetryOptions 50 | ): Promise { 51 | let lastError: Error | null = null; 52 | 53 | for (let attempt = 1; attempt <= options.maxAttempts; attempt++) { 54 | try { 55 | const value = await source.get(key); 56 | return value; 57 | } catch (error) { 58 | lastError = error as Error; 59 | if (attempt < options.maxAttempts) { 60 | await this.delay(options.delayMs); 61 | } 62 | } 63 | } 64 | 65 | console.warn(`Failed to get config "${key}" after ${options.maxAttempts} attempts. Last error: ${lastError?.message}`); 66 | return null; 67 | } 68 | 69 | /** 70 | * 获取配置值 71 | * @param key 配置键 72 | * @param retryOptions 重试选项，可选 73 | * @throws {ConfigurationError} 当所有配置源都无法获取值时抛出 74 | */ 75 | public async get(key: string, retryOptions?: Partial): Promise { 76 | const options = { ...this.defaultRetryOptions, ...retryOptions }; 77 | 78 | for (const source of this.configSources) { 79 | const value = await this.getWithRetry(source, key, options); 80 | if (value !== null) { 81 | return value; 82 | } 83 | } 84 | 85 | throw new ConfigurationError( 86 | `Configuration key "${key}" not found in any source after ${options.maxAttempts} attempts` 87 | ); 88 | } 89 | 90 | /** 91 | * 获取所有已注册的配置源 92 | */ 93 | public getSources(): IConfigSource[] { 94 | return [...this.configSources]; 95 | } 96 | 97 | /** 98 | * 清除所有配置源 99 | */ 100 | public clearSources(): void { 101 | this.configSources = []; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/scrapers/twitter.scraper.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ContentScraper, 3 | ScraperOptions, 4 | ScrapedContent, 5 | } from "./interfaces/scraper.interface"; 6 | import path from "path"; 7 | import fs from "fs"; 8 | import dotenv from "dotenv"; 9 | import { formatDate } from "../utils/common"; 10 | import { ConfigManager } from "../utils/config/config-manager"; 11 | 12 | dotenv.config(); 13 | 14 | export class TwitterScraper implements ContentScraper { 15 | private xApiBearerToken: string | undefined; 16 | 17 | constructor() { 18 | this.refresh(); 19 | } 20 | 21 | async refresh(): Promise { 22 | await this.validateConfig(); 23 | this.xApiBearerToken = await ConfigManager.getInstance().get( 24 | "X_API_BEARER_TOKEN" 25 | ); 26 | } 27 | 28 | async validateConfig(): Promise { 29 | if (!(await ConfigManager.getInstance().get("X_API_BEARER_TOKEN"))) { 30 | throw new Error( 31 | "X API Bearer Token is not set, please set X_API_BEARER_TOKEN in .env file" 32 | ); 33 | } 34 | } 35 | 36 | async scrape( 37 | sourceId: string, 38 | options?: ScraperOptions 39 | ): Promise { 40 | const usernameMatch = sourceId.match(/x\.com\/([^\/]+)/); 41 | if (!usernameMatch) { 42 | throw new Error("Invalid Twitter source ID format"); 43 | } 44 | 45 | const username = usernameMatch[1]; 46 | console.log(`Processing Twitter user: ${username}`); 47 | 48 | try { 49 | const query = `from:${username} -filter:replies within_time:24h`; 50 | const apiUrl = `https://api.twitterapi.io/twitter/tweet/advanced_search?query=${encodeURIComponent( 51 | query 52 | )}&queryType=Top`; 53 | 54 | const response = await fetch(apiUrl, { 55 | headers: { 56 | "X-API-Key": `${this.xApiBearerToken}`, 57 | }, 58 | }); 59 | 60 | if (!response.ok) { 61 | const errorMsg = `Failed to fetch tweets: ${response.statusText}`; 62 | throw new Error(errorMsg); 63 | } 64 | 65 | const tweets = await response.json(); 66 | const scrapedContent: ScrapedContent[] = tweets.tweets 67 | .slice(0, 10) 68 | .map((tweet: any) => ({ 69 | id: tweet.id, 70 | title: tweet.text.split("\n")[0], 71 | content: tweet.text, 72 | url: `https://x.com/${username}/status/${tweet.id}`, 73 | publishDate: formatDate(tweet.createdAt), 74 | score: 0, 75 | metadata: { 76 | platform: "twitter", 77 | username, 78 | }, 79 | })); 80 | 81 | if (scrapedContent.length > 0) { 82 | console.log( 83 | `Successfully fetched ${scrapedContent.length} tweets from ${username}` 84 | ); 85 | } else { 86 | console.log(`No tweets found for ${username}`); 87 | } 88 | 89 | return scrapedContent; 90 | } catch (error) { 91 | const errorMsg = error instanceof Error ? error.message : String(error); 92 | console.error(`Error fetching tweets for ${username}:`, error); 93 | throw error; 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/api/xunfei.api.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import { ConfigManager } from "../utils/config/config-manager"; 3 | 4 | interface XunfeiMessage { 5 | role: "system" | "user" | "assistant"; 6 | content: string; 7 | } 8 | 9 | interface XunfeiResponse { 10 | code: number; 11 | message: string; 12 | sid: string; 13 | choices: { 14 | message: { 15 | role: "assistant" | "user"; 16 | content: string; 17 | }; 18 | index: number; 19 | }[]; 20 | usage: { 21 | prompt_tokens: number; 22 | completion_tokens: number; 23 | total_tokens: number; 24 | }; 25 | } 26 | 27 | export class XunfeiAPI { 28 | private baseURL = "https://spark-api-open.xf-yun.com/v1/chat/completions"; 29 | private token!: string; 30 | private defaultModel = "4.0Ultra"; 31 | 32 | constructor() { 33 | this.refresh(); 34 | } 35 | 36 | async refresh() { 37 | this.token = await ConfigManager.getInstance().get("XUNFEI_API_KEY"); 38 | if (!this.token) { 39 | throw new Error("Xunfei API key is not set"); 40 | } 41 | } 42 | 43 | /** 44 | * Send a message to the Xunfei API and get a response 45 | * @param content The message content to send 46 | * @param systemPrompt Optional system prompt to set context 47 | * @param enableWebSearch Optional flag to enable web search 48 | * @returns Promise The assistant's response text 49 | */ 50 | async sendMessage( 51 | content: string, 52 | systemPrompt?: string, 53 | enableWebSearch?: boolean 54 | ): Promise { 55 | try { 56 | const messages: XunfeiMessage[] = []; 57 | if (systemPrompt) { 58 | messages.push({ 59 | role: "system", 60 | content: systemPrompt, 61 | }); 62 | } 63 | 64 | messages.push({ 65 | role: "user", 66 | content, 67 | }); 68 | 69 | const response = await axios.post( 70 | this.baseURL, 71 | { 72 | model: this.defaultModel, 73 | messages, 74 | stream: false, 75 | ...(enableWebSearch && { 76 | tools: [ 77 | { 78 | type: "web_search", 79 | web_search: { 80 | enable: true, 81 | }, 82 | }, 83 | ], 84 | }), 85 | }, 86 | { 87 | headers: { 88 | "Content-Type": "application/json", 89 | Authorization: `Bearer ${this.token}`, 90 | }, 91 | } 92 | ); 93 | 94 | if (response.data.code !== 0) { 95 | throw new Error(`API Error: ${response.data.message}`); 96 | } 97 | 98 | if (!response.data.choices || response.data.choices.length === 0) { 99 | throw new Error("No response choices available"); 100 | } 101 | 102 | return response.data.choices[0].message.content; 103 | } catch (error) { 104 | if (axios.isAxiosError(error)) { 105 | throw new Error( 106 | `Failed to send message: ${ 107 | error.response?.data?.message || error.message 108 | }` 109 | ); 110 | } 111 | throw error; 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/templates/article.ejs: -------------------------------------------------------------------------------- 1 |
12 |
19 |
28 | 📑 目录 29 |
30 | <% articles.forEach((article, index) => { %> 31 |
40 | <%= index + 1 %>. 41 | 48 | <%= article.title %> 49 | 50 |
51 | <% }); %> 52 |
53 | <% articles.forEach((article, index) => { %> 54 |
55 |
65 | <%= article.title %> 66 |
67 |
77 | 📅 87 | <%= article.publishDate %> 88 |
89 |
101 | <% 102 | // Split content by double newlines to separate paragraphs 103 | const paragraphs = article.content 104 | .replace(//g, '\n\n') // 将标记转换为双换行 105 | .split(/\n\s*\n/) // 按照一个或多个空行分割段落 106 | .map(p => p.trim()) 107 | .filter(p => p.length > 0); 108 | 109 | paragraphs.forEach((paragraph, i) => { 110 | %> 111 |
<%= paragraph %>
112 | <% }); %> 113 |
114 |
115 | <% }); %> 116 |
117 | -------------------------------------------------------------------------------- /src/services/weixin-hellogithub.workflow.ts: -------------------------------------------------------------------------------- 1 | import { Workflow } from "./interfaces/workflow.interface"; 2 | import { HelloGithubScraper } from "../scrapers/hellogithub.scraper"; 3 | import { WeixinPublisher } from "../publishers/weixin.publisher"; 4 | import { AliWanX21ImageGenerator } from "../utils/gen-image/aliwanx2.1.image"; 5 | import path from "path"; 6 | import fs from "fs"; 7 | import ejs from "ejs"; 8 | 9 | export class WeixinHelloGithubWorkflow implements Workflow { 10 | private scraper: HelloGithubScraper; 11 | private publisher: WeixinPublisher; 12 | private templatePath: string; 13 | private imageGenerator: AliWanX21ImageGenerator; 14 | 15 | constructor() { 16 | this.scraper = new HelloGithubScraper(); 17 | this.publisher = new WeixinPublisher(); 18 | this.imageGenerator = new AliWanX21ImageGenerator(); 19 | this.templatePath = path.join( 20 | __dirname, 21 | "../templates/hellogithub-weixin.ejs" 22 | ); 23 | } 24 | 25 | /** 26 | * 刷新工作流所需的资源和配置 27 | */ 28 | public async refresh(): Promise { 29 | await this.publisher.refresh(); 30 | await this.imageGenerator.refresh(); 31 | } 32 | 33 | /** 34 | * 执行工作流的主要处理逻辑 35 | */ 36 | public async process(): Promise { 37 | try { 38 | console.log("开始执行 HelloGithub 工作流..."); 39 | 40 | // 1. 获取热门项目数据 41 | console.log("1. 获取热门项目数据..."); 42 | const hotItems = await this.scraper.getHotItems(1); 43 | const items = await Promise.all( 44 | hotItems.slice(0, 20).map(async (item) => { 45 | console.log(`正在获取项目详情: ${item.title}`); 46 | return await this.scraper.getItemDetail(item.itemId); 47 | }) 48 | ); 49 | 50 | // 2. 生成封面图片 51 | console.log("2. 生成封面图片..."); 52 | const prompt = 53 | "GitHub AI 开源项目精选，展示代码和人工智能的融合，使用现代科技风格，蓝色和绿色为主色调"; 54 | const taskId = await this.imageGenerator 55 | .generateImage(prompt, "1440*768") 56 | .then((res) => res.output.task_id); 57 | 58 | console.log(`[封面图片] 生成任务ID: ${taskId}`); 59 | const imageUrl = await this.imageGenerator 60 | .waitForCompletion(taskId) 61 | .then((res) => res.results?.[0]?.url || ""); 62 | 63 | // 上传封面图片获取 mediaId 64 | console.log("3. 上传封面图片..."); 65 | const mediaId = await this.publisher.uploadImage(imageUrl); 66 | 67 | // 4. 渲染内容 68 | console.log("4. 渲染内容..."); 69 | const template = fs.readFileSync(this.templatePath, "utf-8"); 70 | const firstItem = items[0]; 71 | const title = `GitHub AI 热榜第一名：${firstItem.name} | 本周精选`; 72 | const html = ejs.render( 73 | template, 74 | { 75 | title, 76 | items, 77 | renderDate: new Date().toISOString().split("T")[0], 78 | }, 79 | { 80 | rmWhitespace: true, 81 | } 82 | ); 83 | 84 | // 5. 发布到微信 85 | console.log("5. 准备发布到微信..."); 86 | await this.publisher.publish( 87 | html, 88 | `本期精选 GitHub 热门 AI 开源项目，第一名 ${firstItem.name} 项目备受瞩目，发现最新最酷的人工智能开源工具`, 89 | `本期精选 GitHub 热门 AI 开源项目，第一名 ${firstItem.name} 项目备受瞩目，发现最新最酷的人工智能开源工具`, 90 | mediaId 91 | ); 92 | 93 | console.log("工作流执行完成！"); 94 | } catch (error: any) { 95 | console.error("工作流执行失败:", error.message); 96 | throw error; 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/summarizer/together-ai.summarizer.ts: -------------------------------------------------------------------------------- 1 | import Together from "together-ai"; 2 | import { ContentSummarizer, Summary } from "./interfaces/summarizer.interface"; 3 | import { ConfigManager } from "../utils/config/config-manager"; 4 | import { 5 | getSummarizerSystemPrompt, 6 | getSummarizerUserPrompt, 7 | getTitleSystemPrompt, 8 | getTitleUserPrompt 9 | } from "../prompts/summarizer.prompt"; 10 | 11 | const MODEL_NAME = "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo"; 12 | 13 | export class TogetherAISummarizer implements ContentSummarizer { 14 | private together!: Together; 15 | 16 | constructor() { 17 | this.refresh(); 18 | } 19 | 20 | async refresh(): Promise { 21 | await this.validateConfig(); 22 | this.together = new Together(); 23 | } 24 | 25 | async validateConfig(): Promise { 26 | if (!(await ConfigManager.getInstance().get("TOGETHER_API_KEY"))) { 27 | throw new Error("TOGETHER_API_KEY is not set"); 28 | } 29 | } 30 | 31 | async summarize( 32 | content: string, 33 | options?: Record 34 | ): Promise { 35 | try { 36 | const completion = await this.together.chat.completions.create({ 37 | model: MODEL_NAME, 38 | messages: [ 39 | { 40 | role: "system", 41 | content: getSummarizerSystemPrompt(), 42 | }, 43 | { 44 | role: "user", 45 | content: getSummarizerUserPrompt({ 46 | content, 47 | language: options?.language, 48 | minLength: options?.minLength, 49 | maxLength: options?.maxLength, 50 | }), 51 | }, 52 | ], 53 | response_format: { 54 | type: "json_object", 55 | schema: { 56 | title: "string", 57 | content: "string", 58 | keywords: "array", 59 | score: "number" 60 | }, 61 | }, 62 | }); 63 | 64 | const rawJSON = completion?.choices?.[0]?.message?.content; 65 | if (!rawJSON) { 66 | throw new Error("未获取到有效的摘要结果"); 67 | } 68 | 69 | const summary = JSON.parse(rawJSON) as Summary; 70 | 71 | // 验证必要字段 72 | if ( 73 | !summary.title || 74 | !summary.content || 75 | !Array.isArray(summary.keywords) || 76 | typeof summary.score !== 'number' 77 | ) { 78 | throw new Error("摘要结果格式不正确"); 79 | } 80 | 81 | return summary; 82 | } catch (error) { 83 | console.error("生成摘要时出错:", error); 84 | throw error; 85 | } 86 | } 87 | 88 | async generateTitle( 89 | content: string, 90 | options?: Record 91 | ): Promise { 92 | try { 93 | const completion = await this.together.chat.completions.create({ 94 | model: MODEL_NAME, 95 | messages: [ 96 | { 97 | role: "system", 98 | content: getTitleSystemPrompt(), 99 | }, 100 | { 101 | role: "user", 102 | content: getTitleUserPrompt({ 103 | content, 104 | language: options?.language, 105 | }), 106 | }, 107 | ], 108 | }); 109 | 110 | const title = completion?.choices?.[0]?.message?.content; 111 | if (!title) { 112 | throw new Error("未获取到有效的标题"); 113 | } 114 | 115 | return title; 116 | } catch (error) { 117 | console.error("生成标题时出错:", error); 118 | throw error; 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TrendPublish 2 | 3 | 一个基于 AI 的趋势发现和内容发布系统，支持多源数据采集、智能总结和自动发布到微信公众号。 4 | 5 | > 🌰 示例公众号：**AISPACE科技空间** 6 | 7 | > 即刻关注，体验 AI 智能创作的内容～ 8 | 9 | ## 🌟 主要功能 10 | 11 | - 🤖 多源数据采集 12 | 13 | - Twitter/X 内容抓取 14 | - 网站内容抓取 (基于 FireCrawl) 15 | - 支持自定义数据源配置 16 | 17 | - 🧠 AI 智能处理 18 | 19 | - 使用 DeepseekAI Together 千问 万象 讯飞 进行内容总结 20 | - 关键信息提取 21 | - 智能标题生成 22 | 23 | - 📢 自动发布 24 | 25 | - 微信公众号文章发布 26 | - 自定义文章模板 27 | - 定时发布任务 28 | 29 | - 📱 通知系统 30 | - Bark 通知集成 31 | - 任务执行状态通知 32 | - 错误告警 33 | 34 | ## 🛠 技术栈 35 | 36 | - **运行环境**: Node.js + TypeScript 37 | - **AI 服务**: DeepseekAI Together 千问 万象 讯飞 38 | - **数据源**: 39 | - Twitter/X API 40 | - FireCrawl 41 | - **定时任务**: node-cron 42 | - **模板引擎**: EJS 43 | - **开发工具**: 44 | - nodemon (热重载) 45 | - TypeScript 46 | 47 | ## 📦 项目结构 48 | 49 | ``` 50 | src/ 51 | ├── controllers/ # 控制器层，处理请求 52 | ├── data-sources/ # 数据源配置 53 | ├── publishers/ # 发布器实现 54 | ├── scrapers/ # 数据采集实现 55 | ├── services/ # 业务逻辑层 56 | ├── summarizer/ # AI 总结实现 57 | ├── templates/ # 文章模板 58 | └── utils/ # 工具函数 59 | ``` 60 | 61 | ## 🚀 快速开始 62 | 63 | ### 环境要求 64 | 65 | - Node.js (v22+) 66 | - npm 67 | - TypeScript 68 | 69 | ### 安装 70 | 71 | 1. 克隆项目 72 | 73 | ```bash 74 | git clone https://github.com/OpenAISpace/ai-trend-publish 75 | ``` 76 | 77 | 2. 安装依赖 78 | 79 | ```bash 80 | npm install 81 | ``` 82 | 83 | 3. 配置环境变量 84 | 85 | ```bash 86 | cp .env.example .env 87 | # 编辑 .env 文件配置必要的环境变量 88 | ``` 89 | 90 | ## ⚙️ 环境变量配置 91 | 92 | 在 `.env` 文件中配置以下必要的环境变量： 93 | 94 | ```bash 95 | 如果需要使用数据库配置（先从数据库查找配置key，然后再env寻找）： 96 | DB_HOST=xxxx 97 | DB_PORT=xxxx 98 | DB_USER=xxxx 99 | DB_PASSWORD=xxxx 100 | DB_DATABASE=xxxx 101 | 102 | 103 | 微信文章获取的必备环境： 104 | 105 | # DeepseekAI API 配置 https://api-docs.deepseek.com/ 获取 106 | DEEPSEEK_API_KEY=your_api_key 107 | 108 | # FireCrawl 配置 https://www.firecrawl.dev/ 获取 109 | FIRE_CRAWL_API_KEY=your_api_key 110 | 111 | # Twitter API 配置 https://twitterapi.io/ 获取 112 | X_API_BEARER_TOKEN=your_api_key 113 | 114 | # 千问 https://bailian.console.aliyun.com/ 获取 115 | DASHSCOPE_API_KEY=your_api_key 116 | 117 | # 微信公众号配置 118 | WEIXIN_APP_ID=your_app_id 119 | WEIXIN_APP_SECRET=your_app_secret 120 | 121 | 122 | 可选环境: 123 | 124 | # Bark 通知配置 125 | BARK_KEY=your_key 126 | 127 | # 获取图片 API 配置 https://getimg.cc/ 获取 128 | GETIMG_API_KEY=your_api_key 129 | 130 | TOGETHER_API_KEY=your_api_key 131 | 132 | ``` 133 | 134 | 4. 启动项目 135 | 136 | ```bash 137 | # 测试模式 138 | npm run test 139 | 140 | # 运行 141 | npm run start 142 | 143 | 详细运行时间见 src\controllers\cron.ts 144 | ``` 145 | 146 | ## 📦 部署指南 147 | 148 | ### 方式一：直接部署 149 | 150 | 1. 在服务器上安装 Node.js (v20+) 和 PM2 151 | 152 | ```bash 153 | # 安装 PM2 154 | npm install -g pm2 155 | ``` 156 | 157 | 2. 构建项目 158 | 159 | ```bash 160 | npm run build 161 | ``` 162 | 163 | 3. 使用 PM2 启动服务 164 | 165 | ```bash 166 | pm2 start dist/index.js --name ai-trend-publish 167 | ``` 168 | 169 | 170 | ### CI/CD 自动部署 171 | 172 | 项目已配置 GitHub Actions 自动部署流程： 173 | 174 | 1. 推送代码到 main 分支会自动触发部署 175 | 2. 也可以在 GitHub Actions 页面手动触发部署 176 | 3. 确保在 GitHub Secrets 中配置以下环境变量： 177 | - `SERVER_HOST`: 服务器地址 178 | - `SERVER_USER`: 服务器用户名 179 | - `SSH_PRIVATE_KEY`: SSH 私钥 180 | - 其他必要的环境变量（参考 .env.example） 181 | 182 | 183 | 184 | 185 | ## 🤝 贡献指南 186 | 187 | 1. Fork 本仓库 188 | 2. 创建特性分支 (`git checkout -b feature/amazing-feature`) 189 | 3. 提交更改 (`git commit -m 'Add some amazing feature'`) 190 | 4. 推送到分支 (`git push origin feature/amazing-feature`) 191 | 5. 提交 Pull Request 192 | 193 | ## 📄 许可证 194 | 195 | 本项目采用 MIT 许可证 - 详见 [LICENSE](LICENSE) 文件 196 | -------------------------------------------------------------------------------- /src/utils/bark.notify.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import dotenv from "dotenv"; 3 | import { ConfigManager } from "./config/config-manager"; 4 | 5 | dotenv.config(); 6 | 7 | export class BarkNotifier { 8 | private barkUrl!: string; 9 | 10 | constructor() { 11 | this.refresh(); 12 | } 13 | 14 | async refresh(): Promise { 15 | await this.validateConfig(); 16 | this.barkUrl = await ConfigManager.getInstance().get("BARK_URL"); 17 | } 18 | 19 | async validateConfig(): Promise { 20 | if (!(await ConfigManager.getInstance().get("BARK_URL"))) { 21 | throw new Error("Bark 通知未配置"); 22 | } 23 | } 24 | 25 | /** 26 | * 发送 Bark 通知 27 | * @param title 通知标题 28 | * @param content 通知内容 29 | * @param options 通知选项 30 | */ 31 | async notify( 32 | title: string, 33 | content: string, 34 | options: { 35 | level?: "active" | "timeSensitive" | "passive"; 36 | sound?: string; 37 | icon?: string; 38 | group?: string; 39 | url?: string; 40 | isArchive?: boolean; 41 | } = {} 42 | ): Promise { 43 | try { 44 | if (!this.barkUrl) { 45 | console.warn("Bark 通知未配置，跳过发送"); 46 | return false; 47 | } 48 | 49 | const params = new URLSearchParams(); 50 | 51 | // 添加必要参数 52 | params.append("title", title); 53 | params.append("body", content); 54 | 55 | // 添加可选参数 56 | if (options.level) { 57 | params.append("level", options.level); 58 | } 59 | if (options.sound) { 60 | params.append("sound", options.sound); 61 | } 62 | if (options.icon) { 63 | params.append("icon", options.icon); 64 | } 65 | if (options.group) { 66 | params.append("group", options.group); 67 | } 68 | if (options.url) { 69 | params.append("url", options.url); 70 | } 71 | if (options.isArchive !== undefined) { 72 | params.append("isArchive", options.isArchive.toString()); 73 | } 74 | 75 | // 发送通知 76 | const response = await axios.get( 77 | `${this.barkUrl}/${encodeURIComponent(title)}/${encodeURIComponent( 78 | content 79 | )}?${params.toString()}` 80 | ); 81 | 82 | if (response.status === 200) { 83 | return true; 84 | } 85 | 86 | console.error("Bark 通知发送失败:", response.data); 87 | return false; 88 | } catch (error) { 89 | console.error("Bark 通知发送出错:", error); 90 | return false; 91 | } 92 | } 93 | 94 | /** 95 | * 发送成功通知 96 | * @param title 通知标题 97 | * @param content 通知内容 98 | */ 99 | async success(title: string, content: string): Promise { 100 | return this.notify(title, content, { 101 | level: "active", 102 | sound: "success", 103 | group: "success", 104 | }); 105 | } 106 | 107 | /** 108 | * 发送错误通知 109 | * @param title 通知标题 110 | * @param content 通知内容 111 | */ 112 | async error(title: string, content: string): Promise { 113 | return this.notify(title, content, { 114 | level: "timeSensitive", 115 | sound: "error", 116 | group: "error", 117 | }); 118 | } 119 | 120 | /** 121 | * 发送警告通知 122 | * @param title 通知标题 123 | * @param content 通知内容 124 | */ 125 | async warning(title: string, content: string): Promise { 126 | return this.notify(title, content, { 127 | level: "timeSensitive", 128 | sound: "warning", 129 | group: "warning", 130 | }); 131 | } 132 | 133 | /** 134 | * 发送信息通知 135 | * @param title 通知标题 136 | * @param content 通知内容 137 | */ 138 | async info(title: string, content: string): Promise { 139 | return this.notify(title, content, { 140 | level: "passive", 141 | group: "info", 142 | }); 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /src/utils/gen-image/aliwanx2.1.image.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import { ConfigManager } from "../config/config-manager"; 3 | 4 | interface ApiResponse { 5 | output: { 6 | task_status: "PENDING" | "SUCCEEDED" | "FAILED"; 7 | task_id: string; 8 | results?: Array<{ 9 | url: string; 10 | orig_prompt?: string; 11 | actual_prompt?: string; 12 | }>; 13 | }; 14 | request_id: string; 15 | } 16 | 17 | export class AliWanX21ImageGenerator { 18 | private apiKey!: string; 19 | private baseUrl = 20 | "https://dashscope.aliyuncs.com/api/v1/services/aigc/text2image/image-synthesis"; 21 | private readonly model = "wanx2.1-t2i-turbo"; 22 | 23 | constructor() { 24 | this.refresh(); 25 | } 26 | 27 | async refresh() { 28 | const apiKey = await ConfigManager.getInstance().get( 29 | "DASHSCOPE_API_KEY" 30 | ); 31 | if (!apiKey) { 32 | throw new Error("DASHSCOPE_API_KEY environment variable is not set"); 33 | } 34 | this.apiKey = apiKey; 35 | } 36 | 37 | /** 38 | * 生成图片 39 | * @param prompt 提示词 40 | * @param size 图片尺寸 41 | * @param n 生成数量 42 | * @returns 图片生成结果 43 | */ 44 | async generateImage( 45 | prompt: string, 46 | size: string = "1024*1024", 47 | n: number = 1 48 | ): Promise { 49 | try { 50 | const response = await axios.post( 51 | this.baseUrl, 52 | { 53 | model: this.model, 54 | input: { prompt }, 55 | parameters: { 56 | size, 57 | n, 58 | seed: Math.floor(Math.random() * 4294967290) + 1, 59 | }, 60 | }, 61 | { 62 | headers: { 63 | "X-DashScope-Async": "enable", 64 | Authorization: `Bearer ${this.apiKey}`, 65 | "Content-Type": "application/json", 66 | }, 67 | } 68 | ); 69 | 70 | return response.data; 71 | } catch (error) { 72 | if (axios.isAxiosError(error)) { 73 | throw new Error( 74 | `Image generation failed: ${ 75 | error.response?.data?.message || error.message 76 | }` 77 | ); 78 | } 79 | throw error; 80 | } 81 | } 82 | 83 | async checkTaskStatus(taskId: string): Promise { 84 | try { 85 | const response = await axios.get( 86 | `https://dashscope.aliyuncs.com/api/v1/tasks/${taskId}`, 87 | { 88 | headers: { 89 | Authorization: `Bearer ${this.apiKey}`, 90 | }, 91 | } 92 | ); 93 | 94 | return response.data.output; 95 | } catch (error) { 96 | if (axios.isAxiosError(error)) { 97 | throw new Error( 98 | `Task status check failed: ${ 99 | error.response?.data?.message || error.message 100 | }` 101 | ); 102 | } 103 | throw error; 104 | } 105 | } 106 | 107 | async waitForCompletion( 108 | taskId: string, 109 | maxAttempts: number = 30, 110 | interval: number = 2000 111 | ): Promise { 112 | let attempts = 0; 113 | 114 | while (attempts < maxAttempts) { 115 | const status = await this.checkTaskStatus(taskId); 116 | 117 | if (status.task_status === "SUCCEEDED") { 118 | return status; 119 | } 120 | 121 | if (status.task_status === "FAILED") { 122 | throw new Error("Image generation task failed"); 123 | } 124 | 125 | await new Promise((resolve) => setTimeout(resolve, interval)); 126 | attempts++; 127 | } 128 | 129 | throw new Error("Timeout waiting for image generation"); 130 | } 131 | } 132 | 133 | // Example usage: 134 | // const generator = new AliWanX21ImageGenerator(); 135 | // const response = await generator.generateImage('一间有着精致窗户的花店，漂亮的木质门，摆放着花朵'); 136 | // const result = await generator.waitForCompletion(response.output.task_id); 137 | -------------------------------------------------------------------------------- /src/scrapers/hellogithub.scraper.ts: -------------------------------------------------------------------------------- 1 | import * as cheerio from "cheerio"; 2 | import { 3 | AIGithubItem, 4 | AIGithubItemDetail, 5 | } from "../render/interfaces/aigithub.type"; 6 | 7 | export class HelloGithubScraper { 8 | private static readonly BASE_URL = "https://hellogithub.com"; 9 | private static readonly API_URL = "https://abroad.hellogithub.com/v1"; 10 | 11 | /** 12 | * 获取热门仓库列表 13 | * @param page - 页码 14 | * @returns 仓库列表 15 | */ 16 | public async getHotItems(page: number = 1): Promise { 17 | try { 18 | const url = `${HelloGithubScraper.API_URL}/?sort_by=featured&page=${page}&rank_by=newest&tid=juBLV86qa5`; 19 | const response = await fetch(url); 20 | const data = await response.json(); 21 | 22 | if (!data.success) { 23 | throw new Error("Failed to fetch hot items"); 24 | } 25 | 26 | return data.data.map((item: any) => ({ 27 | itemId: item.item_id, 28 | author: item.author, 29 | title: item.title, 30 | })); 31 | } catch (error: unknown) { 32 | if (error instanceof Error) { 33 | console.error("Failed to fetch hot items:", error); 34 | throw new Error(`Failed to fetch hot items: ${error.message}`); 35 | } 36 | throw new Error("Failed to fetch hot items: Unknown error"); 37 | } 38 | } 39 | 40 | /** 41 | * 从 HelloGithub 获取项目详情 42 | * @param itemId - 项目ID 43 | * @returns 项目详情 44 | */ 45 | public async getItemDetail(itemId: string): Promise { 46 | try { 47 | const url = `${HelloGithubScraper.BASE_URL}/repository/${itemId}`; 48 | const response = await fetch(url); 49 | const html = await response.text(); 50 | const $ = cheerio.load(html); 51 | 52 | // 提取 __NEXT_DATA__ 中的数据 53 | const nextData = JSON.parse($("#__NEXT_DATA__").text()); 54 | const repoData = nextData.props.pageProps.repo; 55 | 56 | // 提取标签 57 | const tags = repoData.tags.map((tag: { name: string }) => tag.name); 58 | 59 | // 提取相关链接 60 | const relatedUrls = []; 61 | 62 | // 提取 GitHub 仓库链接 63 | const githubUrl = repoData.url; 64 | console.log("GitHub URL:", githubUrl); 65 | 66 | // 提取其他链接 67 | if (repoData.homepage && repoData.homepage !== githubUrl) { 68 | console.log("Found homepage:", repoData.homepage); 69 | relatedUrls.push({ url: repoData.homepage, title: "官网" }); 70 | } 71 | if (repoData.document && repoData.document !== githubUrl) { 72 | console.log("Found document:", repoData.document); 73 | relatedUrls.push({ url: repoData.document, title: "文档" }); 74 | } 75 | if (repoData.download && repoData.download !== githubUrl) { 76 | console.log("Found download:", repoData.download); 77 | relatedUrls.push({ url: repoData.download, title: "下载" }); 78 | } 79 | if (repoData.online && repoData.online !== githubUrl) { 80 | console.log("Found online demo:", repoData.online); 81 | relatedUrls.push({ url: repoData.online, title: "演示" }); 82 | } 83 | // 计算上周获得的 star 数 84 | const starHistory = repoData.star_history; 85 | const lastWeekStars = starHistory ? starHistory.increment || 0 : 0; 86 | 87 | return { 88 | itemId, 89 | author: repoData.author, 90 | title: repoData.title, 91 | name: repoData.name, 92 | url: repoData.url, 93 | description: repoData.summary, 94 | language: repoData.primary_lang, 95 | totalStars: repoData.stars, 96 | totalIssues: repoData.open_issues, 97 | totalForks: repoData.forks, 98 | contributors: repoData.contributors, 99 | lastWeekStars, 100 | tags, 101 | license: repoData.license, 102 | relatedUrls, 103 | }; 104 | } catch (error: unknown) { 105 | if (error instanceof Error) { 106 | console.error("Failed to fetch project details:", error); 107 | throw new Error(`Failed to fetch project details: ${error.message}`); 108 | } 109 | throw new Error("Failed to fetch project details: Unknown error"); 110 | } 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/summarizer/deepseek-ai.summarizer.ts: -------------------------------------------------------------------------------- 1 | import { OpenAI } from "openai"; 2 | import { ContentSummarizer, Summary } from "./interfaces/summarizer.interface"; 3 | import { ConfigManager } from "../utils/config/config-manager"; 4 | import { 5 | getSummarizerSystemPrompt, 6 | getSummarizerUserPrompt, 7 | getTitleSystemPrompt, 8 | getTitleUserPrompt 9 | } from "../prompts/summarizer.prompt"; 10 | 11 | const MAX_RETRIES = 3; 12 | const RETRY_DELAY = 1000; // 1 second 13 | const MODEL_NAME = "deepseek-chat"; 14 | 15 | export class DeepseekAISummarizer implements ContentSummarizer { 16 | private client!: OpenAI; 17 | 18 | constructor() { 19 | this.refresh(); 20 | } 21 | 22 | async refresh(): Promise { 23 | await this.validateConfig(); 24 | this.client = new OpenAI({ 25 | apiKey: await ConfigManager.getInstance().get("DEEPSEEK_API_KEY"), 26 | baseURL: "https://api.deepseek.com", 27 | }); 28 | } 29 | 30 | async validateConfig(): Promise { 31 | if (!(await ConfigManager.getInstance().get("DEEPSEEK_API_KEY"))) { 32 | throw new Error("DeepSeek API key is required"); 33 | } 34 | } 35 | 36 | private async retryOperation(operation: () => Promise): Promise { 37 | for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { 38 | try { 39 | return await operation(); 40 | } catch (error) { 41 | if (attempt === MAX_RETRIES) { 42 | throw error; 43 | } 44 | await new Promise((resolve) => 45 | setTimeout(resolve, RETRY_DELAY * attempt) 46 | ); 47 | } 48 | } 49 | throw new Error("Operation failed after max retries"); 50 | } 51 | 52 | async summarize( 53 | content: string, 54 | options?: Record 55 | ): Promise { 56 | if (!content) { 57 | throw new Error("Content is required for summarization"); 58 | } 59 | 60 | return this.retryOperation(async () => { 61 | const response = await this.client.chat.completions.create({ 62 | model: MODEL_NAME, 63 | messages: [ 64 | { 65 | role: "system", 66 | content: getSummarizerSystemPrompt() 67 | }, 68 | { 69 | role: "user", 70 | content: getSummarizerUserPrompt({ 71 | content, 72 | language: options?.language, 73 | minLength: options?.minLength, 74 | maxLength: options?.maxLength, 75 | }) 76 | }, 77 | ], 78 | response_format: { type: "json_object" }, 79 | }); 80 | 81 | const completion = response.choices[0]?.message?.content; 82 | if (!completion) { 83 | throw new Error("未获取到有效的摘要结果"); 84 | } 85 | 86 | try { 87 | const summary = JSON.parse(completion) as Summary; 88 | if ( 89 | !summary.title || 90 | !summary.content || 91 | !Array.isArray(summary.keywords) || 92 | typeof summary.score !== 'number' 93 | ) { 94 | throw new Error("摘要结果格式不正确"); 95 | } 96 | return summary; 97 | } catch (error) { 98 | throw new Error( 99 | `解析摘要结果失败: ${ 100 | error instanceof Error ? error.message : "未知错误" 101 | }` 102 | ); 103 | } 104 | }); 105 | } 106 | 107 | async generateTitle( 108 | content: string, 109 | options?: Record 110 | ): Promise { 111 | return this.retryOperation(async () => { 112 | const response = await this.client.chat.completions.create({ 113 | model: MODEL_NAME, 114 | messages: [ 115 | { 116 | role: "system", 117 | content: getTitleSystemPrompt() 118 | }, 119 | { 120 | role: "user", 121 | content: getTitleUserPrompt({ 122 | content, 123 | language: options?.language, 124 | }) 125 | }, 126 | ], 127 | }); 128 | 129 | const title = response.choices[0]?.message?.content; 130 | if (!title) { 131 | throw new Error("未获取到有效的标题"); 132 | } 133 | return title; 134 | }); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /src/summarizer/qianwen-ai.summarizer.ts: -------------------------------------------------------------------------------- 1 | import OpenAI from "openai"; 2 | import { ContentSummarizer, Summary } from "./interfaces/summarizer.interface"; 3 | import { ConfigManager } from "../utils/config/config-manager"; 4 | import { 5 | getSummarizerSystemPrompt, 6 | getSummarizerUserPrompt, 7 | getTitleSystemPrompt, 8 | getTitleUserPrompt 9 | } from "../prompts/summarizer.prompt"; 10 | 11 | const MAX_RETRIES = 3; 12 | const RETRY_DELAY = 1000; // 1 second 13 | const MODEL_NAME = "qwen-max"; 14 | 15 | export class QianwenAISummarizer implements ContentSummarizer { 16 | private client!: OpenAI; 17 | 18 | constructor() { 19 | this.refresh(); 20 | } 21 | 22 | async refresh(): Promise { 23 | await this.validateConfig(); 24 | this.client = new OpenAI({ 25 | apiKey: await ConfigManager.getInstance().get("DASHSCOPE_API_KEY"), 26 | baseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", 27 | }); 28 | } 29 | 30 | async validateConfig(): Promise { 31 | if (!(await ConfigManager.getInstance().get("DASHSCOPE_API_KEY"))) { 32 | throw new Error("DashScope API key is required"); 33 | } 34 | } 35 | 36 | private async retryOperation(operation: () => Promise): Promise { 37 | for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { 38 | try { 39 | return await operation(); 40 | } catch (error) { 41 | if (attempt === MAX_RETRIES) { 42 | throw error; 43 | } 44 | await new Promise((resolve) => 45 | setTimeout(resolve, RETRY_DELAY * attempt) 46 | ); 47 | console.error(`Retry attempt ${attempt} failed:`, error); 48 | } 49 | } 50 | throw new Error("Operation failed after max retries"); 51 | } 52 | 53 | async summarize( 54 | content: string, 55 | options?: Record 56 | ): Promise { 57 | if (!content) { 58 | throw new Error("Content is required for summarization"); 59 | } 60 | 61 | return this.retryOperation(async () => { 62 | const response = await this.client.chat.completions.create({ 63 | model: MODEL_NAME, 64 | messages: [ 65 | { 66 | role: "system", 67 | content: getSummarizerSystemPrompt() 68 | }, 69 | { 70 | role: "user", 71 | content: getSummarizerUserPrompt({ 72 | content, 73 | language: options?.language, 74 | minLength: options?.minLength, 75 | maxLength: options?.maxLength, 76 | }) 77 | }, 78 | ], 79 | response_format: { type: "json_object" }, 80 | temperature: 0.7 81 | }); 82 | 83 | const completion = response.choices[0]?.message?.content; 84 | if (!completion) { 85 | throw new Error("未获取到有效的摘要结果"); 86 | } 87 | 88 | try { 89 | const summary = JSON.parse(completion) as Summary; 90 | if ( 91 | !summary.title || 92 | !summary.content 93 | ) { 94 | throw new Error("摘要结果格式不正确"); 95 | } 96 | return summary; 97 | } catch (error) { 98 | throw new Error( 99 | `解析摘要结果失败: ${ 100 | error instanceof Error ? error.message : "未知错误" 101 | }` 102 | ); 103 | } 104 | }); 105 | } 106 | 107 | async generateTitle( 108 | content: string, 109 | options?: Record 110 | ): Promise { 111 | return this.retryOperation(async () => { 112 | const response = await this.client.chat.completions.create({ 113 | model: MODEL_NAME, 114 | messages: [ 115 | { 116 | role: "system", 117 | content: getTitleSystemPrompt() 118 | }, 119 | { 120 | role: "user", 121 | content: getTitleUserPrompt({ 122 | content, 123 | language: options?.language, 124 | }) 125 | }, 126 | ], 127 | temperature: 0.7, 128 | max_tokens: 100, 129 | }); 130 | 131 | const title = response.choices[0]?.message?.content; 132 | if (!title) { 133 | throw new Error("未获取到有效的标题"); 134 | } 135 | return title; 136 | }); 137 | } 138 | } -------------------------------------------------------------------------------- /src/scrapers/fireCrawl.scraper.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ContentScraper, 3 | ScraperOptions, 4 | ScrapedContent, 5 | } from "./interfaces/scraper.interface"; 6 | import dotenv from "dotenv"; 7 | import { z } from "zod"; 8 | import FirecrawlApp from "firecrawl"; 9 | import { formatDate } from "../utils/common"; 10 | import { ConfigManager } from "../utils/config/config-manager"; 11 | 12 | dotenv.config(); 13 | 14 | // 使用 zod 定义数据结构 15 | const StorySchema = z.object({ 16 | headline: z.string(), 17 | content: z.string(), 18 | link: z.string(), 19 | date_posted: z.string(), 20 | }); 21 | 22 | const StoriesSchema = z.object({ 23 | stories: z.array(StorySchema), 24 | }); 25 | 26 | type FireCrawlResponse = { 27 | success: boolean; 28 | error?: string; 29 | extract?: z.infer; 30 | }; 31 | 32 | export class FireCrawlScraper implements ContentScraper { 33 | private app!: FirecrawlApp; 34 | 35 | constructor() { 36 | this.refresh(); 37 | } 38 | 39 | async refresh(): Promise { 40 | await this.validateConfig(); 41 | this.app = new FirecrawlApp({ 42 | apiKey: await ConfigManager.getInstance().get("FIRE_CRAWL_API_KEY"), 43 | }); 44 | } 45 | 46 | async validateConfig(): Promise { 47 | if (!(await ConfigManager.getInstance().get("FIRE_CRAWL_API_KEY"))) { 48 | throw new Error("FIRE_CRAWL_API_KEY 环境变量未设置"); 49 | } 50 | } 51 | 52 | private generateId(url: string): string { 53 | const timestamp = Date.now(); 54 | const random = Math.floor(Math.random() * 10000); 55 | const urlHash = url.split("").reduce((acc, char) => { 56 | return ((acc << 5) - acc + char.charCodeAt(0)) | 0; 57 | }, 0); 58 | return `fc_${timestamp}_${random}_${Math.abs(urlHash)}`; 59 | } 60 | 61 | async scrape( 62 | sourceId: string, 63 | options?: ScraperOptions 64 | ): Promise { 65 | try { 66 | const currentDate = new Date().toLocaleDateString(); 67 | 68 | // 构建提取提示词 69 | const promptForFirecrawl = ` 70 | Return only today's AI or LLM related story or post headlines and links in JSON format from the page content. 71 | They must be posted today, ${currentDate}. The format should be: 72 | { 73 | "stories": [ 74 | { 75 | "headline": "headline1", 76 | "content":"content1" 77 | "link": "link1", 78 | "date_posted": "YYYY-MM-DD HH:mm:ss", 79 | }, 80 | ... 81 | ] 82 | } 83 | If there are no AI or LLM stories from today, return {"stories": []}. 84 | 85 | The source link is ${sourceId}. 86 | If a story link is not absolute, prepend ${sourceId} to make it absolute. 87 | Return only pure JSON in the specified format (no extra text, no markdown, no \\\\). 88 | The content should be about 500 words, which can summarize the full text and the main point. 89 | Translate all into Chinese. 90 | !! 91 | `; 92 | 93 | // 使用 FirecrawlApp 进行抓取 94 | const scrapeResult = await this.app.scrapeUrl(sourceId, { 95 | formats: ["extract"], 96 | extract: { 97 | prompt: promptForFirecrawl, 98 | schema: StoriesSchema, 99 | }, 100 | }); 101 | 102 | if (!scrapeResult.success || !scrapeResult.extract?.stories) { 103 | throw new Error(scrapeResult.error || "未获取到有效内容"); 104 | } 105 | 106 | // 使用 zod 验证返回数据 107 | const validatedData = StoriesSchema.parse(scrapeResult.extract); 108 | 109 | // 转换为 ScrapedContent 格式 110 | console.log( 111 | `[FireCrawl] 从 ${sourceId} 获取到 ${validatedData.stories.length} 条内容` 112 | ); 113 | return validatedData.stories.map((story) => ({ 114 | id: this.generateId(story.link), 115 | title: story.headline, 116 | content: story.content, 117 | url: story.link, 118 | publishDate: formatDate(story.date_posted), 119 | score: 0, 120 | metadata: { 121 | source: "fireCrawl", 122 | originalUrl: story.link, 123 | datePosted: story.date_posted, 124 | }, 125 | })); 126 | } catch (error) { 127 | console.error("FireCrawl抓取失败:", error); 128 | throw error; 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/data-sources/getCronSources.ts: -------------------------------------------------------------------------------- 1 | import { ConfigManager } from "../utils/config/config-manager"; 2 | import { EnvConfigSource } from "../utils/config/sources/env-config.source"; 3 | import { MySQLDB } from "../utils/db/mysql.db"; 4 | 5 | export type NewsType = "AI" | "Tech" | "Crypto" | "All"; 6 | export type NewsPlatform = "firecrawl" | "twitter"; 7 | 8 | interface SourceItem { 9 | identifier: string; 10 | } 11 | 12 | interface PlatformSources { 13 | firecrawl: SourceItem[]; 14 | twitter: SourceItem[]; 15 | } 16 | 17 | type SourceConfig = Record; 18 | 19 | interface CronSource { 20 | id: number; 21 | NewsType: NewsType; 22 | NewsPlatform: NewsPlatform; 23 | identifier: string; 24 | } 25 | 26 | // 本地源配置 27 | export const sourceConfigs: SourceConfig = { 28 | AI: { 29 | firecrawl: [ 30 | { identifier: "https://www.anthropic.com/news" }, 31 | { identifier: "https://news.ycombinator.com/" }, 32 | { 33 | identifier: 34 | "https://www.reuters.com/technology/artificial-intelligence/", 35 | }, 36 | { identifier: "https://simonwillison.net/" }, 37 | { identifier: "https://buttondown.com/ainews/archive/" }, 38 | { identifier: "https://www.aibase.com/zh/daily" }, 39 | { identifier: "https://www.aibase.com/zh/news" }, 40 | ], 41 | twitter: [ 42 | { identifier: "https://x.com/OpenAIDevs" }, 43 | { identifier: "https://x.com/xai" }, 44 | { identifier: "https://x.com/alexalbert__" }, 45 | { identifier: "https://x.com/leeerob" }, 46 | { identifier: "https://x.com/v0" }, 47 | { identifier: "https://x.com/aisdk" }, 48 | { identifier: "https://x.com/firecrawl_dev" }, 49 | { identifier: "https://x.com/AIatMeta" }, 50 | { identifier: "https://x.com/googleaidevs" }, 51 | { identifier: "https://x.com/MistralAI" }, 52 | { identifier: "https://x.com/Cohere" }, 53 | { identifier: "https://x.com/karpathy" }, 54 | { identifier: "https://x.com/ylecun" }, 55 | { identifier: "https://x.com/sama" }, 56 | { identifier: "https://x.com/EMostaque" }, 57 | { identifier: "https://x.com/DrJimFan" }, 58 | { identifier: "https://x.com/nickscamara_" }, 59 | { identifier: "https://x.com/CalebPeffer" }, 60 | { identifier: "https://x.com/akshay_pachaar" }, 61 | { identifier: "https://x.com/ericciarla" }, 62 | { identifier: "https://x.com/amasad" }, 63 | { identifier: "https://x.com/nutlope" }, 64 | { identifier: "https://x.com/rauchg" }, 65 | { identifier: "https://x.com/vercel" }, 66 | { identifier: "https://x.com/LangChainAI" }, 67 | { identifier: "https://x.com/llama_index" }, 68 | { identifier: "https://x.com/pinecone" }, 69 | { identifier: "https://x.com/modal_labs" }, 70 | { identifier: "https://x.com/huggingface" }, 71 | { identifier: "https://x.com/weights_biases" }, 72 | { identifier: "https://x.com/replicate" }, 73 | ], 74 | }, 75 | Tech: { 76 | firecrawl: [], 77 | twitter: [], 78 | }, 79 | Crypto: { 80 | firecrawl: [], 81 | twitter: [], 82 | }, 83 | All: { 84 | firecrawl: [], 85 | twitter: [], 86 | }, 87 | } as const; 88 | 89 | export const getCronSources = async (): Promise => { 90 | const configManager = ConfigManager.getInstance(); 91 | configManager.addSource(new EnvConfigSource()); 92 | try { 93 | const mysql = await MySQLDB.getInstance({ 94 | host: await configManager.get("DB_HOST"), 95 | port: await configManager.get("DB_PORT"), 96 | user: await configManager.get("DB_USER"), 97 | password: await configManager.get("DB_PASSWORD"), 98 | database: await configManager.get("DB_DATABASE"), 99 | }); 100 | const dbSources = await mysql.query( 101 | "SELECT * FROM cron_sources" 102 | ); 103 | 104 | // 如果成功获取数据库数据，合并本地和数据库数据 105 | const mergedSources = { ...sourceConfigs }; 106 | 107 | // 遍历数据库数据并合并到本地配置中 108 | if (Array.isArray(dbSources)) { 109 | for (const source of dbSources) { 110 | const { NewsType, NewsPlatform, identifier } = source; 111 | const newsTypeKey = NewsType as keyof SourceConfig; 112 | const platformKey = NewsPlatform as keyof PlatformSources; 113 | 114 | if ( 115 | newsTypeKey in mergedSources && 116 | platformKey in mergedSources[newsTypeKey] 117 | ) { 118 | const platform = mergedSources[newsTypeKey][platformKey]; 119 | // 检查是否已存在相同的identifier 120 | const exists = platform.some( 121 | (item) => item.identifier === identifier 122 | ); 123 | if (!exists) { 124 | platform.push({ identifier }); 125 | } 126 | } 127 | } 128 | } 129 | 130 | return mergedSources; 131 | } catch (error) { 132 | console.error("Failed to get cron sources from database:", error); 133 | // 数据库不可用时返回本地配置 134 | return sourceConfigs; 135 | } 136 | }; 137 | -------------------------------------------------------------------------------- /src/render/aibench/renderer.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | import ejs from "ejs"; 4 | import { AIBenchTemplate } from "../interfaces/aibench.template"; 5 | import { ModelPerformance } from "../../api/livebench.api"; 6 | 7 | interface ModelScore { 8 | name: string; 9 | score: number; 10 | organization?: string; 11 | reasoning?: number; 12 | coding?: number; 13 | math?: number; 14 | dataAnalysis?: number; 15 | language?: number; 16 | if?: number; 17 | } 18 | 19 | interface CategoryData { 20 | name: string; 21 | icon: string; 22 | models: ModelScore[]; 23 | } 24 | 25 | export class AIBenchRenderer { 26 | private static readonly CATEGORIES = [ 27 | { key: "Global Average", name: "总体评分", icon: "🌟" }, 28 | { key: "Reasoning Average", name: "推理能力", icon: "🧠" }, 29 | { key: "Coding Average", name: "编程能力", icon: "💻" }, 30 | { key: "Mathematics Average", name: "数学能力", icon: "📐" }, 31 | { key: "Data Analysis Average", name: "数据分析", icon: "📊" }, 32 | { key: "Language Average", name: "语言能力", icon: "📝" }, 33 | { key: "IF Average", name: "推理框架", icon: "🔍" }, 34 | ]; 35 | 36 | private template!: string; 37 | private weixinTemplate!: string; 38 | 39 | constructor() { 40 | this.loadTemplate(); 41 | this.loadWeixinTemplate(); 42 | } 43 | 44 | private loadTemplate(): void { 45 | const templatePath = path.join(__dirname, "../../templates/aibench.ejs"); 46 | if (!fs.existsSync(templatePath)) { 47 | throw new Error(`Template file not found at ${templatePath}`); 48 | } 49 | this.template = fs.readFileSync(templatePath, "utf-8"); 50 | } 51 | 52 | private loadWeixinTemplate(): void { 53 | const templatePath = path.join( 54 | __dirname, 55 | "../../templates/aibench-weixin.ejs" 56 | ); 57 | if (!fs.existsSync(templatePath)) { 58 | throw new Error(`Weixin template file not found at ${templatePath}`); 59 | } 60 | this.weixinTemplate = fs.readFileSync(templatePath, "utf-8"); 61 | } 62 | 63 | public render(data: AIBenchTemplate): string { 64 | try { 65 | return ejs.render(this.template, data, { 66 | rmWhitespace: true, 67 | }); 68 | } catch (error) { 69 | console.error("Error rendering AI benchmark template:", error); 70 | throw error; 71 | } 72 | } 73 | 74 | public renderForWeixin(data: AIBenchTemplate): string { 75 | try { 76 | return ejs.render(this.weixinTemplate, data, { 77 | rmWhitespace: true, 78 | }); 79 | } catch (error) { 80 | console.error("Error rendering AI benchmark weixin template:", error); 81 | throw error; 82 | } 83 | } 84 | 85 | public async renderToFile( 86 | data: AIBenchTemplate, 87 | outputPath: string 88 | ): Promise { 89 | try { 90 | const html = this.render(data); 91 | const outputDir = path.dirname(outputPath); 92 | 93 | if (!fs.existsSync(outputDir)) { 94 | fs.mkdirSync(outputDir, { recursive: true }); 95 | } 96 | 97 | await fs.promises.writeFile(outputPath, html); 98 | } catch (error) { 99 | console.error("Error writing rendered template to file:", error); 100 | throw error; 101 | } 102 | } 103 | 104 | public static render(data: { 105 | [key: string]: ModelPerformance; 106 | }): AIBenchTemplate { 107 | const categories: CategoryData[] = []; 108 | const globalTop10: ModelScore[] = []; 109 | 110 | // Process global rankings first 111 | const globalScores = Object.entries(data) 112 | .map(([name, performance]) => ({ 113 | name, 114 | organization: performance.organization, 115 | score: performance.metrics["Global Average"] || 0, 116 | reasoning: performance.metrics["Reasoning Average"] || 0, 117 | coding: performance.metrics["Coding Average"] || 0, 118 | math: performance.metrics["Mathematics Average"] || 0, 119 | dataAnalysis: performance.metrics["Data Analysis Average"] || 0, 120 | language: performance.metrics["Language Average"] || 0, 121 | if: performance.metrics["IF Average"] || 0, 122 | })) 123 | .filter((model) => model.score > 0) 124 | .sort((a, b) => b.score - a.score); 125 | 126 | // Get top 10 models for global ranking 127 | globalTop10.push(...globalScores.slice(0, 10)); 128 | 129 | // Process each category 130 | this.CATEGORIES.forEach((category) => { 131 | const categoryScores = Object.entries(data) 132 | .map(([name, performance]) => ({ 133 | name, 134 | organization: performance.organization, 135 | score: performance.metrics[category.key] || 0, 136 | })) 137 | .filter((model) => model.score > 0) 138 | .sort((a, b) => b.score - a.score); 139 | 140 | categories.push({ 141 | name: category.name, 142 | icon: category.icon, 143 | models: categoryScores, 144 | }); 145 | }); 146 | 147 | return { 148 | title: "AI模型能力评测榜单", 149 | updateTime: new Date().toLocaleString("zh-CN", { 150 | timeZone: "Asia/Shanghai", 151 | hour12: false, 152 | }), 153 | categories, 154 | globalTop10, 155 | }; 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/api/deepseek.api.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import { ConfigManager } from "../utils/config/config-manager"; 3 | 4 | interface BalanceInfo { 5 | currency: string; 6 | total_balance: string; 7 | granted_balance: string; 8 | topped_up_balance: string; 9 | } 10 | 11 | interface DeepseekBalanceResponse { 12 | is_available: boolean; 13 | balance_infos: BalanceInfo[]; 14 | } 15 | 16 | interface ChatMessage { 17 | role: "system" | "user" | "assistant"; 18 | content: string; 19 | } 20 | 21 | interface ChatCompletionRequest { 22 | model: string; 23 | messages: ChatMessage[]; 24 | temperature?: number; 25 | top_p?: number; 26 | max_tokens?: number; 27 | stream?: boolean; 28 | } 29 | 30 | interface ChatCompletionResponse { 31 | id: string; 32 | object: string; 33 | created: number; 34 | model: string; 35 | choices: { 36 | index: number; 37 | message: ChatMessage; 38 | finish_reason: string; 39 | }[]; 40 | usage: { 41 | prompt_tokens: number; 42 | completion_tokens: number; 43 | total_tokens: number; 44 | }; 45 | } 46 | 47 | export class DeepseekAPI { 48 | private baseURL = "https://api.deepseek.com"; 49 | private token!: string; 50 | private defaultModel = "deepseek-chat"; 51 | 52 | constructor() { 53 | this.refresh(); 54 | } 55 | 56 | async refresh() { 57 | this.token = await ConfigManager.getInstance().get("DEEPSEEK_API_KEY"); 58 | if (!this.token) { 59 | throw new Error("DeepSeek API key is not set"); 60 | } 61 | } 62 | 63 | /** 64 | * Create a chat completion using Deepseek's chat API 65 | * @param messages Array of messages in the conversation 66 | * @param options Optional parameters for the chat completion 67 | * @returns Promise The chat completion response 68 | * @throws Error if the API request fails 69 | */ 70 | async createChatCompletion( 71 | messages: ChatMessage[], 72 | options: Partial> = {} 73 | ): Promise { 74 | try { 75 | const response = await axios.post( 76 | `${this.baseURL}/v1/chat/completions`, 77 | { 78 | model: this.defaultModel, 79 | messages, 80 | temperature: options.temperature ?? 0.7, 81 | top_p: options.top_p ?? 1, 82 | max_tokens: options.max_tokens ?? 2000, 83 | stream: options.stream ?? false, 84 | }, 85 | { 86 | headers: { 87 | "Content-Type": "application/json", 88 | Authorization: `Bearer ${this.token}`, 89 | }, 90 | } 91 | ); 92 | 93 | return response.data; 94 | } catch (error) { 95 | if (axios.isAxiosError(error)) { 96 | throw new Error( 97 | `Failed to create chat completion: ${ 98 | error.response?.data?.message || error.message 99 | }` 100 | ); 101 | } 102 | throw error; 103 | } 104 | } 105 | 106 | /** 107 | * Simple method to get a response for a single message 108 | * @param content The message content 109 | * @param systemPrompt Optional system prompt to set context 110 | * @returns Promise The assistant's response text 111 | */ 112 | async sendMessage(content: string, systemPrompt?: string): Promise { 113 | const messages: ChatMessage[] = []; 114 | 115 | if (systemPrompt) { 116 | messages.push({ 117 | role: "system", 118 | content: systemPrompt, 119 | }); 120 | } 121 | 122 | messages.push({ 123 | role: "user", 124 | content, 125 | }); 126 | 127 | const response = await this.createChatCompletion(messages); 128 | return response.choices[0].message.content; 129 | } 130 | 131 | /** 132 | * Get the current balance information from Deepseek account 133 | * @returns Promise The balance information including availability and balance details 134 | * @throws Error if the API request fails 135 | */ 136 | async getBalance(): Promise { 137 | try { 138 | const response = await axios.get( 139 | `${this.baseURL}/user/balance`, 140 | { 141 | headers: { 142 | Accept: "application/json", 143 | Authorization: `Bearer ${this.token}`, 144 | }, 145 | } 146 | ); 147 | 148 | return response.data; 149 | } catch (error) { 150 | if (axios.isAxiosError(error)) { 151 | throw new Error( 152 | `Failed to get Deepseek balance: ${ 153 | error.response?.data?.message || error.message 154 | }` 155 | ); 156 | } 157 | throw error; 158 | } 159 | } 160 | 161 | /** 162 | * Get the total balance in CNY (convenience method) 163 | * @returns Promise The total balance in CNY 164 | * @throws Error if the API request fails or CNY balance is not found 165 | */ 166 | async getCNYBalance(): Promise { 167 | const response = await this.getBalance(); 168 | const cnyBalance = response.balance_infos.find( 169 | (info) => info.currency === "CNY" 170 | ); 171 | if (!cnyBalance) { 172 | throw new Error("CNY balance not found in response"); 173 | } 174 | return parseFloat(cnyBalance.total_balance); 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /src/render/test/test.aibench.template.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | import { AIBenchRenderer } from "../aibench/renderer"; 4 | import { ModelPerformance } from "../../api/livebench.api"; 5 | 6 | // 示例数据 7 | const modelData = { 8 | "o1-2024-12-17-high": { 9 | metrics: { 10 | "Reasoning Average": 91.58, 11 | "Coding Average": 69.69, 12 | "Mathematics Average": 80.32, 13 | "Data Analysis Average": 65.47, 14 | "Language Average": 65.39, 15 | "IF Average": 81.55, 16 | "Global Average": 76.32, 17 | }, 18 | organization: "Unknown", 19 | }, 20 | "o3-mini-2025-01-31-high": { 21 | metrics: { 22 | "Reasoning Average": 89.58, 23 | "Coding Average": 82.74, 24 | "Mathematics Average": 77.29, 25 | "Data Analysis Average": 70.64, 26 | "Language Average": 50.68, 27 | "IF Average": 84.36, 28 | "Global Average": 75.97, 29 | }, 30 | organization: "Unknown", 31 | }, 32 | "deepseek-r1": { 33 | metrics: { 34 | "Reasoning Average": 83.17, 35 | "Coding Average": 66.74, 36 | "Mathematics Average": 80.71, 37 | "Data Analysis Average": 69.78, 38 | "Language Average": 48.53, 39 | "IF Average": 80.51, 40 | "Global Average": 72.34, 41 | }, 42 | organization: "Unknown", 43 | }, 44 | "o3-mini-2025-01-31-medium": { 45 | metrics: { 46 | "Reasoning Average": 86.33, 47 | "Coding Average": 65.38, 48 | "Mathematics Average": 72.37, 49 | "Data Analysis Average": 66.56, 50 | "Language Average": 46.26, 51 | "IF Average": 83.16, 52 | "Global Average": 71, 53 | }, 54 | organization: "Unknown", 55 | }, 56 | "gemini-2.0-flash-thinking-exp-01-21": { 57 | metrics: { 58 | "Reasoning Average": 78.17, 59 | "Coding Average": 53.49, 60 | "Mathematics Average": 75.85, 61 | "Data Analysis Average": 69.37, 62 | "Language Average": 42.18, 63 | "IF Average": 82.47, 64 | "Global Average": 68.53, 65 | }, 66 | organization: "Unknown", 67 | }, 68 | "gemini-2.0-pro-exp-02-05": { 69 | metrics: { 70 | "Reasoning Average": 60.08, 71 | "Coding Average": 63.49, 72 | "Mathematics Average": 70.97, 73 | "Data Analysis Average": 68.02, 74 | "Language Average": 44.85, 75 | "IF Average": 83.38, 76 | "Global Average": 66.24, 77 | }, 78 | organization: "Unknown", 79 | }, 80 | "gemini-exp-1206": { 81 | metrics: { 82 | "Reasoning Average": 57, 83 | "Coding Average": 63.41, 84 | "Mathematics Average": 72.36, 85 | "Data Analysis Average": 63.16, 86 | "Language Average": 51.29, 87 | "IF Average": 77.34, 88 | "Global Average": 64.87, 89 | }, 90 | organization: "Unknown", 91 | }, 92 | "o3-mini-2025-01-31-low": { 93 | metrics: { 94 | "Reasoning Average": 69.83, 95 | "Coding Average": 61.46, 96 | "Mathematics Average": 63.06, 97 | "Data Analysis Average": 62.04, 98 | "Language Average": 38.25, 99 | "IF Average": 80.06, 100 | "Global Average": 63.49, 101 | }, 102 | organization: "Unknown", 103 | }, 104 | "gemini-2.0-flash": { 105 | metrics: { 106 | "Reasoning Average": 55.25, 107 | "Coding Average": 53.92, 108 | "Mathematics Average": 65.62, 109 | "Data Analysis Average": 67.55, 110 | "Language Average": 40.69, 111 | "IF Average": 85.79, 112 | "Global Average": 63.24, 113 | }, 114 | organization: "Unknown", 115 | }, 116 | "qwen2.5-max": { 117 | metrics: { 118 | "Reasoning Average": 51.42, 119 | "Coding Average": 64.41, 120 | "Mathematics Average": 58.35, 121 | "Data Analysis Average": 67.93, 122 | "Language Average": 56.28, 123 | "IF Average": 75.35, 124 | "Global Average": 62.89, 125 | }, 126 | organization: "Unknown", 127 | }, 128 | } as { [key: string]: ModelPerformance }; 129 | 130 | async function main() { 131 | try { 132 | // 使用渲染器处理数据 133 | const result = AIBenchRenderer.render(modelData); 134 | 135 | // 创建输出目录 136 | const tempDir = path.join(__dirname, "../../../temp"); 137 | if (!fs.existsSync(tempDir)) { 138 | fs.mkdirSync(tempDir, { recursive: true }); 139 | } 140 | 141 | // 实例化渲染器并生成HTML 142 | const renderer = new AIBenchRenderer(); 143 | const outputPath = path.join(tempDir, "aibench_preview.html"); 144 | 145 | // 渲染并保存结果 146 | await renderer.renderToFile(result, outputPath); 147 | console.log(`预览文件已生成：${outputPath}`); 148 | 149 | // 输出一些基本信息用于验证 150 | console.log("\n=== AI模型评测数据 ==="); 151 | console.log(`总计模型数：${result.globalTop10.length}`); 152 | console.log("\n--- 全局排名前3名 ---"); 153 | result.globalTop10.slice(0, 3).forEach((model, index) => { 154 | console.log(`${index + 1}. ${model.name} (${model.score.toFixed(2)}分)`); 155 | }); 156 | 157 | console.log("\n--- 各项能力最高分 ---"); 158 | result.categories.forEach((category) => { 159 | if (category.models.length > 0) { 160 | const topModel = category.models[0]; 161 | console.log( 162 | `${category.name}: ${topModel.name} (${topModel.score.toFixed(2)}分)` 163 | ); 164 | } 165 | }); 166 | } catch (error) { 167 | console.error("生成预览时出错:", error); 168 | } 169 | } 170 | 171 | // 运行程序 172 | main().catch(console.error); 173 | -------------------------------------------------------------------------------- /src/templates/aibench-weixin.ejs: -------------------------------------------------------------------------------- 1 | 2 |
13 |
22 | <%= title %> 23 |
24 | 25 |
33 | 更新时间：<%= updateTime %> 34 |
35 | 36 | 37 |
38 |
49 | 🏆 全局排名 Top 10 50 |
51 | 52 | <% globalTop10.forEach((model, index) => { %> 53 |
63 |
70 |
71 |
72 | <%= index + 1 %> 76 | <%= model.name %> 79 |
80 |
88 | <%= model.organization || '-' %> 89 |
90 |
91 |
92 | <%= model.score.toFixed(2) %> 101 |
102 |
103 |
104 | <% }); %> 105 |
106 | 107 | 108 | <% categories.forEach(category => { %> 109 |
110 |
121 | <%= category.icon %> <%= category.name %> 122 |
123 | 124 | <% category.models.forEach((model, index) => { %> 125 |
135 |
142 |
143 |
144 | <%= index + 1 %> 148 | <%= model.name %> 151 |
152 |
160 | <%= model.organization || '-' %> 161 |
162 |
163 |
164 | <%= model.score.toFixed(2) %> 173 |
174 |
175 |
176 | <% }); %> 177 |
178 | <% }); %> 179 | 180 |
190 | 数据来源：LiveBench.AI 191 |
192 |
193 | -------------------------------------------------------------------------------- /src/publishers/weixin.publisher.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ContentPublisher, 3 | PublishResult, 4 | } from "./interfaces/publisher.interface"; 5 | import dotenv from "dotenv"; 6 | import axios from "axios"; 7 | import { ConfigManager } from "../utils/config/config-manager"; 8 | import { AliWanX21ImageGenerator } from "../utils/gen-image/aliwanx2.1.image"; 9 | 10 | dotenv.config(); 11 | 12 | interface WeixinToken { 13 | access_token: string; 14 | expires_in: number; 15 | expiresAt: Date; 16 | } 17 | 18 | interface WeixinDraft { 19 | media_id: string; 20 | article_id?: string; 21 | } 22 | 23 | export class WeixinPublisher implements ContentPublisher { 24 | private accessToken: WeixinToken | null = null; 25 | private appId: string | undefined; 26 | private appSecret: string | undefined; 27 | 28 | constructor() { 29 | this.refresh(); 30 | } 31 | 32 | async refresh(): Promise { 33 | await this.validateConfig(); 34 | this.appId = await ConfigManager.getInstance().get("WEIXIN_APP_ID"); 35 | this.appSecret = await ConfigManager.getInstance().get("WEIXIN_APP_SECRET"); 36 | } 37 | 38 | async validateConfig(): Promise { 39 | if ( 40 | !(await ConfigManager.getInstance().get("WEIXIN_APP_ID")) || 41 | !(await ConfigManager.getInstance().get("WEIXIN_APP_SECRET")) 42 | ) { 43 | throw new Error( 44 | "微信公众号配置不完整，请检查 WEIXIN_APP_ID 和 WEIXIN_APP_SECRET" 45 | ); 46 | } 47 | } 48 | 49 | private async ensureAccessToken(): Promise { 50 | // 检查现有token是否有效 51 | if ( 52 | this.accessToken && 53 | this.accessToken.expiresAt > new Date(Date.now() + 60000) // 预留1分钟余量 54 | ) { 55 | return this.accessToken.access_token; 56 | } 57 | 58 | // 获取新token 59 | const url = `https://api.weixin.qq.com/cgi-bin/token?grant_type=client_credential&appid=${this.appId}&secret=${this.appSecret}`; 60 | 61 | try { 62 | const response = await axios.get(url); 63 | const { access_token, expires_in } = response.data; 64 | 65 | if (!access_token) { 66 | throw new Error( 67 | "获取access_token失败: " + JSON.stringify(response.data) 68 | ); 69 | } 70 | 71 | this.accessToken = { 72 | access_token, 73 | expires_in, 74 | expiresAt: new Date(Date.now() + expires_in * 1000), 75 | }; 76 | 77 | return access_token; 78 | } catch (error) { 79 | console.error("获取微信access_token失败:", error); 80 | throw error; 81 | } 82 | } 83 | 84 | private async uploadDraft( 85 | article: string, 86 | title: string, 87 | digest: string, 88 | mediaId: string 89 | ): Promise { 90 | const token = await this.ensureAccessToken(); 91 | const url = `https://api.weixin.qq.com/cgi-bin/draft/add?access_token=${token}`; 92 | 93 | const articles = [ 94 | { 95 | title: title, 96 | author: "刘耀文", 97 | digest: digest, 98 | content: article, 99 | thumb_media_id: mediaId, 100 | need_open_comment: 1, 101 | only_fans_can_comment: 0, 102 | }, 103 | ]; 104 | 105 | try { 106 | const response = await axios.post(url, { 107 | articles, 108 | }); 109 | 110 | if (response.data.errcode) { 111 | throw new Error(`上传草稿失败: ${response.data.errmsg}`); 112 | } 113 | 114 | return { 115 | media_id: response.data.media_id, 116 | }; 117 | } catch (error) { 118 | console.error("上传微信草稿失败:", error); 119 | throw error; 120 | } 121 | } 122 | /** 123 | * 上传图片到微信 124 | * @param imageUrl 图片URL 125 | * @returns 图片ID 126 | */ 127 | async uploadImage(imageUrl: string): Promise { 128 | if (!imageUrl) { 129 | // 如果图片URL为空，则返回一个默认的图片ID 130 | return "SwCSRjrdGJNaWioRQUHzgF68BHFkSlb_f5xlTquvsOSA6Yy0ZRjFo0aW9eS3JJu_"; 131 | } 132 | const imageBuffer = await axios.get(imageUrl, { 133 | responseType: "arraybuffer", 134 | }); 135 | 136 | const token = await this.ensureAccessToken(); 137 | const url = `https://api.weixin.qq.com/cgi-bin/material/add_material?access_token=${token}&type=image`; 138 | 139 | try { 140 | // 创建FormData并添加图片数据 141 | const formData = new FormData(); 142 | formData.append( 143 | "media", 144 | new Blob([imageBuffer.data], { type: "image/jpeg" }), 145 | `image_${Math.random().toString(36).substring(2, 8)}.jpg` 146 | ); 147 | 148 | const response = await axios.post(url, formData, { 149 | headers: { 150 | "Content-Type": "multipart/form-data", 151 | Accept: "*/*", 152 | }, 153 | }); 154 | 155 | if (response.data.errcode) { 156 | throw new Error(`上传图片失败: ${response.data.errmsg}`); 157 | } 158 | 159 | return response.data.media_id; 160 | } catch (error) { 161 | console.error("上传微信图片失败:", error); 162 | throw error; 163 | } 164 | } 165 | 166 | /** 167 | * 发布文章到微信 168 | * @param article 文章内容 169 | * @param title 文章标题 170 | * @param digest 文章摘要 171 | * @param mediaId 图片ID 172 | * @returns 发布结果 173 | */ 174 | async publish( 175 | article: string, 176 | title: string, 177 | digest: string, 178 | mediaId: string 179 | ): Promise { 180 | try { 181 | // 上传草稿 182 | const draft = await this.uploadDraft(article, title, digest, mediaId); 183 | return { 184 | publishId: draft.media_id, 185 | status: "draft", 186 | publishedAt: new Date(), 187 | platform: "weixin", 188 | url: `https://mp.weixin.qq.com/s/${draft.media_id}`, 189 | }; 190 | } catch (error) { 191 | console.error("微信发布失败:", error); 192 | throw error; 193 | } 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /src/utils/db/mysql.db.ts: -------------------------------------------------------------------------------- 1 | import mysql, { Pool, PoolConnection, PoolOptions } from "mysql2/promise"; 2 | 3 | export class MySQLDB { 4 | private static instance: MySQLDB; 5 | private pool: Pool; 6 | 7 | private constructor(config: PoolOptions) { 8 | this.pool = mysql.createPool({ 9 | ...config, 10 | waitForConnections: true, 11 | connectionLimit: 10, 12 | queueLimit: 0, 13 | enableKeepAlive: true, 14 | keepAliveInitialDelay: 0, 15 | }); 16 | } 17 | 18 | public static async getInstance(config?: PoolOptions): Promise { 19 | if (!MySQLDB.instance && config) { 20 | MySQLDB.instance = new MySQLDB(config); 21 | // 测试连接 22 | try { 23 | const connection = await MySQLDB.instance.pool.getConnection(); 24 | connection.release(); 25 | } catch (error) { 26 | throw new Error(`Failed to initialize database connection: ${error}`); 27 | } 28 | } else if (!MySQLDB.instance) { 29 | throw new Error( 30 | "MySQL configuration is required for first initialization" 31 | ); 32 | } 33 | return MySQLDB.instance; 34 | } 35 | 36 | /** 37 | * 执行查询并返回结果 38 | * @param sql SQL查询语句 39 | * @param params 查询参数 40 | * @returns 查询结果 41 | */ 42 | public async query(sql: string, params?: any[]): Promise { 43 | let connection: PoolConnection | null = null; 44 | try { 45 | connection = await this.pool.getConnection(); 46 | const [rows] = await connection.query(sql, params); 47 | 48 | return rows as T[]; 49 | } catch (error) { 50 | throw new Error(`Database query error: ${error}`); 51 | } finally { 52 | if (connection) { 53 | connection.release(); 54 | } 55 | } 56 | } 57 | 58 | /** 59 | * 执行单条查询并返回第一个结果 60 | * @param sql SQL查询语句 61 | * @param params 查询参数 62 | * @returns 单个查询结果 63 | */ 64 | public async queryOne( 65 | sql: string, 66 | params?: any[] 67 | ): Promise { 68 | const results = await this.query(sql, params); 69 | return results.length > 0 ? results[0] : null; 70 | } 71 | 72 | /** 73 | * 执行插入操作 74 | * @param table 表名 75 | * @param data 要插入的数据对象 76 | * @returns 插入结果 77 | */ 78 | public async insert( 79 | table: string, 80 | data: Record 81 | ): Promise { 82 | const keys = Object.keys(data); 83 | const values = Object.values(data); 84 | const sql = `INSERT INTO ${table} (${keys.join(", ")}) VALUES (${keys 85 | .map(() => "?") 86 | .join(", ")})`; 87 | 88 | let connection: PoolConnection | null = null; 89 | try { 90 | connection = await this.pool.getConnection(); 91 | const [result] = await connection.query(sql, values); 92 | return (result as any).insertId; 93 | } catch (error) { 94 | throw new Error(`Database insert error: ${error}`); 95 | } finally { 96 | if (connection) { 97 | connection.release(); 98 | } 99 | } 100 | } 101 | 102 | /** 103 | * 执行更新操作 104 | * @param table 表名 105 | * @param data 要更新的数据对象 106 | * @param where WHERE条件 107 | * @returns 受影响的行数 108 | */ 109 | public async update( 110 | table: string, 111 | data: Record, 112 | where: Record 113 | ): Promise { 114 | const setClause = Object.keys(data) 115 | .map((key) => `${key} = ?`) 116 | .join(", "); 117 | const whereClause = Object.keys(where) 118 | .map((key) => `${key} = ?`) 119 | .join(" AND "); 120 | const sql = `UPDATE ${table} SET ${setClause} WHERE ${whereClause}`; 121 | const params = [...Object.values(data), ...Object.values(where)]; 122 | 123 | let connection: PoolConnection | null = null; 124 | try { 125 | connection = await this.pool.getConnection(); 126 | const [result] = await connection.query(sql, params); 127 | return (result as any).affectedRows; 128 | } catch (error) { 129 | throw new Error(`Database update error: ${error}`); 130 | } finally { 131 | if (connection) { 132 | connection.release(); 133 | } 134 | } 135 | } 136 | 137 | /** 138 | * 执行删除操作 139 | * @param table 表名 140 | * @param where WHERE条件 141 | * @returns 受影响的行数 142 | */ 143 | public async delete( 144 | table: string, 145 | where: Record 146 | ): Promise { 147 | const whereClause = Object.keys(where) 148 | .map((key) => `${key} = ?`) 149 | .join(" AND "); 150 | const sql = `DELETE FROM ${table} WHERE ${whereClause}`; 151 | const params = Object.values(where); 152 | 153 | let connection: PoolConnection | null = null; 154 | try { 155 | connection = await this.pool.getConnection(); 156 | const [result] = await connection.query(sql, params); 157 | return (result as any).affectedRows; 158 | } catch (error) { 159 | throw new Error(`Database delete error: ${error}`); 160 | } finally { 161 | if (connection) { 162 | connection.release(); 163 | } 164 | } 165 | } 166 | 167 | /** 168 | * 开始事务 169 | * @returns 事务连接 170 | */ 171 | public async beginTransaction(): Promise { 172 | const connection = await this.pool.getConnection(); 173 | await connection.beginTransaction(); 174 | return connection; 175 | } 176 | 177 | /** 178 | * 在事务中执行查询 179 | * @param connection 事务连接 180 | * @param sql SQL查询语句 181 | * @param params 查询参数 182 | * @returns 查询结果 183 | */ 184 | public async queryWithTransaction( 185 | connection: PoolConnection, 186 | sql: string, 187 | params?: any[] 188 | ): Promise { 189 | const [rows] = await connection.query(sql, params); 190 | return rows as T[]; 191 | } 192 | 193 | /** 194 | * 提交事务 195 | * @param connection 事务连接 196 | */ 197 | public async commit(connection: PoolConnection): Promise { 198 | try { 199 | await connection.commit(); 200 | } finally { 201 | connection.release(); 202 | } 203 | } 204 | 205 | /** 206 | * 回滚事务 207 | * @param connection 事务连接 208 | */ 209 | public async rollback(connection: PoolConnection): Promise { 210 | try { 211 | await connection.rollback(); 212 | } finally { 213 | connection.release(); 214 | } 215 | } 216 | 217 | /** 218 | * 关闭数据库连接池 219 | */ 220 | public async close(): Promise { 221 | if (this.pool) { 222 | await this.pool.end(); 223 | } 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /src/utils/content-rank/content-ranker.ts: -------------------------------------------------------------------------------- 1 | import { ScrapedContent } from "../../scrapers/interfaces/scraper.interface"; 2 | import { ConfigManager } from "../config/config-manager"; 3 | 4 | export interface RankResult { 5 | id: string; 6 | score: number; 7 | } 8 | 9 | export interface APIConfig { 10 | apiKey: string; 11 | modelName?: string; 12 | } 13 | 14 | const MAX_RETRIES = 3; 15 | const RETRY_DELAY = 1000; // 1 second 16 | const DEFAULT_MODEL_NAME = "deepseek-r1"; 17 | 18 | // API Provider interfaces and implementations 19 | interface APIProvider { 20 | initialize(): Promise; 21 | callAPI(messages: any[]): Promise; 22 | } 23 | 24 | class DashScopeProvider implements APIProvider { 25 | private readonly API_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"; 26 | 27 | constructor( 28 | private readonly config: APIConfig 29 | ) { 30 | this.config.modelName = this.config.modelName || "deepseek-r1"; 31 | } 32 | 33 | async initialize(): Promise { 34 | // 验证必要的配置 35 | if (!this.config.apiKey) { 36 | throw new Error("DashScope API key is required"); 37 | } 38 | } 39 | 40 | async callAPI(messages: any[]): Promise { 41 | const response = await fetch(`${this.API_BASE_URL}/chat/completions`, { 42 | method: 'POST', 43 | headers: { 44 | 'Content-Type': 'application/json', 45 | 'Authorization': `Bearer ${this.config.apiKey}`, 46 | 'User-Agent': 'TrendFinder/1.0.0', 47 | 'Accept': '*/*' 48 | }, 49 | body: JSON.stringify({ 50 | model: this.config.modelName, 51 | messages, 52 | temperature: 0.3 53 | }) 54 | }); 55 | 56 | if (!response.ok) { 57 | const errorText = await response.text(); 58 | console.error('API Error Response:', errorText); 59 | throw new Error(`API request failed with status ${response.status}: ${errorText}`); 60 | } 61 | 62 | const data = await response.json(); 63 | console.log('API Response:', JSON.stringify(data, null, 2)); 64 | return data; 65 | } 66 | } 67 | 68 | class DeepSeekProvider implements APIProvider { 69 | private readonly API_BASE_URL = "https://api.deepseek.com/v1"; 70 | 71 | constructor( 72 | private readonly config: APIConfig 73 | ) { 74 | this.config.modelName = this.config.modelName || "deepseek-chat"; 75 | } 76 | 77 | async initialize(): Promise { 78 | // 验证必要的配置 79 | if (!this.config.apiKey) { 80 | throw new Error("DeepSeek API key is required"); 81 | } 82 | } 83 | 84 | async callAPI(messages: any[]): Promise { 85 | const response = await fetch(`${this.API_BASE_URL}/chat/completions`, { 86 | method: 'POST', 87 | headers: { 88 | 'Content-Type': 'application/json', 89 | 'Authorization': `Bearer ${this.config.apiKey}`, 90 | 'User-Agent': 'TrendFinder/1.0.0' 91 | }, 92 | body: JSON.stringify({ 93 | model: this.config.modelName, 94 | messages, 95 | temperature: 0.3 96 | }) 97 | }); 98 | 99 | if (!response.ok) { 100 | const errorText = await response.text(); 101 | console.error('API Error Response:', errorText); 102 | throw new Error(`API request failed with status ${response.status}: ${errorText}`); 103 | } 104 | 105 | const data = await response.json(); 106 | console.log('API Response:', JSON.stringify(data, null, 2)); 107 | return data; 108 | } 109 | } 110 | 111 | export interface ContentRankerConfig { 112 | provider: "dashscope" | "deepseek"; 113 | apiKey: string; 114 | modelName?: string; 115 | } 116 | 117 | export class ContentRanker { 118 | private apiProvider: APIProvider; 119 | private initialized = false; 120 | 121 | constructor(config: ContentRankerConfig) { 122 | const apiConfig: APIConfig = { 123 | apiKey: config.apiKey, 124 | modelName: config.modelName 125 | }; 126 | 127 | // 根据配置创建对应的provider 128 | if (config.provider === "deepseek") { 129 | this.apiProvider = new DeepSeekProvider(apiConfig); 130 | } else { 131 | this.apiProvider = new DashScopeProvider(apiConfig); 132 | } 133 | } 134 | 135 | private async ensureInitialized(): Promise { 136 | if (!this.initialized) { 137 | await this.apiProvider.initialize(); 138 | this.initialized = true; 139 | } 140 | } 141 | 142 | private async retryOperation(operation: () => Promise): Promise { 143 | for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { 144 | try { 145 | return await operation(); 146 | } catch (error) { 147 | console.error(`Attempt ${attempt} failed with error:`, error); 148 | if (attempt === MAX_RETRIES) { 149 | throw error; 150 | } 151 | await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY * attempt)); 152 | } 153 | } 154 | throw new Error("Operation failed after max retries"); 155 | } 156 | 157 | private getSystemPrompt(): string { 158 | return `你是一个专业的科技内容评估专家，特别专注于AI和前沿科技领域。你的任务是评估文章的重要性、创新性和技术价值。 159 | 160 | 评分标准（总分100分）： 161 | 162 | 1. 技术创新与前沿性 (35分) 163 | - 技术的创新程度和突破性 164 | - 是否涉及最新的技术发展和研究进展 165 | - 技术方案的可行性和实用价值 166 | - AI/科技领域的前沿趋势相关度 167 | 168 | 2. 技术深度与专业性 (25分) 169 | - 技术原理的解释深度 170 | - 技术实现细节的完整性 171 | - 专业术语使用的准确性 172 | - 技术方案的可实施性 173 | 174 | 3. 行业影响力 (20分) 175 | - 对AI/科技行业的潜在影响 176 | - 商业价值和市场潜力 177 | - 技术应用场景的广泛性 178 | - 解决实际问题的效果 179 | 180 | 4. 时效性与竞争格局 (20分) 181 | - 内容的时效性和新闻价值 182 | - 与竞品/竞争技术的对比分析 183 | - 技术发展趋势的预测 184 | - 市场竞争态势的分析 185 | 186 | 请仔细阅读文章，并按照以下格式返回评分结果： 187 | 文章ID: 分数 188 | 文章ID: 分数 189 | ... 190 | 191 | 注意事项： 192 | 1. 分数范围为0-100，精确到小数点后一位 193 | 2. 每篇文章占一行 194 | 3. 只返回ID和分数，不要有其他文字说明 195 | 4. 分数要有区分度，避免所有文章分数过于接近 196 | 5. 重点关注技术创新性和行业影响力 197 | 6. 对于纯市场新闻类文章，技术创新分数应相对较低 198 | 7. 对于深度技术文章，应在技术深度上给予更高权重`; 199 | } 200 | 201 | private getUserPrompt(contents: ScrapedContent[]): string { 202 | return contents.map(content => ( 203 | `文章ID: ${content.id}\n` + 204 | `标题: ${content.title}\n` + 205 | `发布时间: ${content.publishDate}\n` + 206 | `内容:\n${content.content}\n` + 207 | `---\n` 208 | )).join('\n'); 209 | } 210 | 211 | private parseRankingResult(result: string): RankResult[] { 212 | const lines = result.trim().split('\n'); 213 | return lines.map(line => { 214 | // Remove any potential Chinese characters and extra spaces 215 | const cleanedLine = line.replace(/文章ID[:：]?/i, '').trim(); 216 | 217 | // Match either space-separated or colon-separated formats 218 | const match = cleanedLine.match(/^(\S+)(?:[\s:：]+)(\d+(?:\.\d+)?)$/); 219 | 220 | if (!match) { 221 | throw new Error(`Invalid format for line: ${line}`); 222 | } 223 | 224 | const [, id, scoreStr] = match; 225 | const score = parseFloat(scoreStr); 226 | 227 | if (isNaN(score)) { 228 | throw new Error(`Invalid score format for line: ${line}`); 229 | } 230 | 231 | return { id, score }; 232 | }); 233 | } 234 | 235 | public async rankContents(contents: ScrapedContent[]): Promise { 236 | if (!contents.length) { 237 | return []; 238 | } 239 | 240 | await this.ensureInitialized(); 241 | 242 | return this.retryOperation(async () => { 243 | try { 244 | const messages = [ 245 | { 246 | role: "system", 247 | content: this.getSystemPrompt(), 248 | }, 249 | { 250 | role: "user", 251 | content: this.getUserPrompt(contents), 252 | }, 253 | ]; 254 | const response = await this.apiProvider.callAPI(messages); 255 | const result = response.choices?.[0]?.message?.content; 256 | 257 | if (!result) { 258 | throw new Error("未获取到有效的评分结果"); 259 | } 260 | 261 | return this.parseRankingResult(result); 262 | } catch (error) { 263 | console.error("API调用失败:", error); 264 | if (error instanceof Error) { 265 | console.error("详细错误信息:", error.message); 266 | console.error("错误堆栈:", error.stack); 267 | } 268 | throw error; 269 | } 270 | }); 271 | } 272 | 273 | public async rankContentsBatch( 274 | contents: ScrapedContent[], 275 | batchSize: number = 5 276 | ): Promise { 277 | const results: RankResult[] = []; 278 | 279 | for (let i = 0; i < contents.length; i += batchSize) { 280 | const batch = contents.slice(i, i + batchSize); 281 | const batchResults = await this.rankContents(batch); 282 | results.push(...batchResults); 283 | 284 | // 添加延迟以避免API限制 285 | if (i + batchSize < contents.length) { 286 | await new Promise(resolve => setTimeout(resolve, 1000)); 287 | } 288 | } 289 | 290 | return results; 291 | } 292 | } 293 | 294 | 295 | 296 | -------------------------------------------------------------------------------- /src/templates/hellogithub-weixin.ejs: -------------------------------------------------------------------------------- 1 | <% function formatNumber(num) { return num >= 1000 ? (num / 1000).toFixed(1) + 2 | 'k' : num; } %> 3 | 4 |
14 |
23 | GitHub AI 相关热门仓库 24 |
25 |
33 | 更新时间：<%= renderDate %> 34 |
35 | 36 |
45 |
55 | 📑 目录 56 |
57 | <% items.forEach((item, index) => { %> 58 |
69 | <%= index + 1 %> 83 |
92 | 93 | <%= item.name %> 94 | 95 | 106 | <%= item.title %> 107 | 108 |
109 |
110 | <% }); %> 111 |
112 | 113 | <% items.forEach((item, index) => { %> 114 |
124 |
134 | <%= index < 3 ? ['🏆','🥈','🥉'][index] : (index + 1) %> 149 | <%= item.name %> 150 |
151 | 152 |
166 | <%= item.title %> 167 |
168 | 169 |
183 | 191 | ⭐️ 201 | <%= formatNumber(item.totalStars) %> 202 | 203 | <% if (item.lastWeekStars > 0) { %> 204 | 216 | 📈 226 | 本周新增 <%= formatNumber(item.lastWeekStars) %> 227 | 228 | <% } %> 229 |
230 | 231 |
244 | 253 | 👤：<%= item.author %> 256 | 257 | 266 | 💻：<%= item.language %> 269 | 270 |
271 | 272 |
284 | <%= item.description %> 285 |
286 | 287 | <% if (item.tags && item.tags.length > 0) { %> 288 |
289 | <% item.tags.forEach(tag => { %> 290 | <%= tag %> 301 | <% }); %> 302 |
303 | <% } %> 304 | 305 |
312 |
313 | 📎 GitHub 314 |
315 |
316 | <%= item.url %> 317 |
318 | 319 | <% if (item.relatedUrls && item.relatedUrls.length > 0) { %> 320 |
321 |
322 | 🔗 相关链接 323 |
324 | <% item.relatedUrls.forEach(link => { %> 325 |
334 | <%= link.title %>： 337 | <%= link.url %> 340 |
341 | <% }); %> 342 |
343 | <% } %> 344 |
345 |
346 | <% }); %> 347 | 348 |
351 | 数据来源：HelloGitHub 352 |
353 |
354 | -------------------------------------------------------------------------------- /src/templates/aibench.ejs: -------------------------------------------------------------------------------- 1 |
12 |
21 | <%= title %> 22 |
23 | 24 |
32 | Last Updated: <%= updateTime %> 33 |
34 | 35 |
36 |
46 | 🏆 全局排名 Top 10 47 |
48 |
49 | 57 | 58 | 59 | 68 | 77 | 86 | 95 | 104 | 113 | 122 | 131 | 140 | 141 | 142 | 143 | <% globalTop10.forEach((model, index) => { %> 144 | 147 | 156 | 165 | 175 | 185 | 195 | 205 | 215 | 225 | 235 | 236 | <% }); %> 237 | 238 |
66 | Model 67 | 75 | Organization 76 | 84 | Global 85 | 93 | Reasoning 94 | 102 | Coding 103 | 111 | Math 112 | 120 | Data Analysis 121 | 129 | Language 130 | 138 | IF 139 |
154 | <%= model.name %> 155 | 163 | <%= model.organization || '-' %> 164 | 173 | <%= model.score.toFixed(2) %> 174 | 183 | <%= model.reasoning?.toFixed(2) || '-' %> 184 | 193 | <%= model.coding?.toFixed(2) || '-' %> 194 | 203 | <%= model.math?.toFixed(2) || '-' %> 204 | 213 | <%= model.dataAnalysis?.toFixed(2) || '-' %> 214 | 223 | <%= model.language?.toFixed(2) || '-' %> 224 | 233 | <%= model.if?.toFixed(2) || '-' %> 234 |
239 |
240 |
241 | 242 | <% categories.forEach(category => { %> 243 |
244 |
254 | <%= category.icon %> <%= category.name %> 255 |
256 |
257 | 265 | 266 | 267 | 276 | 285 | 294 | 295 | 296 | 297 | <% category.models.forEach((model, index) => { %> 298 | 301 | 310 | 319 | 329 | 330 | <% }); %> 331 | 332 |
274 | Model 275 | 283 | Organization 284 | 292 | Score 293 |
308 | <%= model.name %> 309 | 317 | <%= model.organization || '-' %> 318 | 327 | <%= model.score.toFixed(2) %> 328 |
333 |
334 |
335 | <% }); %> 336 | 337 |
347 |

Source: LiveBench.ai

348 |

349 | Note: Data is for reference only. Actual performance may vary depending on 350 | specific use cases. 351 |

352 |
353 |
354 | -------------------------------------------------------------------------------- /src/services/weixin-article.workflow.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ContentScraper, 3 | ScrapedContent, 4 | } from "../scrapers/interfaces/scraper.interface"; 5 | import { ContentSummarizer } from "../summarizer/interfaces/summarizer.interface"; 6 | import { ContentPublisher } from "../publishers/interfaces/publisher.interface"; 7 | import { WeixinPublisher } from "../publishers/weixin.publisher"; 8 | import { DeepseekAISummarizer } from "../summarizer/deepseek-ai.summarizer"; 9 | import { BarkNotifier } from "../utils/bark.notify"; 10 | import dotenv from "dotenv"; 11 | import { TwitterScraper } from "../scrapers/twitter.scraper"; 12 | import { FireCrawlScraper } from "../scrapers/fireCrawl.scraper"; 13 | import { getCronSources } from "../data-sources/getCronSources"; 14 | import cliProgress from "cli-progress"; 15 | import { WeixinTemplate } from "../render/interfaces/template.interface"; 16 | import { WeixinTemplateRenderer } from "../render/weixin/renderer"; 17 | import { AliWanX21ImageGenerator } from "../utils/gen-image/aliwanx2.1.image"; 18 | import { DeepseekAPI } from "../api/deepseek.api"; 19 | import { ContentRanker, RankResult } from "../utils/content-rank/content-ranker"; 20 | import { QianwenAISummarizer } from "../summarizer/qianwen-ai.summarizer"; 21 | import { ConfigManager } from "../utils/config/config-manager"; 22 | 23 | dotenv.config(); 24 | 25 | export class WeixinWorkflow { 26 | private scraper: Map; 27 | private summarizer: ContentSummarizer; 28 | private publisher: ContentPublisher; 29 | private notifier: BarkNotifier; 30 | private renderer: WeixinTemplateRenderer; 31 | private imageGenerator: AliWanX21ImageGenerator; 32 | private deepSeekClient: DeepseekAPI; 33 | private stats = { 34 | success: 0, 35 | failed: 0, 36 | contents: 0, 37 | }; 38 | 39 | constructor() { 40 | this.scraper = new Map(); 41 | this.scraper.set("fireCrawl", new FireCrawlScraper()); 42 | this.scraper.set("twitter", new TwitterScraper()); 43 | this.summarizer = new QianwenAISummarizer(); 44 | this.publisher = new WeixinPublisher(); 45 | this.notifier = new BarkNotifier(); 46 | this.renderer = new WeixinTemplateRenderer(); 47 | this.imageGenerator = new AliWanX21ImageGenerator(); 48 | this.deepSeekClient = new DeepseekAPI(); 49 | } 50 | 51 | async refresh(): Promise { 52 | await this.notifier.refresh(); 53 | await this.summarizer.refresh(); 54 | await this.publisher.refresh(); 55 | await this.scraper.get("fireCrawl")?.refresh(); 56 | await this.scraper.get("twitter")?.refresh(); 57 | await this.imageGenerator.refresh(); 58 | await this.deepSeekClient.refresh(); 59 | } 60 | 61 | private async scrapeSource( 62 | type: string, 63 | source: { identifier: string }, 64 | scraper: ContentScraper 65 | ): Promise { 66 | try { 67 | console.log(`[${type}] 抓取: ${source.identifier}`); 68 | const contents = await scraper.scrape(source.identifier); 69 | this.stats.success++; 70 | return contents; 71 | } catch (error) { 72 | this.stats.failed++; 73 | const message = error instanceof Error ? error.message : String(error); 74 | console.error(`[${type}] ${source.identifier} 抓取失败:`, message); 75 | await this.notifier.warning( 76 | `${type}抓取失败`, 77 | `源: ${source.identifier}\n错误: ${message}` 78 | ); 79 | return []; 80 | } 81 | } 82 | 83 | private async processContent(content: ScrapedContent): Promise { 84 | try { 85 | const summary = await this.summarizer.summarize(JSON.stringify(content)); 86 | content.title = summary.title; 87 | content.content = summary.content; 88 | content.score = summary.score; 89 | content.metadata.keywords = summary.keywords; 90 | } catch (error) { 91 | const message = error instanceof Error ? error.message : String(error); 92 | console.error(`[内容处理] ${content.id} 处理失败:`, message); 93 | await this.notifier.warning( 94 | "内容处理失败", 95 | `ID: ${content.id}\n保留原始内容` 96 | ); 97 | content.title = content.title || "无标题"; 98 | content.content = content.content || "内容处理失败"; 99 | content.metadata.keywords = content.metadata.keywords || []; 100 | } 101 | } 102 | 103 | async process(): Promise { 104 | try { 105 | console.log("=== 开始执行微信工作流 ==="); 106 | await this.notifier.info("工作流开始", "开始执行内容抓取和处理"); 107 | 108 | // 检查 API 额度 109 | // deepseek 110 | const deepSeekBalance = await this.deepSeekClient.getCNYBalance(); 111 | console.log("DeepSeek余额：", deepSeekBalance); 112 | if (deepSeekBalance < 1.0) { 113 | this.notifier.warning("DeepSeek", "余额小于一元"); 114 | } 115 | // 1. 获取数据源 116 | const sourceConfigs = await getCronSources(); 117 | 118 | const sourceIds = sourceConfigs.AI; 119 | const totalSources = 120 | sourceIds.firecrawl.length + sourceIds.twitter.length; 121 | console.log(`[数据源] 发现 ${totalSources} 个数据源`); 122 | 123 | const progress = new cliProgress.SingleBar( 124 | {}, 125 | cliProgress.Presets.shades_classic 126 | ); 127 | progress.start(totalSources, 0); 128 | let currentProgress = 0; 129 | 130 | // 2. 抓取内容 131 | const allContents: ScrapedContent[] = []; 132 | 133 | // FireCrawl sources 134 | const fireCrawlScraper = this.scraper.get("fireCrawl"); 135 | if (!fireCrawlScraper) throw new Error("FireCrawlScraper not found"); 136 | 137 | for (const source of sourceIds.firecrawl) { 138 | const contents = await this.scrapeSource( 139 | "FireCrawl", 140 | source, 141 | fireCrawlScraper 142 | ); 143 | allContents.push(...contents); 144 | progress.update(++currentProgress); 145 | } 146 | 147 | // Twitter sources 148 | const twitterScraper = this.scraper.get("twitter"); 149 | if (!twitterScraper) throw new Error("TwitterScraper not found"); 150 | 151 | for (const source of sourceIds.twitter) { 152 | const contents = await this.scrapeSource( 153 | "Twitter", 154 | source, 155 | twitterScraper 156 | ); 157 | allContents.push(...contents); 158 | progress.update(++currentProgress); 159 | } 160 | progress.stop(); 161 | 162 | this.stats.contents = allContents.length; 163 | if (this.stats.contents === 0) { 164 | const message = "未获取到任何内容"; 165 | console.error(`[工作流] ${message}`); 166 | await this.notifier.error("工作流终止", message); 167 | return; 168 | } 169 | 170 | // 3. 内容处理 171 | console.log(`\n[内容处理] 处理 ${allContents.length} 条内容`); 172 | const summaryProgress = new cliProgress.SingleBar( 173 | {}, 174 | cliProgress.Presets.shades_classic 175 | ); 176 | summaryProgress.start(allContents.length, 0); 177 | 178 | // 批量处理内容 179 | const batchSize = 1; 180 | for (let i = 0; i < allContents.length; i += batchSize) { 181 | const batch = allContents.slice(i, i + batchSize); 182 | await Promise.all( 183 | batch.map(async (content) => { 184 | await this.processContent(content); 185 | summaryProgress.increment(); 186 | }) 187 | ); 188 | } 189 | summaryProgress.stop(); 190 | 191 | 192 | 193 | // 4. 内容排序 194 | console.log(`[内容排序] 开始排序 ${allContents.length} 条内容`); 195 | let rankedContents: RankResult[] = []; 196 | try { 197 | const configManager = ConfigManager.getInstance(); 198 | const ranker = new ContentRanker({ 199 | provider: "deepseek", 200 | apiKey: await configManager.get("DEEPSEEK_API_KEY") as string, 201 | modelName: "deepseek-reasoner" 202 | }); 203 | rankedContents = await ranker.rankContents(allContents); 204 | 205 | console.log("内容排序完成", rankedContents); 206 | } catch (error) { 207 | console.error("内容排序失败:", error); 208 | await this.notifier.error("内容排序失败", "请检查API额度"); 209 | } 210 | 211 | // 分数更新 212 | console.log(`[分数更新] 开始更新 ${allContents.length} 条内容`); 213 | if (rankedContents.length > 0) { 214 | for (const content of allContents) { 215 | const rankedContent = rankedContents.find( 216 | (ranked) => ranked.id === content.id 217 | ); 218 | if (rankedContent) { 219 | content.score = rankedContent.score; 220 | } 221 | } 222 | } 223 | 224 | // 按照score排序 225 | allContents.sort((a, b) => b.score - a.score); 226 | 227 | // 取出前5条 228 | const topContents = allContents.slice(0, 5); 229 | 230 | // 4. 生成并发布 231 | console.log("\n[模板生成] 生成微信文章"); 232 | const templateData: WeixinTemplate[] = topContents.map((content) => ({ 233 | id: content.id, 234 | title: content.title, 235 | content: content.content, 236 | url: content.url, 237 | publishDate: content.publishDate, 238 | metadata: content.metadata, 239 | keywords: content.metadata.keywords, 240 | })); 241 | 242 | // 将所有标题总结成一个标题，然后让AI生成一个最具有吸引力的标题 243 | const summaryTitle = await this.summarizer.generateTitle( 244 | allContents.map((content) => content.title).join(" | ") 245 | ).then((title) => { 246 | // 限制标题长度 为 64 个字符 247 | return title.slice(0, 64); 248 | }); 249 | 250 | console.log(`[标题生成] 生成标题: ${summaryTitle}`); 251 | 252 | // 生成封面图片 253 | const taskId = await this.imageGenerator 254 | .generateImage("AI新闻日报的封面", "1440*768") 255 | .then((res) => res.output.task_id); 256 | console.log(`[封面图片] 封面图片生成任务ID: ${taskId}`); 257 | const imageUrl = await this.imageGenerator 258 | .waitForCompletion(taskId) 259 | .then((res) => res.results?.[0]?.url) 260 | .then((url) => { 261 | if (!url) { 262 | return ""; 263 | } 264 | return url; 265 | }); 266 | 267 | // 上传封面图片 268 | const mediaId = await this.publisher.uploadImage(imageUrl); 269 | 270 | const renderedTemplate = this.renderer.render(templateData); 271 | console.log("[发布] 发布到微信公众号"); 272 | const publishResult = await this.publisher.publish( 273 | renderedTemplate, 274 | `${new Date().toLocaleDateString()} AI速递 | ${summaryTitle}`, 275 | summaryTitle, 276 | mediaId 277 | ); 278 | 279 | // 5. 完成报告 280 | const summary = ` 281 | 工作流执行完成 282 | - 数据源: ${totalSources} 个 283 | - 成功: ${this.stats.success} 个 284 | - 失败: ${this.stats.failed} 个 285 | - 内容: ${this.stats.contents} 条 286 | - 发布: ${publishResult.status}`.trim(); 287 | 288 | console.log(`=== ${summary} ===`); 289 | 290 | if (this.stats.failed > 0) { 291 | await this.notifier.warning("工作流完成(部分失败)", summary); 292 | } else { 293 | await this.notifier.success("工作流完成", summary); 294 | } 295 | } catch (error) { 296 | const message = error instanceof Error ? error.message : String(error); 297 | console.error("[工作流] 执行失败:", message); 298 | await this.notifier.error("工作流失败", message); 299 | throw error; 300 | } 301 | } 302 | } 303 | -------------------------------------------------------------------------------- /src/api/livebench.api.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import { XunfeiAPI } from "./xunfei.api"; 3 | 4 | interface CategoryMapping { 5 | [key: string]: string[]; 6 | } 7 | 8 | interface ModelScore { 9 | [key: string]: number; 10 | } 11 | 12 | interface Metrics { 13 | [key: string]: number; 14 | } 15 | 16 | export interface ModelPerformance { 17 | metrics: Metrics; 18 | organization: string; 19 | } 20 | 21 | interface ModelInfo { 22 | scores: ModelScore; 23 | organization?: string; 24 | } 25 | 26 | interface ModelScores { 27 | [modelName: string]: ModelInfo; 28 | } 29 | 30 | export class LiveBenchAPI { 31 | private static readonly BASE_URL = "https://livebench.ai"; 32 | private categoryMapping: CategoryMapping = {}; 33 | private xunfeiAPI: XunfeiAPI; 34 | 35 | constructor() { 36 | this.xunfeiAPI = new XunfeiAPI(); 37 | } 38 | 39 | async refresh() { 40 | await this.xunfeiAPI.refresh(); 41 | } 42 | 43 | private async getModelOrganization(modelName: string): Promise { 44 | const maxRetries = 3; 45 | let retryCount = 0; 46 | 47 | while (retryCount < maxRetries) { 48 | try { 49 | const prompt = `请搜索这个AI模型名称 "${modelName}" 属于哪个组织或公司。只需要返回组织名称 不要多余输出！！ 请联网搜索！！！！`; 50 | const systemPrompt = `我给你一个大模型名字 请联网搜索所属组织，只需要返回组织名称 不要多余输出！,或者在下面的信息查找chatgpt-4o-latest-0903: https://openai.com/index/hello-gpt-4o/ (OpenAI)chatgpt-4o-latest-2025-01-29: https://help.openai.com/en/articles/9624314-model-release-notes (OpenAI)claude-3-5-sonnet-20240620: https://www.anthropic.com/news/claude-3-5-sonnet (Anthropic)claude-3-5-sonnet-20241022: https://www.anthropic.com/news/3-5-models-and-computer-use (Anthropic)claude-3-5-haiku-20241022: https://www.anthropic.com/claude/haiku (Anthropic)claude-3-haiku-20240307: https://www.anthropic.com/claude (Anthropic)claude-3-opus-20240229: https://www.anthropic.com/claude (Anthropic)claude-3-sonnet-20240229: https://www.anthropic.com/claude (Anthropic)command-r: https://docs.cohere.com/docs/models (Cohere)command-r-08-2024: https://docs.cohere.com/docs/models (Cohere)command-r-plus: https://docs.cohere.com/docs/models (Cohere)command-r-plus-04-2024: https://cohere.com/blog/command-r-plus-microsoft-azure (Cohere)command-r-plus-08-2024: https://docs.cohere.com/docs/models (Cohere)deepseek-coder-v2: https://huggingface.co/deepseek-ai/DeepSeek-V2 (DeepSeek)deepseek-v2.5: https://huggingface.co/deepseek-ai/DeepSeek-V2.5 (DeepSeek)deepseek-v2.5-1210: https://api-docs.deepseek.com/news/news1210 (DeepSeek)deepseek-v3: https://api-docs.deepseek.com/news/news1226 (DeepSeek)deepseek-r1: https://huggingface.co/deepseek-ai/DeepSeek-R1 (DeepSeek)deepseek-r1-distill-qwen-32b: https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B (DeepSeek)deepseek-r1-distill-llama-70b: https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B (DeepSeek)dracarys-72b-instruct: https://huggingface.co/abacusai/Dracarys-72B-Instruct (AbacusAI)dracarys-llama-3.1-70b-instruct: https://huggingface.co/abacusai/Dracarys-Llama-3.1-70B-Instruct (AbacusAI)dracarys2-72b-instruct: https://huggingface.co/abacusai/Dracarys2-72B-Instruct (AbacusAI)dracarys2-llama-3.1-70b-instruct: https://huggingface.co/abacusai/Dracarys2-Llama-3.1-70B-Instruct (AbacusAI)gemini-1.5-flash-001: https://console.cloud.google.com/vertex-ai/publishers/google/model-garden/gemini-1.5-flash-001 (Google)gemini-1.5-flash-002: https://developers.googleblog.com/en/updated-production-ready-gemini-models-reduced-15-pro-pricing-increased-rate-limits-and-more/ (Google)gemini-1.5-flash-8b-exp-0827: https://ai.google.dev/gemini-api/docs/models/experimental-models (Google)gemini-1.5-flash-8b-exp-0924: https://ai.google.dev/gemini-api/docs/models/gemini#gemini-1.5-flash-8b (Google)gemini-1.5-flash-api-0514: https://console.cloud.google.com/vertex-ai/publishers/google/model-garden/gemini-1.5-flash-preview-0514 (Google)gemini-1.5-flash-exp-0827: https://ai.google.dev/gemini-api/docs/models/experimental-models (Google)gemini-1.5-pro-001: https://console.cloud.google.com/vertex-ai/publishers/google/model-garden/gemini-1.5-pro-001 (Google)gemini-1.5-pro-002: https://developers.googleblog.com/en/updated-production-ready-gemini-models-reduced-15-pro-pricing-increased-rate-limits-and-more/ (Google)gemini-1.5-pro-api-0514: https://console.cloud.google.com/vertex-ai/publishers/google/model-garden/gemini-1.5-flash-preview-0514 (Google)gemini-1.5-pro-exp-0801: https://ai.google.dev/gemini-api/docs/models/experimental-models (Google)gemini-1.5-pro-exp-0827: https://ai.google.dev/gemini-api/docs/models/experimental-models (Google)gemini-2.0-flash-exp: https://cloud.google.com/vertex-ai/generative-ai/docs/gemini-v2 (Google)gemini-2.0-flash: https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/ (Google)gemini-2.0-flash-thinking-exp-1219: https://ai.google.dev/gemini-api/docs/thinking-mode (Google)gemini-2.0-flash-thinking-exp-01-21: https://ai.google.dev/gemini-api/docs/thinking-mode (Google)gemini-2.0-flash-lite-preview-02-05: https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/ (Google)gemini-2.0-pro-exp-02-05: https://blog.google/technology/google-deepmind/gemini-model-updates-february-2025/ (Google)gemini-exp-1114: https://ai.google.dev/gemini-api/docs/models/experimental-models (Google)gemini-exp-1121: https://ai.google.dev/gemini-api/docs/models/experimental-models (Google)gemini-exp-1206: https://ai.google.dev/gemini-api/docs/models/experimental-models (Google)gemma-2-27b-it: https://huggingface.co/google/gemma-2-27b (Google)gemma-2-9b-it: https://huggingface.co/google/gemma-2-9b (Google)gpt-3.5-turbo-0125: https://openai.com/index/new-embedding-models-and-api-updates/ (OpenAI)gpt-4-0125-preview: https://openai.com/index/new-models-and-developer-products-announced-at-devday/ (OpenAI)gpt-4-0613: https://openai.com/index/new-models-and-developer-products-announced-at-devday/ (OpenAI)gpt-4-turbo-2024-04-09: https://openai.com/index/new-models-and-developer-products-announced-at-devday/ (OpenAI)gpt-4o-2024-05-13: https://openai.com/index/hello-gpt-4o/ (OpenAI)gpt-4o-2024-08-06: https://openai.com/index/hello-gpt-4o/ (OpenAI)gpt-4o-2024-11-20: https://openai.com/index/hello-gpt-4o/ (OpenAI)gpt-4o-mini-2024-07-18: https://openai.com/index/hello-gpt-4o/ (OpenAI)grok-2: https://x.ai/blog/grok-2 (xAI)grok-2-mini: https://x.ai/blog/grok-2 (xAI)grok-2-1212: https://x.ai/blog/grok-1212 (xAI)lama-3.1-nemotron-70b-instruct: https://build.nvidia.com/nvidia/llama-3_1-nemotron-70b-instruct (NVIDIA)meta-llama-3.1-405b-instruct-turbo: https://www.together.ai/blog/meta-llama-3-1 (Meta)meta-llama-3.1-70b-instruct-turbo: https://www.together.ai/blog/meta-llama-3-1 (Meta)meta-llama-3.1-8b-instruct-turbo: https://www.together.ai/blog/meta-llama-3-1 (Meta)llama-3.3-70b-instruct-turbo: https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct (Meta)mistral-large-2407: https://huggingface.co/mistralai/Mistral-Large-Instruct-2407 (Mistral AI)mistral-large-2411: https://huggingface.co/mistralai/Mistral-Large-Instruct-2411 (Mistral AI)mistral-small-2402: https://docs.mistral.ai/getting-started/models/ (Mistral AI)mistral-small-2409: https://huggingface.co/mistralai/Mistral-Small-Instruct-2409 (Mistral AI)mistral-small-2501: https://mistral.ai/en/news/mistral-small-3 (Mistral AI)mixtral-8x22b-instruct-v0.1: https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1 (Mistral AI)o1-mini-2024-09-12: https://platform.openai.com/docs/guides/reasoning (OpenAI)o1-preview-2024-09-12: https://platform.openai.com/docs/guides/reasoning (OpenAI)o1-2024-12-17: https://openai.com/o1/ (OpenAI)o1-2024-12-17-high: https://openai.com/o1/ (OpenAI)o1-2024-12-17-low: https://openai.com/o1/ (OpenAI)o3-mini-2025-01-31-high: https://openai.com/index/openai-o3-mini/ (OpenAI)o3-mini-2025-01-31-low: https://openai.com/index/openai-o3-mini/ (OpenAI)o3-mini-2025-01-31-medium: https://openai.com/index/openai-o3-mini/ (OpenAI)o3-mini-2025-01-31: https://openai.com/index/openai-o3-mini/ (OpenAI)open-mistral-nemo: https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407 (Mistral AI)phi-3-medium-128k-instruct: https://huggingface.co/microsoft/Phi-3-medium-128k-instruct (Microsoft)phi-3-medium-4k-instruct: https://huggingface.co/microsoft/Phi-3-medium-4k-instruct (Microsoft)phi-3-mini-128k-instruct: https://huggingface.co/microsoft/Phi-3-mini-128k-instruct (Microsoft)phi-3-mini-4k-instruct: https://huggingface.co/microsoft/Phi-3-mini-4k-instruct (Microsoft)phi-3-small-128k-instruct: https://huggingface.co/microsoft/Phi-3-small-128k-instruct (Microsoft)phi-3-small-8k-instruct: https://huggingface.co/microsoft/Phi-3-small-8k-instruct (Microsoft)phi-3.5-mini-instruct: https://huggingface.co/microsoft/Phi-3.5-mini-instruct (Microsoft)phi-3.5-moe-instruct: https://huggingface.co/microsoft/Phi-3.5-MoE-instruct (Microsoft)phi-4: https://huggingface.co/microsoft/Phi-4 (Microsoft)qwen2.5-72b-instruct-turbo: https://huggingface.co/Qwen/Qwen2.5-72B-Instruct (Alibaba)qwen2.5-7b-instruct-turbo: https://huggingface.co/Qwen/Qwen2.5-7B-Instruct (Alibaba)qwen2.5-coder-32b-instruct: https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct (Alibaba)qwen2.5-max: https://qwenlm.github.io/blog/qwen2.5-max/ (Alibaba)step-2-16k-202411: https://www.stepfun.com/#step2 (StepFun)grok-beta: https://x.ai/blog/api (xAI)amazon.nova-lite-v1:0: https://aws.amazon.com/ai/generative-ai/nova/ (Amazon)amazon.nova-micro-v1:0: https://aws.amazon.com/ai/generative-ai/nova/ (Amazon)amazon.nova-pro-v1:0: https://aws.amazon.com/ai/generative-ai/nova/ (Amazon)qwq-32b-preview: https://huggingface.co/Qwen/QWQ-32B-Preview (Alibaba)olmo-2-1124-13b-instruct: https://huggingface.co/allenai/OLMo-2-1124-13B-Instruct (AllenAI)learnlm-1.5-pro-experimental: https://ai.google.dev/gemini-api/docs/learnlm (Google)`; 51 | const response = await this.xunfeiAPI.sendMessage( 52 | prompt, 53 | systemPrompt, 54 | true 55 | ); 56 | 57 | // 如果响应为空或无效，返回 Unknown 58 | if (!response || typeof response !== "string") { 59 | console.warn( 60 | `Invalid response for model ${modelName}, returning Unknown` 61 | ); 62 | return "Unknown"; 63 | } 64 | 65 | const cleanedResponse = response.trim(); 66 | // 如果清理后的响应为空，返回 Unknown 67 | return cleanedResponse || "Unknown"; 68 | } catch (error) { 69 | retryCount++; 70 | console.error( 71 | `Attempt ${retryCount}/${maxRetries} - Error getting organization for model ${modelName}:`, 72 | error 73 | ); 74 | 75 | // 如果是最后一次重试，返回 Unknown 76 | if (retryCount === maxRetries) { 77 | console.warn( 78 | `Max retries reached for model ${modelName}, returning Unknown` 79 | ); 80 | return "Unknown"; 81 | } 82 | 83 | // 等待一段时间后重试 84 | await new Promise((resolve) => setTimeout(resolve, 1000 * retryCount)); 85 | } 86 | } 87 | 88 | return "Unknown"; 89 | } 90 | 91 | private async fetchCategories(): Promise { 92 | try { 93 | const response = await axios.get( 94 | `${LiveBenchAPI.BASE_URL}/categories_2024_11_25.json` 95 | ); 96 | this.categoryMapping = response.data; 97 | } catch (error) { 98 | console.error("Error fetching categories:", error); 99 | throw new Error("Failed to fetch LiveBench categories"); 100 | } 101 | } 102 | 103 | private async fetchScores(): Promise { 104 | try { 105 | const response = await axios.get( 106 | `${LiveBenchAPI.BASE_URL}/table_2024_11_25.csv` 107 | ); 108 | const rows = response.data.trim().split("\n"); 109 | const headers = rows[0].split(","); 110 | const modelScores: ModelScores = {}; 111 | 112 | const totalRows = rows.length - 1; 113 | for (let i = 1; i < rows.length; i++) { 114 | const values = rows[i].split(","); 115 | const modelName = values[0]; 116 | const scores: ModelScore = {}; 117 | 118 | for (let j = 1; j < headers.length; j++) { 119 | const metric = headers[j]; 120 | const score = parseFloat(values[j]); 121 | if (!isNaN(score)) { 122 | scores[metric] = score; 123 | } 124 | } 125 | 126 | const organization = await this.getModelOrganization(modelName); 127 | modelScores[modelName] = { 128 | scores, 129 | organization, 130 | }; 131 | 132 | // 显示进度 133 | const progress = ((i / totalRows) * 100).toFixed(1); 134 | console.log(`Processing models: ${progress}% (${i}/${totalRows})`); 135 | } 136 | 137 | return modelScores; 138 | } catch (error) { 139 | console.error("Error fetching scores:", error); 140 | throw new Error("Failed to fetch LiveBench scores"); 141 | } 142 | } 143 | 144 | private calculateCategoryAverages( 145 | modelInfo: ModelInfo, 146 | categories: CategoryMapping 147 | ): ModelPerformance { 148 | const metrics: Metrics = {}; 149 | const scores = modelInfo.scores; 150 | 151 | // Calculate category averages 152 | for (const [category, categoryMetrics] of Object.entries(categories)) { 153 | const validScores = categoryMetrics 154 | .map((metric) => scores[metric]) 155 | .filter((score) => !isNaN(score)); 156 | 157 | if (validScores.length > 0) { 158 | const sum = validScores.reduce((acc, score) => acc + score, 0); 159 | metrics[`${category} Average`] = Number( 160 | (sum / validScores.length).toFixed(2) 161 | ); 162 | } else { 163 | metrics[`${category} Average`] = 0; 164 | } 165 | } 166 | 167 | const allScores = Object.values(scores).filter((score) => !isNaN(score)); 168 | if (allScores.length > 0) { 169 | const globalSum = allScores.reduce((acc, score) => acc + score, 0); 170 | metrics["Global Average"] = Number( 171 | (globalSum / allScores.length).toFixed(2) 172 | ); 173 | } else { 174 | metrics["Global Average"] = 0; 175 | } 176 | 177 | return { 178 | metrics, 179 | organization: modelInfo.organization || "Unknown", 180 | }; 181 | } 182 | 183 | public async getModelPerformance( 184 | modelName?: string 185 | ): Promise<{ [key: string]: ModelPerformance }> { 186 | try { 187 | await this.fetchCategories(); 188 | const modelScores = await this.fetchScores(); 189 | const result: { [key: string]: ModelPerformance } = {}; 190 | 191 | if (modelName) { 192 | if (modelScores[modelName]) { 193 | result[modelName] = this.calculateCategoryAverages( 194 | modelScores[modelName], 195 | this.categoryMapping 196 | ); 197 | } else { 198 | throw new Error(`Model ${modelName} not found`); 199 | } 200 | } else { 201 | for (const [model, scores] of Object.entries(modelScores)) { 202 | result[model] = this.calculateCategoryAverages( 203 | scores, 204 | this.categoryMapping 205 | ); 206 | } 207 | } 208 | 209 | return result; 210 | } catch (error) { 211 | console.error("Error in getModelPerformance:", error); 212 | throw error; 213 | } 214 | } 215 | 216 | public async getTopPerformers( 217 | limit: number = 5, 218 | sortBy: string = "Global Average" 219 | ): Promise<{ [key: string]: ModelPerformance }> { 220 | const allPerformance = await this.getModelPerformance(); 221 | 222 | const modelRankings = Object.entries(allPerformance) 223 | .map(([model, performance]) => { 224 | const avgScore = performance.metrics[sortBy] || 0; 225 | return { model, performance, avgScore }; 226 | }) 227 | .sort((a, b) => b.avgScore - a.avgScore) 228 | .slice(0, limit); 229 | 230 | const result: { [key: string]: ModelPerformance } = {}; 231 | modelRankings.forEach(({ model, performance }) => { 232 | result[model] = performance; 233 | }); 234 | 235 | return result; 236 | } 237 | } 238 | --------------------------------------------------------------------------------