├── oven ├── baker │ ├── .actiontrigger │ ├── .browserslistrc │ ├── babel.config.js │ ├── src │ │ ├── styles │ │ │ └── variables.less │ │ ├── plugins │ │ │ └── vuetify.js │ │ ├── router │ │ │ └── index.js │ │ ├── main.js │ │ ├── views │ │ │ └── BakeTweet.vue │ │ ├── components │ │ │ └── TranslationBox.vue │ │ └── App.vue │ ├── vue.config.js │ ├── .gitignore │ ├── .eslintrc.js │ ├── public │ │ └── index.html │ └── package.json ├── .gitignore ├── start.sh ├── configen.sh ├── requirements.txt ├── start_oven.py ├── .config.py ├── app.py ├── tests.py ├── api_models.py ├── oven.py ├── api.py └── tid_code.py ├── fridge ├── fridge-src │ ├── .actiontrigger │ ├── src │ │ ├── main │ │ │ ├── resources │ │ │ │ ├── application-test.properties │ │ │ │ ├── banner.txt │ │ │ │ ├── application.properties │ │ │ │ └── logback │ │ │ │ │ ├── logback-spring-dev.xml │ │ │ │ │ ├── logback-spring-test.xml │ │ │ │ │ └── logback-spring-prod.xml │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── enkanrec │ │ │ │ └── twitkitFridge │ │ │ │ ├── steady │ │ │ │ ├── yui │ │ │ │ │ ├── repository │ │ │ │ │ │ ├── EnkanTwitterRepository.java │ │ │ │ │ │ ├── EnkanConfigRepository.java │ │ │ │ │ │ ├── EnkanTaskRepository.java │ │ │ │ │ │ └── EnkanTranslateRepository.java │ │ │ │ │ └── entity │ │ │ │ │ │ ├── EnkanConfigEntity.java │ │ │ │ │ │ ├── EnkanTwitterEntity.java │ │ │ │ │ │ ├── EnkanTranslateEntity.java │ │ │ │ │ │ └── EnkanTaskEntity.java │ │ │ │ └── bean │ │ │ │ │ ├── DataSourceConfig.java │ │ │ │ │ └── JpaConfigYui.java │ │ │ │ ├── api │ │ │ │ ├── form │ │ │ │ │ ├── TidForm.java │ │ │ │ │ ├── NamespaceForm.java │ │ │ │ │ ├── CommentForm.java │ │ │ │ │ ├── TranslateForm.java │ │ │ │ │ ├── TaskCreationForm.java │ │ │ │ │ ├── BaseFridgeForm.java │ │ │ │ │ ├── BaseJsonWarp.java │ │ │ │ │ └── JsonDataFridgeForm.java │ │ │ │ ├── ws │ │ │ │ │ ├── DisconnectOutListener.java │ │ │ │ │ ├── ConnectInListener.java │ │ │ │ │ ├── WSClientPool.java │ │ │ │ │ ├── FridgeWSServer.java │ │ │ │ │ └── RequestListener.java │ │ │ │ ├── response │ │ │ │ │ ├── AffectedCountResponse.java │ │ │ │ │ └── StandardResponse.java │ │ │ │ └── rest │ │ │ │ │ ├── KVConfigController.java │ │ │ │ │ └── ExceptionControllerAdvice.java │ │ │ │ ├── monitor │ │ │ │ ├── BaseMonitor.java │ │ │ │ ├── PrometheusBean.java │ │ │ │ ├── InterceptorMonitor.java │ │ │ │ ├── BulkMonitor.java │ │ │ │ └── WebSocketMonitor.java │ │ │ │ ├── interceptor │ │ │ │ ├── InterceptorBean.java │ │ │ │ └── MonitorInterceptor.java │ │ │ │ ├── TwitkitFridgeApplication.java │ │ │ │ ├── GDP.java │ │ │ │ ├── service │ │ │ │ ├── kvConfig │ │ │ │ │ ├── KVConfigService.java │ │ │ │ │ └── KVConfigServiceImpl.java │ │ │ │ └── task │ │ │ │ │ └── TaskService.java │ │ │ │ └── util │ │ │ │ └── JsonUtil.java │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── enkanrec │ │ │ └── twitkitFridge │ │ │ ├── helper │ │ │ └── MvcHelper.java │ │ │ ├── wsClient │ │ │ └── WSTestClient.java │ │ │ └── KVConfigTest.java │ ├── .gitignore │ └── pom.xml ├── .gitignore ├── configen.sh ├── start.sh ├── build.sh ├── .application.properties ├── db_init.sh ├── test_init.sh ├── doc │ └── WebSocket使用方式说明.md ├── README.md └── schema │ ├── db_init.sql │ └── test_init.sql ├── koishi-app ├── src │ ├── .actiontrigger │ ├── index.ts │ ├── maid.ts │ ├── utils.ts │ ├── translator.ts │ ├── watcher.ts │ └── twitter.ts ├── .gitignore ├── start.sh ├── configen.sh ├── tsconfig.json ├── package.json ├── .koishi.config.js └── README.md ├── maid ├── .gitignore ├── start.sh ├── requirements.txt ├── configen.sh ├── select_target.sh ├── .config.py ├── start_maid.py ├── rsshub_client.py ├── api_models.py ├── twitter_client.py ├── README.md ├── maid_api.py └── twitter_util.py ├── .github └── workflows │ ├── oven.yml │ ├── fridge.yml │ └── koishi-app.yml ├── README_zh.md ├── .gitignore └── README.md /oven/baker/.actiontrigger: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fridge/fridge-src/.actiontrigger: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /koishi-app/src/.actiontrigger: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fridge/.gitignore: -------------------------------------------------------------------------------- 1 | /application.properties -------------------------------------------------------------------------------- /maid/.gitignore: -------------------------------------------------------------------------------- 1 | /config.py 2 | __pycache__ 3 | .vscode -------------------------------------------------------------------------------- /oven/.gitignore: -------------------------------------------------------------------------------- 1 | /config.py 2 | __pycache__ 3 | static/* -------------------------------------------------------------------------------- /oven/baker/.browserslistrc: -------------------------------------------------------------------------------- 1 | > 1% 2 | last 2 versions 3 | -------------------------------------------------------------------------------- /koishi-app/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | dist/ 3 | /koishi.config.js -------------------------------------------------------------------------------- /maid/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | basepath=$(cd `dirname $0`; pwd) 3 | python3 $basepath/start_maid.py 4 | -------------------------------------------------------------------------------- /oven/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | basepath=$(cd `dirname $0`; pwd) 3 | python3 $basepath/start_oven.py 4 | -------------------------------------------------------------------------------- /koishi-app/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | basepath=$(cd `dirname $0`; pwd) 3 | cd $basepath 4 | npm run dev 5 | -------------------------------------------------------------------------------- /maid/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | coloredlogs 3 | python-dateutil 4 | flask_restx 5 | waitress 6 | tweepy -------------------------------------------------------------------------------- /oven/baker/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: ["@vue/cli-plugin-babel/preset"] 3 | }; 4 | -------------------------------------------------------------------------------- /maid/configen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | basepath=$(cd `dirname $0`; pwd) 3 | cp $basepath/.config.py $basepath/config.py -------------------------------------------------------------------------------- /oven/baker/src/styles/variables.less: -------------------------------------------------------------------------------- 1 | @body-font-family: 'Microsoft YaHei', sans-serif; 2 | // TODO does not work -------------------------------------------------------------------------------- /oven/configen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | basepath=$(cd `dirname $0`; pwd) 3 | cp $basepath/.config.py $basepath/config.py -------------------------------------------------------------------------------- /oven/requirements.txt: -------------------------------------------------------------------------------- 1 | flask_restx 2 | python-dateutil 3 | pillow 4 | crc8 5 | coloredlogs 6 | waitress 7 | pychrome 8 | pdf2image -------------------------------------------------------------------------------- /koishi-app/configen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | basepath=$(cd `dirname $0`; pwd) 3 | cp $basepath/.koishi.config.js $basepath/koishi.config.js -------------------------------------------------------------------------------- /fridge/configen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | basepath=$(cd `dirname $0`; pwd) 3 | cp $basepath/.application.properties $basepath/application.properties -------------------------------------------------------------------------------- /fridge/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | basepath=$(cd `dirname $0`; pwd) 3 | java -jar "${basepath}/fridge-src/target/twitkit-fridge-0.0.1.jar" 4 | -------------------------------------------------------------------------------- /fridge/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | basepath=$(cd `dirname $0`; pwd) 3 | cd "${basepath}/fridge-src" 4 | mvn clean 5 | mvn package #-Dmaven.test.skip=true 6 | -------------------------------------------------------------------------------- /fridge/.application.properties: -------------------------------------------------------------------------------- 1 | # 数据源 2 | spring.datasource.yui.jdbc-url=jdbc:mysql://localhost:3306/fridge?characterEncoding=utf-8&useSSL=true&serverTimezone=Asia/Shanghai 3 | spring.datasource.yui.username=fridge 4 | spring.datasource.yui.password=fridge 5 | -------------------------------------------------------------------------------- /oven/baker/vue.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | publicPath: "./", 3 | devServer: { 4 | proxy: { 5 | "/api_proxy": { 6 | target: "http://localhost:5000/" 7 | } 8 | } 9 | }, 10 | transpileDependencies: ["vuetify"] 11 | }; 12 | -------------------------------------------------------------------------------- /oven/baker/src/plugins/vuetify.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | import Vuetify from "vuetify/lib"; 3 | import zhHans from "vuetify/es5/locale/zh-Hans"; 4 | 5 | Vue.use(Vuetify); 6 | 7 | export default new Vuetify({ 8 | lang: { 9 | locales: { zhHans }, 10 | current: "zh-Hans" 11 | } 12 | }); 13 | -------------------------------------------------------------------------------- /oven/baker/src/router/index.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | import VueRouter from "vue-router"; 3 | 4 | Vue.use(VueRouter); 5 | 6 | const routes = []; 7 | 8 | const router = new VueRouter({ 9 | mode: "history", 10 | base: process.env.BASE_URL, 11 | routes 12 | }); 13 | 14 | export default router; 15 | -------------------------------------------------------------------------------- /oven/start_oven.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from waitress import serve 4 | from app import app 5 | 6 | import coloredlogs 7 | import config 8 | import logging 9 | import threading 10 | 11 | 12 | if __name__ == '__main__': 13 | serve(app, host=config.API_SERVER_HOST, port=config.API_SERVER_PORT) 14 | -------------------------------------------------------------------------------- /oven/baker/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | dist 4 | 5 | # local env files 6 | .env.local 7 | .env.*.local 8 | 9 | # Log files 10 | npm-debug.log* 11 | yarn-debug.log* 12 | yarn-error.log* 13 | 14 | # Editor directories and files 15 | .idea 16 | .vscode 17 | *.suo 18 | *.ntvs* 19 | *.njsproj 20 | *.sln 21 | *.sw? 22 | -------------------------------------------------------------------------------- /oven/baker/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | import App from "./App.vue"; 3 | import router from "./router"; 4 | import vuetify from "./plugins/vuetify"; 5 | import "material-design-icons-iconfont/dist/material-design-icons.css"; 6 | 7 | Vue.config.productionTip = false; 8 | 9 | new Vue({ 10 | router, 11 | vuetify, 12 | render: h => h(App) 13 | }).$mount("#app"); 14 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/resources/application-test.properties: -------------------------------------------------------------------------------- 1 | # 日志路径 2 | enkanRec.logging.path=./test_log 3 | 4 | # 数据源 5 | spring.datasource.yui.driver-class-name=com.mysql.cj.jdbc.Driver 6 | spring.datasource.yui.jdbc-url=jdbc:mysql://localhost:3306/fridge_test?characterEncoding=utf-8&useSSL=true&serverTimezone=Asia/Shanghai 7 | spring.datasource.yui.username=fridge_test 8 | spring.datasource.yui.password=fridge_test 9 | -------------------------------------------------------------------------------- /oven/baker/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { 4 | node: true 5 | }, 6 | extends: ["plugin:vue/essential", "eslint:recommended", "@vue/prettier"], 7 | parserOptions: { 8 | parser: "babel-eslint" 9 | }, 10 | rules: { 11 | "no-console": process.env.NODE_ENV === "production" ? "error" : "off", 12 | "no-debugger": process.env.NODE_ENV === "production" ? "error" : "off" 13 | } 14 | }; 15 | -------------------------------------------------------------------------------- /maid/select_target.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | if [ -z "$1" ];then 3 | echo 'No target specified!' 4 | exit 1 5 | fi 6 | curl --request POST 'http://127.0.0.1:8220/api/db/kv/set' \ 7 | --header 'content-type: application/json' \ 8 | --data-raw '{ 9 | "taskId": "00000000-0000-0000-0000-000000000000", 10 | "forwardFrom": "manually-set", 11 | "timestamp": "'$(date +%FT%T.%3N%:z)'", 12 | "data": { 13 | "twid":"'$1'" 14 | } 15 | } 16 | ' 17 | -------------------------------------------------------------------------------- /koishi-app/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowSyntheticDefaultImports": true, 4 | "declaration": true, 5 | "outDir": "./dist", 6 | "removeComments": true, 7 | "strict": true, 8 | "strictNullChecks": false, 9 | // "module": "es2015", 10 | // "target": "es5", 11 | // "lib": [ 12 | // "dom", 13 | // "es5", 14 | // "es2015" 15 | // ] 16 | "noFallthroughCasesInSwitch": false 17 | } 18 | } -------------------------------------------------------------------------------- /koishi-app/src/index.ts: -------------------------------------------------------------------------------- 1 | export const name = 'translator-bot' 2 | import { Context } from 'koishi-core' 3 | import watcher from './watcher' 4 | import cmd from './cmd' 5 | import { config } from './utils' 6 | 7 | export { 8 | watcher 9 | } 10 | 11 | export function apply (ctx: Context, argv?: config) { 12 | // if (argv.ispro && ctx.app.version.coolqEdition !== "pro") argv.ispro = false 13 | argv.prefix = argv.prefix || "#" 14 | ctx.plugin(watcher, argv) 15 | ctx.plugin(cmd, argv) 16 | } -------------------------------------------------------------------------------- /maid/.config.py: -------------------------------------------------------------------------------- 1 | FRIDGE_API_BASE = 'http://127.0.0.1:8220/api' 2 | APP_API_BASE = 'http://127.0.0.1:8223/api' 3 | OVEN_API_BASE = 'http://127.0.0.1:8221/api' 4 | 5 | API_SERVER_HOST = '127.0.0.1' 6 | API_SERVER_PORT = 8222 7 | 8 | CONSUMER_KEY = "" 9 | CONSUMER_SECRET = "" 10 | ACCESS_TOKEN_KEY = "" 11 | ACCESS_TOKEN_SECRET = "" 12 | 13 | BILIBILI_UID = None 14 | UPDATE_INTERVAL = 60 15 | 16 | LOG_DEBUG = False 17 | LOG_FILE = None 18 | 19 | MAX_HISTORY_TWEETS = 10 20 | 21 | PUSH_RETWEETS = True 22 | PUSH_REPLIES = True -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/steady/yui/repository/EnkanTwitterRepository.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/29 4 | */ 5 | package com.enkanrec.twitkitFridge.steady.yui.repository; 6 | 7 | import com.enkanrec.twitkitFridge.steady.yui.entity.EnkanTwitterEntity; 8 | import org.springframework.data.jpa.repository.JpaRepository; 9 | 10 | public interface EnkanTwitterRepository extends JpaRepository { 11 | 12 | EnkanTwitterEntity findByTwitterUid(String twitterUid); 13 | } 14 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/form/TidForm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/3 4 | */ 5 | package com.enkanrec.twitkitFridge.api.form; 6 | 7 | import lombok.Data; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.ToString; 10 | 11 | import javax.validation.constraints.NotNull; 12 | 13 | /** 14 | * Class : TidForm 15 | * Usage : 16 | */ 17 | @Data 18 | @ToString 19 | @EqualsAndHashCode 20 | public class TidForm { 21 | 22 | /** 23 | * 任务id 24 | */ 25 | @NotNull 26 | private Integer tid; 27 | } 28 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/monitor/BaseMonitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/31 4 | */ 5 | package com.enkanrec.twitkitFridge.monitor; 6 | 7 | /** 8 | * Class : BaseMonitor 9 | * Usage : 10 | */ 11 | public abstract class BaseMonitor { 12 | 13 | public static final String TAG_HTTP_METHOD = "method"; 14 | 15 | public static final String TAG_HTTP_HANDLER = "handler"; 16 | 17 | public static final String TAG_HTTP_URI = "uri"; 18 | 19 | public static final String TAG_HTTP_STATUS_CODE = "code"; 20 | } 21 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | ______ __ __ __ 2 | /\__ _\ __ /\ \__ /\ \ __ /\ \__ 3 | \/_/\ \/ __ __ __ /\_\ \ \ ,_\ \ \ \/'\ /\_\ \ \ ,_\ 4 | \ \ \ /\ \/\ \/\ \ \/\ \ \ \ \/ \ \ , < \/\ \ \ \ \/ 5 | \ \ \ \ \ \_/ \_/ \ \ \ \ \ \ \_ \ \ \\`\ \ \ \ \ \ \_ 6 | \ \_\ \ \___x___/' \ \_\ \ \__\ \ \_\ \_\ \ \_\ \ \__\ 7 | \/_/ \/__//__/ \/_/ \/__/ \/_/\/_/ \/_/ \/__/ 8 | (Enkan Record - Twitkit fridge) 9 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/form/NamespaceForm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/7 4 | */ 5 | package com.enkanrec.twitkitFridge.api.form; 6 | 7 | import lombok.Data; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.ToString; 10 | 11 | import javax.validation.constraints.NotNull; 12 | 13 | /** 14 | * Class : NamespaceForm 15 | * Usage : 16 | */ 17 | @Data 18 | @ToString 19 | @EqualsAndHashCode 20 | public class NamespaceForm { 21 | 22 | /** 23 | * 任务id 24 | */ 25 | @NotNull 26 | private String namespace; 27 | } 28 | -------------------------------------------------------------------------------- /maid/start_maid.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from waitress import serve 4 | from maid_api import app 5 | from maid import Maid 6 | 7 | import coloredlogs 8 | import config 9 | import logging 10 | import threading 11 | 12 | 13 | if __name__ == '__main__': 14 | api_server_thread = threading.Thread( 15 | target=serve, args=(app,), kwargs={ 16 | 'host': config.API_SERVER_HOST, 17 | 'port': config.API_SERVER_PORT 18 | }, daemon=True) 19 | api_server_thread.start() 20 | 21 | maid_ = Maid() 22 | maid_.run() 23 | 24 | api_server_thread.join() 25 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/form/CommentForm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/3 4 | */ 5 | package com.enkanrec.twitkitFridge.api.form; 6 | 7 | import lombok.Data; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.ToString; 10 | 11 | import javax.validation.constraints.NotNull; 12 | 13 | /** 14 | * Class : CommentForm 15 | * Usage : 16 | */ 17 | @Data 18 | @ToString 19 | @EqualsAndHashCode 20 | public class CommentForm { 21 | 22 | /** 23 | * 任务id 24 | */ 25 | @NotNull 26 | private Integer tid; 27 | 28 | /** 29 | * 备注内容 30 | */ 31 | @NotNull 32 | private String comment; 33 | } 34 | -------------------------------------------------------------------------------- /fridge/db_init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | BASEPATH=$(cd `dirname $0`; pwd) 3 | if [ -x "/usr/bin/pwgen" ]; then 4 | PASS=`pwgen 16` 5 | else 6 | PASS=`uuidgen` 7 | fi 8 | if [ -n "$1" ]; then 9 | DB="$1" 10 | else 11 | DB="$USER" 12 | fi 13 | 14 | sudo mysql << MYSQL_SCRIPT 15 | CREATE DATABASE $DB; 16 | CREATE USER '$DB'@'localhost' IDENTIFIED BY '$PASS'; 17 | GRANT ALL PRIVILEGES ON $DB.* TO '$DB'@'localhost'; 18 | FLUSH PRIVILEGES; 19 | USE $DB; 20 | SOURCE $BASEPATH/schema/db_init.sql; 21 | MYSQL_SCRIPT 22 | 23 | echo "# 数据源 24 | spring.datasource.yui.jdbc-url=jdbc:mysql://localhost:3306/$DB?characterEncoding=utf-8&useSSL=true&serverTimezone=Asia/Shanghai 25 | spring.datasource.yui.username=$DB 26 | spring.datasource.yui.password=$PASS" -------------------------------------------------------------------------------- /.github/workflows/oven.yml: -------------------------------------------------------------------------------- 1 | name: Oven 2 | 3 | on: 4 | push: 5 | branches: 6 | - dev 7 | paths: 8 | - 'oven/baker/**' 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | - uses: actions/setup-node@v1 16 | with: 17 | node-version: '12.x' 18 | - name: Build with yarn 19 | run: | 20 | npm install yarn -g 21 | yarn --cwd oven/baker install 22 | yarn --cwd oven/baker build 23 | - name: Moving builds to artifacts directory 24 | run: | 25 | mkdir -p artifacts/oven 26 | cp -r oven/baker/dist artifacts/oven 27 | - uses: actions/upload-artifact@v1 28 | with: 29 | name: oven 30 | path: artifacts/oven 31 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/form/TranslateForm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/3 4 | */ 5 | package com.enkanrec.twitkitFridge.api.form; 6 | 7 | import lombok.Data; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.ToString; 10 | 11 | import javax.validation.constraints.NotNull; 12 | 13 | /** 14 | * Class : TranslateForm 15 | * Usage : 16 | */ 17 | @Data 18 | @ToString 19 | @EqualsAndHashCode 20 | public class TranslateForm { 21 | 22 | /** 23 | * 任务id 24 | */ 25 | @NotNull 26 | private Integer tid; 27 | 28 | /** 29 | * 翻译完毕文本 30 | */ 31 | @NotNull 32 | private String trans; 33 | 34 | /** 35 | * 烤推出图地址 36 | */ 37 | @NotNull 38 | private String img; 39 | } 40 | -------------------------------------------------------------------------------- /.github/workflows/fridge.yml: -------------------------------------------------------------------------------- 1 | name: Fridge 2 | 3 | on: 4 | push: 5 | branches: 6 | - dev 7 | paths: 8 | - 'fridge/fridge-src/**' 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Set up JDK 1.8 16 | uses: actions/setup-java@v1 17 | with: 18 | java-version: 1.8 19 | - name: Build with Maven 20 | run: mvn -B package --file fridge/fridge-src/pom.xml -Dmaven.test.skip=true 21 | - name: Moving builds to artifacts directory 22 | run: | 23 | mkdir -p artifacts/fridge 24 | cp fridge/fridge-src/target/*.jar artifacts/fridge 25 | - uses: actions/upload-artifact@v1 26 | with: 27 | name: fridge 28 | path: artifacts/fridge 29 | -------------------------------------------------------------------------------- /.github/workflows/koishi-app.yml: -------------------------------------------------------------------------------- 1 | name: koishi-app 2 | 3 | on: 4 | push: 5 | branches: 6 | - dev 7 | paths: 8 | - 'koishi-app/src/**' 9 | - 'koishi-app/tsconfig.json' 10 | 11 | jobs: 12 | build: 13 | 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - uses: actions/setup-node@v1 19 | with: 20 | node-version: '12.x' 21 | - name: Build with tsc 22 | run: | 23 | npm install --prefix koishi-app 24 | npm run build --prefix koishi-app 25 | - name: Moving builds to artifacts directory 26 | run: | 27 | mkdir -p artifacts/koishi-app 28 | cp -r koishi-app/dist artifacts/koishi-app 29 | - uses: actions/upload-artifact@v1 30 | with: 31 | name: koishi-app 32 | path: artifacts/koishi-app -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/steady/yui/repository/EnkanConfigRepository.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/30 4 | */ 5 | package com.enkanrec.twitkitFridge.steady.yui.repository; 6 | 7 | import com.enkanrec.twitkitFridge.steady.yui.entity.EnkanConfigEntity; 8 | import org.springframework.data.jpa.repository.JpaRepository; 9 | 10 | import java.util.List; 11 | 12 | /** 13 | * Class : EnkanConfigRepository 14 | * Usage : 15 | */ 16 | public interface EnkanConfigRepository extends JpaRepository { 17 | 18 | EnkanConfigEntity findByNamespaceAndConfigKey(String namespace, String key); 19 | 20 | List findAllByNamespaceAndConfigKey(String namespace, String key); 21 | 22 | List findAllByNamespace(String namespace); 23 | 24 | void deleteAllByNamespace(String namespace); 25 | } 26 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/monitor/PrometheusBean.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/31 4 | */ 5 | package com.enkanrec.twitkitFridge.monitor; 6 | 7 | import io.prometheus.client.exporter.MetricsServlet; 8 | import io.prometheus.client.hotspot.DefaultExports; 9 | import org.springframework.boot.web.servlet.ServletRegistrationBean; 10 | import org.springframework.context.annotation.Bean; 11 | import org.springframework.context.annotation.Configuration; 12 | 13 | /** 14 | * Class : PrometheusBean 15 | * Usage : 注册Prometheus监控Bean 16 | */ 17 | @Configuration 18 | public class PrometheusBean { 19 | 20 | @SuppressWarnings({"rawtypes", "unchecked"}) 21 | @Bean 22 | public ServletRegistrationBean prometheusRegistrationBean(){ 23 | DefaultExports.initialize(); 24 | return new ServletRegistrationBean(new MetricsServlet(), "/metrics"); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /oven/baker/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | <%= htmlWebpackPlugin.options.title %> 9 | 10 | 11 | 12 | 13 | 16 |
17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /README_zh.md: -------------------------------------------------------------------------------- 1 | # TwitKit 2 | 3 | [README](README.md) | [中文文档](README_zh.md) 4 | 5 | TwitKit是一个面向推文翻译工作的开源套件。基于微服务架构设计，组件间均使用REST API通信。 6 | 7 | ### 主要贡献者 8 | 9 | [k9yyy](https://github.com/k9yyy)：oven、maid组件 10 | 11 | [Rinka](https://github.com/rinkako)：fridge组件 12 | 13 | [MisakaHonoka](https://github.com/y2361547758)：koishi-app组件 14 | 15 | [Inactive Virus](https://github.com/inactive-virus)：项目管理和shell scripts 16 | 17 | ## 后端组件 18 | 19 | ### fridge 20 | 21 | 数据库微服务，提供推文内容与元信息的存储服务。 22 | 现支持mysql/mariadb存储后端。 23 | 24 | ### oven 25 | 26 | 图像处理微服务，提供文字→图像的转换，同时附加可识别元素到图像中，供检测发布使用。 27 | 28 | ### maid 29 | 30 | 更新检测微服务，通过Twitter API检测推文更新； 31 | 同时提供检测Bilibili动态是否发布了成品图的功能。 32 | 33 | ## 前端组件 34 | 35 | ### koishi-app 36 | 37 | QQBot前端，基于[koishi框架](https://koishi.js.org)实现与[酷Q机器人](https://cqp.cc/)的通信，提供QQ上的更新通知和译文提交、图片返回服务。 38 | 39 | ## 安装 40 | 41 | 请参阅各子文件夹的`README.md`；仓库中提供了用于加快部署速度的shell script，若需使用，请为相应用户赋予sudo权限，并建议在部署完成后撤销。 42 | -------------------------------------------------------------------------------- /oven/.config.py: -------------------------------------------------------------------------------- 1 | ## 渲染设置 2 | CHROME_REMOTE_DEBUGGING_URL = 'http://127.0.0.1:9222' 3 | DEFAULT_PPI = 144 4 | ZH_FONT = 'Noto Sans CJK SC, Segoe UI Symbol' 5 | JA_FONT = 'Noto Sans CJK JP, Segoe UI Symbol' 6 | LOAD_TIME_LIMIT = 60 7 | 8 | # 监听设置 9 | API_SERVER_HOST = '127.0.0.1' 10 | API_SERVER_PORT = 8221 11 | 12 | ## 二维码设置 13 | ## （以下定位数值对应的均为相对于浏览器渲染时的像素值） 14 | 15 | # 二维码宽高 16 | TID_CODE_WIDTH = 10 17 | TID_CODE_HEIGHT = 10 18 | 19 | # 二维码位置（设为负数从右/下起） 20 | TID_CODE_POS_X = -2 21 | TID_CODE_POS_Y = -2 22 | 23 | # 二维码key（多个实例产生的图在同一账号发布时避免冲突用） 24 | TID_CODE_KEY = 0 25 | 26 | ## URL设置 27 | EXT_STATIC_BASE_URL = 'https://example.com/timg' 28 | INT_BASE_URL = f'http://127.0.0.1:{API_SERVER_PORT}' 29 | 30 | ## Fridge API URL 31 | FRIDGE_API_BASE = 'http://127.0.0.1:8220/api' 32 | 33 | ## Maid API URL 34 | MAID_API_BASE = 'http://127.0.0.1:8222/api' 35 | 36 | ## 日志设置 37 | LOG_DEBUG = False 38 | LOG_FILE = None 39 | 40 | ## 一个遗留问题，未来将移除 41 | VIEWPORT_WIDTH = 480 -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/interceptor/InterceptorBean.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/31 4 | */ 5 | package com.enkanrec.twitkitFridge.interceptor; 6 | 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.web.servlet.config.annotation.InterceptorRegistry; 10 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; 11 | 12 | /** 13 | * Class : InterceptorBean 14 | * Usage : 15 | */ 16 | @Configuration 17 | public class InterceptorBean implements WebMvcConfigurer { 18 | 19 | @Bean 20 | public MonitorInterceptor monitorInterceptorBean() { 21 | return new MonitorInterceptor(); 22 | } 23 | 24 | @Override 25 | public void addInterceptors(InterceptorRegistry registry) { 26 | registry.addInterceptor(this.monitorInterceptorBean()).addPathPatterns("/api/**"); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /fridge/test_init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | if [ -z "$1" ]; then 3 | BASEPATH=$(cd `dirname $0`; pwd) 4 | sudo mysql << MYSQL_SCRIPT 5 | DROP DATABASE IF EXISTS fridge_test; 6 | DROP USER IF EXISTS 'fridge_test'@'localhost'; 7 | CREATE DATABASE fridge_test; 8 | CREATE USER 'fridge_test'@'localhost' IDENTIFIED BY 'fridge_test'; 9 | GRANT ALL PRIVILEGES ON fridge_test.* TO 'fridge_test'@'localhost'; 10 | FLUSH PRIVILEGES; 11 | USE fridge_test; 12 | SOURCE $BASEPATH/schema/test_init.sql; 13 | MYSQL_SCRIPT 14 | echo "Test database created." 15 | echo "Database: fridge_test" 16 | echo "Username: fridge_test" 17 | echo "Password: fridge_test" 18 | 19 | elif [ "$1" = "--remove" ]; then 20 | BASEPATH=$(cd `dirname $0`; pwd) 21 | sudo mysql << MYSQL_SCRIPT 22 | DROP DATABASE IF EXISTS fridge_test; 23 | DROP USER IF EXISTS 'fridge_test'@'localhost'; 24 | MYSQL_SCRIPT 25 | echo "Test database removed." 26 | 27 | else 28 | echo "Wrong argument." 29 | exit 1 30 | fi 31 | -------------------------------------------------------------------------------- /fridge/doc/WebSocket使用方式说明.md: -------------------------------------------------------------------------------- 1 | ## WebSocket使用方式说明 2 | 3 | #### 协议 4 | 使用SocketIO协议来完成通讯。建立连接后，请求和返回都通过长连接进行。 5 | 6 | #### 请求 7 | 请求体是一个JSON，固定发送到事件`twitkit_request`，事件内容是一个JSON，结构： 8 | 9 | |名称|格式|必传|备注| 10 | |:--|:--|:--|:--| 11 | |forwardFrom|string|Y|一个字符串，标定是哪个服务调用了此接口，如`twitkit-app`| 12 | |timestamp|string|Y|请求发出时间戳，ISO8601格式，标准样例`2020-01-29T14:23:23.233+08:00`| 13 | |of|string|Y|服务类型，即要请求的服务在RestAPI清单里的URI的第**3**个部分| 14 | |command|string|Y|服务URI，即要请求的服务在RestAPI清单里的URI的第**4**个部分| 15 | |data|dict|Y|请求参数包，即要请求的服务在RestAPI清单里的**data**字段| 16 | 17 | 以`/api/db/task/get`为例，其请求体的一个例子是： 18 | 19 | ```json 20 | { 21 | "forwardFrom": "twitkit-app", 22 | "timestamp": "2020-01-29T14:40:00.000+08:00", 23 | "of": "task", 24 | "command": "get", 25 | "data": { 26 | "tid": 1001 27 | } 28 | } 29 | ``` 30 | 31 | 此JSON作为事件体，通过SocketIO协议发送到DB模块，代码形如：`socketio.emit("twitkit_request", "请求体JSON所dump得到的字符串")` 32 | 33 | #### 响应 34 | 响应体是一个JSON字符串，固定发送到事件`twitkit_response`，客户端通过订阅这个事件来监听返回。一个客户端只会收到属于它自己的返回，不会收到其它客户端的返回。 35 | 36 | 响应体结构和RestAPI的响应结构完成一致。 -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/steady/bean/DataSourceConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/30 4 | */ 5 | package com.enkanrec.twitkitFridge.steady.bean; 6 | 7 | import org.springframework.beans.factory.annotation.Qualifier; 8 | import org.springframework.boot.context.properties.ConfigurationProperties; 9 | import org.springframework.boot.jdbc.DataSourceBuilder; 10 | import org.springframework.context.annotation.Bean; 11 | import org.springframework.context.annotation.Configuration; 12 | import org.springframework.context.annotation.Primary; 13 | 14 | import javax.sql.DataSource; 15 | 16 | /** 17 | * Class : DataSourceConfig 18 | * Usage : 配置数据源在Spring的托管Bean 19 | */ 20 | @Configuration 21 | public class DataSourceConfig { 22 | 23 | @Primary 24 | @Bean(name = "yuiDataSource") 25 | @Qualifier("yuiDataSource") 26 | @ConfigurationProperties(prefix = "spring.datasource.yui") 27 | public DataSource yuiDataSource() { 28 | return DataSourceBuilder.create().build(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/TwitkitFridgeApplication.java: -------------------------------------------------------------------------------- 1 | package com.enkanrec.twitkitFridge; 2 | 3 | import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; 7 | import org.springframework.context.annotation.Bean; 8 | 9 | import java.time.format.DateTimeFormatter; 10 | import java.util.TimeZone; 11 | 12 | @SpringBootApplication 13 | public class TwitkitFridgeApplication { 14 | 15 | public static void main(String[] args) { 16 | SpringApplication.run(TwitkitFridgeApplication.class, args); 17 | } 18 | 19 | @Bean 20 | public Jackson2ObjectMapperBuilderCustomizer jsonCustomizer() { 21 | return builder -> builder.timeZone(TimeZone.getTimeZone("Asia/Shanghai")) 22 | .serializers(new LocalDateTimeSerializer(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/form/TaskCreationForm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/5 4 | */ 5 | package com.enkanrec.twitkitFridge.api.form; 6 | 7 | import lombok.Data; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.ToString; 10 | 11 | import javax.validation.constraints.NotNull; 12 | 13 | /** 14 | * Class : TaskCreationForm 15 | * Usage : 16 | */ 17 | @Data 18 | @ToString 19 | @EqualsAndHashCode 20 | public class TaskCreationForm { 21 | private String url; 22 | 23 | @NotNull 24 | private String content; 25 | 26 | @NotNull 27 | private String media; 28 | 29 | @NotNull 30 | private String pub_date; 31 | 32 | @NotNull 33 | private String status_id; 34 | 35 | private Integer ref = null; 36 | 37 | private String extra = null; 38 | 39 | @NotNull 40 | private String user_twitter_uid; 41 | 42 | @NotNull 43 | private String user_name; 44 | 45 | @NotNull 46 | private String user_display; 47 | 48 | @NotNull 49 | private String user_avatar; 50 | } 51 | -------------------------------------------------------------------------------- /koishi-app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@enkanrec/koishi-plugin-twitkit-app", 3 | "version": "1.1.0", 4 | "description": "Twitter translator tools plugin for koishi", 5 | "main": "dist/index.js", 6 | "scripts": { 7 | "test": "tsc -b", 8 | "build": "tsc -b", 9 | "dev": "koishi run -- -r ts-node/register", 10 | "prod": "koishi run" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "git@github.com:EnkanRec/TwitKit.git" 15 | }, 16 | "keywords": [ 17 | "bot", 18 | "qqbot", 19 | "twitter", 20 | "koishi-plugin", 21 | "translator-tools" 22 | ], 23 | "author": "MisakaHonoka", 24 | "license": "MIT", 25 | "dependencies": { 26 | "axios": "^0.19.2", 27 | "koishi": "^1.12.0", 28 | "koishi-core": "^1.12.0", 29 | "koishi-plugin-common": "^2.2.0", 30 | "koishi-utils": "^1.1.0" 31 | }, 32 | "devDependencies": { 33 | "@types/node": "^13.13.0", 34 | "@types/ws": "^7.2.4", 35 | "ts-node": "^8.6.2", 36 | "typescript": "^3.8.3" 37 | }, 38 | "publishConfig": { 39 | "registry": "https://npm.pkg.github.com/" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /koishi-app/.koishi.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logLevel: 2, 3 | type: "ws", 4 | server: "ws://localhost:6700", 5 | selfId: undefined, 6 | nickname: '', 7 | commandPrefix: '#', 8 | plugins: [ 9 | ["common", { 10 | admin: false, 11 | broadcast: false, 12 | contextify: false, 13 | echo: false, 14 | exec: false, 15 | exit: false, 16 | info: false, 17 | help: true 18 | }], 19 | [".", { 20 | prefix: '#', // 快捷指令前缀 21 | ispro: false, // 是否发图 22 | cmd: { 23 | host: { 24 | store: "http://localhost:8220", 25 | translator: "http://localhost:8221", 26 | maid: "http://localhost:8222", 27 | }, 28 | group: [], // 监听命令的群组，留空监听所有人 29 | private: false, // 是否允许私聊上班 30 | friend: false, // 是否允许好友上班 31 | cut: 8 // 消息预览截断长度 32 | }, 33 | watcher: { 34 | port: 8223, // 监听端口 35 | target: { // 推送目标 36 | discuss: [], 37 | private: [], 38 | group: [] 39 | } 40 | } 41 | }], 42 | ] 43 | } -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/ws/DisconnectOutListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/8 4 | */ 5 | package com.enkanrec.twitkitFridge.api.ws; 6 | 7 | import com.corundumstudio.socketio.SocketIOClient; 8 | import com.corundumstudio.socketio.listener.DisconnectListener; 9 | import com.enkanrec.twitkitFridge.monitor.WebSocketMonitor; 10 | import lombok.extern.slf4j.Slf4j; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.stereotype.Component; 13 | 14 | /** 15 | * Class : DisconnectOutListener 16 | * Usage : 17 | */ 18 | @Slf4j 19 | @Component 20 | public class DisconnectOutListener implements DisconnectListener { 21 | 22 | @Autowired 23 | private WebSocketMonitor monitor; 24 | 25 | @Override 26 | public void onDisconnect(SocketIOClient client) { 27 | log.info("A participant disconnected from Fridge-Server: " + client.getSessionId()); 28 | WSClientPool.remove(client.getSessionId().toString()); 29 | this.monitor.disconnectOutCounter.inc(); 30 | this.monitor.activeConnectionCounter.dec(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/GDP.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/8 4 | */ 5 | package com.enkanrec.twitkitFridge; 6 | 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.boot.ApplicationArguments; 9 | import org.springframework.core.annotation.Order; 10 | import org.springframework.stereotype.Component; 11 | 12 | import javax.annotation.PostConstruct; 13 | import java.util.Set; 14 | 15 | /** 16 | * Class : GDP 17 | * Usage : Global data package 18 | */ 19 | @Component 20 | @Order(0) 21 | public class GDP { 22 | 23 | @Autowired 24 | private ApplicationArguments appArguments; 25 | 26 | public static boolean EnableWebSocket = false; 27 | 28 | @PostConstruct 29 | private void init() { 30 | Set rawTags = this.appArguments.getOptionNames(); 31 | if (rawTags != null) { 32 | rawTags.forEach(tag -> { 33 | switch (tag) { 34 | case "enable-websocket": 35 | GDP.EnableWebSocket = true; 36 | break; 37 | } 38 | }); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/ws/ConnectInListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/8 4 | */ 5 | package com.enkanrec.twitkitFridge.api.ws; 6 | 7 | import com.corundumstudio.socketio.SocketIOClient; 8 | import com.corundumstudio.socketio.listener.ConnectListener; 9 | import com.enkanrec.twitkitFridge.monitor.WebSocketMonitor; 10 | import lombok.extern.slf4j.Slf4j; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.stereotype.Component; 13 | 14 | /** 15 | * Class : ConnectInListener 16 | * Usage : 17 | */ 18 | @Slf4j 19 | @Component 20 | public class ConnectInListener implements ConnectListener { 21 | 22 | @Autowired 23 | private WebSocketMonitor monitor; 24 | 25 | @Override 26 | public void onConnect(SocketIOClient client) { 27 | log.info(String.format("A client connected to Fridge-Server: %s (%s | %s)", 28 | client.getSessionId(), client.getRemoteAddress().toString(), client.getTransport().getValue())); 29 | WSClientPool.add(client.getSessionId().toString(), client); 30 | this.monitor.connectInCounter.inc(); 31 | this.monitor.activeConnectionCounter.inc(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/monitor/InterceptorMonitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/31 4 | */ 5 | package com.enkanrec.twitkitFridge.monitor; 6 | 7 | import io.prometheus.client.Counter; 8 | import io.prometheus.client.Summary; 9 | import org.springframework.stereotype.Component; 10 | 11 | /** 12 | * Class : InterceptorMonitor 13 | * Usage : HTTP拦截器指标数据的包装 14 | */ 15 | @Component 16 | public class InterceptorMonitor extends BaseMonitor { 17 | 18 | public final Summary responseTimeInMs = Summary 19 | .build() 20 | .name("twitkit_fridge_http_response_time_milliseconds") 21 | .labelNames(BaseMonitor.TAG_HTTP_METHOD, BaseMonitor.TAG_HTTP_HANDLER, 22 | BaseMonitor.TAG_HTTP_URI, BaseMonitor.TAG_HTTP_STATUS_CODE) 23 | .help("HTTP Request completed time in milliseconds") 24 | .register(); 25 | 26 | public final Counter exceptionCounter = Counter 27 | .build() 28 | .name("twitkit_fridge_http_exception_count") 29 | .labelNames(BaseMonitor.TAG_HTTP_METHOD, BaseMonitor.TAG_HTTP_URI) 30 | .help("HTTP Request exception counter") 31 | .register(); 32 | } 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/** 5 | !**/src/test/** 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | 30 | ### VS Code ### 31 | .vscode/ 32 | ### Maven template 33 | target/ 34 | pom.xml.tag 35 | pom.xml.releaseBackup 36 | pom.xml.versionsBackup 37 | pom.xml.next 38 | release.properties 39 | dependency-reduced-pom.xml 40 | buildNumber.properties 41 | .mvn/timing.properties 42 | # https://github.com/takari/maven-wrapper#usage-without-binary-jar 43 | .mvn/wrapper/maven-wrapper.jar 44 | 45 | ### Java template 46 | # Compiled class file 47 | *.class 48 | 49 | # Log file 50 | *.log 51 | 52 | # BlueJ files 53 | *.ctxt 54 | 55 | # Mobile Tools for Java (J2ME) 56 | .mtj.tmp/ 57 | 58 | # Package Files # 59 | *.jar 60 | *.war 61 | *.nar 62 | *.ear 63 | *.zip 64 | *.tar.gz 65 | *.rar 66 | 67 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 68 | hs_err_pid* 69 | 70 | /.mvn/ 71 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/response/AffectedCountResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/5 4 | */ 5 | package com.enkanrec.twitkitFridge.api.response; 6 | 7 | import com.fasterxml.jackson.annotation.JsonIgnore; 8 | import lombok.Data; 9 | import lombok.EqualsAndHashCode; 10 | import lombok.ToString; 11 | 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | 15 | /** 16 | * Class : AffectedCountResponse 17 | * Usage : 18 | */ 19 | @Data 20 | @ToString(callSuper = true) 21 | @EqualsAndHashCode(callSuper = true) 22 | public class AffectedCountResponse extends StandardResponse { 23 | 24 | private static final String KEY_AFFECTED_COUNT = "affected"; 25 | 26 | @JsonIgnore 27 | private int affectedCount; 28 | 29 | public static AffectedCountResponse of(int affected) { 30 | AffectedCountResponse acr = new AffectedCountResponse(); 31 | acr.affectedCount = affected; 32 | Map payload = new HashMap<>(); 33 | payload.put(KEY_AFFECTED_COUNT, affected); 34 | acr.setData(payload); 35 | acr.setCode(StandardResponse.CODE_SUCCESS); 36 | acr.setMessage(StandardResponse.MESSAGE_SUCCESS); 37 | return acr; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/service/kvConfig/KVConfigService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/30 4 | */ 5 | package com.enkanrec.twitkitFridge.service.kvConfig; 6 | 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.stereotype.Service; 9 | 10 | import java.util.Collection; 11 | import java.util.Map; 12 | 13 | /** 14 | * Class : ConfigService 15 | * Usage : 16 | */ 17 | public interface KVConfigService { 18 | 19 | void setOneDefault(String key, String value) throws Exception; 20 | 21 | String getOneDefault(String key); 22 | 23 | void setManyDefault(Map configs) throws Exception; 24 | 25 | Map getManyDefault(Collection keys); 26 | 27 | void setOne(String namespace, String key, String value) throws Exception; 28 | 29 | String getOne(String namespace, String key); 30 | 31 | void setMany(String namespace, Map configs) throws Exception; 32 | 33 | Map getMany(String namespace, Collection keys); 34 | 35 | Map getAll(String namespace); 36 | 37 | void clearNamespace(String namespace); 38 | 39 | Map getAll(); 40 | } 41 | -------------------------------------------------------------------------------- /fridge/fridge-src/.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/** 5 | !**/src/test/** 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | 30 | ### VS Code ### 31 | .vscode/ 32 | ### Maven template 33 | target/ 34 | pom.xml.tag 35 | pom.xml.releaseBackup 36 | pom.xml.versionsBackup 37 | pom.xml.next 38 | release.properties 39 | dependency-reduced-pom.xml 40 | buildNumber.properties 41 | .mvn/timing.properties 42 | # https://github.com/takari/maven-wrapper#usage-without-binary-jar 43 | .mvn/wrapper/maven-wrapper.jar 44 | 45 | ### Java template 46 | # Compiled class file 47 | *.class 48 | 49 | # Log file 50 | *.log 51 | 52 | # BlueJ files 53 | *.ctxt 54 | 55 | # Mobile Tools for Java (J2ME) 56 | .mtj.tmp/ 57 | 58 | # Package Files # 59 | *.jar 60 | *.war 61 | *.nar 62 | *.ear 63 | *.zip 64 | *.tar.gz 65 | *.rar 66 | 67 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 68 | hs_err_pid* 69 | 70 | /.mvn/ 71 | /web_server/ 72 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TwitKit 2 | 3 | [README](README.md) | [中文文档](README_zh.md) 4 | 5 | > WARNING: Machine-translation detected. 6 | 7 | TwitKit is an open source suite for tweet translation. Based on microservices architecture design, components communicate with each other with REST API. 8 | 9 | ## backend components 10 | 11 | ### fridge 12 | Database microservice which provides storage services for tweet content and metadata. 13 | Mysql/mariadb storage backend is now supported. 14 | 15 | ### oven 16 | Image processing microservice which provides text-to-image conversion while attaching recognizable elements to the image for publish detection. 17 | 18 | ### maid 19 | Update detection microservice to detect tweet updates via the Twitter API. 20 | Also provides the ability to detect whether Bilibili Dynamics has released an output image. 21 | 22 | ## front-end components 23 | 24 | ### koishi-app 25 | QQBot front-end, based on [koishi](https://koishi.js.org) framework to communicate with CoolQ robot, provides QQ update notification, translation submission and image return services. 26 | 27 | ## install 28 | See README.md in each subfolder; shell scripts are available in the repository for quick deployments. To use them, please give sudo privileges to the appropriate users and recommend revoking them after deployment is complete. 29 | -------------------------------------------------------------------------------- /oven/baker/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "twitkit-oven-baker", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "serve": "vue-cli-service serve", 7 | "build": "vue-cli-service build", 8 | "lint": "vue-cli-service lint" 9 | }, 10 | "dependencies": { 11 | "axios": "^0.19.2", 12 | "cookie": "^0.4.0", 13 | "core-js": "^3.6.4", 14 | "element-resize-event": "^3.0.3", 15 | "escape-html": "^1.0.3", 16 | "material-design-icons-iconfont": "^5.0.1", 17 | "twemoji-parser": "^12.1.3", 18 | "uuid": "^7.0.1", 19 | "vue": "^2.6.11", 20 | "vue-router": "^3.1.5", 21 | "vuetify": "^2.2.11" 22 | }, 23 | "devDependencies": { 24 | "@vue/cli-plugin-babel": "^4.2.0", 25 | "@vue/cli-plugin-eslint": "^4.2.0", 26 | "@vue/cli-plugin-router": "^4.2.3", 27 | "@vue/cli-service": "^4.2.0", 28 | "@vue/eslint-config-prettier": "^6.0.0", 29 | "babel-eslint": "^10.0.3", 30 | "eslint": "^6.7.2", 31 | "eslint-plugin-prettier": "^3.1.1", 32 | "eslint-plugin-vue": "^6.1.2", 33 | "less": "^3.0.4", 34 | "less-loader": "^5.0.0", 35 | "prettier": "^1.19.1", 36 | "sass": "^1.19.0", 37 | "sass-loader": "^8.0.0", 38 | "vue-cli-plugin-vuetify": "^2.0.5", 39 | "vue-template-compiler": "^2.6.11", 40 | "vuetify-loader": "^1.3.0" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/steady/yui/repository/EnkanTaskRepository.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/4 4 | */ 5 | package com.enkanrec.twitkitFridge.steady.yui.repository; 6 | 7 | import com.enkanrec.twitkitFridge.steady.yui.entity.EnkanTaskEntity; 8 | import org.springframework.data.jpa.repository.JpaRepository; 9 | import org.springframework.data.jpa.repository.Lock; 10 | import org.springframework.data.jpa.repository.Query; 11 | 12 | import javax.persistence.LockModeType; 13 | import java.util.Collection; 14 | import java.util.List; 15 | 16 | /** 17 | * Class : EnkanTaskRepository 18 | * Usage : 19 | */ 20 | public interface EnkanTaskRepository extends JpaRepository { 21 | 22 | boolean existsByStatusId(String statusId); 23 | 24 | EnkanTaskEntity findByUrl(String url); 25 | 26 | EnkanTaskEntity findByStatusId(String statusId); 27 | 28 | List findByStatusIdIn(Collection statusIds); 29 | 30 | @Lock(LockModeType.PESSIMISTIC_WRITE) 31 | @Query("SELECT et FROM EnkanTaskEntity et WHERE et.tid = :tid") 32 | EnkanTaskEntity findByTidForUpdate(Integer tid); 33 | 34 | EnkanTaskEntity findFirstByHidedIsFalseOrderByTidDesc(); 35 | 36 | EnkanTaskEntity findFirstByOrderByTidDesc(); 37 | 38 | List findAllByTidGreaterThanAndHidedIsFalse(Integer tid); 39 | 40 | EnkanTaskEntity findTop10ByTidOrderByNewdate(Integer tid); 41 | } 42 | -------------------------------------------------------------------------------- /oven/baker/src/views/BakeTweet.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 51 | 52 | 64 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/steady/yui/repository/EnkanTranslateRepository.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/4 4 | */ 5 | package com.enkanrec.twitkitFridge.steady.yui.repository; 6 | 7 | import com.enkanrec.twitkitFridge.steady.yui.entity.EnkanTaskEntity; 8 | import com.enkanrec.twitkitFridge.steady.yui.entity.EnkanTranslateEntity; 9 | import org.springframework.data.jpa.repository.JpaRepository; 10 | import org.springframework.data.jpa.repository.Modifying; 11 | import org.springframework.data.jpa.repository.Query; 12 | import org.springframework.transaction.annotation.Transactional; 13 | 14 | import java.util.List; 15 | 16 | /** 17 | * Class : EnkanTranslateRepository 18 | * Usage : 19 | */ 20 | public interface EnkanTranslateRepository extends JpaRepository { 21 | 22 | @Transactional 23 | @Query(nativeQuery = true, value = "SELECT * FROM enkan_translate t, (SELECT tid, max(version) maxVersion FROM enkan_translate AS et GROUP BY et.tid) tr WHERE t.tid = tr.tid AND t.version = tr.maxVersion") 24 | List getTranslationsWithLatestVersion(); 25 | 26 | EnkanTranslateEntity findFirstByTaskOrderByVersionDesc(EnkanTaskEntity task); 27 | 28 | @Modifying 29 | @Transactional 30 | @Query("DELETE FROM EnkanTranslateEntity et WHERE et.task.tid = :tid") 31 | int bulkDeleteByTid(Integer tid); 32 | 33 | // List findAllByTidIn(Collection tidCandidates); 34 | } 35 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/monitor/BulkMonitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Project Seele Workflow 3 | * Author : Rinka 4 | * Date : 2020/2/16 5 | */ 6 | package com.enkanrec.twitkitFridge.monitor; 7 | 8 | import io.prometheus.client.Counter; 9 | import org.springframework.stereotype.Component; 10 | 11 | /** 12 | * Class : BulkMonitor 13 | * Usage : 批量入库接口的定制监控 14 | */ 15 | @Component 16 | public class BulkMonitor extends BaseMonitor { 17 | public final Counter totalCounter = Counter 18 | .build() 19 | .name("twitkit_fridge_bulk_raw_count") 20 | .help("Twitkit fridge bulk service without bloom cache total requests counter") 21 | .register(); 22 | 23 | public final Counter bloomQueryCounter = Counter 24 | .build() 25 | .name("twitkit_fridge_bulk_bloom_total_count") 26 | .help("Twitkit fridge bulk service bloom cache total query counter") 27 | .register(); 28 | 29 | public final Counter bloomHitCounter = Counter 30 | .build() 31 | .name("twitkit_fridge_bulk_bloom_hit_count") 32 | .help("Twitkit fridge bulk service bloom cache hit counter") 33 | .register(); 34 | 35 | public final Counter bloomFalsePositiveCounter = Counter 36 | .build() 37 | .name("twitkit_fridge_bulk_bloom_false_positive_count") 38 | .help("Twitkit fridge bulk service bloom cache hit but not exist counter") 39 | .register(); 40 | } 41 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/ws/WSClientPool.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/8 4 | */ 5 | package com.enkanrec.twitkitFridge.api.ws; 6 | 7 | import com.corundumstudio.socketio.SocketIOClient; 8 | import lombok.extern.slf4j.Slf4j; 9 | 10 | import java.util.Map; 11 | import java.util.concurrent.ConcurrentHashMap; 12 | 13 | /** 14 | * Class : WSClientPool 15 | * Usage : 16 | */ 17 | @Slf4j 18 | public class WSClientPool { 19 | private static ConcurrentHashMap pool = new ConcurrentHashMap<>(); 20 | 21 | public static void add(String clientId, SocketIOClient clientRef) { 22 | WSClientPool.pool.put(clientId, clientRef); 23 | } 24 | 25 | public static SocketIOClient get(String clientId) { 26 | return WSClientPool.pool.get(clientId); 27 | } 28 | 29 | public static SocketIOClient remove(String clientId) { 30 | return WSClientPool.pool.remove(clientId); 31 | } 32 | 33 | public static void clearAndDisconnect() { 34 | log.info("client pool will be clear soon"); 35 | for (Map.Entry cached : WSClientPool.pool.entrySet()) { 36 | try { 37 | cached.getValue().disconnect(); 38 | log.info("client is disconnected elegant, " + cached.getKey()); 39 | } catch (Exception de) { 40 | log.warn("cannot disconnect client elegant, " + cached.getKey()); 41 | } 42 | } 43 | WSClientPool.pool.clear(); 44 | log.info("client pool is cleared"); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /koishi-app/src/maid.ts: -------------------------------------------------------------------------------- 1 | import { Context, Logger } from 'koishi-core' 2 | import { request } from './utils' 3 | import axios from 'axios' 4 | 5 | let host: string 6 | let logger: Logger 7 | 8 | class response { // oven return 9 | code: number 10 | message: string 11 | error?: any 12 | addedTid?: number[] 13 | rootTid?: number 14 | } 15 | 16 | async function rest(url: string, data: any): Promise { 17 | logger.debug("POST " + url) 18 | logger.debug(data) 19 | try { 20 | const res = await axios.post(host + url, new request(data)) 21 | if (res.data.code === 0) { 22 | logger.debug("Return %d: %s", res.data.code, res.data.message || "") 23 | logger.debug(res.data) 24 | return res.data 25 | } else { 26 | logger.warn("Error %d: %s", res.data.code, res.data.message) 27 | return null 28 | } 29 | } catch (e) { 30 | if (e.response) { 31 | logger.error("Internet error: %d", e.response.status) 32 | logger.debug(e.response.data) 33 | } else logger.error(e) 34 | } 35 | return null 36 | } 37 | 38 | async function addTask(url: string): Promise { 39 | const res = await rest("/api/maid/addtask", { url }) 40 | if (!res || !res.rootTid || !res.addedTid) return null 41 | return res.addedTid 42 | } 43 | 44 | function init(ctx: Context, Host: string) { 45 | logger = ctx.logger("app:maid") // 初始化logger 46 | host = Host // 初始化maid的Host 47 | logger.info("maid ready") 48 | } 49 | 50 | export default { 51 | addTask, 52 | init 53 | } -------------------------------------------------------------------------------- /fridge/README.md: -------------------------------------------------------------------------------- 1 | # TwitKit Fridge 2 | 3 | TwitKit的数据库后端，因为是存**生肉**的，所以是冰箱。 4 | 5 | ## 依赖 6 | 7 | ### 运行依赖 8 | 9 | - mysql或mariadb 10 | - jre8 11 | 12 | ### 编译依赖 13 | 14 | - jdk8 15 | - maven 16 | 17 | 在Ubuntu 18.04下： 18 | 19 | ```shell 20 | sudo apt install openjdk-8-jre-headless openjdk-8-jdk-headless maven -y 21 | ``` 22 | 23 | - 在`maven`的依赖关系中，虚包由这些包填实: `default-jre-headless`, `openjdk-11-jre-headless`, `openjdk-8-jre-headless`； 24 | 也就是必须指明`openjdk-8-jre-headless`，否则就会安装`openjdk-11-jre-headless`，另一方面，要执行**编译**的话又必须依赖jdk，因此需要同时安装这三个包。 25 | 另外，目前来看jdk11除了编译警告较多以外，不影响使用。 26 | 27 | ## 快速安装 28 | 29 | 如果当前用户具有sudo权限，并且数据库加载了`unix_socket`插件（Ubuntu 18.04的默认配置），则可以使用根目录下的脚本进行快速安装。安装完成后建议取消sudo权限。 30 | 31 | - `test_init.sh`用于快速建立测试库，增加`--remove`参数用于移除。 32 | - `build.sh`用于运行编译和运行测试。 33 | 34 | - `configen.sh`用于生成模板配置文件。 35 | 36 | - `db_init.sh`用于快速部署生产使用的数据库，执行`db_init.sh [name]`会产生同名的数据库、用户和一串随机密码（优先使用pwgen，未找到则会使用uuidgen），并将其按照配置文件的格式打印出来；没有提供参数则会使用当前登录的用户名（显然，如果root用户不带参数运行此脚本会导致报错） 37 | 38 | - `start.sh`用于运行程序。 39 | 40 | ## 手动安装 41 | 42 | 1. 为了确保数据库后端的可用性，编译时默认会进行测试，请建立数据库、用户名和密码都为`fridge_test`的测试环境，并在库中运行`schema/test_init.sql`初始化。 43 | 44 | 2. 进入`src/fridge_src`目录后执行下列命令来编译，生成到`fridge-src/target`中。如要跳过测试，取消注释`maven.test.skip`参数。 45 | 46 | ``` 47 | mvn clean 48 | mvn package #-Dmaven.test.skip=true 49 | ``` 50 | 51 | 3. 为生产使用建立数据库和用户，然后用`schema/db_init.sql`初始化； 52 | 53 | 4. 在**工作目录**下建立`application.properties`文件（可使用`configen.sh`），并按以下格式写入数据库参数，替换<>的内容： 54 | 55 | ``` 56 | spring.datasource.yui.jdbc-url=jdbc:mysql://localhost:3306/?characterEncoding=u 57 | tf-8&useSSL=true&serverTimezone=Asia/Shanghai 58 | spring.datasource.yui.username= 59 | spring.datasource.yui.password= 60 | ``` 61 | 62 | 5. 通过`java -jar`执行编译出的jar包。 63 | 64 | 65 | -------------------------------------------------------------------------------- /oven/baker/src/components/TranslationBox.vue: -------------------------------------------------------------------------------- 1 | 36 | 37 | 63 | 64 | -------------------------------------------------------------------------------- /maid/rsshub_client.py: -------------------------------------------------------------------------------- 1 | import dateutil.parser 2 | import requests 3 | import re 4 | 5 | from collections import namedtuple 6 | from xml.etree import ElementTree 7 | 8 | 9 | RSSHUB_BILIBILI_URL = 'https://rsshub.app/bilibili/user/dynamic/{}' 10 | 11 | FeedItem = namedtuple('FeedItem', 'feed_title content media_list url pub_date') 12 | 13 | 14 | def extract_content_media(content): 15 | media_list = [] 16 | for match in re.findall(r'()', content): 17 | media_list.append(match[1]) 18 | content.replace(match[0], '') 19 | content = re.sub('
', '\n', content, flags=re.IGNORECASE) 20 | content = re.sub('<.*?>', '', content) 21 | return content.strip(), media_list 22 | 23 | 24 | def get_new_feed(feed_url): 25 | r = requests.get(feed_url) 26 | if r.status_code != 200: 27 | raise Exception(f'RSSHub请求不成功，状态码：{r.status_code}') 28 | rss = ElementTree.fromstring(r.content) 29 | if rss.tag != 'rss': 30 | raise ValueError('RSS格式有误') 31 | channel = rss.find('channel') 32 | feed_title = channel.find('title').text 33 | ret = [] 34 | for item in channel: 35 | if item.tag != 'item': 36 | continue 37 | content = item.find('description').text 38 | content, media_list = extract_content_media(content) 39 | pub_date = dateutil.parser.parse(item.find('pubDate').text) 40 | url = item.find('guid').text 41 | ret.append(FeedItem( 42 | feed_title=feed_title, 43 | content=content, 44 | media_list=media_list, 45 | url=url, 46 | pub_date=pub_date)) 47 | return ret[::-1] 48 | 49 | 50 | def get_new_bilibili_status(bilibili_uid): 51 | new_feed_items = get_new_feed(RSSHUB_BILIBILI_URL.format(bilibili_uid)) 52 | return new_feed_items 53 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/service/task/TaskService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/4 4 | */ 5 | package com.enkanrec.twitkitFridge.service.task; 6 | 7 | import com.enkanrec.twitkitFridge.api.form.TaskCreationForm; 8 | import com.enkanrec.twitkitFridge.steady.yui.entity.EnkanTaskEntity; 9 | 10 | import java.util.List; 11 | 12 | /** 13 | * Class : TaskService 14 | * Usage : 15 | */ 16 | public interface TaskService { 17 | 18 | TaskServiceImpl.CreateTaskReplay addTask(TaskCreationForm twitter); 19 | 20 | List addTaskByBulk(List twitters); 21 | 22 | List addTaskByBulkWithCache(List twitters); 23 | 24 | Boolean removeTask(Integer tid); 25 | 26 | TaskServiceImpl.TaskReplay getOneLatestOfVisible(); 27 | 28 | TaskServiceImpl.TranslatedTask getOneLatestOfVisibleWithTranslation(); 29 | 30 | TaskServiceImpl.TaskReplay getOneLatest(); 31 | 32 | TaskServiceImpl.TranslatedTask getOneWithTranslation(Integer tid); 33 | 34 | List getManyFromTidWithTranslation(Integer tid); 35 | 36 | TaskServiceImpl.TaskReplay updateComment(Integer tid, String comment); 37 | 38 | TaskServiceImpl.TaskReplay hide(Integer tid); 39 | 40 | TaskServiceImpl.TaskReplay visible(Integer tid); 41 | 42 | TaskServiceImpl.TaskReplay setPublished(Integer tid); 43 | 44 | TaskServiceImpl.TaskReplay setUnpublished(Integer tid); 45 | 46 | Integer removeAllTranslations(Integer tid); 47 | 48 | TaskServiceImpl.TranslatedTask addTranslation(Integer tid, String translation, String img); 49 | 50 | TaskServiceImpl.TranslatedTask rollbackTranslation(Integer tid); 51 | 52 | TaskServiceImpl.VersionedTranslatedTask getAllTranslation(Integer tid); 53 | } 54 | -------------------------------------------------------------------------------- /oven/app.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, request, jsonify, make_response, abort 2 | from flask_restx import Resource, Api 3 | from oven import tweet_page_bp 4 | from api import oven_api 5 | from requests import post, options 6 | 7 | import logging 8 | import sys 9 | import json 10 | import config 11 | import coloredlogs 12 | 13 | app = Flask(__name__, static_url_path='/static') 14 | api = Api(app) 15 | 16 | if config.LOG_FILE: 17 | log_handler = logging.FileHandler(config.LOG_FILE) 18 | logging.getLogger().addHandler(log_handler) 19 | 20 | coloredlogs.install( 21 | level=logging.DEBUG if config.LOG_DEBUG else logging.INFO) 22 | 23 | app.register_blueprint(tweet_page_bp, url_prefix='/internal') 24 | api.add_namespace(oven_api, path='/api/oven') 25 | 26 | 27 | @app.route('/api_proxy//', methods=['POST', 'OPTIONS']) 28 | def proxy(module, path): 29 | if module == 'maid': 30 | url = f'{config.MAID_API_BASE}/{path}' 31 | elif module == 'fridge': 32 | url = f'{config.FRIDGE_API_BASE}/{path}' 33 | else: 34 | abort(404) 35 | 36 | if request.method == 'POST': 37 | post_data = request.json 38 | resp = post(url, json=post_data) 39 | response = jsonify(resp.json()) 40 | else: 41 | resp = options(url) 42 | response = make_response() 43 | 44 | return response, resp.status_code 45 | 46 | 47 | @app.after_request 48 | def after_request(response): 49 | if request.path.startswith('/api/oven'): 50 | try: 51 | response_data = json.loads(response.get_data()) 52 | except: 53 | return response 54 | if 'code' not in response_data: 55 | response_data['code'] = response.status_code 56 | status_code = 200 57 | response.set_data(json.dumps(response_data)) 58 | return response 59 | 60 | 61 | if __name__ == '__main__': 62 | app.run(debug=True) 63 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/monitor/WebSocketMonitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/9 4 | */ 5 | package com.enkanrec.twitkitFridge.monitor; 6 | 7 | import io.prometheus.client.Counter; 8 | import io.prometheus.client.Gauge; 9 | import io.prometheus.client.Summary; 10 | import org.springframework.stereotype.Component; 11 | 12 | /** 13 | * Class : WebSocketMonitor 14 | * Usage : WebSocket指标数据的包装 15 | */ 16 | @Component 17 | public class WebSocketMonitor extends BaseMonitor { 18 | 19 | public final Summary responseTimeInMs = Summary 20 | .build() 21 | .name("twitkit_fridge_socketio_response_time_milliseconds") 22 | .labelNames(BaseMonitor.TAG_HTTP_HANDLER, BaseMonitor.TAG_HTTP_URI, BaseMonitor.TAG_HTTP_STATUS_CODE) 23 | .help("Websocket Request via socketIO completed time in milliseconds") 24 | .register(); 25 | 26 | public final Counter exceptionCounter = Counter 27 | .build() 28 | .name("twitkit_fridge_socketio_exception_count") 29 | .labelNames(BaseMonitor.TAG_HTTP_METHOD, BaseMonitor.TAG_HTTP_URI) 30 | .help("Websocket Request via socketIO exception counter") 31 | .register(); 32 | 33 | public final Counter connectInCounter = Counter 34 | .build() 35 | .name("twitkit_fridge_socketio_connected_count") 36 | .help("SocketIO connected in counter") 37 | .register(); 38 | 39 | public final Counter disconnectOutCounter = Counter 40 | .build() 41 | .name("twitkit_fridge_socketio_disconnected_count") 42 | .help("SocketIO disconnected out counter") 43 | .register(); 44 | 45 | public final Gauge activeConnectionCounter = Gauge 46 | .build() 47 | .name("twitkit_fridge_socketio_active_connection_count") 48 | .help("SocketIO active connection counter") 49 | .register(); 50 | 51 | } 52 | -------------------------------------------------------------------------------- /oven/tests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import tid_code 3 | import tempfile 4 | import os 5 | 6 | from PIL import Image 7 | 8 | 9 | class TestTidCode(unittest.TestCase): 10 | 11 | def test_checksum(self): 12 | self.assertEqual(tid_code.calculate_checksum(0x1234), 0xF1) 13 | self.assertEqual(tid_code.calculate_checksum(0x556832), 0x27) 14 | 15 | def test_encode(self): 16 | self.assertEqual( 17 | tid_code.encode(0x1234, key=0), 18 | [ 19 | True, False, True, False, True, False, True, False, # 1111 F 20 | False, True, False, True, False, True, True, False, # 0001 1 21 | False, True, False, True, False, True, False, True, # 0000 0 22 | False, True, False, True, False, True, False, True, # 0000 0 23 | False, True, False, True, False, True, True, False, # 0001 1 24 | False, True, False, True, True, False, False, True, # 0010 2 25 | False, True, False, True, True, False, True, False, # 0011 3 26 | False, True, True, False, False, True, False, True, # 0100 4 27 | ]) 28 | 29 | def test_decode(self): 30 | self.assertEqual(tid_code.decode(tid_code.encode(0x5678)), 0x5678) 31 | self.assertEqual(tid_code.decode(tid_code.encode(0x2333)), 0x2333) 32 | self.assertEqual(tid_code.decode(tid_code.encode(0xffaa45)), 0xffaa45) 33 | 34 | def test_integration(self): 35 | self.assertEqual( 36 | tid_code.decode_from_image(tid_code.generate_code_image(20, 20, 37 | 2233)), 38 | 2233) 39 | 40 | def test_image_integration(self): 41 | im = Image.new('RGBA', (1000, 1000)) 42 | test_tid = 1000 43 | tid_code.add_code_to_image(im, test_tid, 0, 0, 20, 20) 44 | tid = tid_code.read_code_from_image(im, 0, 0, 20, 20) 45 | self.assertEqual(test_tid, tid) 46 | 47 | 48 | if __name__ == '__main__': 49 | unittest.main() 50 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/response/StandardResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/30 4 | */ 5 | package com.enkanrec.twitkitFridge.api.response; 6 | 7 | import lombok.Data; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.NoArgsConstructor; 10 | import lombok.ToString; 11 | 12 | import java.io.Serializable; 13 | import java.time.ZonedDateTime; 14 | import java.time.format.DateTimeFormatter; 15 | 16 | /** 17 | * Class : StandardResponse 18 | * Usage : 19 | */ 20 | @Data 21 | @ToString 22 | @NoArgsConstructor 23 | @EqualsAndHashCode 24 | public class StandardResponse implements Serializable { 25 | private static final long serialVersionUID = 1L; 26 | 27 | public static final int CODE_SUCCESS = 0; 28 | public static final int CODE_NOTFOUND = 404; 29 | public static final int CODE_EXCEPTION = 500; 30 | 31 | public static final String MESSAGE_SUCCESS = "OK"; 32 | 33 | private int code; 34 | 35 | private String message; 36 | 37 | private String timestamp = ZonedDateTime.now().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); 38 | 39 | private Object data; 40 | 41 | public StandardResponse(int code, String message) { 42 | this(code, message, null); 43 | } 44 | 45 | public StandardResponse(int code, String message, Object payload) { 46 | this.code = code; 47 | this.message = message; 48 | this.data = payload; 49 | } 50 | 51 | public static StandardResponse ok() { 52 | return new StandardResponse(CODE_SUCCESS, MESSAGE_SUCCESS); 53 | } 54 | 55 | public static StandardResponse ok(Object payload) { 56 | return new StandardResponse(CODE_SUCCESS, MESSAGE_SUCCESS, payload); 57 | } 58 | 59 | public static StandardResponse exception(String message) { 60 | return new StandardResponse(CODE_EXCEPTION, message); 61 | } 62 | 63 | public static StandardResponse exception(Throwable ex) { 64 | return new StandardResponse(CODE_EXCEPTION, ex.getMessage()); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/form/BaseFridgeForm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/30 4 | */ 5 | package com.enkanrec.twitkitFridge.api.form; 6 | 7 | import com.enkanrec.twitkitFridge.interceptor.MonitorInterceptor; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.Getter; 10 | import lombok.Setter; 11 | import lombok.ToString; 12 | import lombok.extern.slf4j.Slf4j; 13 | import org.slf4j.MDC; 14 | 15 | import javax.validation.constraints.NotNull; 16 | import java.io.Serializable; 17 | import java.util.UUID; 18 | 19 | /** 20 | * Class : BaseFridgeForm 21 | * Usage : 标准请求格式 22 | */ 23 | @Deprecated 24 | @Slf4j 25 | @ToString 26 | @EqualsAndHashCode 27 | public class BaseFridgeForm implements Serializable { 28 | private static final long serialVersionUID = 1L; 29 | 30 | /** 31 | * 从哪个外部服务发起的请求 32 | */ 33 | @Getter 34 | @NotNull 35 | private String forwardFrom; 36 | 37 | /** 38 | * 请求的客户端时间戳（ISO8601） 39 | */ 40 | @Getter 41 | @Setter 42 | @NotNull 43 | private String timestamp; 44 | 45 | /** 46 | * 请求的request id 47 | */ 48 | @Getter 49 | private String taskId; 50 | 51 | /** 52 | * WebSocket模式下的命令所在模块名 53 | */ 54 | @Getter 55 | @Setter 56 | private String of = null; 57 | 58 | /** 59 | * WebSocket模式下的命令参数 60 | */ 61 | @Getter 62 | @Setter 63 | private String command = null; 64 | 65 | public void setTaskId(String requestId) { 66 | this.taskId = requestId; 67 | if (requestId == null) { 68 | requestId = "SELF_" + UUID.randomUUID().toString(); 69 | log.warn("RequestId is not pass in, generated by fridge"); 70 | } 71 | MDC.put(MonitorInterceptor.LOG_KEY_REQUEST_ID, requestId); 72 | log.info(String.format("Request id generated: %s", requestId)); 73 | } 74 | 75 | public void setForwardFrom(String forwardFrom) { 76 | log.info(String.format("Request form built, forward from: %s", forwardFrom)); 77 | this.forwardFrom = forwardFrom; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # 默认配置项 2 | spring.profiles.active=dev 3 | 4 | # 日志配置 5 | logging.config=classpath:logback/logback-spring-${spring.profiles.active}.xml 6 | 7 | # 监听端口 8 | server.port=8220 9 | 10 | # 异常处理 11 | server.error.include-exception=true 12 | 13 | # Banner 14 | spring.main.banner-mode=console 15 | 16 | # 访问日志配置 17 | server.tomcat.basedir=web_server 18 | server.tomcat.accesslog.enabled=true 19 | server.tomcat.accesslog.rotate=true 20 | server.tomcat.accesslog.rename-on-rotate=true 21 | server.tomcat.accesslog.max-days=7 22 | server.tomcat.accesslog.directory=access_log 23 | server.tomcat.accesslog.encoding=UTF-8 24 | server.tomcat.accesslog.pattern=[%{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}t][%I] %{X-Forwarded-For}i %h %l %u "%r" %s %b (%D ms) 25 | 26 | # jpa配置 27 | spring.jpa.properties.show-sql=true 28 | spring.jpa.properties.database-platform=mysql 29 | spring.jpa.properties.database=mysql 30 | spring.jpa.properties.hibernate.ddl-auto=validate 31 | spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.MySQL57Dialect 32 | 33 | ## 下面为连接池的补充设置，应用到上面所有数据源中 34 | spring.datasource.type=com.zaxxer.hikari.HikariDataSource 35 | ## 最小连接池数量 36 | spring.datasource.hikari.minimum-idle=5 37 | ## 池中最大连接数 38 | spring.datasource.hikari.maximum-pool-size=25 39 | ## 此属性控制从池返回的连接的默认自动提交行为,默认值：true 40 | spring.datasource.hikari.auto-commit=true 41 | ## 空闲连接存活最大时间，默认600000（10分钟） 42 | spring.datasource.hikari.idle-timeout=600000 43 | 44 | ## 连接池名字 45 | spring.datasource.hikari.pool-name=YuiHikariCP 46 | ## 此属性控制池中连接的最长生命周期，值0表示无限生命周期，默认1800000即30分钟 47 | spring.datasource.hikari.max-lifetime=1800000 48 | ## 数据库连接超时时间,默认30秒，即30000 49 | spring.datasource.hikari.connection-timeout=30000 50 | spring.datasource.hikari.connection-test-query=SELECT 1 51 | 52 | # 数据源 53 | spring.datasource.yui.driver-class-name=com.mysql.cj.jdbc.Driver 54 | spring.datasource.yui.jdbc-url=jdbc:mysql://localhost:3306/fridge?characterEncoding=utf-8&useSSL=true&serverTimezone=Asia/Shanghai 55 | spring.datasource.yui.username=fridge 56 | spring.datasource.yui.password=fridge 57 | 58 | # 日志路径 59 | enkanRec.logging.path=./log -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/form/BaseJsonWarp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/13 4 | */ 5 | package com.enkanrec.twitkitFridge.api.form; 6 | 7 | import com.enkanrec.twitkitFridge.interceptor.MonitorInterceptor; 8 | import lombok.EqualsAndHashCode; 9 | import lombok.Getter; 10 | import lombok.Setter; 11 | import lombok.ToString; 12 | import lombok.extern.slf4j.Slf4j; 13 | import org.slf4j.MDC; 14 | 15 | import javax.validation.constraints.NotNull; 16 | import java.io.Serializable; 17 | import java.util.UUID; 18 | 19 | /** 20 | * Class : BaseJsonWarp 21 | * Usage : 22 | */ 23 | @Slf4j 24 | @ToString 25 | @EqualsAndHashCode 26 | public class BaseJsonWarp implements Serializable { 27 | private static final long serialVersionUID = 1L; 28 | 29 | /** 30 | * 从哪个外部服务发起的请求 31 | */ 32 | @Getter 33 | @NotNull 34 | private String forwardFrom; 35 | 36 | /** 37 | * 请求的客户端时间戳（ISO8601） 38 | */ 39 | @Getter 40 | @Setter 41 | @NotNull 42 | private String timestamp; 43 | 44 | /** 45 | * 请求的request id 46 | */ 47 | @Getter 48 | private String taskId; 49 | 50 | /** 51 | * WebSocket模式下的命令所在模块名 52 | */ 53 | @Getter 54 | @Setter 55 | private String of = null; 56 | 57 | /** 58 | * WebSocket模式下的命令参数 59 | */ 60 | @Getter 61 | @Setter 62 | private String command = null; 63 | 64 | @Getter 65 | @Setter 66 | private InnerTy data; 67 | 68 | public void setTaskId(String requestId) { 69 | this.taskId = requestId; 70 | if (requestId == null) { 71 | requestId = "SELF_" + UUID.randomUUID().toString(); 72 | log.warn("RequestId is not pass in, generated by fridge"); 73 | } 74 | MDC.put(MonitorInterceptor.LOG_KEY_REQUEST_ID, requestId); 75 | log.info(String.format("Request id generated: %s", requestId)); 76 | } 77 | 78 | public void setForwardFrom(String forwardFrom) { 79 | log.info(String.format("Request form built, forward from: %s", forwardFrom)); 80 | this.forwardFrom = forwardFrom; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/ws/FridgeWSServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/8 4 | */ 5 | package com.enkanrec.twitkitFridge.api.ws; 6 | 7 | import com.corundumstudio.socketio.Configuration; 8 | import com.corundumstudio.socketio.SocketIOServer; 9 | import com.enkanrec.twitkitFridge.GDP; 10 | import lombok.extern.slf4j.Slf4j; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.beans.factory.annotation.Value; 13 | import org.springframework.stereotype.Component; 14 | 15 | import javax.annotation.PostConstruct; 16 | import javax.annotation.PreDestroy; 17 | 18 | /** 19 | * Class : FridgeWSServer 20 | * Usage : 21 | */ 22 | @Slf4j 23 | @Component 24 | public class FridgeWSServer { 25 | public static final String REQUEST_EVT = "twitkit_request"; 26 | public static final String RESPONSE_EVT = "twitkit_response"; 27 | 28 | @Value("${server.port}") 29 | private Integer listenPort; 30 | 31 | private SocketIOServer server; 32 | 33 | @Autowired 34 | private ConnectInListener connectInListener; 35 | @Autowired 36 | private DisconnectOutListener disconnectOutListener; 37 | @Autowired 38 | private RequestListener requestListener; 39 | 40 | @PostConstruct 41 | public void init() { 42 | if (GDP.EnableWebSocket) { 43 | log.info("`EnableWebSocket` is true, providing service by REST and SocketIO"); 44 | Configuration config = new Configuration(); 45 | config.setPort(this.listenPort); 46 | config.setHostname("localhost"); 47 | this.server = new SocketIOServer(config); 48 | this.server.addConnectListener(this.connectInListener); 49 | this.server.addDisconnectListener(this.disconnectOutListener); 50 | this.server.addEventListener(FridgeWSServer.REQUEST_EVT, String.class, this.requestListener); 51 | this.server.start(); 52 | log.info("SocketIO server is started"); 53 | } else { 54 | log.info("`EnableWebSocket` is false, providing service by REST only"); 55 | } 56 | } 57 | 58 | @PreDestroy 59 | public void disposing() { 60 | log.info("WS server is disposing"); 61 | WSClientPool.clearAndDisconnect(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/rest/KVConfigController.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/30 4 | */ 5 | package com.enkanrec.twitkitFridge.api.rest; 6 | 7 | import com.enkanrec.twitkitFridge.api.form.BaseJsonWarp; 8 | import com.enkanrec.twitkitFridge.api.form.NamespaceForm; 9 | import com.enkanrec.twitkitFridge.api.response.StandardResponse; 10 | import com.enkanrec.twitkitFridge.service.kvConfig.KVConfigService; 11 | import org.springframework.web.bind.annotation.*; 12 | 13 | import javax.validation.Valid; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | /** 18 | * Class : KVConfigController 19 | * Usage : KV设置存储器对外接口 20 | */ 21 | @RestController 22 | @RequestMapping("/api/db/kv") 23 | public class KVConfigController { 24 | 25 | private final KVConfigService service; 26 | 27 | public KVConfigController(KVConfigService service) { 28 | this.service = service; 29 | } 30 | 31 | /** 32 | * 获取全部配置项 33 | */ 34 | @ResponseBody 35 | @RequestMapping(value = "/getall", method = RequestMethod.POST) 36 | public StandardResponse getAllKVConfigs(@Valid @RequestBody BaseJsonWarp form) { 37 | return StandardResponse.ok(this.service.getAll()); 38 | } 39 | 40 | /** 41 | * 更新默认命名空间下的设置项 42 | */ 43 | @ResponseBody 44 | @RequestMapping(value = "/set", method = RequestMethod.POST) 45 | public StandardResponse setKVConfigsByDefaultNamespace(@Valid @RequestBody BaseJsonWarp> form) throws Exception { 46 | this.service.setManyDefault(form.getData()); 47 | return StandardResponse.ok(""); 48 | } 49 | 50 | /** 51 | * 获取默认命名空间下的设置项 52 | */ 53 | @ResponseBody 54 | @RequestMapping(value = "/get", method = RequestMethod.POST) 55 | public StandardResponse getKVConfigsByDefaultNamespace(@Valid @RequestBody BaseJsonWarp> form) { 56 | List params = form.getData(); 57 | Map result = this.service.getManyDefault(params); 58 | return StandardResponse.ok(result); 59 | } 60 | 61 | /** 62 | * 清空一个命名空间下的所有配置项 63 | */ 64 | @ResponseBody 65 | @RequestMapping(value = "/clear", method = RequestMethod.POST) 66 | public StandardResponse clearConfigOfNamespace(@Valid @RequestBody BaseJsonWarp form) { 67 | this.service.clearNamespace(form.getData().getNamespace()); 68 | return StandardResponse.ok(); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/rest/ExceptionControllerAdvice.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/3 4 | */ 5 | package com.enkanrec.twitkitFridge.api.rest; 6 | 7 | import com.enkanrec.twitkitFridge.api.response.StandardResponse; 8 | import com.enkanrec.twitkitFridge.monitor.InterceptorMonitor; 9 | import com.enkanrec.twitkitFridge.util.JsonUtil; 10 | import com.fasterxml.jackson.core.JsonProcessingException; 11 | import lombok.extern.slf4j.Slf4j; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.web.bind.annotation.ControllerAdvice; 14 | import org.springframework.web.bind.annotation.ExceptionHandler; 15 | import org.springframework.web.bind.annotation.ResponseBody; 16 | 17 | import javax.servlet.http.HttpServletRequest; 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | 21 | /** 22 | * Class : ExceptionControllerAdvice 23 | * Usage : 异常处理路由 24 | */ 25 | @Slf4j 26 | @ControllerAdvice 27 | public class ExceptionControllerAdvice { 28 | 29 | @Autowired 30 | private InterceptorMonitor monitor; 31 | 32 | /** 33 | * 捕获全部异常 34 | */ 35 | @ResponseBody 36 | @ExceptionHandler(Exception.class) 37 | public StandardResponse exceptionHandler(HttpServletRequest request, Exception e) { 38 | StandardResponse sr = new StandardResponse(); 39 | Map hint = new HashMap<>(); 40 | String method = request.getMethod(); 41 | String path = request.getPathInfo(); 42 | if (method == null) { 43 | method = "null"; 44 | } 45 | if (path == null) { 46 | path = "null"; 47 | } 48 | hint.put("msg", e.getMessage()); 49 | hint.put("path", path); 50 | hint.put("method", method); 51 | String formatted; 52 | this.monitor.exceptionCounter.labels(method, path).inc(); 53 | try { 54 | formatted = JsonUtil.Mapper.writeValueAsString(hint); 55 | } catch (JsonProcessingException ex) { 56 | log.error("cannot json dump exception hint, " + ex.getMessage()); 57 | formatted = "___EXCEPTION_HANDLER_FAULT___"; 58 | } 59 | log.error(String.format("Rest Exception: %s", formatted)); 60 | sr.setMessage(formatted); 61 | if (e instanceof org.springframework.web.servlet.NoHandlerFoundException) { 62 | sr.setCode(StandardResponse.CODE_NOTFOUND); 63 | } else { 64 | sr.setCode(StandardResponse.CODE_EXCEPTION); 65 | } 66 | return sr; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /maid/api_models.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, make_response 2 | from flask_restplus import Api, fields, Namespace 3 | import config 4 | 5 | app = Flask(__name__, root_path='/api/maid') 6 | api = Api(app, description='Maid API') 7 | 8 | 9 | def make_request_model(data_fields: dict, label: str): 10 | base_fields = { 11 | 'forwardFrom': fields.String( 12 | required=True, 13 | description='一个字符串，标定是哪个服务调用了此接口', 14 | example='twitkit-app'), 15 | 'timestamp': fields.DateTime( 16 | dt_format='iso8601', 17 | required=True, 18 | description='请求发出时间戳，ISO8601格式', 19 | example='2020-01-29T14:23:23.233+08:00'), 20 | 'taskId': fields.String( 21 | required=True, 22 | description='一个UUID，是一个任务的上下文唯一标识符', 23 | example='123e4567-e89b-12d3-a456-426655440000') 24 | } 25 | data_model = api.model(f'{label}RequestDataModel', data_fields) 26 | 27 | req_fields = base_fields.copy() 28 | req_fields['data'] = fields.Nested( 29 | data_fields, description='请求数据', required=True) 30 | 31 | bake_model = api.model(f'{label}RequestModel', req_fields) 32 | 33 | 34 | def make_response_model(resp_fields: dict, label: str): 35 | resp_fields.update({ 36 | 'code': fields.Integer(description='返回码，0为成功，其他情况另外注明', 37 | required=True), 38 | 'message': fields.String(description='备注消息', required=True), 39 | }) 40 | return api.model(f'{label}ResponseModel', resp_fields) 41 | 42 | 43 | addtask_model = make_request_model({ 44 | 'url': fields.String( 45 | required=True, 46 | description='要加入到任务列表中的推文URL', 47 | example='https://twitter.com/magireco/status/1233776691064868865'), 48 | }, 'addtask') 49 | 50 | gettweet_model = make_request_model({ 51 | 'url': fields.String( 52 | required=True, 53 | description='要请求的推文URL', 54 | example='https://twitter.com/magireco/status/1233776691064868865'), 55 | }, 'gettweet') 56 | 57 | addtask_resp_model = make_response_model({ 58 | 'addedTid': fields.List( 59 | fields.String, 60 | description='已入库的任务的tid列表', 61 | example=[1001, 1002]), 62 | 'rootTid': fields.Integer( 63 | description='请求插入的推文本身的tid', 64 | example=1001), 65 | }, 'addtask') 66 | 67 | gettweet_resp_model = make_response_model({ 68 | 'tweets': fields.Raw( 69 | description='得到的推文'), 70 | 'rootStatusId': fields.Integer( 71 | description='请求插入的推文本身的推特推文ID', 72 | example=1001), 73 | }, 'addtask') -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/interceptor/MonitorInterceptor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/9 4 | * Contact: gzlinjia@corp.netease.com 5 | */ 6 | package com.enkanrec.twitkitFridge.interceptor; 7 | 8 | import com.enkanrec.twitkitFridge.monitor.InterceptorMonitor; 9 | import lombok.extern.slf4j.Slf4j; 10 | import org.slf4j.MDC; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.web.method.HandlerMethod; 13 | import org.springframework.web.servlet.HandlerInterceptor; 14 | 15 | import javax.servlet.http.HttpServletRequest; 16 | import javax.servlet.http.HttpServletResponse; 17 | import java.lang.reflect.Method; 18 | import java.util.UUID; 19 | 20 | /** 21 | * Class : MonitorInterceptor 22 | * Usage : 监控拦截器，在请求进入时注入监控信息，并在结束时更新统计指标 23 | */ 24 | @Slf4j 25 | public class MonitorInterceptor implements HandlerInterceptor { 26 | 27 | private static final String REQ_PARAM_TIMING = "__inject_cost_timing"; 28 | private static final String REQ_REQUEST_ID = "__inject_request_id"; 29 | 30 | public static final String LOG_KEY_REQUEST_ID = "requestId"; 31 | 32 | @Autowired 33 | private InterceptorMonitor monitor; 34 | 35 | /** 36 | * 所有`/api/*`的请求在这里带上她的开始时间戳 37 | */ 38 | @Override 39 | public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { 40 | request.setAttribute(REQ_PARAM_TIMING, System.currentTimeMillis()); 41 | return true; 42 | } 43 | 44 | /** 45 | * 所有`/api/*`的请求在结束后记录一下访问情况指标 46 | */ 47 | @Override 48 | public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception { 49 | Long timingAttr = (Long) request.getAttribute(REQ_PARAM_TIMING); 50 | long completedTime = System.currentTimeMillis() - timingAttr; 51 | String handlerLabel = handler.toString(); 52 | if (handler instanceof HandlerMethod) { 53 | Method method = ((HandlerMethod) handler).getMethod(); 54 | handlerLabel = method.getDeclaringClass().getSimpleName() + "." + method.getName(); 55 | } 56 | this.monitor.responseTimeInMs.labels(request.getMethod(), handlerLabel, request.getRequestURI(), Integer.toString(response.getStatus())) 57 | .observe(completedTime); 58 | String requestId = MDC.get(LOG_KEY_REQUEST_ID); 59 | log.info(String.format("Request id is removed: %s, path: %s", requestId, request.getRequestURI())); 60 | MDC.remove(LOG_KEY_REQUEST_ID); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /oven/api_models.py: -------------------------------------------------------------------------------- 1 | from flask_restx import fields, Namespace 2 | import config 3 | 4 | oven_api = Namespace('api', description='烤推出图API') 5 | 6 | 7 | def make_request_model(data_fields: dict, label: str): 8 | base_fields = { 9 | 'forwardFrom': fields.String( 10 | required=True, 11 | description='一个字符串，标定是哪个服务调用了此接口', 12 | example='twitkit-app'), 13 | 'timestamp': fields.DateTime( 14 | dt_format='iso8601', 15 | required=True, 16 | description='请求发出时间戳，ISO8601格式', 17 | example='2020-01-29T14:23:23.233+08:00'), 18 | 'taskId': fields.String( 19 | required=True, 20 | description='一个UUID，是一个任务的上下文唯一标识符', 21 | example='123e4567-e89b-12d3-a456-426655440000') 22 | } 23 | data_model = oven_api.model(f'{label}RequestDataModel', data_fields) 24 | 25 | req_fields = base_fields 26 | req_fields['data'] = fields.Nested( 27 | data_model, description='请求数据', required=True) 28 | 29 | return oven_api.model(f'{label}RequestModel', req_fields) 30 | 31 | 32 | def make_response_model(resp_fields: dict, label: str): 33 | resp_fields.update({ 34 | 'code': fields.Integer(description='返回码，0为成功，其他情况另外注明', 35 | required=True), 36 | 'message': fields.String(description='备注消息', required=True), 37 | }) 38 | return oven_api.model(f'{label}ResponseModel', resp_fields) 39 | 40 | 41 | bake_model = make_request_model({ 42 | 'tid': fields.Integer( 43 | description='任务ID（从任务列表烤推时，传此参数）', 44 | example=2233), 45 | 'url': fields.String( 46 | description='推文URL（URL烤推时，传此参数）', 47 | example='https://twitter.com/magireco/status/1233776691064868865'), 48 | 'transText': fields.String( 49 | description='推文译文，Plain Text格式，不传则使用数据库中的译文', 50 | example='译文'), 51 | 'ppi': fields.Integer( 52 | default=config.DEFAULT_PPI, 53 | description='生成图像PPI', 54 | example=144) 55 | }, 'bake') 56 | 57 | check_model = make_request_model({ 58 | 'imageUrl': fields.String( 59 | required=True, 60 | description='要查询tid的图片的URL', 61 | example='http://localhost:5000/static/' 62 | '4f5b954b59360a1aeb8dfd9df0ed60ca.png') 63 | }, 'check') 64 | 65 | bake_response_model = make_response_model({ 66 | 'processTime': fields.Integer(description='处理用时（毫秒）'), 67 | 'resultUrl': fields.String(description='输出图片URL') 68 | }, 'bake') 69 | 70 | check_response_model = make_response_model({ 71 | 'processTime': fields.Integer(description='处理用时（毫秒）'), 72 | 'tid': fields.Integer( 73 | description='结果tid（仅处理成功时；若图片中不含有效的tid信息则返回-1）') 74 | }, 'check') 75 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/resources/logback/logback-spring-dev.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | twitkit-fridge 6 | 7 | 8 | 9 | 10 | 11 | %yellow(%date{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Asia/Shanghai}) %highlight(%-5level) [%thread][%X{requestId}] %cyan(%logger{50}) %magenta([%file:%line]) - %msg%n 12 | 13 | UTF-8 14 | 15 | 16 | 17 | 18 | 19 | 20 | INFO 21 | 22 | 23 | 24 | %date{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Asia/Shanghai} %-5level [%thread][%X{requestId}] %logger{50} [%file:%line] - %msg%n 25 | 26 | UTF-8 27 | 28 | 29 | ${logging.path}/fridge.run.%d{yyyy-MM-dd}.log 30 | 7 31 | 1GB 32 | 33 | 34 | 35 | 36 | 37 | 38 | ERROR 39 | 40 | 41 | 42 | %date{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Asia/Shanghai} %-5level [%thread][%X{requestId}] %logger{50} [%file:%line] - %msg%n 43 | 44 | 45 | 46 | ${logging.path}/fridge.error.%d{yyyy-MM-dd}.log 47 | 30 48 | 1GB 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/resources/logback/logback-spring-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | twitkit-fridge 6 | 7 | 8 | 9 | 10 | 11 | %yellow(%date{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Asia/Shanghai}) %highlight(%-5level) [%thread][%X{requestId}] %cyan(%logger{50}) %magenta([%file:%line]) - %msg%n 12 | 13 | UTF-8 14 | 15 | 16 | 17 | 18 | 19 | 20 | INFO 21 | 22 | 23 | 24 | %date{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Asia/Shanghai} %-5level [%thread][%X{requestId}] %logger{50} [%file:%line] - %msg%n 25 | 26 | UTF-8 27 | 28 | 29 | ${logging.path}/fridge.run.%d{yyyy-MM-dd}.log 30 | 7 31 | 1GB 32 | 33 | 34 | 35 | 36 | 37 | 38 | ERROR 39 | 40 | 41 | 42 | %date{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Asia/Shanghai} %-5level [%thread][%X{requestId}] %logger{50} [%file:%line] - %msg%n 43 | 44 | 45 | 46 | ${logging.path}/fridge.error.%d{yyyy-MM-dd}.log 47 | 30 48 | 1GB 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/test/java/com/enkanrec/twitkitFridge/helper/MvcHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/7 4 | */ 5 | package com.enkanrec.twitkitFridge.helper; 6 | 7 | import com.enkanrec.twitkitFridge.api.response.StandardResponse; 8 | import com.enkanrec.twitkitFridge.util.JsonUtil; 9 | import com.fasterxml.jackson.core.type.TypeReference; 10 | import org.junit.Assert; 11 | import org.springframework.http.MediaType; 12 | import org.springframework.mock.web.MockHttpServletResponse; 13 | import org.springframework.test.web.servlet.MockMvc; 14 | import org.springframework.test.web.servlet.MvcResult; 15 | import org.springframework.test.web.servlet.RequestBuilder; 16 | import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; 17 | import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; 18 | import org.springframework.test.web.servlet.result.MockMvcResultMatchers; 19 | 20 | import java.nio.charset.StandardCharsets; 21 | import java.time.ZonedDateTime; 22 | import java.time.format.DateTimeFormatter; 23 | import java.util.HashMap; 24 | import java.util.List; 25 | import java.util.Map; 26 | 27 | /** 28 | * Class : MvcHelper 29 | * Usage : 30 | */ 31 | public class MvcHelper { 32 | 33 | private String baseUrl; 34 | private MockMvc mvc; 35 | 36 | public MvcHelper(String baseUrl, MockMvc mvc) { 37 | this.baseUrl = baseUrl; 38 | this.mvc = mvc; 39 | } 40 | 41 | public T apiPost(String uri, Object postData, Class returnHint) throws Exception { 42 | Map jsonData = new HashMap<>(); 43 | jsonData.put("forwardFrom", "tester"); 44 | jsonData.put("timestamp", ZonedDateTime.now().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); 45 | if (postData != null) { 46 | jsonData.put("data", postData); 47 | } 48 | String jString = JsonUtil.dumps(jsonData); 49 | RequestBuilder request = MockMvcRequestBuilders.post(this.baseUrl + uri) 50 | .contentType(MediaType.APPLICATION_JSON) 51 | .content(jString); 52 | MvcResult future = this.mvc 53 | .perform(request) 54 | .andExpect(MockMvcResultMatchers.status().isOk()) 55 | .andReturn(); 56 | MockHttpServletResponse result = future.getResponse(); 57 | String cnt = result.getContentAsString(StandardCharsets.UTF_8); 58 | StandardResponse resp = JsonUtil.parseRaw(cnt, new TypeReference() {}); 59 | Assert.assertNotNull(resp); 60 | Assert.assertEquals(StandardResponse.CODE_SUCCESS, resp.getCode()); 61 | Assert.assertEquals(StandardResponse.MESSAGE_SUCCESS, resp.getMessage()); 62 | return (T) resp.getData(); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /maid/twitter_client.py: -------------------------------------------------------------------------------- 1 | import tweepy 2 | import config 3 | import logging 4 | import traceback 5 | import re 6 | 7 | consumer_key = config.CONSUMER_KEY 8 | consumer_secret = config.CONSUMER_SECRET 9 | access_token_key = config.ACCESS_TOKEN_KEY 10 | access_token_secret = config.ACCESS_TOKEN_SECRET 11 | 12 | 13 | auth = tweepy.OAuthHandler(consumer_key, consumer_secret) 14 | auth.set_access_token(access_token_key, access_token_secret) 15 | 16 | twitter_api = tweepy.API(auth) 17 | 18 | 19 | def username_to_uid(username) -> int: 20 | user = twitter_api.get_user(screen_name=username) 21 | return user.id 22 | 23 | 24 | def get_history_tweets(username, max_count=200, min_tweet_id=None): 25 | new_tweets = [] 26 | while True: 27 | range_args = {} 28 | if min_tweet_id: 29 | range_args["since_id"] = min_tweet_id 30 | if new_tweets: 31 | range_args["max_id"] = new_tweets[-1].id - 1 32 | 33 | count = min(max_count, 200) 34 | new_tweets_ = twitter_api.user_timeline( 35 | username, tweet_mode='extended', count=count, **range_args) 36 | max_count -= count 37 | new_tweets += new_tweets_ 38 | if not new_tweets_ or max_count <= 0: 39 | break 40 | 41 | return new_tweets 42 | 43 | 44 | def get_tweet_by_id(id: int): 45 | return twitter_api.get_status(id, tweet_mode='extended') 46 | 47 | 48 | def get_tweet_by_url(url): 49 | search_result = re.search(r'twitter.com\/[\w]{1,15}/status/([0-9]+)', url) 50 | if search_result: 51 | status_id = int(search_result.group(1)) 52 | else: 53 | return False 54 | return get_tweet_by_id(status_id) 55 | 56 | 57 | class RealtimeUpdateStreamListener(tweepy.StreamListener): 58 | 59 | def __init__(self, uid, callback): 60 | """callback函数接受一个参数，为一个tweepy的Status对象""" 61 | self.callback = callback 62 | self.uid = uid 63 | self.api = twitter_api 64 | logging.info(f"开始监听推特用户{uid}") 65 | super().__init__() 66 | 67 | def on_status(self, status): 68 | try: 69 | if status.user.id == self.uid: 70 | logging.info(f"收到实时动态：{status.text}") 71 | self.callback(status) 72 | except Exception as e: 73 | logging.warning(f"on_status发生错误：{e}") 74 | logging.warning(traceback.format_exc()) 75 | 76 | def on_error(self, status_code): 77 | raise Exception(f"发生错误：{status_code}") 78 | 79 | 80 | def run_realtime_update(username, callback): 81 | uid = username_to_uid(username) 82 | rusl = RealtimeUpdateStreamListener(uid, callback) 83 | stream = tweepy.Stream( 84 | auth=twitter_api.auth, listener=rusl, tweet_mode='extended') 85 | stream.filter(follow=[str(uid)], is_async=False) 86 | 87 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/resources/logback/logback-spring-prod.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | twitkit-fridge 6 | 7 | 8 | 9 | 10 | 11 | ERROR 12 | 13 | 14 | 15 | %yellow(%date{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Asia/Shanghai}) %highlight(%-5level) [%thread][%X{requestId}] %cyan(%logger{50}) %magenta([%file:%line]) - %msg%n 16 | 17 | UTF-8 18 | 19 | 20 | 21 | 22 | 23 | 24 | INFO 25 | 26 | 27 | 28 | %date{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Asia/Shanghai} %-5level [%thread][%X{requestId}] %logger{50} [%file:%line] - %msg%n 29 | 30 | UTF-8 31 | 32 | 33 | ${logging.path}/fridge.run.%d{yyyy-MM-dd}.log 34 | 7 35 | 1GB 36 | 37 | 38 | 39 | 40 | 41 | 42 | ERROR 43 | 44 | 45 | 46 | %date{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Asia/Shanghai} %-5level [%thread][%X{requestId}] %logger{50} [%file:%line] - %msg%n 47 | 48 | 49 | 50 | ${logging.path}/fridge.error.%d{yyyy-MM-dd}.log 51 | 30 52 | 1GB 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /koishi-app/src/utils.ts: -------------------------------------------------------------------------------- 1 | export class config { 2 | prefix?: string // 快捷指令前缀 3 | ispro?: boolean // 是否发图 4 | cmd?: config_cmd // 指令模块 5 | watcher?: config_watcher// 监听器模块 6 | } 7 | 8 | class host { 9 | store?: string // 数据库 10 | maid?: string // 推特监视器 11 | translator?: string // 烤推机 12 | } 13 | 14 | export class config_cmd { 15 | host?: host // 微服务域名 16 | group?: number[] // 允许使用指令的群号 17 | private?: boolean // 是否允许私聊指令 18 | friend?: boolean // 是否允许好友指令 19 | cut?: number // 消息预览截断长度 20 | } 21 | 22 | export class target { 23 | discuss?: number[] // 讨论组 24 | private?: number[] // 私聊 25 | group?: number[] // 群 26 | } 27 | 28 | export class config_watcher { 29 | port?: number // 接收更新推送端口 30 | target?: target // 更新推送目标 31 | } 32 | 33 | /** 34 | * guid: "{"GUID"}" 35 | * GUID: hex{8}"-"hex{4}"-"hex{4}"-"hex{4}"-"hex{12} 36 | * hex: [0-9A-Fa-f] 37 | */ 38 | export type uuid = string 39 | export function verifyUuid(str: uuid): boolean { 40 | return /[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}/.test(str) 41 | } 42 | 43 | const chars = "0123456789abcdef" 44 | function randstr(count: number = 1) { 45 | let msg: string = "" 46 | while (msg.length < count) msg += chars[Math.random() * 16 | 0] 47 | return msg 48 | } 49 | 50 | export function genUuid(): uuid { 51 | return randstr(8) + '-' + randstr(4) + '-' + randstr(4) + '-' + randstr(4) + '-' + randstr(12) 52 | } 53 | 54 | /** 55 | * guid: "{"GUID"}" 56 | * GUID: \d+"-"(0\d|1[0-2])"-"([0-2]\d|3[0-1])"T"([0-1]\d|2[0-3])":"[0-5]\d":"[0-5]\d"."\d{3}"+"(0\d|1[0-3])":"[0,3]0 57 | */ 58 | export type ISO8601 = string 59 | export function verifyDatetime(str: string): boolean { 60 | return /\d+-(0\d|1[0-2])-([0-2]\d|3[0-1])T([0-1]\d|2[0-3]):[0-5]\d(:[0-5]\d(\.\d{1,3})?)?(\+(0\d|1[0-3]):[0,3]0)?/.test(str) 61 | } 62 | 63 | export class request { 64 | forwardFrom: string 65 | timestamp: ISO8601 66 | taskId: uuid 67 | data: any 68 | 69 | constructor (Data: any, Forward?: string, Time?: ISO8601) { 70 | this.forwardFrom = Forward || "twitkit-app" 71 | this.timestamp = verifyDatetime(Time) ? Time : new Date().toISOString() 72 | this.taskId = genUuid() 73 | this.data = Data 74 | } 75 | 76 | static parse(Data: request): request { 77 | if (typeof Data.forwardFrom === "undefined") throw "'forwardFrom' undefined" 78 | if (!Data.timestamp || !verifyDatetime(Data.timestamp)) throw "'timestamp' not match" 79 | return Data 80 | } 81 | } 82 | 83 | export class response { 84 | code: number 85 | msg: string 86 | data?: any 87 | 88 | constructor (Msg: string, Code: number = 0, Data: any = undefined) { 89 | this.msg = Msg 90 | this.code = Code 91 | if (typeof Data !== "undefined") this.data = Data 92 | } 93 | 94 | public toString(this: response): string { 95 | return JSON.stringify(this) 96 | } 97 | } -------------------------------------------------------------------------------- /fridge/fridge-src/src/test/java/com/enkanrec/twitkitFridge/wsClient/WSTestClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/8 4 | */ 5 | package com.enkanrec.twitkitFridge.wsClient; 6 | 7 | import com.enkanrec.twitkitFridge.api.ws.FridgeWSServer; 8 | import com.enkanrec.twitkitFridge.util.JsonUtil; 9 | import com.fasterxml.jackson.core.JsonProcessingException; 10 | import io.socket.client.IO; 11 | import io.socket.client.Socket; 12 | import io.socket.engineio.client.transports.Polling; 13 | import io.socket.engineio.client.transports.WebSocket; 14 | import lombok.extern.slf4j.Slf4j; 15 | 16 | import java.time.ZonedDateTime; 17 | import java.time.format.DateTimeFormatter; 18 | import java.util.Arrays; 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | /** 23 | * Class : WSTestClient 24 | * Usage : 25 | */ 26 | @Slf4j 27 | public class WSTestClient { 28 | 29 | public static void main(String[] args) throws Exception { 30 | WSTestClient wsc = new WSTestClient(); 31 | wsc.integrations(); 32 | } 33 | 34 | private Socket webSocket; 35 | 36 | private void integrations() throws Exception { 37 | IO.Options options = new IO.Options(); 38 | options.forceNew = true; 39 | options.reconnection = true; 40 | options.transports = new String[]{WebSocket.NAME, Polling.NAME}; 41 | webSocket = IO.socket("http://localhost:10103", options); 42 | 43 | webSocket.on(Socket.EVENT_CONNECT, args -> log.info("connected in with her handshake: " + Arrays.toString(args))) 44 | .on(Socket.EVENT_DISCONNECT, args -> log.warn("disconnected in with her handshake: " + Arrays.toString(args))) 45 | .on(Socket.EVENT_RECONNECTING, args -> log.warn("reconnecting: " + Arrays.toString(args))) 46 | .on(Socket.EVENT_RECONNECT, args -> log.warn("reconnected: " + Arrays.toString(args))) 47 | .on(FridgeWSServer.RESPONSE_EVT, args -> { 48 | log.info(Arrays.toString(args)); 49 | }); 50 | 51 | webSocket.connect(); 52 | 53 | while (!webSocket.connected()) { 54 | Thread.sleep(1000); 55 | System.out.println("waiting: " + webSocket.connected()); 56 | } 57 | 58 | Map getPayload = new HashMap<>(); 59 | getPayload.put("tid", 1001); 60 | webSocket.emit(FridgeWSServer.REQUEST_EVT, buildPayload("task", "get", getPayload)); 61 | 62 | while (true) { 63 | Thread.sleep(10000); 64 | System.out.println("beat: " + webSocket.connected()); 65 | } 66 | } 67 | 68 | private String buildPayload(String of, String command, Object data) throws JsonProcessingException { 69 | Map payload = new HashMap<>(); 70 | payload.put("of", of); 71 | payload.put("command", command); 72 | payload.put("forwardFrom", "tester.socketIO"); 73 | payload.put("timestamp", ZonedDateTime.now().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); 74 | payload.put("data", data); 75 | return JsonUtil.dumps(payload); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /fridge/schema/db_init.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | DROP TABLE IF EXISTS enkan_config; 4 | DROP TABLE IF EXISTS enkan_task; 5 | DROP TABLE IF EXISTS enkan_translate; 6 | DROP TABLE IF EXISTS enkan_twitter; 7 | 8 | CREATE TABLE `enkan_config` ( 9 | `id` int(11) NOT NULL AUTO_INCREMENT, 10 | `namespace` varchar(191) NOT NULL DEFAULT '___DEFAULT___', 11 | `config_key` varchar(191) NOT NULL, 12 | `config_value` text, 13 | `newdate` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, 14 | `updatetime` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 15 | PRIMARY KEY (`id`), 16 | UNIQUE KEY `key_namespace_key` (`namespace`,`config_key`), 17 | KEY `key_updatetime` (`updatetime`) 18 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 19 | 20 | CREATE TABLE `enkan_task` ( 21 | `tid` int(11) NOT NULL AUTO_INCREMENT COMMENT '任务id', 22 | `status_id` varchar(191) NOT NULL COMMENT '推特生成的推文唯一id', 23 | `url` varchar(767) NOT NULL COMMENT '推文URL', 24 | `content` longtext NOT NULL COMMENT '推文内容', 25 | `media` text NOT NULL COMMENT '媒体地址', 26 | `published` tinyint(2) NOT NULL DEFAULT '0' COMMENT '是否已发布', 27 | `hided` tinyint(2) NOT NULL DEFAULT '0' COMMENT '是否隐藏', 28 | `comment` varchar(1023) NOT NULL DEFAULT '' COMMENT '备注', 29 | `newdate` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, 30 | `updatetime` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 31 | `twitter_uid` varchar(191) DEFAULT NULL COMMENT '指向该推文主动用户', 32 | `ref_tid` int(11) DEFAULT NULL COMMENT '指向转发/引用的推文id\n', 33 | `pub_date` datetime DEFAULT NULL COMMENT 'API返回的发推时间', 34 | `extra` longtext COMMENT 'json储存动态字段，任意扩展，不需要索引的东西都可以丢进来\n\n\n\n', 35 | PRIMARY KEY (`tid`), 36 | UNIQUE KEY `key_uid_statusid` (`twitter_uid`,`status_id`) USING BTREE, 37 | KEY `key_newdate` (`newdate`), 38 | KEY `key_updatetime` (`updatetime`), 39 | KEY `key_url` (`url`) USING BTREE 40 | ) ENGINE=InnoDB AUTO_INCREMENT=1000 DEFAULT CHARSET=utf8mb4; 41 | 42 | CREATE TABLE `enkan_translate` ( 43 | `zzid` int(11) NOT NULL AUTO_INCREMENT, 44 | `tid` int(11) NOT NULL COMMENT '推文id', 45 | `version` int(11) NOT NULL DEFAULT '0' COMMENT '版本号', 46 | `translation` longtext NOT NULL COMMENT '翻译内容', 47 | `img` varchar(2047) NOT NULL COMMENT '烤推机生成的图地址', 48 | `newdate` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, 49 | `updatetime` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 50 | PRIMARY KEY (`zzid`), 51 | UNIQUE KEY `key_tid_version` (`tid`,`version`), 52 | KEY `key_updatetime` (`updatetime`), 53 | KEY `key_newdate` (`newdate`) 54 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 55 | 56 | CREATE TABLE `enkan_twitter` ( 57 | `uid` int(11) NOT NULL AUTO_INCREMENT, 58 | `twitter_uid` varchar(191) NOT NULL, 59 | `name` varchar(511) NOT NULL, 60 | `display` varchar(511) NOT NULL, 61 | `avatar` varchar(2047) NOT NULL, 62 | `updatetime` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 63 | `newdate` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, 64 | PRIMARY KEY (`uid`), 65 | UNIQUE KEY `key_twitteruid` (`twitter_uid`) 66 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 67 | 68 | COMMIT; 69 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/steady/bean/JpaConfigYui.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/30 4 | */ 5 | package com.enkanrec.twitkitFridge.steady.bean; 6 | 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.beans.factory.annotation.Qualifier; 9 | import org.springframework.boot.autoconfigure.orm.jpa.HibernateProperties; 10 | import org.springframework.boot.autoconfigure.orm.jpa.HibernateSettings; 11 | import org.springframework.boot.autoconfigure.orm.jpa.JpaProperties; 12 | import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; 13 | import org.springframework.context.annotation.Bean; 14 | import org.springframework.context.annotation.Configuration; 15 | import org.springframework.context.annotation.Primary; 16 | import org.springframework.data.jpa.repository.config.EnableJpaRepositories; 17 | import org.springframework.orm.jpa.JpaTransactionManager; 18 | import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; 19 | import org.springframework.transaction.PlatformTransactionManager; 20 | import org.springframework.transaction.annotation.EnableTransactionManagement; 21 | 22 | import javax.persistence.EntityManager; 23 | import javax.sql.DataSource; 24 | import java.util.Map; 25 | 26 | /** 27 | * Class : JpaConfigYui 28 | * Usage : Yui数据库的数据源配置项 29 | */ 30 | @Configuration 31 | @EnableTransactionManagement 32 | @EnableJpaRepositories( 33 | entityManagerFactoryRef = "entityManagerFactoryYui", 34 | transactionManagerRef = "transactionManagerYui", 35 | basePackages = {"com.enkanrec.twitkitFridge.steady.yui.repository"}) 36 | public class JpaConfigYui { 37 | 38 | @Autowired 39 | @Qualifier("yuiDataSource") 40 | private DataSource yuiDataSource; 41 | 42 | @Autowired 43 | private JpaProperties jpaProperties; 44 | 45 | @Autowired 46 | private HibernateProperties hibernateProperties; 47 | 48 | @Primary 49 | @Bean(name = "entityManagerYui") 50 | public EntityManager entityManager(EntityManagerFactoryBuilder builder) { 51 | return entityManagerFactoryYui(builder).getObject().createEntityManager(); 52 | } 53 | 54 | @Primary 55 | @Bean(name = "entityManagerFactoryYui") 56 | public LocalContainerEntityManagerFactoryBean entityManagerFactoryYui(EntityManagerFactoryBuilder builder) { 57 | return builder 58 | .dataSource(yuiDataSource) 59 | .packages("com.enkanrec.twitkitFridge.steady.yui.entity") 60 | .persistenceUnit("yuiPersistenceUnit") 61 | .properties(getVendorProperties()) 62 | .build(); 63 | } 64 | 65 | private Map getVendorProperties() { 66 | return hibernateProperties.determineHibernateProperties(jpaProperties.getProperties(), new HibernateSettings()); 67 | } 68 | 69 | @Primary 70 | @Bean(name = "transactionManagerYui") 71 | public PlatformTransactionManager transactionManagerYui(EntityManagerFactoryBuilder builder) { 72 | return new JpaTransactionManager(entityManagerFactoryYui(builder).getObject()); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/test/java/com/enkanrec/twitkitFridge/KVConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/3 4 | */ 5 | package com.enkanrec.twitkitFridge; 6 | 7 | import com.enkanrec.twitkitFridge.helper.MvcHelper; 8 | import org.junit.Assert; 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | import org.junit.runner.RunWith; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.boot.test.context.SpringBootTest; 14 | import org.springframework.test.context.TestPropertySource; 15 | import org.springframework.test.context.junit4.SpringRunner; 16 | import org.springframework.test.context.web.WebAppConfiguration; 17 | import org.springframework.test.web.servlet.setup.MockMvcBuilders; 18 | import org.springframework.transaction.annotation.Transactional; 19 | import org.springframework.web.context.WebApplicationContext; 20 | 21 | import java.time.ZonedDateTime; 22 | import java.time.format.DateTimeFormatter; 23 | import java.util.ArrayList; 24 | import java.util.HashMap; 25 | import java.util.List; 26 | import java.util.Map; 27 | 28 | /** 29 | * Class : KVConfigTest 30 | * Usage : 31 | */ 32 | @SuppressWarnings("all") 33 | @RunWith(SpringRunner.class) 34 | @SpringBootTest 35 | @WebAppConfiguration 36 | @TestPropertySource("classpath:application-test.properties") 37 | public class KVConfigTest { 38 | 39 | private static final String BASE_URL = "/api/db/kv"; 40 | 41 | @Autowired 42 | private WebApplicationContext ctx; 43 | 44 | private MvcHelper helper; 45 | 46 | @Before 47 | public void setUp() { 48 | this.helper = new MvcHelper(BASE_URL, MockMvcBuilders.webAppContextSetup(ctx).build()); 49 | } 50 | 51 | @Transactional 52 | @Test 53 | public void getAll() throws Exception { 54 | Map respData = helper.apiPost("/getall", null, Map.class); 55 | Assert.assertEquals("iroha", respData.get("test.default.yachiyo.love")); 56 | Assert.assertEquals("五十铃怜", respData.get("test.default.rika")); 57 | Assert.assertEquals("❤①+123AB\uF8FF", respData.get("___TEST_MB4___#test.mb4.emoji")); 58 | Assert.assertNull(respData.get("test.not.exist.one")); 59 | } 60 | 61 | @Transactional 62 | @Test 63 | public void setDefaultAndGet() throws Exception { 64 | String currentTs = ZonedDateTime.now().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); 65 | Map data = new HashMap<>(); 66 | data.put("test.currentTsStr", currentTs); 67 | data.put("test.mb4", "❤①+123AB"); 68 | String respData = helper.apiPost("/set", data, String.class); 69 | Assert.assertEquals("", respData); 70 | 71 | List data2 = new ArrayList<>(); 72 | data2.add("test.currentTsStr"); 73 | data2.add("test.not.exist.SetDefaultAndGet"); 74 | data2.add("test.mb4"); 75 | Map respData2 = helper.apiPost("/get", data2, Map.class); 76 | Assert.assertEquals(currentTs, respData2.get("test.currentTsStr")); 77 | Assert.assertNull(respData2.get("test.not.exist.SetDefaultAndGet")); 78 | Assert.assertEquals("❤①+123AB\uF8FF", respData2.get("test.mb4")); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /koishi-app/src/translator.ts: -------------------------------------------------------------------------------- 1 | import { Context, Logger } from 'koishi-core' 2 | import { Twitter } from './twitter' 3 | import { request } from './utils' 4 | import axios from 'axios' 5 | 6 | let host: string 7 | let logger: Logger 8 | 9 | class response { // oven return 10 | code: number 11 | message: string 12 | error?: any 13 | processTime?: number 14 | resultUrl?: string 15 | tid?: number 16 | } 17 | 18 | /** 19 | * @description 封装api请求，但仅用于bake/bakeurl 20 | * @param url api地址 21 | * @param data 数据 22 | * @returns response.resultUrl 23 | */ 24 | async function rest(url: string, data: any): Promise { 25 | logger.debug("POST " + url) 26 | logger.debug(data) 27 | try { 28 | const res = await axios.post(host + url, new request(data)) 29 | if (res.data.code === 0) { 30 | logger.debug(res.data) 31 | logger.debug("Return %d: %s", res.data.code, res.data.message || "") 32 | logger.debug("Finish in %d s", res.data.processTime) 33 | return res.data.resultUrl 34 | } else { 35 | logger.warn("Error %d: %s", res.data.code, res.data.message) 36 | return null 37 | } 38 | } catch (e) { 39 | if (e.response) { 40 | logger.error("Internet error: %d", e.response.status) 41 | logger.debug(e.response.data) 42 | } else logger.error(e) 43 | } 44 | return null 45 | } 46 | 47 | /** 48 | * @description 请求一个推文的烤图结果 49 | * @param tw 推文内容 50 | * @param tw.id 推文id 51 | * @param tw.trans 推文翻译 52 | * @returns 烤推结果图片url 53 | */ 54 | function get(tw: Twitter): Promise { 55 | logger.debug("Tid: %d", tw.id) 56 | return rest("/api/oven/bake", { 57 | tid: tw.id, 58 | transText: tw.trans 59 | }) 60 | } 61 | 62 | /** 63 | * @description 请求特定url的烤图结果 64 | * @param url 特定推特url 65 | * @param trans 翻译 66 | * @returns 烤推结果图片url 67 | */ 68 | function getByUrl(url: string, trans: string): Promise { 69 | logger.debug("url: " + url) 70 | return rest("/api/oven/bake", { 71 | url, 72 | transText: trans 73 | }) 74 | } 75 | 76 | /** 77 | * @description 检查图片是否是烤推发布 78 | * @param url 需要检查的图片url 79 | * @returns 检测出的推文id 80 | * @deprecated 暂未使用 81 | */ 82 | async function check(url: string): Promise { 83 | let res = await axios.post(host + "/api/oven/check", new request({ 84 | // uuid: utils.genUuid(), 85 | imageUrl: url 86 | })) 87 | if (res.status !== 200) { 88 | logger.error("Internet error: %d", res.status) 89 | return null 90 | } 91 | logger.debug("Return %d: %s", res.data.code, res.data.message) 92 | if (res.data.code === 0) { 93 | logger.debug("Found tid %d", res.data.tid) 94 | return res.data.tid 95 | } else { 96 | logger.warn("Error: %s", res.data.error) 97 | return null 98 | } 99 | } 100 | 101 | function init(ctx: Context, Host: string): void { 102 | logger = ctx.logger("app:translator") 103 | host = Host 104 | logger.info("translator client ready") 105 | } 106 | 107 | export default { 108 | init, 109 | get, 110 | getByUrl, 111 | check 112 | } -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/steady/yui/entity/EnkanConfigEntity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/3 4 | */ 5 | package com.enkanrec.twitkitFridge.steady.yui.entity; 6 | 7 | import lombok.ToString; 8 | 9 | import javax.persistence.*; 10 | import java.sql.Timestamp; 11 | import java.util.Objects; 12 | 13 | /** 14 | * Class : EnkanConfigEntity 15 | * Usage : 16 | */ 17 | @Entity 18 | @ToString 19 | @Table(name = "enkan_config", schema = "yui") 20 | public class EnkanConfigEntity { 21 | private int id; 22 | private String namespace; 23 | private String configKey; 24 | private String configValue; 25 | private Timestamp newdate; 26 | private Timestamp updatetime; 27 | 28 | @Id 29 | @Column(name = "id", nullable = false) 30 | @GeneratedValue(strategy = GenerationType.IDENTITY) 31 | public int getId() { 32 | return id; 33 | } 34 | 35 | public void setId(int id) { 36 | this.id = id; 37 | } 38 | 39 | @Basic 40 | @Column(name = "namespace", nullable = false, length = 255) 41 | public String getNamespace() { 42 | return namespace; 43 | } 44 | 45 | public void setNamespace(String namespace) { 46 | this.namespace = namespace; 47 | } 48 | 49 | @Basic 50 | @Column(name = "config_key", nullable = false, length = 255) 51 | public String getConfigKey() { 52 | return configKey; 53 | } 54 | 55 | public void setConfigKey(String configKey) { 56 | this.configKey = configKey; 57 | } 58 | 59 | @Basic 60 | @Column(name = "config_value", nullable = true, length = -1) 61 | public String getConfigValue() { 62 | return configValue; 63 | } 64 | 65 | public void setConfigValue(String configValue) { 66 | this.configValue = configValue; 67 | } 68 | 69 | @Basic 70 | @Column(name = "newdate", nullable = false, updatable = false, insertable = false) 71 | public Timestamp getNewdate() { 72 | return newdate; 73 | } 74 | 75 | public void setNewdate(Timestamp newdate) { 76 | this.newdate = newdate; 77 | } 78 | 79 | @Basic 80 | @Column(name = "updatetime", nullable = false, updatable = false, insertable = false) 81 | public Timestamp getUpdatetime() { 82 | return updatetime; 83 | } 84 | 85 | public void setUpdatetime(Timestamp updatetime) { 86 | this.updatetime = updatetime; 87 | } 88 | 89 | @Override 90 | public boolean equals(Object o) { 91 | if (this == o) return true; 92 | if (o == null || getClass() != o.getClass()) return false; 93 | EnkanConfigEntity that = (EnkanConfigEntity) o; 94 | return id == that.id && 95 | Objects.equals(namespace, that.namespace) && 96 | Objects.equals(configKey, that.configKey) && 97 | Objects.equals(configValue, that.configValue) && 98 | Objects.equals(newdate, that.newdate) && 99 | Objects.equals(updatetime, that.updatetime); 100 | } 101 | 102 | @Override 103 | public int hashCode() { 104 | return Objects.hash(id, namespace, configKey, configValue, newdate, updatetime); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/steady/yui/entity/EnkanTwitterEntity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/29 4 | */ 5 | package com.enkanrec.twitkitFridge.steady.yui.entity; 6 | 7 | import lombok.ToString; 8 | 9 | import javax.persistence.*; 10 | import java.sql.Timestamp; 11 | import java.util.Objects; 12 | 13 | /** 14 | * Class : EnkanTwitterEntity 15 | * Usage : 16 | */ 17 | @ToString 18 | @Entity 19 | @Table(name = "enkan_twitter", schema = "yui") 20 | public class EnkanTwitterEntity { 21 | private int uid; 22 | private String twitterUid; 23 | private String name; 24 | private String display; 25 | private String avatar; 26 | private Timestamp updatetime; 27 | private Timestamp newdate; 28 | 29 | @Id 30 | @Column(name = "uid", nullable = false) 31 | @GeneratedValue(strategy = GenerationType.IDENTITY) 32 | public int getUid() { 33 | return uid; 34 | } 35 | 36 | public void setUid(int uid) { 37 | this.uid = uid; 38 | } 39 | 40 | @Basic 41 | @Column(name = "twitter_uid", nullable = false, length = 255) 42 | public String getTwitterUid() { 43 | return twitterUid; 44 | } 45 | 46 | public void setTwitterUid(String twitterUid) { 47 | this.twitterUid = twitterUid; 48 | } 49 | 50 | @Basic 51 | @Column(name = "name", nullable = false, length = 511) 52 | public String getName() { 53 | return name; 54 | } 55 | 56 | public void setName(String name) { 57 | this.name = name; 58 | } 59 | 60 | @Basic 61 | @Column(name = "display", nullable = false, length = 511) 62 | public String getDisplay() { 63 | return display; 64 | } 65 | 66 | public void setDisplay(String display) { 67 | this.display = display; 68 | } 69 | 70 | @Basic 71 | @Column(name = "avatar", nullable = false, length = 2047) 72 | public String getAvatar() { 73 | return avatar; 74 | } 75 | 76 | public void setAvatar(String avatar) { 77 | this.avatar = avatar; 78 | } 79 | 80 | @Override 81 | public boolean equals(Object o) { 82 | if (this == o) return true; 83 | if (o == null || getClass() != o.getClass()) return false; 84 | EnkanTwitterEntity that = (EnkanTwitterEntity) o; 85 | return uid == that.uid && 86 | Objects.equals(twitterUid, that.twitterUid) && 87 | Objects.equals(name, that.name) && 88 | Objects.equals(display, that.display) && 89 | Objects.equals(avatar, that.avatar) && 90 | Objects.equals(updatetime, that.updatetime) && 91 | Objects.equals(newdate, that.newdate); 92 | } 93 | 94 | @Override 95 | public int hashCode() { 96 | return Objects.hash(uid, twitterUid, name, display, avatar, updatetime, newdate); 97 | } 98 | 99 | @Basic 100 | @Column(name = "updatetime", nullable = false, insertable = false, updatable = false) 101 | public Timestamp getUpdatetime() { 102 | return updatetime; 103 | } 104 | 105 | public void setUpdatetime(Timestamp updatetime) { 106 | this.updatetime = updatetime; 107 | } 108 | 109 | @Basic 110 | @Column(name = "newdate", nullable = false, insertable = false, updatable = false) 111 | public Timestamp getNewdate() { 112 | return newdate; 113 | } 114 | 115 | public void setNewdate(Timestamp newdate) { 116 | this.newdate = newdate; 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/util/JsonUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/30 4 | */ 5 | package com.enkanrec.twitkitFridge.util; 6 | 7 | import com.fasterxml.jackson.core.JsonProcessingException; 8 | import com.fasterxml.jackson.core.type.TypeReference; 9 | import com.fasterxml.jackson.databind.JsonNode; 10 | import com.fasterxml.jackson.databind.ObjectMapper; 11 | import com.fasterxml.jackson.databind.node.*; 12 | 13 | import java.util.*; 14 | 15 | /** 16 | * Class : JsonUtil 17 | * Usage : 18 | */ 19 | public class JsonUtil { 20 | 21 | public static final ObjectMapper Mapper = new ObjectMapper(); 22 | 23 | public static String dumps(Object dumper) throws JsonProcessingException { 24 | return JsonUtil.Mapper.writeValueAsString(dumper); 25 | } 26 | 27 | public static T parseRaw(String jString, TypeReference valueTypeRef) throws JsonProcessingException { 28 | return JsonUtil.Mapper.readValue(jString, valueTypeRef); 29 | } 30 | 31 | public static T parse(String jString, Class outerHint) throws JsonProcessingException { 32 | if (Map.class.isAssignableFrom(outerHint) || List.class.isAssignableFrom(outerHint)) { 33 | return (T) JsonUtil.parse(jString); 34 | } else { 35 | return null; 36 | } 37 | } 38 | 39 | public static Object parse(String jString) throws JsonProcessingException { 40 | if (jString == null) { 41 | return null; 42 | } 43 | if (jString.startsWith("{")) { 44 | ObjectNode scoped = JsonUtil.Mapper.readValue(jString, ObjectNode.class); 45 | return JsonUtil.parseMap(scoped); 46 | } else if (jString.startsWith("[")) { 47 | ArrayNode scoped = JsonUtil.Mapper.readValue(jString, ArrayNode.class); 48 | return JsonUtil.parseList(scoped); 49 | } else { 50 | return jString; 51 | } 52 | } 53 | 54 | public static Map parseMap(ObjectNode jNode) throws JsonProcessingException { 55 | Map scoped = new HashMap<>(); 56 | Iterator> iter = jNode.fields(); 57 | while (iter.hasNext()) { 58 | Map.Entry iterItem = iter.next(); 59 | String key = iterItem.getKey(); 60 | Object value = iterItem.getValue(); 61 | value = getTinyNode(value); 62 | scoped.put(key, value); 63 | } 64 | return scoped; 65 | } 66 | 67 | public static List parseList(ArrayNode jArray) throws JsonProcessingException { 68 | List list = new ArrayList<>(); 69 | for (int i = 0; i < jArray.size(); i++) { 70 | Object value = jArray.get(i); 71 | value = getTinyNode(value); 72 | list.add(value); 73 | } 74 | return list; 75 | } 76 | 77 | private static Object getTinyNode(Object value) throws JsonProcessingException { 78 | if (value instanceof TextNode) { 79 | value = ((TextNode) value).asText(); 80 | } else if (value instanceof ArrayNode) { 81 | value = parseList((ArrayNode) value); 82 | } else if (value instanceof ObjectNode) { 83 | value = parseMap((ObjectNode) value); 84 | } else if (value instanceof NumericNode) { 85 | value = ((NumericNode) value).numberValue(); 86 | } else if (value instanceof BooleanNode) { 87 | value = ((BooleanNode) value).booleanValue(); 88 | } else if (value instanceof NullNode) { 89 | value = null; 90 | } 91 | return value; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/steady/yui/entity/EnkanTranslateEntity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/4 4 | */ 5 | package com.enkanrec.twitkitFridge.steady.yui.entity; 6 | 7 | import com.fasterxml.jackson.annotation.JsonIgnore; 8 | import lombok.ToString; 9 | 10 | import javax.persistence.*; 11 | import java.sql.Timestamp; 12 | import java.util.Objects; 13 | 14 | /** 15 | * Class : EnkanTranslateEntity 16 | * Usage : 17 | */ 18 | @Entity 19 | @ToString(exclude = {"task"}) 20 | @Table(name = "enkan_translate", schema = "yui") 21 | public class EnkanTranslateEntity { 22 | private int zzid; 23 | private int version; 24 | private String translation; 25 | private String img; 26 | private Timestamp newdate; 27 | private Timestamp updatetime; 28 | 29 | private EnkanTaskEntity task; 30 | 31 | @Id 32 | @Column(name = "zzid", nullable = false) 33 | @GeneratedValue(strategy = GenerationType.IDENTITY) 34 | public int getZzid() { 35 | return zzid; 36 | } 37 | 38 | public void setZzid(int zzid) { 39 | this.zzid = zzid; 40 | } 41 | 42 | @Basic 43 | @Column(name = "version", nullable = false) 44 | public int getVersion() { 45 | return version; 46 | } 47 | 48 | public void setVersion(int version) { 49 | this.version = version; 50 | } 51 | 52 | @Basic 53 | @Column(name = "translation", nullable = false, length = -1) 54 | public String getTranslation() { 55 | return translation; 56 | } 57 | 58 | public void setTranslation(String translation) { 59 | this.translation = translation; 60 | } 61 | 62 | @Basic 63 | @Column(name = "img", nullable = false, length = 2047) 64 | public String getImg() { 65 | return img; 66 | } 67 | 68 | public void setImg(String img) { 69 | this.img = img; 70 | } 71 | 72 | @Basic 73 | @Column(name = "newdate", nullable = false, insertable = false, updatable = false) 74 | public Timestamp getNewdate() { 75 | return newdate; 76 | } 77 | 78 | public void setNewdate(Timestamp newdate) { 79 | this.newdate = newdate; 80 | } 81 | 82 | @Basic 83 | @Column(name = "updatetime", nullable = false, insertable = false, updatable = false) 84 | public Timestamp getUpdatetime() { 85 | return updatetime; 86 | } 87 | 88 | public void setUpdatetime(Timestamp updatetime) { 89 | this.updatetime = updatetime; 90 | } 91 | 92 | @Override 93 | public boolean equals(Object o) { 94 | if (this == o) return true; 95 | if (o == null || getClass() != o.getClass()) return false; 96 | EnkanTranslateEntity that = (EnkanTranslateEntity) o; 97 | return zzid == that.zzid && 98 | getTask().equals(that.getTask()) && 99 | version == that.version && 100 | Objects.equals(translation, that.translation) && 101 | Objects.equals(img, that.img) && 102 | Objects.equals(newdate, that.newdate) && 103 | Objects.equals(updatetime, that.updatetime); 104 | } 105 | 106 | @Override 107 | public int hashCode() { 108 | return Objects.hash(zzid, this.getTask().getTid(), version, translation, img, newdate, updatetime); 109 | } 110 | 111 | @JsonIgnore 112 | @ManyToOne(cascade = {CascadeType.REFRESH}) 113 | @JoinColumn(name = "tid") 114 | public EnkanTaskEntity getTask() { 115 | return task; 116 | } 117 | 118 | public void setTask(EnkanTaskEntity task) { 119 | this.task = task; 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /maid/README.md: -------------------------------------------------------------------------------- 1 | # TwitKit-Maid 2 | 3 | 获取推特及Bilibili更新用的组件 4 | 5 | ## 环境要求 6 | 7 | * Python 3.6 及以上 8 | * requests 9 | 10 | ## 使用方法 11 | 12 | 1. 安装Python依赖： 13 | 14 | ``` 15 | pip3 install -r requirements.txt 16 | ``` 17 | 18 | 2. 执行`./configen.sh`创建配置文件（`config.py`），将必要的信息填入（具体下述） 19 | 20 | 3. 执行`./start.sh`启动Maid 21 | 22 | 23 | 24 | ### 配置文件选项说明 25 | 26 | 其他组件API设置： 27 | 28 | * `FRIDGE_API_BASE`：Fridge API根URL 29 | * `APP_API_BASE`：App API根URL 30 | * `OVEN_API_BASE`：Oven API根URL 31 | 32 | 监听设置： 33 | 34 | * `API_SERVER_HOST`：监听地址 35 | * `API_SERVER_PORT`：监听端口 36 | 37 | 推特API相关设置： 38 | 39 | * `CONSUMER_KEY`：Consumer Key 40 | * `CONSUMER_SECRET`：Consumer Secret 41 | * `ACCESS_TOKEN_KEY`：Access Token Key 42 | * `ACCESS_TOKEN_SECRET`：Access Token Secret 43 | 44 | RSSHub相关设置： 45 | 46 | * `BILIBILI_UID`：监听的Bilibili账号的UID（考虑改为数据库中的设置）,设`None`可以禁用 47 | * `UPDATE_INTERVAL`：更新间隔（秒） 48 | 49 | 日志设置： 50 | 51 | * `LOG_DEBUG`：是否使用调试日志级别 52 | * `LOG_FILE`：日志文件路径（`None`为不保存日志文件） 53 | 54 | 推送过滤设置： 55 | 56 | * `PUSH_RETWEETS`：是否推送转推（布尔值） 57 | * `PUSH_REPLIES`：是否推送回复（布尔值） 58 | 59 | 其他： 60 | 61 | * `MAX_HISTORY_TWEETS`：启动时获取的最大历史推数（数据库中最新推开始至当前） 62 | 63 | ## API文档 64 | 65 | ### 从URL入库一条推 66 | 67 | 接口地址：`/api/maid/addtask` 68 | 69 | 请求和返回报文均为标准JSON格式 70 | 71 | 提交方法：POST 72 | 73 | #### 请求报文 74 | 75 | 请求报文样例： 76 | 77 | ``` 78 | { 79 | "forwardFrom": ..., 80 | "timestamp": ..., 81 | "taskId": ..., 82 | "data": { 83 | "url": "https://twitter.com/magireco/status/1233776691064868865" 84 | } 85 | } 86 | ``` 87 | （注：更正了`taskId`的位置） 88 | 89 | 参数说明： 90 | 91 | * `url`：推文的URL 92 | 93 | #### 返回报文 94 | 95 | 若请求成功，返回报文内容例： 96 | 97 | ``` 98 | { 99 | "code": 0, 100 | "message": "OK", 101 | "addedTid": [ 102 | 1001, 103 | 1002, 104 | ... 105 | ], 106 | "rootTid": 1001 107 | } 108 | ``` 109 | 110 | 参数说明： 111 | 112 | * `addedTid`：含有已入库的tid的数组 113 | * `rootTid`：请求插入的推文本身的tid 114 | 115 | （`addedTid`中`rootTid`之外的结果是由嵌套产生的） 116 | 117 | 目前可能出现的返回码： 118 | 119 | * `0`：加入任务成功 120 | * `400`：输入格式验证失败 121 | * `500`： 推特API请求失败 122 | * `501`： Fridge API请求失败 123 | 124 | ### 从URL获取一条推（不入库） 125 | 126 | 接口地址：`/api/maid/gettweet` 127 | 128 | 请求和返回报文均为标准JSON格式 129 | 130 | 提交方法：POST 131 | 132 | #### 请求报文 133 | 134 | 请求报文样例： 135 | 136 | ``` 137 | { 138 | "forwardFrom": ..., 139 | "timestamp": ..., 140 | "taskId": ..., 141 | "data": { 142 | "url": "https://twitter.com/magireco/status/1233776691064868865" 143 | } 144 | } 145 | ``` 146 | 147 | 参数说明： 148 | 149 | - `url`：推文的URL 150 | 151 | #### 返回报文 152 | 153 | 若请求成功，返回报文内容例： 154 | 155 | ``` 156 | { 157 | "code": 0, 158 | "message": "OK", 159 | "tweets": { 160 | "1237693889504358401": { 161 | "twitter": { 162 | "statusId": "1237693889504358401", 163 | "url": "https://twitter.com/magireco/status/1237693889504358401", 164 | "content": "内容", 165 | "media": "[\"https://pbs.twimg.com/media/ES0r4KLUMAE-Ywe.jpg\"]", 166 | "refStatusId": null, 167 | "twitterUid": "761743038800474112", 168 | "pubDate": "2020-03-11T19:56:33+09:00", 169 | }, 170 | "user": { 171 | "twitterUid": "761743038800474112", 172 | "name": "magireco", 173 | "display": "マギアレコード公式", 174 | "avatar": "https://pbs.twimg.com/profile_images/1205868817336766465/gqo0RuBq_400x400.jpg" 175 | } 176 | }, 177 | ... 178 | }, 179 | "rootStatusId": 1001 180 | } 181 | ``` 182 | 183 | 参数说明： 184 | 185 | - `tweets`：得到的推（key是推特的Status ID，value是具体推文数据，格式和Fridge返回的类似） 186 | - `twitter.refStatus`：引用的Status ID（引用也一定在返回结果中） 187 | - 其余项意义和格式都和Fridge的一样 188 | - `rootStatusId`：请求插入的推文本身的tid 189 | 190 | （`tweets`中`rootStatusId`之外的结果是由嵌套产生的） 191 | 192 | 目前可能出现的返回码： 193 | 194 | - `0`：获取推文成功 195 | - `400`：输入格式验证失败 196 | - `500`： 推特API请求失败 -------------------------------------------------------------------------------- /oven/baker/src/App.vue: -------------------------------------------------------------------------------- 1 | 47 | 48 | 119 | 120 | -------------------------------------------------------------------------------- /oven/oven.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, request 2 | from PIL import Image 3 | from tid_code import add_code_to_image 4 | from hashlib import md5 5 | from PIL import Image 6 | from pdf2image import convert_from_bytes 7 | from base64 import b64decode 8 | 9 | import pychrome 10 | import urllib.parse 11 | import os 12 | import sys 13 | import logging 14 | import json 15 | import config 16 | import traceback 17 | import time 18 | 19 | browser = pychrome.Browser(url=config.CHROME_REMOTE_DEBUGGING_URL) 20 | 21 | logger = logging.getLogger('oven') 22 | tweet_page_bp = Blueprint( 23 | 'internal', __name__, static_folder='baker/dist/', static_url_path='/') 24 | 25 | STATIC = 'static' 26 | 27 | 28 | def bake_tweet(tid=None, url=None, trans_text=None, ppi=config.DEFAULT_PPI): 29 | 30 | if tid and url: 31 | logger.error('不能同时指定tid和url') 32 | return False 33 | 34 | try: 35 | os.makedirs(STATIC, mode=0o755, exist_ok=True) 36 | except OSError: 37 | logger.error('创建输出目录失败。') 38 | return False 39 | 40 | payload_data = json.dumps({ 41 | 'tid': tid, 42 | 'url': url, 43 | 'transText': trans_text, 44 | 'zhFont': config.ZH_FONT, 45 | 'jaFont': config.JA_FONT, 46 | }) 47 | 48 | logger.debug(f'payload_data: {payload_data}') 49 | 50 | payload_data = urllib.parse.quote(payload_data, safe='') 51 | 52 | tab = browser.new_tab() 53 | tab.start() 54 | tab.Network.enable() 55 | 56 | im = None 57 | try: 58 | im = print_image_from_tab(tab, payload_data, ppi) 59 | except Exception as e: 60 | logger.error(f'输出图片失败：{e}') 61 | logger.error(traceback.format_exc) 62 | finally: 63 | tab.stop() 64 | browser.close_tab(tab) 65 | if not im: 66 | return False 67 | 68 | if not url: 69 | add_code_to_image(im, tid, 70 | config.TID_CODE_POS_X, config.TID_CODE_POS_Y, 71 | config.TID_CODE_WIDTH, config.TID_CODE_HEIGHT) 72 | 73 | actual_filename = f'{md5(im.tobytes()).hexdigest()}.png' 74 | im.save(os.path.join(STATIC, actual_filename)) 75 | 76 | return actual_filename 77 | 78 | 79 | def print_image_from_tab(tab, payload_data, ppi): 80 | 81 | loading_counter = [] 82 | 83 | def request_will_be_sent(**kwargs): 84 | logger.debug(f"载入：{kwargs['request']['url']}") 85 | loading_counter.append(True) 86 | 87 | tab.Network.requestWillBeSent = request_will_be_sent 88 | 89 | def loading_finished(**kwargs): 90 | logger.debug('完成载入') 91 | loading_counter.pop() 92 | 93 | tab.Network.loadingFinished = loading_finished 94 | 95 | tab.Network.setCookie( 96 | name='payload_data', 97 | value=payload_data, 98 | url=config.INT_BASE_URL 99 | ) 100 | 101 | tab.Page.navigate(url=f'{config.INT_BASE_URL}/internal/index.html') 102 | 103 | success_count = 0 104 | begin_time = time.time() 105 | while time.time() - begin_time < config.LOAD_TIME_LIMIT: 106 | tab.wait(0.1) 107 | ready_state = tab.Runtime.evaluate(expression="document.readyState")[ 108 | 'result']['value'] 109 | logger.debug(f'页面状态：{ready_state}') 110 | if ready_state == 'complete' and not loading_counter: 111 | success_count += 1 112 | if success_count == 10: 113 | break 114 | else: 115 | success_count = 0 116 | 117 | layout_metrics = tab.Page.getLayoutMetrics() 118 | content_size = layout_metrics['contentSize'] 119 | 120 | pdf_data = b64decode(tab.Page.printToPDF( 121 | paperWidth=content_size['width']/96, 122 | paperHeight=content_size['height']/96, 123 | marginTop=0, 124 | marginBottom=0, 125 | marginLeft=0, 126 | marginRight=0, 127 | printBackground=True 128 | )['data']) 129 | 130 | return convert_from_bytes(pdf_data, single_file=True)[0] 131 | -------------------------------------------------------------------------------- /oven/api.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from flask_restx import Resource, Api 4 | from flask import request 5 | from datetime import datetime 6 | from api_models import oven_api 7 | from api_models import bake_model, bake_response_model 8 | from api_models import check_model, check_response_model 9 | from oven import bake_tweet 10 | from tid_code import read_code_from_image_url 11 | from traceback import format_exc 12 | 13 | import json 14 | import os 15 | import time 16 | import logging 17 | import config 18 | 19 | 20 | image_path = 'static' 21 | image_url_prefix = f'{config.EXT_STATIC_BASE_URL}/' 22 | logger = logging.getLogger('api') 23 | 24 | 25 | @oven_api.route('/bake') 26 | class GenerateImage(Resource): 27 | @oven_api.expect(bake_model, validate=True) 28 | @oven_api.doc("bakeResponse", model=bake_response_model) 29 | def post(self): 30 | request_base_data = request.json 31 | request_data = request_base_data['data'] 32 | task_id = request_base_data['taskId'] 33 | 34 | if 'url' in request_data == 'tid' in request_data: 35 | return make_response(400, '必须指定url或tid中的一个') 36 | 37 | url = request_data['url'] if 'url' in request_data else None 38 | tid = request_data['tid'] if 'tid' in request_data else None 39 | 40 | logger.info(f'[{task_id}] ' 41 | f'从 {request_base_data["forwardFrom"]} 收到烤图任务：' 42 | f'{url if url else tid}') 43 | 44 | bake_params = { 45 | 'tid': tid, 46 | 'url': url 47 | } 48 | if 'transText' in request_data: 49 | bake_params['trans_text'] = request_data['transText'] 50 | if 'ppi' in request_data: 51 | bake_params['ppi'] = request_data['ppi'] 52 | 53 | start_time = time.time() 54 | filename = bake_tweet(**bake_params) 55 | end_time = time.time() 56 | process_time_ms = int((end_time - start_time) * 1000) 57 | if not filename: 58 | return make_response(500, '后端生成图片发生错误，请联系管理员检查日志') 59 | else: 60 | logger.info(f'[{task_id}] ' 61 | f'烤图任务完成，输出文件：{filename}') 62 | result_url = image_url_prefix + filename 63 | return make_response( 64 | message='OK', 65 | resultUrl=result_url, 66 | processTime=process_time_ms 67 | ) 68 | 69 | 70 | @oven_api.route('/check') 71 | class CheckImageTid(Resource): 72 | @oven_api.expect(check_model, validate=True) 73 | @oven_api.doc("CheckResponse", model=check_response_model) 74 | def post(self): 75 | request_base_data = request.json 76 | request_data = request_base_data['data'] 77 | task_id = request_base_data['taskId'] 78 | logger.info(f'[{task_id}] ' 79 | f'从 {request_base_data["forwardFrom"]} 收到检查tid任务') 80 | url = request_data['imageUrl'] 81 | 82 | start_time = time.time() 83 | try: 84 | tid = read_code_from_image_url( 85 | url, config.TID_CODE_POS_X, config.TID_CODE_POS_Y, 86 | config.TID_CODE_WIDTH, config.TID_CODE_HEIGHT) 87 | message = 'OK' 88 | logger.info(f'[{task_id}] 检查tid任务完成（{tid}）。') 89 | except ValueError as e: 90 | logger.debug(f'[{task_id}] 二维码解码失败：{e}') 91 | tid = -1 92 | message = '没有找到有效的tid二维码' 93 | except Exception as e: 94 | logger.error(f'[{task_id}] 检查tid时发生了错误：{e}') 95 | logger.error(format_exc()) 96 | return make_response(500, '检查tid失败，请联系管理员检查日志') 97 | 98 | end_time = time.time() 99 | process_time_ms = int((end_time - start_time) * 1000) 100 | 101 | return make_response( 102 | message=message, tid=tid, processTime=process_time_ms) 103 | 104 | 105 | def make_response(code=0, message="", **kwargs): 106 | response = {'code': code, 'message': message} 107 | response.update(kwargs) 108 | return response 109 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/form/JsonDataFridgeForm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/30 4 | */ 5 | package com.enkanrec.twitkitFridge.api.form; 6 | 7 | import com.enkanrec.twitkitFridge.util.JsonUtil; 8 | import com.fasterxml.jackson.core.JsonProcessingException; 9 | import lombok.EqualsAndHashCode; 10 | import lombok.Getter; 11 | import lombok.ToString; 12 | import lombok.extern.slf4j.Slf4j; 13 | 14 | import java.lang.reflect.Field; 15 | import java.util.List; 16 | import java.util.Map; 17 | 18 | /** 19 | * Class : JsonDataFridgeForm 20 | * Usage : 标准请求体表单。表单会自动解开data字段成Map和List组合的Json结构体 21 | */ 22 | @Deprecated 23 | @Slf4j 24 | @ToString 25 | @EqualsAndHashCode(callSuper = true) 26 | public class JsonDataFridgeForm extends BaseFridgeForm { 27 | 28 | @Getter 29 | private String data; 30 | 31 | @Getter 32 | private Map mappedData; 33 | 34 | @Getter 35 | private List listedData; 36 | 37 | @Getter 38 | private JsonDataType autoDecodeType; 39 | 40 | public void setData(String data) { 41 | if (data == null) { 42 | this.autoDecodeType = JsonDataType.Null; 43 | this.data = null; 44 | return; 45 | } 46 | this.data = data; 47 | if (data.startsWith("{")) { 48 | this.autoDecodeType = JsonDataType.Map; 49 | this.mappedData = this.dataToMap(); 50 | if (this.getClass() != JsonDataFridgeForm.class) { 51 | this.autoDispatchMappedDataField(); 52 | } 53 | } else { 54 | this.autoDecodeType = JsonDataType.List; 55 | this.listedData = this.dataToList(); 56 | } 57 | this.afterSetData(); 58 | } 59 | 60 | public Map dataToMap() { 61 | Map result = null; 62 | try { 63 | result = JsonUtil.parse(this.data, Map.class); 64 | } catch (JsonProcessingException e) { 65 | log.error("try to parse form data to map but failed: " + e.getMessage()); 66 | } 67 | return result; 68 | } 69 | 70 | public List dataToList() { 71 | List result = null; 72 | try { 73 | result = JsonUtil.parse(this.data, List.class); 74 | } catch (JsonProcessingException e) { 75 | log.error("try to parse form data to list but failed: " + e.getMessage()); 76 | } 77 | return result; 78 | } 79 | 80 | public Map asMap() { 81 | return this.mappedData; 82 | } 83 | 84 | public List asList() { 85 | return this.listedData; 86 | } 87 | 88 | private void autoDispatchMappedDataField() { 89 | for (Map.Entry kvp : this.mappedData.entrySet()) { 90 | try { 91 | Field keyedField = this.getClass().getDeclaredField(kvp.getKey()); 92 | keyedField.setAccessible(true); 93 | keyedField.set(this, kvp.getValue()); 94 | } 95 | catch (Exception ignore) { 96 | // pass 97 | } 98 | } 99 | } 100 | 101 | protected void afterSetData() { } 102 | 103 | public void fromRawString(String jStr) throws Exception { 104 | Map rawMap = JsonUtil.parse(jStr, Map.class); 105 | this.setForwardFrom((String) rawMap.get("forwardFrom")); 106 | this.setTimestamp((String) rawMap.get("timestamp")); 107 | this.setCommand((String) rawMap.get("command")); 108 | this.setOf((String) rawMap.get("of")); 109 | Object rawData = rawMap.get("data"); 110 | if (rawData instanceof Map) { 111 | this.autoDecodeType = JsonDataType.Map; 112 | this.mappedData = (Map) rawData; 113 | if (this.getClass() != JsonDataFridgeForm.class) { 114 | this.autoDispatchMappedDataField(); 115 | } 116 | } else if (rawData instanceof List) { 117 | this.autoDecodeType = JsonDataType.List; 118 | this.listedData = this.dataToList(); 119 | } else { 120 | this.setData(rawData.toString()); 121 | } 122 | } 123 | 124 | public static enum JsonDataType { 125 | Map, 126 | List, 127 | Null 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /koishi-app/README.md: -------------------------------------------------------------------------------- 1 | # twitkit-app 2 | 3 | 基于koishi框架的APP部分，业务逻辑依赖于其他组件 4 | 5 | ## 输入接口 6 | 7 | ### 指令 8 | 9 | 除了使用koishi的commend创建的普通命令，还用middleware实现了仿原bot的短指令 10 | 11 | 涉及的三个变量： 12 | 13 | * 最新一条任务id(last)：数据库持久化变量，既数据表自增id，无缓存 14 | * 最近修改过的翻译(lastTrans)：非持久化变量，用户使用`translate`（带翻译）同时更新，缓存在内存 15 | * 队列头(todo)：数据库持久化变量，由用户使用`clear`命令更新，启动时从数据库获取缓存在内存 16 | 17 | #### 短指令/快捷命令 18 | 19 | 20 | 基本格式 `"指令前缀[任务id][指令符] [附加文字]"` 21 | 22 | 除了附加文字，各部分之间没有空格 23 | 24 | 指令前缀单独配置，默认为 `#` 25 | 26 | | 指令符 | 有id | 例子1 | 等价长指令1 | 无id | 例子2 | 等价长指令2 | 27 | |:-----:|:----:|:-----:|:----------:|:----:|:----:|:----------:| 28 | | <无> | 查看该任务原文或烤图（若存在） | #1000 | translate 1000 | 列出队列所有任务状态 | # | list | 29 | | ! | 不更新翻译刷新烤图 | #1000! | fresh 1000 | 刷新最近修改过的翻译的烤图 | #! | fresh | 30 | | * | 显示推文原文 | #1000! | raw 1000 | 显示最近修改过的翻译服推文原文 | #* | raw | 31 | | ~ | 列出该任务之后的所有任务状态 | #1000~ | list 1000 | 列出队列所有任务状态 | #~ | list | 32 | | ~~ | 列出该任务之后的所有烤推结果（及媒体） | #1000~~ | list-detail 1000 | 列出队列所有烤推结果（及媒体） | #~~ | list-detail | 33 | | / | 设置队列头为该id | #1000/ | clear 1000 | 清空队列（设置队列头为最新的任务id） | #/ | clear | 34 | | - | 隐藏/显示该任务 | #1000- | hide 1000 | 批量隐藏队列里已发布的任务（相当于移出队列） | #- | hide | 35 | | + | 给该任务添加备注 | #1000+[备注] | comment 1000 [备注] | 给最后一条任务添加备注 | #+[备注] | comment [备注] | 36 | | ? | | | | 显示这条帮助 | #? | | 37 | 38 | 其他无指令符指令 39 | 40 | | 参数1 | 参数2 | 功能 | 例子 | 等价长指令 | 41 | |:-----:|:----:|:----:|:----:|:---------:| 42 | | id | 翻译 | 更新该任务翻译 | #1000 [翻译] | translate 1000 [翻译] | 43 | | url | <无> | 将指定链接的推文入库 | # | fetch [url] | 44 | | url | 翻译 | 直接对链接烤图 | # [翻译] | fetch [url] [翻译] | 45 | 46 | 47 | #### 长指令 48 | 49 | 50 | 指令前缀参考koishi指令帮助，默认为空，可以在群聊中直接使用如 `undo` 51 | 52 | 指令前缀 53 | 54 | | 指令 | 使用 | 介绍 | 55 | |:----:|:---:|:----:| 56 | | translate | translate <tid> [trans] | 获取/更新这个id的翻译内容 | 57 | | fetch | fetch <url> [trans] | 将指定链接的推文入库或直接烤图 | 58 | | fresh | fresh [tid] | 刷新这个id的翻译烤图，id为空时使用最近修改过的翻译 | 59 | | raw | raw [tid] | 显示这个id的推文原文，id为空时使用最近修改过的翻译 | 60 | | list | list [tid] | 查看队列某个id后的任务，id为空时使用预设的队列头 | 61 | | list-detail | list-detail [tid] | 批量获取队列某个id后的烤推结果，id为空时使用预设的队列头 | 62 | | clear | clear [tid] | 设置队列头，id为空时，设置成最新的id（清空队列） | 63 | | hide | hide [tid] | 隐藏或显示某个推，id为空时，隐藏所有已烤的推 | 64 | | comment | comment [tid] <text> | 为某个推添加注释，id为空时，加到最近的推 | 65 | | undo | undo [tid] | 撤销某个推的翻译修改，id为空时，撤销最近修改过的翻译 | 66 | | delete| delete <tid> | 删除一个任务，返回是否删除成功，建议使用hide隐藏 | 67 | | how | how | 显示内置帮助 | 68 | 69 | 70 | ### 监听器 71 | 72 | 73 | REST基本请求格式： 74 | 75 | HTTP method: `POST` 76 | 77 | content-type: `application/json` 78 | 79 | body：（均非空） 80 | 81 | | 字段 | 类型 | 注释 | 82 | |:----:|:---:|:----:| 83 | | forwardFrom | string | 请求来源名 | 84 | | timestamp | string | ISO8601格式的日期字符串 | 85 | | taskId | string | 一个uuid，用于调试跟踪执行过程 | 86 | | data | any | 接口需要的数据，见下 | 87 | 88 | 89 | #### 推特更新 90 | 91 | 92 | 路由：`/api/app/twitter` 93 | 94 | data: number[] 所有新到达的任务id 95 | 96 | 通知app有哪些新任务 97 | 98 | 99 | #### 其他更新 100 | 101 | 102 | 路由：`/api/app/other` 103 | 104 | data: dict （无注明既可空说）其他更新的详细信息 105 | 106 | | 字段 | 类型 | 注释 | 107 | |:----:|:---:|:----:| 108 | | tid | number[] | 识别到的烤推发布 | 109 | | title | string | 标题 | 110 | | content |string | 非空，正文 | 111 | | url | string | 原始链接 | 112 | | media | string[] | 媒体图片链接 | 113 | | author | string | 非空，发布人 | 114 | | postDate | string | 发布日期，ISO8601格式 | 115 | 116 | 将非任务更新，推送给app以便在群里通知 117 | 118 | 119 | ## 输出接口 120 | 121 | ### 数据库 122 | 123 | 见 `store.ts` 124 | 125 | ### 烤推机 126 | 127 | 见 `translator.ts` 128 | 129 | ## 配置 130 | 131 | ```js 132 | { 133 | prefix: '#', // 快捷指令前缀 134 | ispro: false, // 是否发图 135 | cmd: { 136 | host: { // 其他组件的REST监听HOST 137 | store: "http://localhost:8220", 138 | translator: "http://localhost:8221", 139 | maid: "http://localhost:8222", 140 | }, 141 | group: [], // 监听命令的群组，留空监听所有人 142 | private: true, // 是否允许私聊上班 143 | friend: true, // 是否允许好友上班 144 | cut: 8 // 消息预览截断长度 145 | }, 146 | watcher: { 147 | port: 8223, // 监视器推送端口 148 | target: { // 监视器更新推送目标 149 | discuss: [], 150 | private: [], 151 | group: [] 152 | } 153 | } 154 | } 155 | ``` 156 | 157 | ## 启动 158 | 159 | 1. 安装[koishi](https://koishi.js.org/) 160 | 2. 安装依赖 `npm install` 161 | 3. 编写配置，参考 `.koishi.config.js` 162 | 4. 编译 `npm run build` 163 | 5. 运行 `koishi run` 或 `npm run dev` 164 | -------------------------------------------------------------------------------- /fridge/schema/test_init.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | DROP TABLE IF EXISTS enkan_config; 4 | DROP TABLE IF EXISTS enkan_task; 5 | DROP TABLE IF EXISTS enkan_translate; 6 | DROP TABLE IF EXISTS enkan_twitter; 7 | 8 | CREATE TABLE `enkan_config` ( 9 | `id` int(11) NOT NULL AUTO_INCREMENT, 10 | `namespace` varchar(191) NOT NULL DEFAULT '___DEFAULT___', 11 | `config_key` varchar(191) NOT NULL, 12 | `config_value` text, 13 | `newdate` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, 14 | `updatetime` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 15 | PRIMARY KEY (`id`), 16 | UNIQUE KEY `key_namespace_key` (`namespace`,`config_key`), 17 | KEY `key_updatetime` (`updatetime`) 18 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 19 | 20 | CREATE TABLE `enkan_task` ( 21 | `tid` int(11) NOT NULL AUTO_INCREMENT COMMENT '任务id', 22 | `status_id` varchar(191) NOT NULL COMMENT '推特生成的推文唯一id', 23 | `url` varchar(767) NOT NULL COMMENT '推文URL', 24 | `content` longtext NOT NULL COMMENT '推文内容', 25 | `media` text NOT NULL COMMENT '媒体地址', 26 | `published` tinyint(2) NOT NULL DEFAULT '0' COMMENT '是否已发布', 27 | `hided` tinyint(2) NOT NULL DEFAULT '0' COMMENT '是否隐藏', 28 | `comment` varchar(1023) NOT NULL DEFAULT '' COMMENT '备注', 29 | `newdate` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, 30 | `updatetime` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 31 | `twitter_uid` varchar(191) DEFAULT NULL COMMENT '指向该推文主动用户', 32 | `ref_tid` int(11) DEFAULT NULL COMMENT '指向转发/引用的推文id\n', 33 | `pub_date` datetime DEFAULT NULL COMMENT 'API返回的发推时间', 34 | `extra` longtext COMMENT 'json储存动态字段，任意扩展，不需要索引的东西都可以丢进来\n\n\n\n', 35 | PRIMARY KEY (`tid`), 36 | UNIQUE KEY `key_uid_statusid` (`twitter_uid`,`status_id`) USING BTREE, 37 | KEY `key_newdate` (`newdate`), 38 | KEY `key_updatetime` (`updatetime`), 39 | KEY `key_url` (`url`) USING BTREE 40 | ) ENGINE=InnoDB AUTO_INCREMENT=1000 DEFAULT CHARSET=utf8mb4; 41 | 42 | CREATE TABLE `enkan_translate` ( 43 | `zzid` int(11) NOT NULL AUTO_INCREMENT, 44 | `tid` int(11) NOT NULL COMMENT '推文id', 45 | `version` int(11) NOT NULL DEFAULT '0' COMMENT '版本号', 46 | `translation` longtext NOT NULL COMMENT '翻译内容', 47 | `img` varchar(2047) NOT NULL COMMENT '烤推机生成的图地址', 48 | `newdate` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, 49 | `updatetime` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 50 | PRIMARY KEY (`zzid`), 51 | UNIQUE KEY `key_tid_version` (`tid`,`version`), 52 | KEY `key_updatetime` (`updatetime`), 53 | KEY `key_newdate` (`newdate`) 54 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 55 | 56 | CREATE TABLE `enkan_twitter` ( 57 | `uid` int(11) NOT NULL AUTO_INCREMENT, 58 | `twitter_uid` varchar(191) NOT NULL, 59 | `name` varchar(511) NOT NULL, 60 | `display` varchar(511) NOT NULL, 61 | `avatar` varchar(2047) NOT NULL, 62 | `updatetime` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 63 | `newdate` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, 64 | PRIMARY KEY (`uid`), 65 | UNIQUE KEY `key_twitteruid` (`twitter_uid`) 66 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 67 | 68 | INSERT INTO `enkan_config` VALUES (1, '___DEFAULT___', 'test.default.yachiyo.love', 'iroha', '2020-02-07 00:12:56', '2020-02-07 00:12:56'); 69 | INSERT INTO `enkan_config` VALUES (2, '___DEFAULT___', 'test.default.rika', '五十铃怜', '2020-02-07 00:13:19', '2020-02-07 00:13:19'); 70 | INSERT INTO `enkan_config` VALUES (3, '___TEST_MB4___', 'test.mb4.emoji', '❤①+123AB', '2020-02-07 00:26:48', '2020-02-07 00:29:46'); 71 | 72 | INSERT INTO `enkan_task` VALUES (1000, '10000', 'URL_0', '内容0', '[]', 0, 0, '', '2020-02-07 00:02:21', '2020-03-01 12:27:41', '123000123', NULL, NULL, NULL); 73 | INSERT INTO `enkan_task` VALUES (1001, '10001', 'URL_1', '内容1', '[]', 0, 0, '', '2020-02-07 00:02:41', '2020-03-01 12:27:42', '123000123', NULL, NULL, NULL); 74 | INSERT INTO `enkan_task` VALUES (1002, '10002', 'URL_2', '内容2🍒', '[\"media_2\"]', 1, 0, '', '2020-02-07 16:12:41', '2020-03-01 12:27:43', '123000123', NULL, NULL, NULL); 75 | INSERT INTO `enkan_task` VALUES (1003, '10003', 'URL_3', '内容3', '[]', 1, 0, '', '2020-02-07 16:59:03', '2020-03-01 12:27:43', '123000123', NULL, NULL, NULL); 76 | INSERT INTO `enkan_task` VALUES (1004, '10004', 'URL_4_HIDED', '内容4', '[]', 0, 1, '', '2020-02-07 17:20:58', '2020-03-01 12:27:46', '123000123', NULL, NULL, NULL); 77 | 78 | INSERT INTO `enkan_translate` VALUES (1, 1000, 0, '翻译A1', '[img1]', '2020-02-07 00:07:56', '2020-02-07 00:07:56'); 79 | INSERT INTO `enkan_translate` VALUES (3, 1000, 1, '翻译A2', '[img2]', '2020-02-07 00:08:18', '2020-02-07 00:08:47'); 80 | INSERT INTO `enkan_translate` VALUES (4, 1000, 2, '翻译A3', '[img3]', '2020-02-07 00:08:46', '2020-02-07 00:08:46'); 81 | INSERT INTO `enkan_translate` VALUES (5, 1001, 0, '翻译B1', '[img_21]', '2020-02-07 00:10:30', '2020-02-07 00:10:30'); 82 | INSERT INTO `enkan_translate` VALUES (6, 1001, 1, '翻译B2', '[img_22]', '2020-02-07 00:10:57', '2020-02-07 00:10:57'); 83 | INSERT INTO `enkan_translate` VALUES (8, 1002, 0, '翻译C1', '[img_31]', '2020-02-07 17:21:34', '2020-02-07 17:21:34'); 84 | 85 | INSERT INTO `enkan_twitter` VALUES (4, '123000123', 'enkanRecGLZ', '圆环纪录攻略组', 'http://111/111.jpg', '2020-03-01 12:42:25', '2020-03-01 12:08:29'); 86 | 87 | COMMIT; 88 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/service/kvConfig/KVConfigServiceImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/1/30 4 | */ 5 | package com.enkanrec.twitkitFridge.service.kvConfig; 6 | 7 | import com.enkanrec.twitkitFridge.steady.yui.entity.EnkanConfigEntity; 8 | import com.enkanrec.twitkitFridge.steady.yui.repository.EnkanConfigRepository; 9 | import lombok.extern.slf4j.Slf4j; 10 | import org.springframework.stereotype.Service; 11 | import org.springframework.transaction.annotation.Transactional; 12 | 13 | import java.util.Collection; 14 | import java.util.HashMap; 15 | import java.util.List; 16 | import java.util.Map; 17 | 18 | /** 19 | * Class : KVConfigServiceImpl 20 | * Usage : 公共键值对配置仓库的交互逻辑 21 | */ 22 | @Slf4j 23 | @Service 24 | public class KVConfigServiceImpl implements KVConfigService { 25 | 26 | private static final String KEY_NAMESPACE_DEFAULT = "___DEFAULT___"; 27 | 28 | private final EnkanConfigRepository repository; 29 | 30 | public KVConfigServiceImpl(EnkanConfigRepository repository) { 31 | this.repository = repository; 32 | } 33 | 34 | @Transactional 35 | @Override 36 | public void setOneDefault(String key, String value) throws Exception { 37 | this.setOne(KEY_NAMESPACE_DEFAULT, key, value); 38 | } 39 | 40 | @Transactional 41 | @Override 42 | public String getOneDefault(String key) { 43 | return this.getOne(KEY_NAMESPACE_DEFAULT, key); 44 | } 45 | 46 | @Transactional 47 | @Override 48 | public void setManyDefault(Map configs) throws Exception { 49 | this.setMany(KEY_NAMESPACE_DEFAULT, configs); 50 | } 51 | 52 | @Transactional 53 | @Override 54 | public Map getManyDefault(Collection keys) { 55 | return this.getMany(KEY_NAMESPACE_DEFAULT, keys); 56 | } 57 | 58 | @Transactional 59 | @Override 60 | public void setOne(String namespace, String key, String value) throws Exception { 61 | if (namespace.contains("#") || key.contains("#")) { 62 | String exKey = String.format("Set config with `#` in namespace or key: [%s][%s]", namespace, key); 63 | log.error(exKey); 64 | throw new Exception(exKey); 65 | } 66 | EnkanConfigEntity ece = this.repository.findByNamespaceAndConfigKey(namespace, key); 67 | if (ece == null) { 68 | EnkanConfigEntity nObj = new EnkanConfigEntity(); 69 | nObj.setNamespace(namespace); 70 | nObj.setConfigKey(key); 71 | nObj.setConfigValue(value); 72 | this.repository.save(nObj); 73 | } else { 74 | ece.setConfigValue(value); 75 | this.repository.save(ece); 76 | } 77 | } 78 | 79 | @Transactional 80 | @Override 81 | public String getOne(String namespace, String key) { 82 | EnkanConfigEntity ece = this.repository.findByNamespaceAndConfigKey(namespace, key); 83 | if (ece == null) { 84 | return null; 85 | } else { 86 | return ece.getConfigValue(); 87 | } 88 | } 89 | 90 | @Transactional 91 | @Override 92 | public void setMany(String namespace, Map configs) throws Exception { 93 | for (Map.Entry kvp : configs.entrySet()) { 94 | Object val = kvp.getValue(); 95 | if (val != null) { 96 | this.setOne(namespace, kvp.getKey(), val.toString()); 97 | } else { 98 | this.setOne(namespace, kvp.getKey(), null); 99 | } 100 | } 101 | } 102 | 103 | @Transactional 104 | @Override 105 | public Map getMany(String namespace, Collection keys) { 106 | Map result = new HashMap<>(); 107 | for (String k : keys) { 108 | String val = this.getOne(namespace, k); 109 | result.put(k, val); 110 | } 111 | return result; 112 | } 113 | 114 | @Transactional 115 | @Override 116 | public Map getAll(String namespace) { 117 | Map result = new HashMap<>(); 118 | List eces = this.repository.findAllByNamespace(namespace); 119 | for (EnkanConfigEntity ece : eces) { 120 | result.put(ece.getConfigKey(), ece.getConfigValue()); 121 | } 122 | return result; 123 | } 124 | 125 | @Transactional 126 | @Override 127 | public void clearNamespace(String namespace) { 128 | this.repository.deleteAllByNamespace(namespace); 129 | } 130 | 131 | @Transactional 132 | @Override 133 | public Map getAll() { 134 | Map result = new HashMap<>(); 135 | List allConfig = repository.findAll(); 136 | allConfig.forEach(c -> { 137 | String namespace = c.getNamespace(); 138 | if (namespace.equals(KEY_NAMESPACE_DEFAULT)) { 139 | result.put(c.getConfigKey(), c.getConfigValue()); 140 | } else { 141 | result.put(namespace + "#" + c.getConfigKey(), c.getConfigValue()); 142 | } 143 | 144 | }); 145 | return result; 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /fridge/fridge-src/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.2.2.RELEASE 9 | 10 | 11 | com.enkanRec 12 | twitkit-fridge 13 | 0.0.1 14 | twitkit-fridge 15 | twitkit-fridge for EnkanRec 16 | 17 | 18 | 1.8 19 | 1.8 20 | 1.8 21 | 1.8 22 | UTF-8 23 | UTF-8 24 | 25 | 26 | 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-data-jpa 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-starter-data-redis 35 | 36 | 37 | org.springframework.boot 38 | spring-boot-starter-jdbc 39 | 40 | 41 | org.springframework.boot 42 | spring-boot-starter-web 43 | 44 | 45 | org.springframework.boot 46 | spring-boot-devtools 47 | runtime 48 | true 49 | 50 | 51 | 52 | 53 | com.corundumstudio.socketio 54 | netty-socketio 55 | 1.7.17 56 | 57 | 58 | 59 | 60 | org.projectlombok 61 | lombok 62 | 1.18.8 63 | 64 | 65 | org.apache.commons 66 | commons-lang3 67 | 3.9 68 | 69 | 70 | com.google.guava 71 | guava 72 | 28.1-jre 73 | 74 | 75 | 76 | 77 | 78 | mysql 79 | mysql-connector-java 80 | runtime 81 | 82 | 83 | 84 | 85 | io.prometheus 86 | simpleclient 87 | 0.8.0 88 | 89 | 90 | io.prometheus 91 | simpleclient_hotspot 92 | 0.8.0 93 | 94 | 95 | io.prometheus 96 | simpleclient_servlet 97 | 0.8.0 98 | 99 | 100 | 101 | 102 | io.socket 103 | socket.io-client 104 | 1.0.0 105 | test 106 | 107 | 108 | junit 109 | junit 110 | 4.12 111 | test 112 | 113 | 114 | org.springframework.boot 115 | spring-boot-starter-test 116 | test 117 | 118 | 119 | org.junit.vintage 120 | junit-vintage-engine 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | org.springframework.boot 130 | spring-boot-maven-plugin 131 | 132 | 133 | org.apache.maven.plugins 134 | maven-surefire-plugin 135 | 2.19 136 | 137 | 138 | 139 | 140 | 141 | -------------------------------------------------------------------------------- /oven/tid_code.py: -------------------------------------------------------------------------------- 1 | from PIL import Image 2 | from typing import List 3 | from crc8 import crc8 4 | 5 | import math 6 | import requests 7 | import config 8 | 9 | 10 | TID_CODE_KEY = config.TID_CODE_KEY & 0xff 11 | 12 | 13 | def calculate_checksum(tid: int) -> int: 14 | bytes_ = [] 15 | for i in range(3): 16 | bytes_.append((tid >> (16 - 8 * i)) & 0xff) 17 | bytes_ = bytes(bytes_) 18 | return int(crc8(bytes_).digest()[0]) 19 | 20 | 21 | def encode(tid: int, key=TID_CODE_KEY) -> List[bool]: 22 | ''' 23 | 返回bool列表，表示编码后的bits。 24 | 参数：tid -- 支持最大24位 25 | ''' 26 | checksum = calculate_checksum(tid) ^ key 27 | data = tid & 0xffffff | (checksum << 24) 28 | encoded = [] 29 | for i in range(32): 30 | bit = (data >> (31 - i)) & 1 31 | if bit: 32 | encoded += [True, False] 33 | else: 34 | encoded += [False, True] 35 | return encoded 36 | 37 | 38 | def decode(encoded_tid: List[bool], key=TID_CODE_KEY) -> int: 39 | ''' 40 | 返回解码后的tid。 41 | 参数：表示bits的bool列表。 42 | 如果输入不合法，抛出ValueError异常。 43 | ''' 44 | if len(encoded_tid) != 64: 45 | raise ValueError('输入长度不符合要求') 46 | decoded_with_checksum = 0 47 | for i in range(0, 64, 2): 48 | if encoded_tid[i] == True and encoded_tid[i+1] == False: 49 | decoded_with_checksum |= 1 << (31 - i // 2) 50 | elif encoded_tid[i] == False and encoded_tid[i+1] == True: 51 | continue 52 | else: 53 | raise ValueError('曼彻斯特编码有误') 54 | tid = decoded_with_checksum & 0xffffff 55 | checksum = (decoded_with_checksum >> 24) ^ key 56 | if calculate_checksum(tid) != checksum: 57 | raise ValueError('校验码有误') 58 | return tid 59 | 60 | 61 | def code_image_properties_check(n_levels, total_bits): 62 | if int(math.log2(n_levels)) != math.log2(n_levels): 63 | raise Exception('levels长度必须是2的整数次方') 64 | if total_bits < 64: 65 | raise Exception("指定的参数容量不足") 66 | 67 | 68 | def generate_code_image(width, height, tid, rows=5, cols=5, 69 | levels=(0x7f, 0xff)) -> Image: 70 | n_level_bits = int(math.log2(len(levels))) 71 | total_bits = rows * cols * n_level_bits * 3 72 | 73 | code_image_properties_check(n_level_bits, total_bits) 74 | 75 | pad_bits = total_bits - 64 76 | encoded_bits = encode(tid) 77 | for i in range(pad_bits): 78 | encoded_bits.append(bool(i & 1)) 79 | levels_matrix = [[[0, 0, 0] for _ in range(cols)] for _ in range(rows)] 80 | for i in range(rows * cols): 81 | for j in range(n_level_bits): 82 | for k in range(3): 83 | if encoded_bits[i + j * total_bits // (3 * n_level_bits) + 84 | k * total_bits // 3]: 85 | levels_matrix[i // cols][i % cols][k] |= 1 << j 86 | 87 | scaled_width = width * cols // math.gcd(width, cols) 88 | scaled_height = height * rows // math.gcd(height, rows) 89 | im = Image.new('RGB', (scaled_width, scaled_height)) 90 | for i in range(scaled_width): 91 | for j in range(scaled_height): 92 | c = int(i / (scaled_width / cols)) 93 | r = int(j / (scaled_height / rows)) 94 | rgb = [levels[value] for value in levels_matrix[r][c]] 95 | im.putpixel((i, j), tuple(rgb)) 96 | im.getpixel 97 | im.thumbnail((width, height), resample=Image.BICUBIC) 98 | return im 99 | 100 | 101 | def decode_from_image(im: Image, rows=5, cols=5, 102 | levels=(0x7f, 0xff)): 103 | '''返回：tid''' 104 | 105 | im.thumbnail((rows, cols), Image.BICUBIC) 106 | 107 | im_w, im_h = im.size 108 | 109 | n_level_bits = int(math.log2(len(levels))) 110 | total_bits = rows * cols * n_level_bits * 3 111 | unpacked_bits = [None] * total_bits 112 | 113 | code_image_properties_check(n_level_bits, total_bits) 114 | 115 | for i in range(cols): 116 | for j in range(rows): 117 | block_center = (i, j) 118 | px = im.getpixel(block_center) 119 | for k in range(3): 120 | level = min(levels, key=lambda x: abs(x - px[k])) 121 | bits = levels.index(level) 122 | for l in range(n_level_bits): 123 | bit = (bits >> l) & 1 124 | unpacked_bits[i + j * cols + 125 | l * total_bits // (3 * n_level_bits) + 126 | k * total_bits // 3] = bool(bit) 127 | unpacked_bits = unpacked_bits[:64] 128 | return decode(unpacked_bits) 129 | 130 | 131 | def get_scaled_area(x, y, w, h, orig_size): 132 | scale = orig_size[0] / config.VIEWPORT_WIDTH 133 | x *= scale 134 | y *= scale 135 | w *= scale 136 | h *= scale 137 | 138 | if x < 0: 139 | x = orig_size[0] + x - w 140 | if y < 0: 141 | y = orig_size[1] + y - h 142 | 143 | return int(x), int(y), int(w + x), int(h + y) 144 | 145 | 146 | def add_code_to_image(image, tid, x, y, w, h): 147 | '''给image打上二维码（in-place）''' 148 | actual_code_area = get_scaled_area(x, y, w, h, image.size) 149 | x0, y0, x1, y1 = actual_code_area 150 | code = generate_code_image(x1 - x0, y1 - y0, tid) 151 | image.paste(code, actual_code_area) 152 | 153 | 154 | def read_code_from_image(image, x, y, w, h): 155 | code = image.crop(get_scaled_area(x, y, w, h, image.size)) 156 | return decode_from_image(code) 157 | 158 | 159 | def read_code_from_image_url(url, x, y, w, h): 160 | im = Image.open(requests.get(url, stream=True).raw) 161 | return read_code_from_image(im, x, y, w, h) 162 | -------------------------------------------------------------------------------- /koishi-app/src/watcher.ts: -------------------------------------------------------------------------------- 1 | import { Context, Logger } from 'koishi-core' 2 | import * as http from 'http' 3 | import { parse } from 'url' 4 | import { ISO8601, verifyDatetime, config, target, config_watcher, request, response } from './utils' 5 | import { tids2msgs } from './twitter' 6 | import translator from './translator' 7 | 8 | let logger: Logger 9 | 10 | class rss { 11 | tid?: number[] 12 | title?: string 13 | content: string 14 | url?: string 15 | media?: string[] 16 | author: string 17 | postDate?: ISO8601 18 | 19 | static parse(Data: rss): rss { 20 | if (typeof Data.content === "undefined" 21 | || typeof Data.author === "undefined" 22 | || typeof Data.postDate !== "undefined" 23 | && !verifyDatetime(Data.postDate) 24 | ) throw "param lost" 25 | return Data 26 | } 27 | } 28 | 29 | async function rss2msg(tw: rss, argv: config): Promise { 30 | let msg: string = "【" + tw.author + "】" 31 | if (tw.title) msg += tw.title 32 | msg += "\n----------------\n内容: " + tw.content 33 | if (tw.media) { 34 | msg += "\n媒体: " 35 | if (!tw.tid) tw.tid = [] 36 | for (const img of tw.media) { 37 | msg += argv.ispro ? "[CQ:image,file=" + img + "]" : img 38 | // 检查是否为烤推发布；未启用 39 | // const tid: number = await translator.check(img) 40 | // if (tid) tw.tid.push(tid) 41 | } 42 | } 43 | if (tw.url) msg += "\n原链接: " + tw.url 44 | if (tw.tid && tw.tid.length) msg += "\n识别到本条发布包含" + tw.tid.length + "条烤推结果: " 45 | + tw.tid.join(', ') + "\n - 发送#-批量隐藏已发推特" 46 | + "\n- 发送" + argv.prefix + "~核对记录的推文是否全部发布" 47 | + "\n- 发送" + argv.prefix + "/将快速搜索起始位置更新为即将到来的下一条推特" 48 | + "\n辛苦了！" 49 | return msg 50 | } 51 | 52 | async function sendmsg(ctx: Context, target: target, msg: string) { 53 | logger.debug(`sending msg: ${msg}`) 54 | logger.debug(`to: ${target}`) 55 | for (const j of target.discuss) await ctx.sender.sendDiscussMsgAsync(j, msg) 56 | for (const j of target.private) await ctx.sender.sendPrivateMsgAsync(j, msg) 57 | for (const j of target.group) await ctx.sender.sendGroupMsgAsync(j, msg) 58 | } 59 | 60 | export default function (ctx: Context, argv: config) { 61 | const watcher: config_watcher = { 62 | port: argv.watcher.port || 1551, 63 | target: { 64 | discuss: argv.watcher.target.discuss || [], 65 | private: argv.watcher.target.private || [], 66 | group: argv.watcher.target.group || [] 67 | } 68 | } 69 | logger = ctx.logger("app:watcher") 70 | logger.debug("watcher server starting...") 71 | const server = http.createServer((req, res) => { 72 | let pathname = parse(req.url).pathname; 73 | logger.debug(req.method + " " + pathname + " HTTP " + req.httpVersion) 74 | res.writeHead(200, { 'Content-Type': 'application/json' }) 75 | let r = /^\/api\/app\/(twitter|other)$/.exec(pathname) 76 | if (req.method === "POST" && r) { 77 | let raw: string = "" 78 | req.on("data", (chunk) => { raw += chunk }) 79 | req.on("end", async () => { 80 | let data: request 81 | logger.debug(raw) 82 | try { 83 | data = request.parse(JSON.parse(raw)) 84 | logger.info("[" + data.forwardFrom + "] " + data.timestamp + " :") 85 | res.end(new response("copy").toString()) 86 | } catch (e) { 87 | logger.warn("Parse data error: " + e) 88 | res.end(new response("Data format error", 400).toString()) 89 | return 90 | } 91 | logger.debug("[" + new Date().toISOString() + "]" + data.taskId) 92 | logger.debug(data.data) 93 | switch (r[1]) { 94 | case "twitter": 95 | let list: number[] = [] 96 | for (const i in data.data) { 97 | if (data.data[i]) { 98 | logger.debug("Event %s, tid: %d", i, data.data[i]) 99 | list.push(data.data[i]) 100 | } 101 | else logger.debug("Event %s, no update", i) 102 | } 103 | let quere: string[] = await tids2msgs(list, argv) 104 | for (const msg of quere) await sendmsg(ctx, watcher.target, msg) 105 | logger.info("Update notice done") 106 | break 107 | case "other": 108 | let rdata: rss 109 | try { 110 | rdata = rss.parse(data.data) 111 | logger.debug("[" + rdata.postDate + "] " + rdata.content) 112 | } catch (e) { 113 | logger.warn("Parse rss data error: " + e) 114 | return 115 | } 116 | const msg: string = await rss2msg(rdata, argv) 117 | sendmsg(ctx, watcher.target, msg) 118 | logger.info("Update notice done") 119 | break 120 | default: 121 | } 122 | }) 123 | } else { 124 | logger.warn("Unknow request: " + pathname) 125 | res.end(new response("Not Found", 404).toString()) 126 | } 127 | }) 128 | try { 129 | server.listen(watcher.port); 130 | logger.success("Listening watcher on port %d", watcher.port) 131 | } catch (e) { 132 | logger.warn("Listen watcher fail on port " + watcher.port + ": " + e) 133 | } 134 | } -------------------------------------------------------------------------------- /koishi-app/src/twitter.ts: -------------------------------------------------------------------------------- 1 | import { ISO8601, config } from './utils' 2 | import store from './store' 3 | export class Twitter { 4 | /** twitter **/ 5 | id: number 6 | statusId: string 7 | url: string 8 | content: string 9 | media: string[] 10 | video?: string[] 11 | published: boolean 12 | type: "更新" | "转推" | "引用" 13 | postDate?: string 14 | refTid: number 15 | comment?: string 16 | /** translation **/ 17 | trans?: string 18 | img?: string 19 | /** users **/ 20 | user: { 21 | twitterUid: string 22 | name: string 23 | display: string 24 | avatar: string 25 | } 26 | oirgUser?: { 27 | twitterUid: string 28 | name: string 29 | display: string 30 | avatar: string 31 | } 32 | /** extra **/ 33 | extra?: any 34 | } 35 | 36 | interface db_twitter { 37 | tid: number 38 | statusId: string 39 | url: string 40 | content: string 41 | media: string 42 | published: boolean 43 | hided: boolean 44 | comment: string 45 | twitterUid: string 46 | refTid: number 47 | pubDate: ISO8601 48 | extra: string 49 | newdate: ISO8601 50 | updatetime: ISO8601 51 | } 52 | 53 | interface db_user { 54 | uid: number 55 | twitterUid: string 56 | name: string 57 | display: string 58 | avatar: string 59 | newdate: ISO8601 60 | updatetime: ISO8601 61 | } 62 | 63 | interface db_translation { 64 | zzid: number 65 | version: number 66 | translation: string 67 | img: string 68 | newdate: ISO8601 69 | updatetime: ISO8601 70 | } 71 | export type dbtw = { 72 | twitter: db_twitter, 73 | user: db_user, 74 | translation?: db_translation 75 | } 76 | 77 | function twURL2user(url: string): string { 78 | const r = /twitter\.com\/([^\/]+)\/.+/.exec(url) 79 | return r ? r[1] : undefined 80 | } 81 | 82 | function makeTwUrl(statusId: string, name: string = "_"): string { 83 | return "https://twitter.com/" + name + "/status/" + statusId 84 | } 85 | 86 | /** 87 | * @description 将数据库几个表整合成本地类 88 | * @param dbtw 推文内容 89 | * @param orig 转发者，非转发推时为空 90 | */ 91 | export function convert(dbtw: dbtw, orig?: dbtw): Twitter { 92 | let tw: Twitter = { 93 | id: dbtw.twitter.tid, 94 | statusId: dbtw.twitter.statusId, 95 | url: dbtw.twitter.url || makeTwUrl(dbtw.twitter.statusId, dbtw.user.name), 96 | content: dbtw.twitter.content, 97 | media: JSON.parse(dbtw.twitter.media), 98 | video: [], 99 | published: dbtw.twitter.published, 100 | type: "更新", 101 | postDate: dbtw.twitter.pubDate, 102 | refTid: dbtw.twitter.refTid, 103 | comment: dbtw.twitter.comment, 104 | trans: dbtw.translation ? dbtw.translation.translation : undefined, 105 | img: dbtw.translation ? dbtw.translation.img : undefined, 106 | user: { 107 | twitterUid: dbtw.user.twitterUid, 108 | name: dbtw.user.name, 109 | display: dbtw.user.display, 110 | avatar: dbtw.user.avatar 111 | }, 112 | extra: undefined 113 | } 114 | try { 115 | tw.extra = JSON.parse(dbtw.twitter.extra) || {} 116 | } catch (e) { 117 | // logger.warn(e) 118 | tw.extra = {} 119 | } 120 | if (orig || !tw.content) { 121 | tw.type = "转推" 122 | if (orig) { 123 | tw.oirgUser = { 124 | twitterUid: orig.user.twitterUid, 125 | name: orig.user.name, 126 | display: orig.user.display, 127 | avatar: orig.user.avatar 128 | } 129 | tw.content = orig.twitter.content 130 | tw.media = JSON.parse(orig.twitter.media) 131 | } 132 | } else if (dbtw.twitter.refTid) { 133 | tw.type = "引用" 134 | } 135 | if (tw.extra.media) { 136 | for (const i of tw.extra.media) if (i.type === "video" && i.video_info && i.video_info.variants) { 137 | let bitrate: number = 0 138 | let link: string 139 | for (const j of i.video_info.variants) if (j.url && (!link || j.bitrate && bitrate < j.bitrate)) { 140 | link = j.url 141 | bitrate = j.bitrate || 0 142 | } 143 | if (link) tw.video.push(link) 144 | } 145 | } 146 | return tw 147 | } 148 | 149 | export function Twitter2msg(tw: Twitter, argv: config): string { 150 | let msg: string = "【" + tw.user.display + "】" 151 | if (tw.type === "转推") { 152 | msg += "转发了【" + tw.oirgUser.display + "】的推 " + argv.prefix + tw.refTid 153 | } else { 154 | msg += "更新了" 155 | } 156 | msg += "\n----------------\n内容: " + tw.content 157 | if (tw.media && tw.media.length) { 158 | msg += "\n媒体: " 159 | for (const img of tw.media) msg += argv.ispro ? "[CQ:image,file=" + img + "]" : img 160 | } 161 | if (tw.video && tw.video.length) { 162 | msg += "\n视频:\n" + tw.video.join("\n") 163 | } 164 | if (tw.type === "引用") msg += "\n引用: " + argv.prefix + tw.refTid 165 | if (tw.comment) msg += "\n备注: " + tw.comment 166 | msg += "\n原链接: " + tw.url + "\n快速嵌字发送: " + argv.prefix + tw.id + " 译文" 167 | return msg 168 | } 169 | 170 | export async function tids2msgs(list: number[], argv: config): Promise { 171 | list = list.sort() 172 | let quere: string[] = [] 173 | for (const i of list) { 174 | const tw: Twitter = await store.getTask(i) 175 | let msg: string 176 | if (tw.type === "转推") { 177 | msg = "【" + tw.user.display + "】转发了【" + tw.oirgUser.display + "】的推 " 178 | + argv.prefix + tw.refTid + "\n----------------\n" 179 | + "原链接: " + tw.url + "\n快速嵌字发送: " + argv.prefix + tw.id + " 译文" 180 | } else { 181 | msg = Twitter2msg(tw, argv) 182 | } 183 | quere.push(msg) 184 | } 185 | return quere 186 | } -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/steady/yui/entity/EnkanTaskEntity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/4 4 | */ 5 | package com.enkanrec.twitkitFridge.steady.yui.entity; 6 | 7 | import com.fasterxml.jackson.annotation.JsonIgnore; 8 | import lombok.ToString; 9 | 10 | import javax.persistence.*; 11 | import java.sql.Timestamp; 12 | import java.util.List; 13 | import java.util.Objects; 14 | 15 | /** 16 | * Class : EnkanTaskEntity 17 | * Usage : 18 | */ 19 | @Entity 20 | @ToString(exclude = {"translations"}) 21 | @Table(name = "enkan_task", schema = "yui") 22 | public class EnkanTaskEntity { 23 | private int tid; 24 | private String statusId; 25 | private String url; 26 | private String content; 27 | private String media; 28 | private boolean published; 29 | private boolean hided; 30 | private String comment; 31 | private String twitterUid; 32 | private Integer refTid; 33 | private Timestamp pubDate; 34 | private String extra; 35 | private Timestamp newdate; 36 | private Timestamp updatetime; 37 | 38 | private List translations; 39 | 40 | @Id 41 | @Column(name = "tid", nullable = false, insertable = false, updatable = false) 42 | @GeneratedValue(strategy = GenerationType.IDENTITY) 43 | public int getTid() { 44 | return tid; 45 | } 46 | 47 | public void setTid(int tid) { 48 | this.tid = tid; 49 | } 50 | 51 | @Basic 52 | @Column(name = "url", length = 767) 53 | public String getUrl() { 54 | return url; 55 | } 56 | 57 | public void setUrl(String url) { 58 | this.url = url; 59 | } 60 | 61 | @Basic 62 | @Column(name = "status_id", length = 255) 63 | public String getStatusId() { 64 | return statusId; 65 | } 66 | 67 | public void setStatusId(String statusId) { 68 | this.statusId = statusId; 69 | } 70 | 71 | @Basic 72 | @Column(name = "content", nullable = false, length = -1) 73 | public String getContent() { 74 | return content; 75 | } 76 | 77 | public void setContent(String content) { 78 | this.content = content; 79 | } 80 | 81 | @Basic 82 | @Column(name = "media", nullable = false, length = -1) 83 | public String getMedia() { 84 | return media; 85 | } 86 | 87 | public void setMedia(String media) { 88 | this.media = media; 89 | } 90 | 91 | @Basic 92 | @Column(name = "published", nullable = false) 93 | public boolean isPublished() { 94 | return published; 95 | } 96 | 97 | public void setPublished(boolean published) { 98 | this.published = published; 99 | } 100 | 101 | @Basic 102 | @Column(name = "hided", nullable = false) 103 | public boolean isHided() { 104 | return hided; 105 | } 106 | 107 | public void setHided(boolean hided) { 108 | this.hided = hided; 109 | } 110 | 111 | @Basic 112 | @Column(name = "comment", nullable = false, length = 1023) 113 | public String getComment() { 114 | return comment; 115 | } 116 | 117 | public void setComment(String comment) { 118 | this.comment = comment; 119 | } 120 | 121 | @Basic 122 | @Column(name = "extra") 123 | public String getExtra() { 124 | return extra; 125 | } 126 | 127 | public void setExtra(String extra) { 128 | this.extra = extra; 129 | } 130 | 131 | @Basic 132 | @Column(name = "pub_date") 133 | public Timestamp getPubDate() { 134 | return pubDate; 135 | } 136 | 137 | public void setPubDate(Timestamp pubDate) { 138 | this.pubDate = pubDate; 139 | } 140 | 141 | @Basic 142 | @Column(name = "twitter_uid") 143 | public String getTwitterUid() { 144 | return twitterUid; 145 | } 146 | 147 | public void setTwitterUid(String twitterUid) { 148 | this.twitterUid = twitterUid; 149 | } 150 | 151 | @Basic 152 | @Column(name = "ref_tid") 153 | public Integer getRefTid() { 154 | return refTid; 155 | } 156 | 157 | public void setRefTid(Integer refTid) { 158 | this.refTid = refTid; 159 | } 160 | 161 | @Basic 162 | @Column(name = "newdate", nullable = false, insertable = false, updatable = false) 163 | public Timestamp getNewdate() { 164 | return newdate; 165 | } 166 | 167 | public void setNewdate(Timestamp newdate) { 168 | this.newdate = newdate; 169 | } 170 | 171 | @Basic 172 | @Column(name = "updatetime", nullable = false, insertable = false, updatable = false) 173 | public Timestamp getUpdatetime() { 174 | return updatetime; 175 | } 176 | 177 | public void setUpdatetime(Timestamp updatetime) { 178 | this.updatetime = updatetime; 179 | } 180 | 181 | @Override 182 | public boolean equals(Object o) { 183 | if (this == o) return true; 184 | if (o == null || getClass() != o.getClass()) return false; 185 | EnkanTaskEntity that = (EnkanTaskEntity) o; 186 | return tid == that.tid && 187 | published == that.published && 188 | Objects.equals(statusId, that.statusId) && 189 | Objects.equals(url, that.url) && 190 | Objects.equals(content, that.content) && 191 | Objects.equals(media, that.media) && 192 | Objects.equals(comment, that.comment) && 193 | Objects.equals(hided, that.hided) && 194 | Objects.equals(extra, that.extra) && 195 | Objects.equals(twitterUid, that.twitterUid) && 196 | Objects.equals(refTid, that.refTid) && 197 | Objects.equals(pubDate, that.pubDate) && 198 | Objects.equals(newdate, that.newdate) && 199 | Objects.equals(updatetime, that.updatetime); 200 | } 201 | 202 | @Override 203 | public int hashCode() { 204 | return Objects.hash(tid, statusId, url, content, media, published, comment, newdate, updatetime, hided, pubDate, extra, refTid, twitterUid); 205 | } 206 | 207 | @JsonIgnore 208 | @OneToMany(mappedBy = "task", cascade = CascadeType.ALL, fetch = FetchType.LAZY) 209 | public List getTranslations() { 210 | return translations; 211 | } 212 | 213 | public void setTranslations(List translations) { 214 | this.translations = translations; 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /maid/maid_api.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from flask import Flask, make_response, request 4 | from flask_restx import Api, fields, Namespace, Resource 5 | from traceback import format_exc 6 | 7 | import maid 8 | import twitter_client 9 | import twitter_util 10 | import logging 11 | import coloredlogs 12 | import config 13 | import json 14 | 15 | app = Flask(__name__) 16 | api = Api(app, description='Maid API') 17 | 18 | 19 | def make_request_model(data_fields: dict, label: str): 20 | base_fields = { 21 | 'forwardFrom': fields.String( 22 | required=True, 23 | description='一个字符串，标定是哪个服务调用了此接口', 24 | example='twitkit-app'), 25 | 'timestamp': fields.DateTime( 26 | dt_format='iso8601', 27 | required=True, 28 | description='请求发出时间戳，ISO8601格式', 29 | example='2020-01-29T14:23:23.233+08:00'), 30 | 'taskId': fields.String( 31 | required=True, 32 | description='一个UUID，是一个任务的上下文唯一标识符', 33 | example='123e4567-e89b-12d3-a456-426655440000') 34 | } 35 | data_model = api.model(f'{label}RequestDataModel', data_fields) 36 | 37 | req_fields = base_fields 38 | req_fields['data'] = fields.Nested( 39 | data_model, description='请求数据', required=True) 40 | 41 | return api.model(f'{label}RequestModel', req_fields) 42 | 43 | 44 | def make_response_model(resp_fields: dict, label: str): 45 | resp_fields.update({ 46 | 'code': fields.Integer(description='返回码，0为成功，其他情况另外注明', 47 | required=True), 48 | 'message': fields.String(description='备注消息', required=True), 49 | }) 50 | return api.model(f'{label}ResponseModel', resp_fields) 51 | 52 | 53 | addtask_model = make_request_model({ 54 | 'url': fields.String( 55 | required=True, 56 | description='要加入到任务列表中的推文URL', 57 | example='https://twitter.com/magireco/status/1233776691064868865'), 58 | }, 'addtask') 59 | 60 | gettweet_model = make_request_model({ 61 | 'url': fields.String( 62 | required=True, 63 | description='要请求的推文URL', 64 | example='https://twitter.com/magireco/status/1233776691064868865'), 65 | }, 'gettweet') 66 | 67 | addtask_resp_model = make_response_model({ 68 | 'addedTid': fields.List( 69 | fields.String, 70 | description='已入库的任务的tid列表', 71 | example=[1001, 1002]), 72 | 'rootTid': fields.Integer( 73 | description='请求插入的推文本身的tid', 74 | example=1001), 75 | }, 'addtask') 76 | 77 | gettweet_resp_model = make_response_model({ 78 | 'tweets': fields.Raw( 79 | description='得到的推文'), 80 | 'rootStatusId': fields.Integer( 81 | description='请求插入的推文本身的推特推文ID', 82 | example=1001), 83 | }, 'addtask') 84 | 85 | 86 | def make_response(code=0, message="", **kwargs): 87 | response = {'code': code, 'message': message} 88 | response.update(kwargs) 89 | return response 90 | 91 | 92 | @api.route('/api/maid/addtask') 93 | class AddTask(Resource): 94 | @api.expect(addtask_model, validate=True) 95 | @api.doc("addtaskResponse", model=addtask_resp_model) 96 | def post(self): 97 | request_base_data = request.json 98 | request_data = request_base_data['data'] 99 | task_label = "[{}][{}]".format( 100 | request_base_data['forwardFrom'], request_base_data['taskId']) 101 | 102 | logging.info(f'{task_label} 请求加入任务：{request_data["url"]}') 103 | 104 | try: 105 | tweet = twitter_client.get_tweet_by_url(request_data['url']) 106 | except Exception as e: 107 | logging.warning(f'{task_label} 请求推特API失败') 108 | logging.warning(format_exc()) 109 | return make_response(500, '请求推特API失败') 110 | 111 | if not tweet: 112 | logging.warning(f'{task_label} 格式不正确') 113 | return make_response(400, 'URL格式不正确') 114 | 115 | try: 116 | inserted = maid.bulk_insert( 117 | twitter_util.convert_tweepy_tweet(tweet), 118 | full_ret=True) 119 | except Exception as e: 120 | logging.warning(f'{task_label} 请求Fridge失败') 121 | logging.warning(format_exc()) 122 | return make_response(500, f'请求Fridge失败：{e}') 123 | 124 | if not inserted: 125 | return make_response(500, '已插入列表为空') 126 | 127 | # root推文是最后插入的，所以tid最大 128 | root_tid = max(inserted) 129 | return make_response( 130 | message='OK', 131 | addedTid=list(inserted), 132 | rootTid=root_tid 133 | ) 134 | 135 | 136 | @api.route('/api/maid/gettweet') 137 | class GetTweet(Resource): 138 | @api.expect(gettweet_model, validate=True) 139 | @api.doc("CheckResponse", model=gettweet_resp_model) 140 | def post(self): 141 | request_base_data = request.json 142 | request_data = request_base_data['data'] 143 | task_label = "[{}][{}]".format( 144 | request_base_data['forwardFrom'], request_base_data['taskId']) 145 | 146 | logging.info(f'{task_label} 请求获取推文：{request_data["url"]}') 147 | 148 | try: 149 | tweet = twitter_client.get_tweet_by_url(request_data['url']) 150 | except Exception as e: 151 | logging.warning(f'{task_label} 请求推特API失败') 152 | logging.warning(format_exc()) 153 | return make_response(500, f'请求推特API失败：{e}') 154 | 155 | converted_tweets = twitter_util.convert_tweepy_tweet( 156 | tweet, two_level_format=True) 157 | 158 | root_status_id = converted_tweets[-1]['twitter']['statusId'] 159 | resp_tweets = {} 160 | for t in converted_tweets: 161 | resp_tweets[t['twitter']['statusId']] = t 162 | 163 | return make_response( 164 | message='OK', 165 | tweets=resp_tweets, 166 | rootId=root_status_id 167 | ) 168 | 169 | 170 | @app.after_request 171 | def after_request(response): 172 | if request.path.startswith('/api/maid'): 173 | try: 174 | response_data = json.loads(response.get_data()) 175 | except: 176 | return response 177 | if 'code' not in response_data: 178 | response_data['code'] = response.status_code 179 | status_code = 200 180 | response.set_data(json.dumps(response_data)) 181 | return response 182 | 183 | 184 | if __name__ == '__main__': 185 | coloredlogs.install( 186 | level=logging.DEBUG if config.LOG_DEBUG else logging.INFO) 187 | app.run(debug=True, port=5001) 188 | -------------------------------------------------------------------------------- /maid/twitter_util.py: -------------------------------------------------------------------------------- 1 | from dateutil import tz 2 | from datetime import datetime 3 | from twitter_client import get_tweet_by_id 4 | from tweepy.error import TweepError 5 | from config import PUSH_REPLIES, PUSH_RETWEETS 6 | from html import unescape 7 | 8 | import json 9 | import re 10 | import logging 11 | 12 | 13 | def utc_to_local(dt: datetime): 14 | dt = dt.replace(tzinfo=tz.tzutc()) 15 | dt = dt.astimezone(tz.tzlocal()) 16 | return dt 17 | 18 | 19 | def convert_tweepy_tweet(tweepy_tweet, two_level_format=False): 20 | ret = [] 21 | 22 | def _convert_tweepy_tweet(tweepy_tweet, is_push_filtered=False): 23 | logging.debug(tweepy_tweet) 24 | ref_tweet = None 25 | if hasattr(tweepy_tweet, 'full_text'): 26 | full_text = tweepy_tweet.full_text 27 | elif hasattr(tweepy_tweet, 'extended_tweet'): 28 | full_text = tweepy_tweet.extended_tweet['full_text'] 29 | else: 30 | full_text = tweepy_tweet.text 31 | 32 | full_text = re.sub(r' https:\/\/t.co\/[A-Za-z0-9]{10}$', '', full_text) 33 | full_text = unescape(full_text) 34 | 35 | ref_id = None 36 | 37 | media_urls = [] 38 | entities_list = [] 39 | if hasattr(tweepy_tweet, 'extended_entities'): 40 | entities_list.append(tweepy_tweet.extended_entities) 41 | elif hasattr(tweepy_tweet, 'extended_tweet') and \ 42 | 'extended_entities' in tweepy_tweet.extended_tweet: 43 | entities_list.append( 44 | tweepy_tweet.extended_tweet['extended_entities']) 45 | else: 46 | entities_list.append(tweepy_tweet.entities) 47 | 48 | entities = {} 49 | for e in entities_list: 50 | for k, v in e.items(): 51 | if k not in entities: 52 | entities[k] = v 53 | else: 54 | if len(v) > len(entities[k]): 55 | entities[k] = v 56 | 57 | if 'media' in entities: 58 | for media in entities['media']: 59 | media_urls.append(media['media_url_https']) 60 | 61 | is_retweet = False 62 | is_reply = False 63 | 64 | if hasattr(tweepy_tweet, 'retweeted_status'): 65 | is_retweet = True 66 | if not PUSH_RETWEETS: 67 | is_push_filtered = True 68 | full_text = None 69 | _convert_tweepy_tweet( 70 | tweepy_tweet.retweeted_status, is_push_filtered) 71 | ref_id = tweepy_tweet.retweeted_status.id 72 | media_urls = [] 73 | elif hasattr(tweepy_tweet, 'quoted_status'): 74 | _convert_tweepy_tweet(tweepy_tweet.quoted_status, is_push_filtered) 75 | ref_id = tweepy_tweet.quoted_status.id 76 | elif hasattr(tweepy_tweet, 'in_reply_to_status_id') \ 77 | and tweepy_tweet.in_reply_to_status_id: 78 | is_reply = True 79 | if not PUSH_REPLIES: 80 | is_push_filtered = True 81 | ref_id = tweepy_tweet.in_reply_to_status_id 82 | try: 83 | _convert_tweepy_tweet( 84 | get_tweet_by_id(ref_id), is_push_filtered) 85 | except TweepError as e: 86 | if e.api_code == 144: 87 | ret.append(make_dummy_tweet( 88 | '找不到此推文，可能已删除', two_level_format, ref_id)) 89 | else: 90 | ret.append(make_dummy_tweet( 91 | e.reason, two_level_format, ref_id)) 92 | 93 | user = tweepy_tweet.user 94 | user_avatar_url = user.profile_image_url_https.replace( 95 | '_normal.jpg', '_400x400.jpg') 96 | tweet_url = 'https://twitter.com/' + \ 97 | f'{user.screen_name}/status/{tweepy_tweet.id}' 98 | 99 | entities.update({ 100 | 'is_reply': is_reply, 101 | 'is_retweet': is_retweet, 102 | }) 103 | 104 | fridge_tweet = { 105 | 'url': tweet_url, 106 | 'content': full_text, 107 | 'media': json.dumps(media_urls), 108 | 'pub_date': utc_to_local(tweepy_tweet.created_at).isoformat(), 109 | 'status_id': str(tweepy_tweet.id), 110 | 'user_twitter_uid': str(user.id), 111 | 'user_name': user.screen_name, 112 | 'user_display': user.name, 113 | 'user_avatar': user_avatar_url, 114 | 'extra': json.dumps(entities), 115 | 'ref': str(ref_id) if ref_id else None, 116 | } if not two_level_format else { 117 | 'twitter': { 118 | 'statusId': str(tweepy_tweet.id), 119 | 'url': tweet_url, 120 | 'content': full_text, 121 | 'media': json.dumps(media_urls), 122 | 'refStatusId': str(ref_id) if ref_id else None, 123 | 'twitterUid': str(user.id), 124 | 'pubDate': utc_to_local(tweepy_tweet.created_at).isoformat(), 125 | 'extra': json.dumps(entities) 126 | }, 127 | 'user': { 128 | 'twitterUid': str(user.id), 129 | 'name': user.screen_name, 130 | 'display': user.name, 131 | 'avatar': user_avatar_url 132 | } 133 | } 134 | fridge_tweet['is_push_filtered'] = is_push_filtered 135 | ret.append(fridge_tweet) 136 | _convert_tweepy_tweet(tweepy_tweet) 137 | return ret 138 | 139 | 140 | def batch_convert_tweepy_tweets(tweepy_tweets: list): 141 | ret = [] 142 | for tweet in tweepy_tweets: 143 | converted = convert_tweepy_tweet(tweet) 144 | ret += converted 145 | return ret 146 | 147 | 148 | def make_dummy_tweet(message, two_level_format=False, status_id=0): 149 | dummy_avatar = 'https://www.gravatar.com/avatar/' + \ 150 | '00000000000000000000000000000000?d=mp&f=y' 151 | dummy_tweet = { 152 | 'url': 'n/a', 153 | 'content': message, 154 | 'media': json.dumps([]), 155 | 'pub_date': datetime.now(tz.tzlocal()).isoformat(), 156 | 'status_id': str(status_id), 157 | 'user_twitter_uid': '0', 158 | 'user_name': '**error**', 159 | 'user_display': '错误', 160 | 'user_avatar': dummy_avatar, 161 | 'extra': json.dumps({}), 162 | 'ref': None, 163 | } if not two_level_format else { 164 | 'twitter': { 165 | 'statusId': str(status_id), 166 | 'url': 'n/a', 167 | 'content': message, 168 | 'media': json.dumps([]), 169 | 'refStatusId': None, 170 | 'twitterUid': '0', 171 | 'pubDate': datetime.now(tz.tzlocal()).isoformat(), 172 | 'extra': json.dumps({}), 173 | }, 174 | 'user': { 175 | 'twitterUid': '0', 176 | 'name': '**error**', 177 | 'display': '错误', 178 | 'avatar': dummy_avatar 179 | } 180 | } 181 | dummy_tweet['is_push_filtered'] = True 182 | return dummy_tweet 183 | -------------------------------------------------------------------------------- /fridge/fridge-src/src/main/java/com/enkanrec/twitkitFridge/api/ws/RequestListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Author : Rinka 3 | * Date : 2020/2/8 4 | */ 5 | package com.enkanrec.twitkitFridge.api.ws; 6 | 7 | import com.corundumstudio.socketio.AckRequest; 8 | import com.corundumstudio.socketio.SocketIOClient; 9 | import com.corundumstudio.socketio.listener.DataListener; 10 | import com.enkanrec.twitkitFridge.api.form.BaseJsonWarp; 11 | import com.enkanrec.twitkitFridge.api.response.StandardResponse; 12 | import com.enkanrec.twitkitFridge.api.rest.KVConfigController; 13 | import com.enkanrec.twitkitFridge.api.rest.TaskController; 14 | import com.enkanrec.twitkitFridge.monitor.WebSocketMonitor; 15 | import com.enkanrec.twitkitFridge.util.JsonUtil; 16 | import com.fasterxml.jackson.core.type.TypeReference; 17 | import lombok.extern.slf4j.Slf4j; 18 | import org.slf4j.MDC; 19 | import org.springframework.beans.factory.annotation.Autowired; 20 | import org.springframework.stereotype.Component; 21 | import org.springframework.web.bind.annotation.RequestMapping; 22 | import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl; 23 | 24 | import javax.annotation.PostConstruct; 25 | import java.lang.reflect.Method; 26 | import java.lang.reflect.Parameter; 27 | import java.lang.reflect.Type; 28 | import java.util.Collections; 29 | import java.util.HashMap; 30 | import java.util.Map; 31 | import java.util.UUID; 32 | 33 | /** 34 | * Class : RequestListener 35 | * Usage : 36 | */ 37 | @SuppressWarnings("all") 38 | @Slf4j 39 | @Component 40 | public class RequestListener implements DataListener { 41 | 42 | private static final String LOG_KEY_REQUEST_ID = "requestId"; 43 | 44 | @Autowired 45 | private KVConfigController kvConfigController; 46 | @Autowired 47 | private TaskController taskController; 48 | 49 | @Autowired 50 | private WebSocketMonitor monitor; 51 | 52 | Map methodAccessMap; 53 | Map formClassAccessMap; 54 | 55 | @PostConstruct 56 | void init() { 57 | Map methodMap = new HashMap<>(); 58 | Map formClassMap = new HashMap<>(); 59 | // method for request 60 | Method[] methods = KVConfigController.class.getMethods(); 61 | for (Method method : methods) { 62 | RequestMapping ano = method.getAnnotation(RequestMapping.class); 63 | if (ano != null) { 64 | String[] rm = ano.value(); 65 | for (String rmUri : rm) { 66 | methodMap.put("kv&" + rmUri, method); 67 | } 68 | } 69 | } 70 | methods = TaskController.class.getMethods(); 71 | for (Method method : methods) { 72 | RequestMapping ano = method.getAnnotation(RequestMapping.class); 73 | if (ano != null) { 74 | String[] rm = ano.value(); 75 | for (String rmUri : rm) { 76 | methodMap.put("task&" + rmUri.substring(1), method); 77 | } 78 | } 79 | } 80 | // web form 81 | for (Method chosenMethod : methodMap.values()) { 82 | Parameter[] ps = chosenMethod.getParameters(); 83 | for (Parameter p : ps) { 84 | Type formType = p.getParameterizedType(); 85 | if (formType.getClass().equals(ParameterizedTypeImpl.class)) { 86 | Class formClazz = ((ParameterizedTypeImpl) formType).getRawType(); 87 | formClassMap.put(chosenMethod, formType); 88 | } else { 89 | log.warn("ignore method with RAW USE of BaseJsonWarp: " + chosenMethod.toString()); 90 | } 91 | } 92 | } 93 | // unmodify the mapping 94 | this.formClassAccessMap = Collections.unmodifiableMap(formClassMap); 95 | this.methodAccessMap = Collections.unmodifiableMap(methodMap); 96 | } 97 | 98 | @Override 99 | public void onData(SocketIOClient client, String data, AckRequest ackSender) throws Exception { 100 | try { 101 | long beginTs = System.currentTimeMillis(); 102 | Map baseForm = JsonUtil.parse(data, Map.class); 103 | String useController = (String) baseForm.get("of"); 104 | String useMethod = (String) baseForm.get("command"); 105 | String requestId = "WS_" + UUID.randomUUID().toString(); 106 | MDC.put(LOG_KEY_REQUEST_ID, requestId); 107 | log.info(String.format("Client request[RequestID:%s SessionID:%s]: Controller:%s UriMethod:%s", requestId, client.getSessionId(), useController, useMethod)); 108 | Object chosenController; 109 | switch (useController) { 110 | case "kv": 111 | chosenController = this.kvConfigController; 112 | break; 113 | case "task": 114 | chosenController = this.taskController; 115 | break; 116 | default: 117 | String hint = "unsupported controller in `of`: " + useController; 118 | log.warn(hint); 119 | client.sendEvent(FridgeWSServer.RESPONSE_EVT, StandardResponse.exception(hint)); 120 | return; 121 | } 122 | String methodKey = useController + "&" + useMethod; 123 | Method chosenMethod = this.methodAccessMap.get(methodKey); 124 | if (chosenMethod != null) { 125 | Type chosenForm = this.formClassAccessMap.get(chosenMethod); 126 | ParameterizedTypeImpl formType = (ParameterizedTypeImpl) chosenForm; 127 | BaseJsonWarp dataWarp = JsonUtil.Mapper.readValue(data, new TypeReference() { 128 | @Override 129 | public Type getType() { 130 | return formType; 131 | } 132 | }); 133 | try { 134 | Object resp = chosenMethod.invoke(chosenController, dataWarp); 135 | String handlerName = chosenController.getClass().getSimpleName() + "." + chosenMethod.getName(); 136 | log.info(String.format("ws request handled [%s], prepare to emit.", handlerName)); 137 | client.sendEvent(FridgeWSServer.RESPONSE_EVT, resp); 138 | String code = String.valueOf(((StandardResponse) resp).getCode()); 139 | this.monitor.responseTimeInMs.labels(handlerName, "/" + useMethod, code) 140 | .observe(System.currentTimeMillis() - beginTs); 141 | } catch (Exception invEx) { 142 | String hint = "exception at invoke: " + invEx.getMessage(); 143 | log.warn(hint); 144 | client.sendEvent(FridgeWSServer.RESPONSE_EVT, StandardResponse.exception(hint)); 145 | this.monitor.exceptionCounter.inc(); 146 | } 147 | } else { 148 | String hint = "unsupported method: " + methodKey; 149 | log.warn(hint); 150 | client.sendEvent(FridgeWSServer.RESPONSE_EVT, StandardResponse.exception(hint)); 151 | return; 152 | } 153 | } finally { 154 | MDC.remove(LOG_KEY_REQUEST_ID); 155 | } 156 | } 157 | } 158 | --------------------------------------------------------------------------------